diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ce655b38..41a937c4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,9 @@ - A definition beside other tags in a sidecar entry, categorical or value, such as `(Definition/MyDef, (Red)), Label/#` (`DEFINITION_INVALID`). An entry that holds only definitions is still valid. - A `HED` key below the second level of an entry that also has a valid one, such as in `Levels` (`SIDECAR_INVALID`). - Units in a `Def` value checked against the definition's placeholder tag with its units: `Def/Freq/3 kHz` with `(Definition/Freq/#, (Frequency/# Hz))` gives `Frequency/3 kHz Hz` (`UNITS_INVALID`). -- **A sidecar string that references `{HED}` is no longer checked for repeated tags and group rules in the sidecar**, since what the `HED` column contributes is known only per row (specification 3.2.9.3 rule 10). `(Red, {HED}), (Red)` was reported as `TAG_EXPRESSION_REPEATED` although a row with `HED=Blue` is valid; such errors are now found when each row is assembled. +- **The sidecar check of curly-brace references fills one referenced column at a time** (specification 3.2.9.3 rule 10, as narrowed for 4.0.0): each annotation of each referenced column is substituted with the other references removed, and the string is checked once with all references removed. Every combination was checked before, which cost the product of the columns' annotation counts (6 references of 10 annotations each: about 1.8 million substitutions for one string; now 61). A substitution with a removal is checked for `TAG_EXPRESSION_REPEATED` only, so the other string checks run in the sidecar only for a string with one referenced column. An error that needs two referenced columns filled, such as the repeat in `({color}, {shade})` when both give `Red`, is no longer reported for the sidecar alone; it is reported when a row with those values is validated. +- **A removed curly-brace reference is a substitution for the repeated-expression check** (specification 3.2.9.3 rules 3 and 10): any referenced column can be `n/a` or blank in a row, so a sidecar that validated clean can now report `TAG_EXPRESSION_REPEATED`. `(Red, {HED}), (Red)`, `(Red, {color}), (Red)` and `(Red, {rt}), (Red)` are errors whatever the annotations of `color` or `rt`, since such a row assembles to `(Red), (Red)`. For `{HED}`, whose content is known only per row, the removal is the only substitution, and a string that references it is not checked for group rules or anything else in the sidecar; those errors are found when each row is assembled. A combination with a removal reports only repeats, and a repeat found by several combinations is reported once. +- **An empty line or a line with the wrong number of cells in a `.tsv` file is `TABULAR_INVALID`** (specification 3.2.10, Appendix B reasons e and f): every such line is reported by its file line (`ec_row`, the first line is 1), with the cell count found and expected for (f), and the rows of the file are then not validated. Before, an empty or blanks-only line was skipped, a short row was padded with `n/a`, a long first data row lost its extra cells, and a long later row raised `HedFileError`; such a file now loads, records the lines in the new `BaseInput.line_problems`, and refuses `assemble` with `HedFileError` `INVALID_FILE_FORMAT`. An empty first line is a blank column name, a line of only blanks in a one-column table is a value (padded, reason d, if the column is annotated), and lines end at LF or CRLF only: a lone CR stays inside its line. A clean file's row numbers are therefore always its file lines. Excel, DataFrame input and a text file read with `has_column_names=False` are not checked. ([#1475](https://github.com/hed-standard/hed-python/issues/1475)) - **`annotation_util.df_to_hed` with `description_tag=True` no longer adds a `Description` tag beside definitions**, which made the sidecar invalid; the description stays in the `Levels` or `Description` text. - In a file without an `onset` column, the `char_index` of an assembled-row issue now counts in the assembled string with `", "` between cells, as for files with onsets (one more per earlier cell than before). diff --git a/hed/errors/error_messages.py b/hed/errors/error_messages.py index ebd83a003..636a6b89c 100644 --- a/hed/errors/error_messages.py +++ b/hed/errors/error_messages.py @@ -267,6 +267,16 @@ def val_error_tabular_column_name_blank(column_number): return f"Column {column_number} of the first line (the column names) of the tabular file has a blank name." +@hed_error(ValidationErrors.TABULAR_LINE_EMPTY, actual_code=ValidationErrors.TABULAR_INVALID) +def val_error_tabular_line_empty(line_number): + return f"Line {line_number} of the tabular file is empty (no characters before its line terminator)." + + +@hed_error(ValidationErrors.TABULAR_LINE_CELL_COUNT, actual_code=ValidationErrors.TABULAR_INVALID) +def val_error_tabular_line_cell_count(line_number, cells, expected): + return f"Line {line_number} of the tabular file has {cells} tab-separated cells, but the first line has {expected}." + + @hed_error(ValidationErrors.TABULAR_VALUE_PADDED, actual_code=ValidationErrors.TABULAR_INVALID) def val_error_tabular_value_padded(value, column_name): return ( diff --git a/hed/errors/error_types.py b/hed/errors/error_types.py index be7a6ed9b..68703a1c1 100644 --- a/hed/errors/error_types.py +++ b/hed/errors/error_types.py @@ -95,6 +95,9 @@ class ValidationErrors: TABULAR_COLUMN_NAME_BLANK = "TABULAR_COLUMN_NAME_BLANK" # Reported as TABULAR_INVALID (reason d): a value with leading or trailing whitespace in an annotated column. TABULAR_VALUE_PADDED = "TABULAR_VALUE_PADDED" + # Reported as TABULAR_INVALID (reasons e and f): a text file's empty line, or a line with the wrong cell count. + TABULAR_LINE_EMPTY = "TABULAR_LINE_EMPTY" + TABULAR_LINE_CELL_COUNT = "TABULAR_LINE_CELL_COUNT" # end internal codes # Still being worked on below this line diff --git a/hed/models/base_input.py b/hed/models/base_input.py index 2c509a851..ab63a0201 100644 --- a/hed/models/base_input.py +++ b/hed/models/base_input.py @@ -18,6 +18,39 @@ from hed.models.df_util import _handle_curly_braces_refs, filter_series_by_onset +def find_line_problems(text) -> tuple[list[str], list[tuple[int, int | None, int]]]: + """Return the cells of a text table's first line and the lines that break the line rules. + + The rules are TABULAR_INVALID (e), a line other than the first that is empty, and (f), a line other than the + first whose number of tab-separated cells differs from the first line's (specification 3.2.10). Lines end at + LF or CRLF; a lone CR is part of its line. A single line ending at the end of the text is not an empty line, + but two are. An empty first line is one blank cell, so it is a blank column name, not (e). A line of only + blanks has one cell. A byte order mark at the start is not part of the first line. + + Parameters: + text (str): The whole text of the file. + + Returns: + tuple: The first line's cells (list[str]) and the problems, each (file line counting the first line as 1, + cells found or None for an empty line, cells expected), in line order. + """ + # Only a CR directly before an LF is part of a line terminator; any other CR is cell content. + text = text.removeprefix("\ufeff").replace("\r\n", "\n") + lines = text.split("\n") + if lines and lines[-1] == "": + lines.pop() # The terminator of the last line, not an empty line. + if not lines: + return [], [] + header = lines[0].split("\t") + problems = [] + for number, line in enumerate(lines[1:], 2): + if line == "": + problems.append((number, None, len(header))) + elif (cells := line.count("\t") + 1) != len(header): + problems.append((number, cells, len(header))) + return header, problems + + class BaseInput: """Superclass representing a basic columnar file.""" @@ -90,6 +123,7 @@ def __init__( self._padded_columns = [] self._blank_columns = [] self._duplicate_columns = [] + self._line_problems = [] input_type = file_type if isinstance(file, str): @@ -205,6 +239,18 @@ def blank_columns(self) -> list[int]: """ return self._blank_columns + @property + def line_problems(self) -> list[tuple[int, int | None, int]]: + """The lines of a text file that are empty or have the wrong number of cells (TABULAR_INVALID e and f). + + Each is (file line counting the first line as 1, cells found or None for an empty line, cells expected), + in line order; see :func:`find_line_problems`. ``validate`` reports them and checks no row. When pandas + cannot read such a file, or reads a header other than its first line, the dataframe has the first line's + columns and no rows. Always empty for Excel and DataFrame input and for a text file read without column + names, to which these rules do not apply. + """ + return self._line_problems + @property def loaded_workbook(self): """The underlying loaded workbooks.""" @@ -539,53 +585,18 @@ def _set_column_names(self, names, header_cells=None): self._dataframe.columns = self._unique_names(names) @staticmethod - def _read_header_cells(file): - """Return the cells of a text file's header record and the file to hand to pandas. - - The header is read by pandas as a data row (``header=None``), so its cells are parsed exactly as the - dataframe's header is (blank lines skipped, BOM removed, a double quote kept as an ordinary character) - but are not yet suffixed. A file-like object is returned to where it was; one that cannot seek or - report its position is read into memory first. + def _read_text(file) -> str: + """Return the whole text of a text file given as a path or a binary or text file-like object. - Parameters: - file (str, os.PathLike or file-like): Path to the text file or a binary or text file-like object. - - Returns: - tuple: The header cells (list[str]) and the file or in-memory copy for pandas. + Bytes are decoded as UTF-8, as pandas reads them; a byte order mark is kept for ``find_line_problems`` + and pandas to remove. """ if isinstance(file, (str, os.PathLike)): - return BaseInput._first_record_cells(file), file - position = None - if hasattr(file, "seekable") and file.seekable(): - try: - position = file.tell() - except OSError: - pass # A text file advanced with next() cannot tell its position, but can still be read. - if position is None: + with open(file, "rb") as fp: + contents = fp.read() + else: contents = file.read() - file = io.BytesIO(contents) if isinstance(contents, bytes) else io.StringIO(contents) - position = 0 - cells = BaseInput._first_record_cells(file) - file.seek(position) - return cells, file - - @staticmethod - def _first_record_cells(file) -> list[str]: - """Return the cells of the first record pandas reads from file, or [] if there is none.""" - try: - first = pd.read_csv( - file, - delimiter="\t", - header=None, - nrows=1, - dtype=str, - keep_default_na=False, - skip_blank_lines=True, - quoting=csv.QUOTE_NONE, - ) - except pd.errors.EmptyDataError: - return [] - return list(first.iloc[0]) if len(first) else [] + return contents.decode("utf-8") if isinstance(contents, bytes) else contents def assemble(self, mapper=None, skip_curly_braces=False) -> pd.DataFrame: """Assembles the HED strings. @@ -605,6 +616,8 @@ def assemble(self, mapper=None, skip_curly_braces=False) -> pd.DataFrame: ``blank_columns``, ``duplicate_columns``). Such a table cannot be assembled correctly (a padded ``HED`` column, for example, would be ignored), so it is refused rather than assembled wrongly. ``validate`` reports these names as TABULAR_INVALID and never reaches assembly for them. + HedFileError: INVALID_FILE_FORMAT if a line of a text file is empty or has the wrong number of cells + (``line_problems``): the dataframe does not hold the file's rows as written. """ self._check_names_for_assembly() if mapper is None: @@ -619,7 +632,15 @@ def assemble(self, mapper=None, skip_curly_braces=False) -> pd.DataFrame: return _handle_curly_braces_refs(all_columns, refs, column_names) def _check_names_for_assembly(self): - """Raise BAD_COLUMN_NAMES if a column name is padded, blank or repeated; see ``assemble``.""" + """Raise if a column name is padded, blank or repeated or a line is empty or ragged; see ``assemble``.""" + if self._line_problems: + lines = [line for line, _, _ in self._line_problems] + raise HedFileError( + HedExceptions.INVALID_FILE_FORMAT, + f"Cannot assemble rows: the file has an empty line or a wrong number of cells at line " + f"{', '.join(str(line) for line in lines)}. Validate the file for the TABULAR_INVALID details.", + self.name, + ) if not (self._padded_columns or self._blank_columns or self._duplicate_columns): return problems = [] @@ -830,46 +851,78 @@ def _load_text_file(self, file, pandas_header): - Handles empty files by creating an empty DataFrame - Converts all data to string type for consistency - Replaces NaN values with "n/a" for consistent handling - - Skips blank lines during parsing + - With column names, keeps every line: an empty line or a line of only blanks is a row, so a row's + index plus 2 is its file line. Without column names, blank lines are skipped and short rows are + padded, as before; the line rules below apply only to a file with column names - Treats only an empty cell and "n/a" as missing (na_values ("", "n/a"), keep_default_na=False), so "NaN", "NULL" and the other pandas default NA strings are values - Handles pandas.errors.EmptyDataError for files with no data - - With a header, the header record is read first so that blank and repeated names are known before - pandas renames them (``blank_columns``, ``duplicate_columns``); the column names are the header - cells exactly as read + - The text is read once and split into lines first (``find_line_problems``): the column names are the + cells of the first line exactly as written, so blank and repeated names are known before pandas + renames them (``blank_columns``, ``duplicate_columns``), and an empty line or a line with the wrong + number of cells is recorded in ``line_problems`` (TABULAR_INVALID e and f) instead of being + skipped, padded or truncated. The file is still loaded: if pandas cannot read it, or reads a + different number of columns than the first line has, the dataframe has the first line's columns + and no rows - Double quotes are ordinary characters, as in hed-javascript and the BIDS validator: a line is split on every tab, a quoted cell keeps its quotes (a quote in a HED string is CHARACTER_INVALID), and a - quoted cell holding a tab is two cells. A row with more cells than the header loses the extra - cells (pandas warns); the BIDS validator reports such a row + quoted cell holding a tab is two cells """ - if isinstance(file, str) and os.path.exists(file) and os.path.getsize(file) == 0: + try: + text = self._read_text(file) + except Exception as e: + raise HedFileError( + HedExceptions.INVALID_FILE_FORMAT, f"Failed to load text file: {str(e)}", self.name + ) from e + if not text.removeprefix("\ufeff"): self._dataframe = pd.DataFrame() # Handle empty file return + # The line rules are rules of a tabular file, whose first line holds the column names (specification + # 3.2.10); a text file read without column names is loaded as before, blank lines skipped. + has_names = pandas_header is not None + header_cells = None + if has_names: + header_cells, self._line_problems = find_line_problems(text) + # pandas reads the same lines as find_line_problems: it ends a line at LF only, so a lone CR stays + # in its cell instead of starting a new row. + text = text.replace("\r\n", "\n") try: - header_cells = None - if pandas_header is not None: - header_cells, file = self._read_header_cells(file) - self._dataframe = pd.read_csv( - file, - delimiter="\t", - header=pandas_header, - skip_blank_lines=True, - dtype=str, - keep_default_na=False, - na_values=("", "n/a"), - quoting=csv.QUOTE_NONE, - index_col=False, - ) - # Only an empty cell and n/a are missing (BIDS). Every other cell text, including nan, NaN, NA, - # None and NULL, which pandas would otherwise read as missing, is a value and is validated as one, - # as it is when a DataFrame is passed in. - self._dataframe = self._dataframe.fillna("n/a") - if header_cells is not None: - self._set_column_names(self._dataframe.columns, header_cells) + with warnings.catch_warnings(): + # pandas warns when it drops the extra cells of a long row; that row is in line_problems. + warnings.simplefilter("ignore", pd.errors.ParserWarning) + self._dataframe = pd.read_csv( + io.StringIO(text), + delimiter="\t", + header=pandas_header, + skip_blank_lines=not has_names, + lineterminator="\n" if has_names else None, + dtype=str, + keep_default_na=False, + na_values=("", "n/a"), + quoting=csv.QUOTE_NONE, + index_col=False, + ) except pd.errors.EmptyDataError: - self._dataframe = pd.DataFrame() # Handle case where file has no data + if not has_names: + self._dataframe = pd.DataFrame() # Handle case where file has no data + return + # pandas finds no columns when the first line is empty (the text is not, see above). That line is + # the column names, one blank name, so the names are still set and reported below, whether or not + # other lines follow. + self._dataframe = None except Exception as e: - raise HedFileError( - HedExceptions.INVALID_FILE_FORMAT, f"Failed to load text file: {str(e)}", self.name - ) from e + if not self._line_problems: + raise HedFileError( + HedExceptions.INVALID_FILE_FORMAT, f"Failed to load text file: {str(e)}", self.name + ) from e + self._dataframe = None + if has_names and (self._dataframe is None or len(self._dataframe.columns) != len(header_cells)): + # The rows are not the file's rows as written; validate reports line_problems and checks no row. + self._dataframe = pd.DataFrame(columns=range(len(header_cells)), dtype=str) + # Only an empty cell and n/a are missing (BIDS). Every other cell text, including nan, NaN, NA, + # None and NULL, which pandas would otherwise read as missing, is a value and is validated as one, + # as it is when a DataFrame is passed in. + self._dataframe = self._dataframe.fillna("n/a") + if has_names: + self._set_column_names(self._dataframe.columns, header_cells) diff --git a/hed/tools/bids/bids_file_group.py b/hed/tools/bids/bids_file_group.py index c7eae0829..177354150 100644 --- a/hed/tools/bids/bids_file_group.py +++ b/hed/tools/bids/bids_file_group.py @@ -211,8 +211,8 @@ def validate_datafiles(self, hed_schema, extra_def_dicts=None, error_handler=Non had_contents = data_obj.contents # A file whose merged sidecar has errors is skipped, unless its header has a padded, blank or - # repeated column name: validate then reports only TABULAR_INVALID (stage 0) and checks no row - # against the sidecar. A file that cannot even be loaded is skipped as it was before loading. + # repeated column name or it has an empty or ragged line: validate then reports only + # TABULAR_INVALID (stage 0) and checks no row against the sidecar. A file that cannot even be loaded is skipped as it was before loading. try: data_obj.set_contents(overwrite=False) except HedFileError: @@ -221,8 +221,13 @@ def validate_datafiles(self, hed_schema, extra_def_dicts=None, error_handler=Non logger.debug(f"Skipping {os.path.basename(data_obj.file_path)}: its merged sidecar has errors") continue contents = data_obj.contents - bad_names = contents.padded_columns or contents.blank_columns or contents.duplicate_columns - if sidecar_has_errors and not bad_names: + stage_zero = ( + contents.padded_columns + or contents.blank_columns + or contents.duplicate_columns + or contents.line_problems + ) + if sidecar_has_errors and not stage_zero: logger.debug(f"Skipping {os.path.basename(data_obj.file_path)}: its merged sidecar has errors") if not had_contents: data_obj.clear_contents() diff --git a/hed/validator/sidecar_validator.py b/hed/validator/sidecar_validator.py index ae6bd195b..a3858e67b 100644 --- a/hed/validator/sidecar_validator.py +++ b/hed/validator/sidecar_validator.py @@ -3,7 +3,6 @@ from __future__ import annotations import copy -import itertools from hed.errors import ColumnErrors, DefinitionErrors, ErrorContext, ErrorHandler, SidecarErrors, ValidationErrors from hed.errors.error_reporter import check_for_any_errors, sort_issues @@ -14,6 +13,7 @@ from hed.models.hed_string import HedString from hed.models.model_constants import COLUMN_REF_PATTERN, DefTagNames, TopTagReturnType from hed.validator.reserved_checker import ReservedChecker +from hed.validator.util.dup_util import DuplicateChecker # todo: Add/improve validation for definitions being in known columns(right now it just assumes they aren't) @@ -31,6 +31,7 @@ def __init__(self, hed_schema): hed_schema (HedSchema): HED schema object to use for validation. """ self._schema = hed_schema + self._duplicate_checker = DuplicateChecker() def validate(self, sidecar, extra_def_dicts=None, name=None, error_handler=None) -> list[dict]: """Validate the input data using the schema @@ -112,29 +113,11 @@ def _validate_sidecar(self, sidecar, extra_def_dicts, error_handler) -> list[dic issues += new_issues error_handler.pop_error_context() # Hed String - # Only do full string checks on full columns, not partial ref columns. Every combination of the - # referenced columns' annotations is checked, whether or not a file uses it (specification - # 3.2.9.3 rule 10). What {HED} contributes is known only per row, so a string that references it - # gets its full-string checks when each row is assembled. - refs = COLUMN_REF_PATTERN.findall(hed_string) - if not is_ref_column and "HED" not in refs: - refs_strings = {data.column_name: data.get_hed_strings() for data in sidecar} - for combination in itertools.product(*[refs_strings[key] for key in refs]): - new_issues = [] - ref_dict = dict(zip(refs, combination, strict=False)) - modified_string = hed_string - for ref in refs: - modified_string = df_util.replace_ref(modified_string, f"{{{ref}}}", ref_dict[ref]) - hed_string_obj = HedString(modified_string, hed_schema=self._schema, def_dict=sidecar_def_dict) - - # A sidecar string is a fragment of a row: a tag it lacks may come from another - # column, so the required-tag check waits for assembly. A unique tag repeated in - # the fragment is repeated in the row, so that check runs here. - error_handler.push_error_context(ErrorContext.HED_STRING, hed_string_obj) - new_issues += hed_validator.run_full_string_checks(hed_string_obj, required_tags=False) - error_handler.add_context_and_filter(new_issues) - issues += new_issues - error_handler.pop_error_context() # Hed string + # Only do full string checks on full columns, not partial ref columns. + if not is_ref_column: + issues += self._validate_ref_combinations( + hed_string, sidecar, sidecar_def_dict, hed_validator, error_handler + ) if len(hed_strings) > 1: error_handler.pop_error_context() # Category key @@ -144,6 +127,78 @@ def _validate_sidecar(self, sidecar, extra_def_dicts, error_handler) -> list[dic return issues + def _validate_ref_combinations(self, hed_string, sidecar, def_dict, hed_validator, error_handler) -> list[dict]: + """Check the required substitutions of the curly-brace references in one sidecar string. + + Parameters: + hed_string (str): The sidecar string, with its curly-brace references. + sidecar (Sidecar): The sidecar the string belongs to. + def_dict (DefinitionDict): The definitions of the sidecar. + hed_validator (HedValidator): The validator for the full-string checks. + error_handler (ErrorHandler): The error handler, with the sidecar column and key context pushed. + + Returns: + list[dict]: The issues found in the substituted strings. + + Notes: + - The annotations of each referenced column are substituted one column at a time, whether or not + a file uses them, with every other reference removed; the string is also checked with all of its + references removed (specification 3.2.9.3 rule 10). A value column's template is substituted + with its # unfilled. Combinations that fill several columns are not checked here: the number of + substitutions is one more than the number of annotations, not their product, and an error that + needs two columns filled is found when a row is assembled. + - Any referenced column can be n/a or blank in a row, which removes the reference (rule 3); for + {HED}, whose content is known only per row, the removal is the only substitution. A + substitution with a removal is checked for TAG_EXPRESSION_REPEATED only; other checks may rely + on a column being filled and wait for the assembled row. So the full string checks run only + for a string with one referenced column. + - The removal is made by df_util.replace_ref with n/a, as in assembly. + - A repeat found by several substitutions is reported once. + """ + # replace_ref replaces every occurrence of a reference at once, so each name is one substitution. + refs = list(dict.fromkeys(COLUMN_REF_PATTERN.findall(hed_string))) + refs_strings = {data.column_name: list(data.get_hed_strings()) for data in sidecar} + filled = [refs_strings.get(ref, []) if ref != "HED" else [] for ref in refs] + # One column filled at a time; None stands for the removal of a reference. The substitutions without + # a removal (one referenced column) go first, so a repeat they find is reported with their string. + combinations = [ + tuple(value if position == filled_position else None for position in range(len(refs))) + for filled_position, values in enumerate(filled) + for value in values + ] + combinations.append((None,) * len(refs)) + + issues = [] + reported_repeats = set() + for combination in combinations: + modified_string = hed_string + for ref, value in zip(refs, combination, strict=True): + modified_string = df_util.replace_ref(modified_string, f"{{{ref}}}", "n/a" if value is None else value) + hed_string_obj = HedString(modified_string, hed_schema=self._schema, def_dict=def_dict) + + if None in combination: + new_issues = self._duplicate_checker.check_for_duplicates(hed_string_obj) + else: + # A sidecar string is a fragment of a row: a tag it lacks may come from another + # column, so the required-tag check waits for assembly. A unique tag repeated in + # the fragment is repeated in the row, so that check runs here. + new_issues = hed_validator.run_full_string_checks(hed_string_obj, required_tags=False) + + kept_issues = [] + for issue in new_issues: + if issue["code"] == ValidationErrors.TAG_EXPRESSION_REPEATED: + repeat_key = issue["message"] + if repeat_key in reported_repeats: + continue + reported_repeats.add(repeat_key) + kept_issues.append(issue) + + error_handler.push_error_context(ErrorContext.HED_STRING, hed_string_obj) + error_handler.add_context_and_filter(kept_issues) + issues += kept_issues + error_handler.pop_error_context() # Hed string + return issues + def validate_structure(self, sidecar, error_handler) -> list[dict]: """Validate the raw structure of this sidecar. diff --git a/hed/validator/spreadsheet_validator.py b/hed/validator/spreadsheet_validator.py index e11eebf9d..bad21604a 100644 --- a/hed/validator/spreadsheet_validator.py +++ b/hed/validator/spreadsheet_validator.py @@ -34,11 +34,13 @@ class SpreadsheetValidator: The stages run in order and each stops the run when it finds an error (never on a warning alone): - 0. **Column names**: a table with a padded, blank or repeated column name gets one TABULAR_INVALID per - such name and nothing else: neither the sidecar nor any row is checked or assembled (specification + 0. **Column names and lines**: a table with a padded, blank or repeated column name gets one TABULAR_INVALID + per such name, and a text file with an empty line or a line with the wrong number of cells one per such + line, and nothing else: neither the sidecar nor any row is checked or assembled (specification 3.2.10). Names are compared exactly. A :class:`~hed.models.base_input.BaseInput` supplies the names it - recorded from its header (``padded_columns``, ``blank_columns``, ``duplicate_columns``); for any other - column source they are found in ``column_names()``. + recorded from its header (``padded_columns``, ``blank_columns``, ``duplicate_columns``) and the lines + (``line_problems``); for any other column source the names are found in ``column_names()`` and there + are no lines to check. 1. **Sidecar**: every template and categorical string of the sidecar (``SidecarValidator``). A sidecar string is a fragment of a row, so the required-tag check waits for assembly. 2. **Column values**: the column structure (mapper issues, sidecar references to columns the table @@ -102,9 +104,10 @@ def validate( ) -> list[dict]: """Run the stages on a column source, stopping at the first stage that finds an error. - A table with a padded, blank or repeated column name is reported by stage 0 alone: one TABULAR_INVALID - per padded name, then per blank name, then per repeated name, with the FILE_NAME context when ``name`` - is given (see ``validate_column_names``). + A table with a padded, blank or repeated column name, or a text file with an empty line or a line with + the wrong number of cells, is reported by stage 0 alone: one TABULAR_INVALID per padded name, then per + blank name, then per repeated name, then per such line, with the FILE_NAME context when ``name`` is given + (see ``validate_column_names`` and ``validate_lines``). Parameters: source (ColumnSource): The table to validate. A BaseInput (TabularInput, SpreadsheetInput) or any @@ -166,16 +169,20 @@ def validate( # Strings and lists of strings need the schema to become definitions. extra_def_dicts = DefinitionDict(extra_def_dicts, hed_schema=self._schema) - # Stage 0: a padded, blank or repeated column name stops the run before the sidecar. + # Stage 0: a padded, blank or repeated column name, or an empty or ragged line, stops the run before the + # sidecar. if isinstance(source, BaseInput): padded, blank, repeated = source.padded_columns, source.blank_columns, source.duplicate_columns + line_problems = source.line_problems else: padded, blank, repeated = check_column_names(source.column_names()) - if padded or blank or repeated: + line_problems = [] + if padded or blank or repeated or line_problems: if name: error_handler.push_error_context(ErrorContext.FILE_NAME, name) try: - return self.validate_column_names(padded, blank, repeated, error_handler) + issues = self.validate_column_names(padded, blank, repeated, error_handler) + return issues + self.validate_lines(line_problems, error_handler) finally: if name: error_handler.pop_error_context() @@ -233,6 +240,33 @@ def validate_column_names(padded, blank, repeated, error_handler) -> list[dict]: error_handler.pop_error_context() return issues + @staticmethod + def validate_lines(line_problems, error_handler) -> list[dict]: + """Stage 0: report the empty lines (e) and the lines with the wrong number of cells (f) as TABULAR_INVALID. + + Each issue carries the ROW context set to the file line, counting the first line as 1, and is kept in + line order (not sorted). + + Parameters: + line_problems (list): (file line, cells found or None for an empty line, cells expected) for each + line, as :attr:`~hed.models.base_input.BaseInput.line_problems` holds them. + error_handler (ErrorHandler): Holds context. + + Returns: + list[dict]: The issues found. + """ + issues = [] + for line_number, cells, expected in line_problems: + error_handler.push_error_context(ErrorContext.ROW, line_number) + if cells is None: + issues += error_handler.format_error_with_context(ValidationErrors.TABULAR_LINE_EMPTY, line_number) + else: + issues += error_handler.format_error_with_context( + ValidationErrors.TABULAR_LINE_CELL_COUNT, line_number, cells, expected + ) + error_handler.pop_error_context() + return issues + def _validate_source(self, source, sidecar, extra_def_dicts, error_handler, row_offset, issues) -> list[dict]: """Stages 2 to 4, run with the FILE_NAME context (if any) already pushed.""" if sidecar is not None: diff --git a/spec_tests/hed-tests b/spec_tests/hed-tests index babc44406..0a07fe008 160000 --- a/spec_tests/hed-tests +++ b/spec_tests/hed-tests @@ -1 +1 @@ -Subproject commit babc44406dfede6590de63bba5bb48792f97c817 +Subproject commit 0a07fe008b1de7c8e645b895168b1bf39d063005 diff --git a/tests/data/validator_tests/bids_events_invalid_columns.tsv b/tests/data/validator_tests/bids_events_invalid_columns.tsv index 335df8425..1399ca2a6 100644 --- a/tests/data/validator_tests/bids_events_invalid_columns.tsv +++ b/tests/data/validator_tests/bids_events_invalid_columns.tsv @@ -1,2 +1,2 @@ onset duration hed_keys next_column -1 Definition/Illegal Definition/Illegal2 +1 Definition/Illegal Definition/Illegal2 diff --git a/tests/models/test_base_input.py b/tests/models/test_base_input.py index f017c331f..94b9baff1 100644 --- a/tests/models/test_base_input.py +++ b/tests/models/test_base_input.py @@ -11,7 +11,7 @@ from hed import schema from hed.errors.error_types import ErrorContext, ValidationErrors from hed.errors.exceptions import HedFileError -from hed.models.base_input import BaseInput +from hed.models.base_input import BaseInput, find_line_problems from hed.models.column_mapper import ColumnMapper from hed.models.definition_dict import DefinitionDict from hed.models.sidecar import Sidecar @@ -264,11 +264,19 @@ def test_text_stream_advanced_with_next(self): self.assertEqual(input_file.columns, ["trial", "trial.1", "HED"]) self.assertEqual(len(input_file.dataframe), 1) - def test_bom_blank_line_and_crlf(self): - text = "\ufeff\r\nonset\tduration\ttrial\ttrial\r\n1\t2\t3\t4\r\n" + def test_bom_and_crlf(self): + text = "\ufeffonset\tduration\ttrial\ttrial\r\n1\t2\t3\t4\r\n" input_file = BaseInput(io.BytesIO(text.encode("utf-8")), file_type=".tsv") self.assertEqual(input_file.duplicate_columns, ["trial"]) self.assertEqual(input_file.columns, ["onset", "duration", "trial", "trial.1"]) + self.assertEqual(input_file.line_problems, []) + + def test_bom_and_empty_first_line(self): + # An empty first line is one blank column name (TABULAR_INVALID c); it is no longer skipped. + text = "\ufeff\r\nonset\tduration\r\n1\t2\r\n" + input_file = BaseInput(io.BytesIO(text.encode("utf-8")), file_type=".tsv") + self.assertEqual(input_file.blank_columns, [0]) + self.assertEqual(input_file.line_problems, [(2, 2, 1), (3, 2, 1)]) def test_names_are_kept_as_read(self): # Names are compared exactly: "trial " beside "trial" is one padded name, not a repeat. @@ -291,10 +299,11 @@ def test_quoted_header_cells(self): def test_quotes_do_not_join_lines(self): # A quote does not open a cell that spans lines: the header is the first line, split on tabs. text = self._text('"trial\nnote"\ttrial\ttrial', ("a\tb\tc",)) - with self.assertWarns(pd.errors.ParserWarning): # the later lines have more cells than the header - input_file = BaseInput(io.BytesIO(text.encode("utf-8")), file_type=".tsv") + input_file = BaseInput(io.BytesIO(text.encode("utf-8")), file_type=".tsv") self.assertEqual(input_file.columns, ['"trial']) self.assertEqual(input_file.duplicate_columns, []) + # The later lines have more cells than the first line (TABULAR_INVALID f). + self.assertEqual(input_file.line_problems, [(2, 3, 1), (3, 3, 1)]) def test_quoted_data_cells(self): text = 'onset\tnote\tHED\n1\t"a, b"\t"Red"\n2\t"open\tBlue\n' @@ -322,10 +331,10 @@ def test_save_refuses_a_tab_in_a_cell(self): input_file.to_csv() def test_quoted_tab_splits_the_cell(self): - # The extra cell of the row is dropped, as hed-javascript drops it; the BIDS validator reports the row. + # The row has one cell more than the first line (TABULAR_INVALID f), as the BIDS validator reports it. text = 'onset\tnote\tHED\n1\t"a\tb"\tRed\n' - with self.assertWarns(pd.errors.ParserWarning): - input_file = BaseInput(io.StringIO(text), file_type=".tsv") + input_file = BaseInput(io.StringIO(text), file_type=".tsv") + self.assertEqual(input_file.line_problems, [(2, 4, 3)]) self.assertEqual(input_file.dataframe.iloc[0].tolist(), ["1", '"a', 'b"']) def test_order_and_count(self): @@ -485,3 +494,110 @@ def test_no_column_names(self): input_file = BaseInput(io.StringIO(" a\t\tb \n1\t2\t3\n"), file_type=".tsv", has_column_names=False) self.assertEqual(input_file.padded_columns, []) self.assertEqual(input_file.blank_columns, []) + + +class TestLineProblems(unittest.TestCase): + """line_problems: TABULAR_INVALID (e) empty line and (f) wrong cell count of a text tabular file.""" + + # (text, header cells, problems as (file line, cells found or None for an empty line, cells expected)) + SPLIT_CASES = [ + ("a\tb\n1\t2", ["a", "b"], []), # no final terminator + ("a\tb\n1\t2\n", ["a", "b"], []), # one final terminator + ("a\tb\n1\t2\n\n", ["a", "b"], [(3, None, 2)]), # two final terminators: an empty last line + ("a\tb\n1\t2\n\n\n", ["a", "b"], [(3, None, 2), (4, None, 2)]), + ("a\tb\n1\t2\n\n3\t4\n", ["a", "b"], [(3, None, 2)]), # empty line between rows + ("a\tb\r\n1\t2\r\n", ["a", "b"], []), # CRLF + ("a\tb\r\n\r\n1\t2\r\n", ["a", "b"], [(2, None, 2)]), + ("a\tb\n1\t2\r\n3\t4\n", ["a", "b"], []), # mixed CRLF and LF + ("a\tb\n1\t2\r3\t4\n", ["a", "b"], [(2, 3, 2)]), # a lone CR is not a line end + ("a\r", ["a\r"], []), # a lone CR at the end of the text is part of the last line + ("a\n\r", ["a"], []), # a line of a lone CR is one cell, not an empty line + ("\na\tb\n", [""], [(2, 2, 1)]), # an empty first line is one blank name (c), not (e) + ("a\n \n", ["a"], []), # blanks-only line in a one-column table: a value, not (f) + ("a\tb\n \n", ["a", "b"], [(2, 1, 2)]), # blanks-only line in a two-column table + ("a\tb\n1\n", ["a", "b"], [(2, 1, 2)]), # short row + ("a\tb\n1\t2\t3\n4\t5\n", ["a", "b"], [(2, 3, 2)]), # long first data row + ("a\tb\n1\t2\n3\t4\t5\n", ["a", "b"], [(3, 3, 2)]), # long later row + ("a\tb\n\t\n", ["a", "b"], []), # tabs only, right length: a row of missing values + ("a\tb\n1\t2\n", ["a", "b"], []), # a byte order mark is not part of the first name + ] + + def test_split_table(self): + for text, header, problems in self.SPLIT_CASES: + with self.subTest(text=text): + self.assertEqual(find_line_problems(text), (header, problems)) + + def test_text_input_records_problems(self): + text = "onset\tHED\n1\tRed\n\n2\n" + for file in (io.StringIO(text), io.BytesIO(text.encode("utf-8"))): + with self.subTest(file=type(file).__name__): + input_file = BaseInput(file, file_type=".tsv") + self.assertEqual(input_file.line_problems, [(3, None, 2), (4, 1, 2)]) + + def test_path_input_records_problems(self): + folder = os.path.join(os.path.dirname(__file__), "../data/temp_line_problems") + os.makedirs(folder, exist_ok=True) + try: + path = os.path.join(folder, "events.tsv") + with open(path, "wb") as fp: + fp.write(b"onset\tHED\r\n1\tRed\r\n\r\n") + self.assertEqual(BaseInput(path).line_problems, [(3, None, 2)]) + finally: + shutil.rmtree(folder) + + def test_long_later_row_does_not_raise(self): + # pandas cannot read this file; the frame is empty and keeps the header. + input_file = BaseInput(io.StringIO("onset\tHED\n1\tRed\n2\tBlue\tx\n"), file_type=".tsv") + self.assertEqual(input_file.line_problems, [(3, 3, 2)]) + self.assertEqual(input_file.columns, ["onset", "HED"]) + self.assertEqual(len(input_file.dataframe), 0) + + def test_empty_first_line(self): + input_file = BaseInput(io.StringIO("\nonset\tHED\n1\tRed\n"), file_type=".tsv") + self.assertEqual(input_file.blank_columns, [0]) + self.assertEqual(input_file.columns, [""]) + self.assertEqual(input_file.line_problems, [(2, 2, 1), (3, 2, 1)]) + + def test_two_empty_first_lines(self): + # pandas finds no columns here; the first line is still one blank column name. + input_file = BaseInput(io.StringIO("\n\nonset\tHED\n1\tRed\n"), file_type=".tsv") + self.assertEqual(input_file.blank_columns, [0]) + self.assertEqual(input_file.columns, [""]) + self.assertEqual(input_file.line_problems, [(2, None, 1), (3, 2, 1), (4, 2, 1)]) + + def test_only_a_line_ending_is_a_blank_column_name(self): + # An empty first line is one blank column name (specification 3.2.10); only an empty file has no columns. + for text in ("\n", "\r\n", "\n"): + with self.subTest(text=text): + input_file = BaseInput(io.StringIO(text), file_type=".tsv") + self.assertEqual(input_file.blank_columns, [0]) + self.assertEqual(input_file.columns, [""]) + self.assertEqual(input_file.line_problems, []) + self.assertEqual(BaseInput(io.StringIO(""), file_type=".tsv").blank_columns, []) + + def test_blanks_only_line_kept_as_value(self): + input_file = BaseInput(io.StringIO("trial\ngo\n \n"), file_type=".tsv") + self.assertEqual(input_file.line_problems, []) + self.assertEqual(input_file.dataframe["trial"].tolist(), ["go", " "]) + + def test_clean_file_unchanged(self): + input_file = BaseInput(io.StringIO("onset\tHED\n1\tRed\n2\t\n"), file_type=".tsv") + self.assertEqual(input_file.line_problems, []) + self.assertEqual(input_file.dataframe.values.tolist(), [["1", "Red"], ["2", "n/a"]]) + + def test_dataframe_and_excel_have_no_line_problems(self): + self.assertEqual(BaseInput(pd.DataFrame([["1", "Red"]], columns=["onset", "HED"])).line_problems, []) + self.assertEqual(_excel_input(["onset", "HED"], ["1", "Red"]).line_problems, []) + + def test_file_without_column_names_not_checked(self): + # The line rules are rules of a tabular file, whose first line holds the column names. + input_file = BaseInput(io.StringIO("Red\tBlue\nSquare\n\n"), file_type=".tsv", has_column_names=False) + self.assertEqual(input_file.line_problems, []) + self.assertEqual(input_file.dataframe.values.tolist(), [["Red", "Blue"], ["Square", "n/a"]]) + + def test_assembly_refused_for_line_problems(self): + input_file = BaseInput(io.StringIO("onset\tHED\n1\tRed\n\n"), file_type=".tsv") + with self.assertRaises(HedFileError) as context: + input_file.assemble() + self.assertEqual(context.exception.code, "INVALID_FILE_FORMAT") + self.assertIn("line 3", context.exception.message) diff --git a/tests/tools/bids/test_bids_file_group.py b/tests/tools/bids/test_bids_file_group.py index 56ed0ca13..263fe9867 100644 --- a/tests/tools/bids/test_bids_file_group.py +++ b/tests/tools/bids/test_bids_file_group.py @@ -157,6 +157,35 @@ def test_padded_or_blank_column_reported_despite_bad_merged_sidecar(self): self.assertEqual(bad_name.get("ec_column"), column) self.assertTrue(bad_name["ec_filename"].endswith("sub-002_task-FacePerception_run-1_events.tsv")) + def test_empty_line_reported_despite_bad_merged_sidecar(self): + """An empty line (TABULAR_INVALID e) is reported by its file line, like a bad column name.""" + source = os.path.realpath( + os.path.join(os.path.dirname(__file__), "../../data/bids_tests/eeg_ds003645s_hed_inheritance") + ) + with tempfile.TemporaryDirectory() as temp_dir: + root_path = os.path.join(temp_dir, "dataset") + shutil.copytree(source, root_path) + events_path = os.path.join(root_path, "sub-002", "sub-002_task-FacePerception_run-1_events.tsv") + with open(events_path, encoding="utf-8", newline="") as fp: + header, second, rest = fp.read().split("\n", 2) + with open(events_path, "w", encoding="utf-8", newline="") as fp: + fp.write(header + "\n" + second + "\n\n" + rest) + + file_paths = io_util.get_file_list( + root_path, extensions=[".tsv", ".json"], exclude_dirs=self.exclude_dirs, name_suffix=["_events"] + ) + events = BidsFileGroup(root_path, file_paths, "events") + group_sidecars = {id(sidecar.contents) for sidecar in events.sidecar_dict.values()} + for data_obj in events.datafile_dict.values(): + if "sub-002" in data_obj.file_path and id(data_obj.sidecar) not in group_sidecars: + data_obj.sidecar.loaded_dict["bad_column"] = {"HED": {"x": "InvalidTagXYZ"}} + issues = events.validate_datafiles(load_schema_version("8.4.0")) + + self.assertEqual(sorted(issue["code"] for issue in issues), ["TABULAR_INVALID", "TAG_INVALID"]) + empty_line = next(issue for issue in issues if issue["code"] == "TABULAR_INVALID") + self.assertEqual(empty_line["ec_row"], 3) + self.assertTrue(empty_line["ec_filename"].endswith("sub-002_task-FacePerception_run-1_events.tsv")) + def test_padded_hed_header_found_without_sidecar(self): """Discovery trims header cells only to find HED: a sidecar-free file headed `HED \\t HED` has HED columns, so it is validated, and each padded name is TABULAR_INVALID.""" @@ -248,6 +277,29 @@ def test_merged_sidecar_with_only_warnings_still_runs_its_files(self): self.assertEqual(len({i["ec_filename"] for i in bad_values}), 3, "all three sub-002 files still ran") self.assertTrue(all("sub-002" in i["ec_filename"] and i["ec_column"] == "stim_file" for i in bad_values)) + def test_sidecar_without_hed_does_not_make_file_hed(self): + # Specification 3.2.10: only a file with a HED column or a sidecar with HED annotations is checked. + tsv_text = "onset\tduration\tresponse\n1.0\t0\ta\n\n" # An empty last line, which is TABULAR_INVALID. + for sidecar_text, expected_codes in ( + ('{"response": {"Description": "A response."}}', []), + ('{"response": {"HED": {"a": "Red"}}}', ["TABULAR_INVALID"]), + ): + with self.subTest(sidecar=sidecar_text), tempfile.TemporaryDirectory() as temp_dir: + root_path = os.path.join(temp_dir, "dataset") + os.makedirs(os.path.join(root_path, "sub-01")) + tsv_path = os.path.join(root_path, "sub-01", "sub-01_task-x_events.tsv") + json_path = os.path.join(root_path, "task-x_events.json") + with open(tsv_path, "w", encoding="utf-8", newline="") as fp: + fp.write(tsv_text) + with open(json_path, "w", encoding="utf-8", newline="") as fp: + fp.write(sidecar_text) + events = BidsFileGroup(root_path, [tsv_path, json_path], "events") + has_hed = [data_obj for data_obj in events.datafile_dict.values() if data_obj.has_hed] + issues = events.validate_datafiles(load_schema_version("8.4.0")) + self.assertEqual(len(events.datafile_dict), 1) + self.assertEqual(len(has_hed), len(expected_codes)) + self.assertEqual([issue["code"] for issue in issues], expected_codes) + if __name__ == "__main__": unittest.main() diff --git a/tests/validator/test_sidecar_validator.py b/tests/validator/test_sidecar_validator.py index fd383fdaf..b451a2f0d 100644 --- a/tests/validator/test_sidecar_validator.py +++ b/tests/validator/test_sidecar_validator.py @@ -2,11 +2,12 @@ import json import os import unittest +from unittest import mock from hed import schema from hed.errors.error_reporter import ErrorHandler from hed.errors.error_types import ErrorContext, SidecarErrors -from hed.models import Sidecar, TabularInput +from hed.models import HedString, Sidecar, TabularInput from hed.validator.sidecar_validator import SidecarValidator @@ -33,11 +34,18 @@ def test_multicategory_refs(self): sidecar = Sidecar(self._multiple_category_refs) issues = sidecar.validate(self.hed_schema) - # 3 issues are expected for repeated tags from stacking lines - self.assertEqual(len(issues), 2) + # Azure repeats only when cat1 and cat2 are both filled (go with go2, stop with go2). The sidecar + # check fills one referenced column at a time (specification 3.2.9.3 rule 10), so it reports nothing; + # the repeat is found when a row with those values is assembled. + self.assertEqual(len(issues), 0) refs = sidecar.get_column_refs() self.assertEqual(len(refs), 2) + events = TabularInput(io.StringIO("cat1\tcat2\tcombo\ngo\tgo2\t3\nstop\tstop2\t3\n"), sidecar=sidecar) + row_issues = events.validate(self.hed_schema) + self.assertEqual([issue["code"] for issue in row_issues], ["TAG_EXPRESSION_REPEATED"]) + self.assertEqual([issue["ec_row"] for issue in row_issues], [2]) + def test_bad_refs(self): sidecar = Sidecar(self._bad_refs_json_filename) issues = sidecar.validate(self.hed_schema) @@ -305,10 +313,6 @@ def test_every_categorical_substitution_checked(self): sidecar = {"event_code": {"HED": {"face": "(Red, {color})"}}, "color": {"HED": {"warm": "Red", "cool": "Blue"}}} self.assertEqual(self.codes(sidecar), ["TAG_EXPRESSION_REPEATED"]) - def test_hed_reference_not_checked_in_sidecar(self): - # What {HED} contributes is known only per row (specification 3.2.9.3 rule 10). - self.assertEqual(self.codes({"c": {"HED": {"a": "(Red, {HED}), (Red)"}}}), []) - def test_definition_beside_tags(self): cases = [ {"cond": {"HED": {"a": "(Definition/MyDef, (Red)), Blue"}}}, @@ -337,3 +341,122 @@ def test_nested_hed_key(self): for sidecar in cases: with self.subTest(sidecar=sidecar): self.assertEqual(self.codes(sidecar), ["SIDECAR_INVALID"]) + + +class TestRemovalSubstitution(unittest.TestCase): + """A removed reference (its column n/a or blank) is a substitution for the repeat check only (HED 8.4.0). + + Specification 3.2.9.3 rule 10 and its explanation: for TAG_EXPRESSION_REPEATED the annotation with a + reference removed (rule 3) is one of the substitutions, for every reference; for {HED} it is the only one. + """ + + COLOR = {"HED": {"warm": "Blue", "cool": "Green"}} + + @classmethod + def setUpClass(cls): + cls.hed_schema = schema.load_schema_version("8.4.0") + + def issues(self, sidecar_dict): + return Sidecar(io.StringIO(json.dumps(sidecar_dict))).validate(self.hed_schema) + + def codes(self, sidecar_dict): + return [issue["code"] for issue in self.issues(sidecar_dict)] + + def test_hed_reference_removed_repeats(self): + sidecar = {"event_code": {"HED": {"face": "(Red, {HED}), (Red)"}}} + self.assertEqual(self.codes(sidecar), ["TAG_EXPRESSION_REPEATED"]) + + def test_categorical_reference_removed_repeats(self): + sidecar = {"event_code": {"HED": {"face": "(Red, {color}), (Red)"}}, "color": self.COLOR} + self.assertEqual(self.codes(sidecar), ["TAG_EXPRESSION_REPEATED"]) + + def test_value_reference_removed_repeats(self): + sidecar = {"event_code": {"HED": {"face": "(Red, {rt}), (Red)"}}, "rt": {"HED": "Label/#"}} + self.assertEqual(self.codes(sidecar), ["TAG_EXPRESSION_REPEATED"]) + + def test_hed_reference_alone_in_group(self): + self.assertEqual(self.codes({"event_code": {"HED": {"face": "Red, ({HED})"}}}), []) + + def test_hed_reference_removed_no_repeat(self): + self.assertEqual(self.codes({"event_code": {"HED": {"face": "(Red, {HED}), (Blue)"}}}), []) + + def test_categorical_reference_removed_no_repeat(self): + sidecar = {"event_code": {"HED": {"face": "(Red, {color}), (Yellow)"}}, "color": self.COLOR} + self.assertEqual(self.codes(sidecar), []) + + def test_duration_group_rule_not_checked_with_removal(self): + # Removing {stim_desc} leaves (Duration/# s) without its inner group; the row with n/a fails when + # assembled, not the sidecar. + sidecar = {"dur": {"HED": "(Duration/# s, {stim_desc})"}, "stim_desc": {"HED": {"a": "(Red)", "b": "(Blue)"}}} + self.assertEqual(self.codes(sidecar), []) + + def test_duration_group_rule_not_checked_for_hed_reference(self): + self.assertEqual(self.codes({"dur": {"HED": "(Duration/# s, {HED})"}}), []) + + def test_categorical_substitution_repeat_kept(self): + sidecar = {"event_code": {"HED": {"face": "(Red, {color})"}}, "color": {"HED": {"warm": "Red", "cool": "Blue"}}} + self.assertEqual(self.codes(sidecar), ["TAG_EXPRESSION_REPEATED"]) + + def test_removal_reports_only_repeats(self): + # Each removal leaves a group that breaks another rule ((Duration/# s) with no inner group, a lone + # (Onset)); only the repeat is reported. + cases = [ + {"dur": {"HED": "(Duration/# s, {HED}), (Red, {HED}), (Red)"}}, + {"event_code": {"HED": {"face": "(Onset, {HED}), (Red, {HED}), (Red)"}}}, + { + "dur": {"HED": "(Duration/# s, {stim_desc}), (Red, {stim_desc}), (Red)"}, + "stim_desc": {"HED": {"a": "(Blue)", "b": "(Green)"}}, + }, + ] + for sidecar in cases: + with self.subTest(sidecar=sidecar): + self.assertEqual(self.codes(sidecar), ["TAG_EXPRESSION_REPEATED"]) + + def test_repeat_reported_once(self): + # Several substitutions find these repeats; each is reported once. + cases = [ + ({"event_code": {"HED": {"face": "Red, Red, {color}"}}, "color": self.COLOR}, 1), + ({"event_code": {"HED": {"face": "(Red, {color}), (Red, {HED})"}}, "color": self.COLOR}, 1), + ( + { + "event_code": {"HED": {"face": "(Red, {color}, {rt}), (Red)"}}, + "color": self.COLOR, + "rt": {"HED": "Label/#"}, + }, + 1, + ), + ] + for sidecar, count in cases: + with self.subTest(sidecar=sidecar): + self.assertEqual(self.codes(sidecar), ["TAG_EXPRESSION_REPEATED"] * count) + + def test_several_references_cost_is_linear(self): + # 8 referenced columns of 10 annotations: 81 substitutions, not 11**8 combinations. + sidecar = {"event_code": {"HED": {"face": ", ".join(f"(Label/G{n}, {{c{n}}})" for n in range(8))}}} + for n in range(8): + sidecar[f"c{n}"] = {"HED": {f"k{level}": f"Label/V{n}-{level}" for level in range(10)}} + strings = [] + original = HedString.__init__ + + def counting(hed_string_obj, *args, **kwargs): + strings.append(1) + original(hed_string_obj, *args, **kwargs) + + with mock.patch.object(HedString, "__init__", counting): + self.assertEqual(self.codes(sidecar), []) + self.assertLess(len(strings), 500) + + def test_two_filled_columns_not_checked_in_sidecar(self): + # The repeat needs color and shade both filled; it is left to row assembly. + sidecar = { + "event_code": {"HED": {"face": "({color}, {shade})"}}, + "color": {"HED": {"warm": "Red"}}, + "shade": {"HED": {"dark": "Red"}}, + } + self.assertEqual(self.codes(sidecar), []) + + def test_repeated_reference_is_one_substitution(self): + # Every occurrence of a reference gets the same value, so 20 occurrences are 3 substitutions, not 3**20. + groups = ", ".join(f"(Label/L{number}, {{color}})" for number in range(20)) + sidecar = {"event_code": {"HED": {"face": groups}}, "color": self.COLOR} + self.assertEqual(self.codes(sidecar), []) diff --git a/tests/validator/test_spreadsheet_validator.py b/tests/validator/test_spreadsheet_validator.py index 192ec219d..2b4f7de85 100644 --- a/tests/validator/test_spreadsheet_validator.py +++ b/tests/validator/test_spreadsheet_validator.py @@ -619,3 +619,83 @@ def test_delay_error_keeps_the_delay_in_its_context(self): issues = self.validate(table, sidecar=False) self.assertEqual([issue["code"] for issue in issues], [ValidationErrors.TEMPORAL_TAG_ERROR]) self.assertIn("Delay/3 month", str(issues[0][ErrorContext.HED_STRING])) + + +class TestTabularLines(unittest.TestCase): + """Stage 0: TABULAR_INVALID (e) empty line and (f) wrong cell count stop the run (specification 3.2.10).""" + + @classmethod + def setUpClass(cls): + cls.schema = load_schema_version("8.4.0") + + def _validate(self, text, sidecar=None): + error_handler = ErrorHandler(check_for_warnings=True) + tabular = TabularInput(io.StringIO(text), sidecar=sidecar, name="events.tsv") + issues = tabular.validate(self.schema, error_handler=error_handler) + self.assertEqual(error_handler.error_context, []) + return issues + + def test_empty_line_reported_and_rows_not_validated(self): + # The row with an invalid tag is not checked once the file has an empty line. + issues = self._validate("onset\tHED\n1.0\tInvalidTagXYZ\n\n2.0\tBlue\n") + self.assertEqual([issue["code"] for issue in issues], [ValidationErrors.TABULAR_INVALID]) + self.assertEqual(issues[0][ErrorContext.ROW], 3) + self.assertEqual(issues[0][ErrorContext.FILE_NAME], "events.tsv") + self.assertIn("Line 3 ", issues[0]["message"]) + self.assertIn("empty", issues[0]["message"]) + + def test_empty_last_line(self): + issues = self._validate("onset\tHED\n1.0\tRed\n\n") + self.assertEqual([issue[ErrorContext.ROW] for issue in issues], [3]) + + def test_every_wrong_cell_count_reported(self): + issues = self._validate("onset\tHED\n1.0\n2.0\tRed\tx\n3.0\tBlue\n") + self.assertEqual([issue["code"] for issue in issues], [ValidationErrors.TABULAR_INVALID] * 2) + self.assertEqual([issue[ErrorContext.ROW] for issue in issues], [2, 3]) + self.assertIn("has 1 tab-separated cells", issues[0]["message"]) + self.assertIn("first line has 2", issues[0]["message"]) + self.assertIn("has 3 tab-separated cells", issues[1]["message"]) + + def test_empty_line_and_wrong_count_together(self): + issues = self._validate("onset\tHED\n1.0\n\n2.0\tRed\n") + self.assertEqual([issue[ErrorContext.ROW] for issue in issues], [2, 3]) + + def test_empty_first_line_is_blank_name(self): + issues = self._validate("\nonset\tHED\n1.0\tRed\n") + self.assertEqual([issue["code"] for issue in issues], [ValidationErrors.TABULAR_INVALID] * 3) + self.assertIn("Column 1 ", issues[0]["message"]) + self.assertEqual([issue.get(ErrorContext.ROW) for issue in issues[1:]], [2, 3]) + + def test_blanks_only_line_in_one_column_table_is_padded_value(self): + sidecar = Sidecar(io.StringIO(json.dumps({"trial": {"HED": {"go": "Experimental-trial"}}}))) + issues = self._validate("trial\ngo\n \n", sidecar=sidecar) + self.assertEqual([issue["code"] for issue in issues], [ValidationErrors.TABULAR_INVALID]) + self.assertIn("begins or ends with whitespace", issues[0]["message"]) + self.assertEqual(issues[0][ErrorContext.ROW], 3) + + def test_clean_files_valid(self): + for text in [ + "onset\tHED\n1.0\tRed\n", + "onset\tHED\r\n1.0\tRed\r\n", + "onset\tHED\n1.0\tRed", + "onset\tHED\n1.0\t\n", + ]: + with self.subTest(text=text): + self.assertEqual(self._validate(text), []) + + def test_row_numbers_are_file_lines(self): + issues = self._validate("onset\tHED\n1.0\tRed\n2.0\tInvalidTagXYZ\n") + self.assertEqual([issue[ErrorContext.ROW] for issue in issues], [3]) + + def test_only_a_line_ending_with_annotating_sidecar(self): + sidecar = Sidecar(io.StringIO(json.dumps({"trial": {"HED": {"go": "Experimental-trial"}}}))) + issues = self._validate("\n", sidecar=sidecar) + self.assertEqual([issue["code"] for issue in issues], [ValidationErrors.TABULAR_INVALID]) + self.assertIn("Column 1 ", issues[0]["message"]) + + def test_lone_cr_does_not_end_a_row(self): + # The lone CR is inside the value of line 2, so pandas also puts the invalid tag on line 3. + issues = self._validate("HED\nRed\rBlue\nInvalidTagXYZ\n") + rows = [issue[ErrorContext.ROW] for issue in issues] + self.assertNotIn(4, rows) + self.assertEqual([issue[ErrorContext.ROW] for issue in issues if "InvalidTagXYZ" in issue["message"]], [3])