From 790ea5a0578ea2f580b45a90d8fb9ec3af91bec0 Mon Sep 17 00:00:00 2001 From: Kay Robbins Date: Sun, 15 Nov 2026 06:39:18 -0600 Subject: [PATCH] Report empty lines and wrong cell counts in tabular files Port hed-typescript's line rules (specification 3.2.10). The header is always line 1, so an empty first line is one blank column name. One line break at the end is not a line; any other empty line is TABULAR_INVALID (emptyTsvLine), and a line whose cell count differs from line 1 is TABULAR_INVALID (tsvCellCount). Row problems are reported by line after header problems and before the sidecar is checked. They are checked only on files that carry HED, including a HED cell in the first non-empty line. Move the hed-tests pin to 0a07fe0; its 11 new TABULAR_INVALID cases pass. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 23 ++++- spec_tests/hed-tests | 2 +- src/bids/tsvParser.js | 127 ++++++++++++++++++------- src/bids/types/tsv.js | 40 +++++++- src/bids/validator/tsvValidator.js | 15 +++ src/issues/data.js | 10 ++ tests/otherTests/tsvParser.spec.js | 143 +++++++++++++++++++++++++++-- types/index.d.ts | 4 + 8 files changed, 316 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c55a5430..40a57bfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -235,13 +235,30 @@ testSpecs`. characters is SIDECAR_INVALID (`paddedSidecarCategoricalKey`, hed-tests `sidecar-invalid-value-key-whitespace`). Before, `go ` was trimmed to `go` and annotated silently. A HED-column cell such as `n/a ` or a blanks-only cell - is still a missing value (`isMissingTsvValue`). A data line that holds a tab, - such as ` \t`, is a row and its cells are checked; an empty line or a line of - blanks without a tab is skipped, as hed-python reads it. Onsets are grouped + is still a missing value (`isMissingTsvValue`). A data line is a row and its + cells are checked, whether or not it holds a tab: ` \t` is a row, and so is a + line of one blank (a one-column table has no wrong cell count). An empty line + is reported, not skipped (see the line rules below). Onsets are grouped without their surrounding whitespace when duplicate tags across rows are checked, so `19` with or without blanks around it is one onset. A categorical value with no key in a column whose annotation holds a `{column}` splice is SIDECAR_KEY_MISSING, as in other columns; it was an INTERNAL_ERROR. +- The line rules of a tabular file (specification 3.2.10, as in hed-typescript, + hed-tests `tabular-invalid-blank-column-name`, `tabular-invalid-empty-line` + and `tabular-invalid-cell-count`): the header is always line 1, so an empty + first line is one blank column name (`blankTsvColumn`) instead of being + skipped. One line break at the end of the contents is not a line; any other + empty line is TABULAR_INVALID (`emptyTsvLine`), and a line whose number of + tab-separated cells differs from line 1 is TABULAR_INVALID (`tsvCellCount`). + Each offending line is reported with its `tsvLine`, in line order, after any + header problem and before the sidecar is checked, and the rows are not + validated further. `{headers, rows}` input is checked too (`rows[0]` is the + header row; a row with no cells is an empty line); `Map` input has no rows to + check. `BidsTsvFile` has the new fields `emptyLines` and `badCellCounts`. + These checks run only on a file that carries HED: a column or sidecar entry + named `HED`, or a cell of the first non-empty line of text data that is `HED` + once trimmed, so an empty first line does not hide the HED column. A blank + column name alone does not. - A partnered library loaded from its unmerged file now gives the same schema as its merged file: - its elements carry `inLibrary`, so redeclaring a tag of the standard partner diff --git a/spec_tests/hed-tests b/spec_tests/hed-tests index babc4440..0a07fe00 160000 --- a/spec_tests/hed-tests +++ b/spec_tests/hed-tests @@ -1 +1 @@ -Subproject commit babc44406dfede6590de63bba5bb48792f97c817 +Subproject commit 0a07fe008b1de7c8e645b895168b1bf39d063005 diff --git a/src/bids/tsvParser.js b/src/bids/tsvParser.js index 1c4a9c07..3d6697c0 100644 --- a/src/bids/tsvParser.js +++ b/src/bids/tsvParser.js @@ -17,34 +17,40 @@ import { isPaddedName, stripNameWhitespace } from '../utils/string' const stripBOM = (str) => str.replace(/^\uFEFF/, '') const normalizeEOL = (str) => str.replace(/\r\n/g, '\n').replace(/\r/g, '\n') -const isContentfulRow = (row) => row && !/^\s*$/.test(row) -// A line with a tab has at least two columns, so it is a row even when every cell is blank or padded. An empty line -// or a line of blanks without a tab is skipped, as pandas reads it in hed-python (skip_blank_lines). -const isTableRow = (row) => row.includes('\t') || isContentfulRow(row) /** - * Split TSV contents into cells: the first row, which is the header, then one array per row after it. + * Split TSV contents into lines (specification 3.2.10). * - * The header cells are the column names and the other cells the values, all exactly as written (specification - * 3.2.9.1 and 3.2.10): nothing is trimmed, as for Map and {headers, rows} input. Header and data rows are found by - * the same rule, so a data line such as ` \t` is a row and its padded value is checked. + * Line breaks are \r\n, \r or \n. One line break at the end of the contents ends the last line and is not itself a + * line; any other empty line is kept, so that it can be reported. Line 1 is always the first line, even when empty. * * @param {string} contents The contents of a TSV file. - * @returns {string[][]} The rows of the file. + * @returns {string[]} The lines of the file; empty when the contents are empty. */ -const splitRows = (contents) => { - const lines = stripBOM(normalizeEOL(contents)).split('\n') - const headerIndex = lines.findIndex(isTableRow) - if (headerIndex < 0) { +const splitLines = (contents) => { + const text = stripBOM(normalizeEOL(contents)) + if (text.length === 0) { return [] } - const dataRows = lines - .slice(headerIndex + 1) - .filter(isTableRow) - .map((str) => str.split('\t')) - return [lines[headerIndex].split('\t'), ...dataRows] + const lines = text.split('\n') + if (lines[lines.length - 1] === '') { + lines.pop() + } + return lines } +/** + * Split TSV contents into cells: the first line, which is the header, then one array per line after it. + * + * The header cells are the column names and the other cells the values, all exactly as written (specification + * 3.2.9.1 and 3.2.10): nothing is trimmed, as for Map and {headers, rows} input. Every line is kept, so that row + * index i is file line i + 1; an empty line becomes the row `['']` (see {@link findRowProblems}). + * + * @param {string} contents The contents of a TSV file. + * @returns {string[][]} The rows of the file. + */ +const splitRows = (contents) => splitLines(contents).map((line) => line.split('\t')) + /** * Determine whether a TSV cell is a missing value: empty or `n/a` once any leading or trailing blank, tab, line feed * or carriage return is removed. @@ -155,31 +161,84 @@ export function findDuplicateHeaders(contents) { } /** - * Return the cells of the first header line of TSV contents (see {@link splitRows}), without splitting the other lines. + * Return the cells of the first line of TSV contents, the header (see {@link splitRows}), without splitting the other + * lines. * * Line breaks are \r\n, \r or \n, as in {@link splitRows}, so the result equals the header row that - * {@link parseTSV} uses. The cells are not trimmed. + * {@link parseTSV} uses. The cells are not trimmed. An empty first line is one blank column name, `['']`. * * @param {string} contents The contents of a TSV file. - * @returns {string[]} The header cells; empty when the contents have no header line. + * @returns {string[]} The header cells; empty when the contents are empty. */ export function parseTSVHeader(contents) { const text = stripBOM(contents) - const lineBreak = /\r\n|\r|\n/g - let start = 0 - while (start <= text.length) { - lineBreak.lastIndex = start - const match = lineBreak.exec(text) - const line = text.slice(start, match ? match.index : text.length) - if (isTableRow(line)) { - return line.split('\t') - } - if (!match) { - break + if (text.length === 0) { + return [] + } + const match = /\r\n|\r|\n/.exec(text) + return text.slice(0, match ? match.index : text.length).split('\t') +} + +/** + * Determine whether a cell of the first non-empty line of TSV contents is `HED` once its leading and trailing + * whitespace is removed. + * + * Such a file carries HED even when a malformed first line hides its HED column (specification 3.2.10), so it is + * validated and the malformed line reported. + * + * @param {string} contents The contents of a TSV file. + * @returns {boolean} Whether the first non-empty line names a HED column. + */ +export function firstNonEmptyLineHasHed(contents) { + const line = splitLines(contents).find((text) => text !== '') + return line !== undefined && line.split('\t').some((cell) => stripNameWhitespace(cell) === 'HED') +} + +/** + * The lines of a tabular file that break the line rules of specification 3.2.10. + * @typedef {Object} TsvRowProblems + * @property {number[]} emptyLines The 1-based numbers of the empty lines other than the first. + * @property {{line: number, cells: number, expected: number}[]} badCellCounts Each other line whose number of + * tab-separated cells differs from the first line's. + */ + +/** + * Find the lines other than the first that are empty or have a different number of cells than the first line + * (specification 3.2.10, TABULAR_INVALID). + * + * An empty line is reported as an empty line only, not also as a wrong cell count. For text contents, an empty line + * is a line with no characters; for {headers, rows} data, a row with no cells, since a row `['']` is one empty + * cell. In {headers, rows} data, rows[0] is the header row and the expected count is the number of headers. + * + * @param {string|OldParsedTSV} data The contents of a TSV file, or parsed TSV data in the old BIDS format. + * @returns {TsvRowProblems} The empty lines and the lines with a wrong cell count, by 1-based file line number. + */ +export function findRowProblems(data) { + const problems = { emptyLines: [], badCellCounts: [] } + let expected + let rows + if (typeof data === 'string') { + const lines = splitLines(data) + if (lines.length === 0) { + return problems } - start = lineBreak.lastIndex + expected = lines[0].split('\t').length + rows = lines.map((line) => (line === '' ? null : line.split('\t'))) + } else { + expected = data.headers.length + rows = data.rows.map((row) => (row.length === 0 ? null : row)) } - return [] + rows.forEach((row, index) => { + if (index === 0) { + return + } + if (row === null) { + problems.emptyLines.push(index + 1) + } else if (row.length !== expected) { + problems.badCellCounts.push({ line: index + 1, cells: row.length, expected }) + } + }) + return problems } /** diff --git a/src/bids/types/tsv.js b/src/bids/types/tsv.js index 87b8d331..dc3d8280 100644 --- a/src/bids/types/tsv.js +++ b/src/bids/types/tsv.js @@ -6,7 +6,15 @@ import isPlainObject from 'lodash/isPlainObject' import { BidsFile } from './file' -import { convertParsedTSVData, findHeaderProblems, isMissingTsvValue, parseTSV, parseTSVHeader } from '../tsvParser' +import { + convertParsedTSVData, + findHeaderProblems, + findRowProblems, + firstNonEmptyLineHasHed, + isMissingTsvValue, + parseTSV, + parseTSVHeader, +} from '../tsvParser' import { BidsSidecar } from './json' import BidsHedTsvValidator from '../validator/tsvValidator' import { IssueError } from '../../issues/issues' @@ -40,6 +48,26 @@ export class BidsTsvFile extends BidsFile { */ blankColumns + /** + * The 1-based numbers of the empty lines other than the first (TABULAR_INVALID). + * @type {number[]} + */ + emptyLines + + /** + * The lines other than the first whose number of cells differs from the first line's (TABULAR_INVALID). + * @type {{line: number, cells: number, expected: number}[]} + */ + badCellCounts + + /** + * Whether a cell of the first non-empty line of text TSV data is `HED` once trimmed, so the file carries HED even + * when its first line is malformed (specification 3.2.10). + * @type {boolean} + * @private + */ + _hedInFirstNonEmptyLine + /** * HED strings in the "HED" column of the TSV data. * @type {string[]} @@ -70,6 +98,12 @@ export class BidsTsvFile extends BidsFile { this.paddedColumns = padded this.blankColumns = blank this.duplicateColumns = duplicate + // A Map has no row structure, so it has no empty lines or wrong cell counts to report. + const { emptyLines, badCellCounts } = + tsvData instanceof Map ? { emptyLines: [], badCellCounts: [] } : findRowProblems(tsvData) + this.emptyLines = emptyLines + this.badCellCounts = badCellCounts + this._hedInFirstNonEmptyLine = typeof tsvData === 'string' && firstNonEmptyLineHasHed(tsvData) this.hedColumnHedStrings = this._parseHedColumn() this.mergedSidecar = new BidsSidecar(name, this.file, mergedDictionary, defManager) } @@ -127,10 +161,12 @@ export class BidsTsvFile extends BidsFile { * @returns {boolean} */ get hasHedData() { - // A padded "HED" header is still the HED column, so the file is checked and its header reported. + // A padded "HED" header is still the HED column, so the file is checked and its header reported. So is a "HED" + // cell in the first non-empty line of text data, which an empty first line would otherwise hide (3.2.10). return ( this.parsedTsv.has('HED') || this.paddedColumns.some((column) => stripNameWhitespace(column) === 'HED') || + this._hedInFirstNonEmptyLine || this.mergedSidecar.hasHedData ) } diff --git a/src/bids/validator/tsvValidator.js b/src/bids/validator/tsvValidator.js index 3be5887d..95825856 100644 --- a/src/bids/validator/tsvValidator.js +++ b/src/bids/validator/tsvValidator.js @@ -74,6 +74,21 @@ export class BidsHedTsvValidator extends BidsValidator { return } + // An empty line or a wrong cell count: report every offending line and do not validate the rows (3.2.10). + const rowIssues = [ + ...this.tsvFile.emptyLines.map((line) => ({ line, issue: generateIssue('emptyTsvLine', { line }) })), + ...this.tsvFile.badCellCounts.map(({ line, cells, expected }) => ({ + line, + issue: generateIssue('tsvCellCount', { line, cells, expected }), + })), + ].sort((a, b) => a.line - b.line) + if (rowIssues.length > 0) { + for (const { line, issue } of rowIssues) { + this.errors.push(BidsHedIssue.fromHedIssue(issue, this.tsvFile.file, { tsvLine: line })) + } + return + } + // Validate the BIDS sidecar if it exists and return if there are errors if (this.tsvFile.mergedSidecar) { const issues = this.tsvFile.mergedSidecar.validate(this.hedSchemas) diff --git a/src/issues/data.js b/src/issues/data.js index b445825e..b6aa2273 100644 --- a/src/issues/data.js +++ b/src/issues/data.js @@ -513,6 +513,16 @@ const issueData = { level: 'error', message: issueMessageTemplate`Column ${'position'} in the header of the tabular file has a blank name.`, }, + emptyTsvLine: { + hedCode: 'TABULAR_INVALID', + level: 'error', + message: issueMessageTemplate`Line ${'line'} of the tabular file is empty; only the first line may be empty.`, + }, + tsvCellCount: { + hedCode: 'TABULAR_INVALID', + level: 'error', + message: issueMessageTemplate`Line ${'line'} of the tabular file has a tab-separated cell count of ${'cells'}, but the first line has a count of ${'expected'}.`, + }, paddedTsvValue: { hedCode: 'TABULAR_INVALID', level: 'error', diff --git a/tests/otherTests/tsvParser.spec.js b/tests/otherTests/tsvParser.spec.js index 94a8b96f..ddcc0669 100644 --- a/tests/otherTests/tsvParser.spec.js +++ b/tests/otherTests/tsvParser.spec.js @@ -9,6 +9,8 @@ import { findDuplicateHeaders, findDuplicateParsedHeaders, findHeaderProblems, + findRowProblems, + firstNonEmptyLineHasHed, isMissingTsvValue, parseTSV, parseTSVHeader, @@ -183,28 +185,83 @@ describe('Padded and blank TSV column names and sidecar keys', () => { ) }) - it('keep a header of tabs only as blank names, not skip it', () => { - const contents = '\n\t\n1\tgo\n' + it('keep a header of tabs only as blank names', () => { + const contents = '\t\n1\tgo\n' assert.deepEqual(parseTSVHeader(contents), ['', '']) assert.deepEqual([...parseTSV(contents).entries()], [['', ['1']]]) }) - it('still skip blank lines before the header', () => { - assert.deepEqual(parseTSVHeader('\n \nonset\tHED\n1\tRed\n'), ['onset', 'HED']) + // Specification 3.2.10: the first line is the header, even when empty; an empty first line is one blank name. + it('take the header from line 1, even when it is empty or blank', () => { + assert.deepEqual(parseTSVHeader('\n \nonset\tHED\n1\tRed\n'), ['']) + assert.deepEqual(parseTSVHeader(' \nonset\tHED\n'), [' ']) + assert.deepEqual(parseTSVHeader('\r\nonset\tHED\r\n'), ['']) + assert.deepEqual(parseTSVHeader('\n'), ['']) + assert.deepEqual(parseTSVHeader(''), []) }) - it('keep a data line with a tab as a row, and skip an empty line or blanks without a tab', () => { - // As pandas reads it in hed-python (skip_blank_lines): ` \t` is a row; '' and ' ' are not. + it('keep every line after the header as a row, empty or not', () => { + // Row i is file line i + 2; the empty line 3 and the one-cell line 4 are reported by findRowProblems. assert.deepEqual( [...parseTSV('trial\tHED\n \t\n\n \ngo\tRed\n').entries()], [ - ['trial', [' ', 'go']], - ['HED', ['', 'Red']], + ['trial', [' ', '', ' ', 'go']], + ['HED', ['', undefined, undefined, 'Red']], ], ) }) }) + describe('findRowProblems', () => { + it.each([ + ['a file with no problems', 'trial\tHED\ngo\tRed\n', [], []], + ['no final line break', 'trial\tHED\ngo\tRed', [], []], + ['a row of tabs with the right count', 'trial\tHED\n\t\n', [], []], + ['an empty line between rows', 'trial\tHED\ngo\tRed\n\nstop\tBlue\n', [3], []], + ['two line breaks at the end', 'trial\tHED\ngo\tRed\n\n', [3], []], + ['CRLF line breaks', 'trial\tHED\r\ngo\tRed\r\n\r\n', [3], []], + ['a short row', 'trial\tHED\ngo\n', [], [{ line: 2, cells: 1, expected: 2 }]], + ['a long row', 'trial\tHED\ngo\tRed\tx\n', [], [{ line: 2, cells: 3, expected: 2 }]], + ['a line of one blank', 'trial\tHED\n \n', [], [{ line: 2, cells: 1, expected: 2 }]], + ['a line of one blank, one column', 'trial\n \n', [], []], + ['an empty line in one column', 'trial\n\ngo\n', [2], []], + [ + 'an empty first line', + '\nresponse\tHED\na\tRed\n', + [], + [ + { line: 2, cells: 2, expected: 1 }, + { line: 3, cells: 2, expected: 1 }, + ], + ], + ['empty contents', '', [], []], + ])('%s', (_, contents, emptyLines, badCellCounts) => { + assert.deepEqual(findRowProblems(contents), { emptyLines, badCellCounts }) + }) + + it('reads old-format rows, where rows[0] is the header and a row with no cells is an empty line', () => { + const headers = ['trial', 'HED'] + assert.deepEqual(findRowProblems({ headers, rows: [headers, ['go', 'Red'], [], ['go'], ['', '']] }), { + emptyLines: [3], + badCellCounts: [{ line: 4, cells: 1, expected: 2 }], + }) + }) + }) + + describe('firstNonEmptyLineHasHed', () => { + it.each([ + ['response\tHED\n', true], + ['\nresponse\tHED\n', true], + ['\n\nresponse\t HED \n', true], + ['\nresponse\ttrial\nHED\tRed\n', false], + [' \nresponse\tHED\n', false], + ['\nresponse\tHEDx\n', false], + ['', false], + ])('%j: %j', (contents, expected) => { + assert.strictEqual(firstNonEmptyLineHasHed(contents), expected) + }) + }) + describe('findHeaderProblems', () => { it.each([ [['onset', 'duration', 'HED'], [], [], []], @@ -278,6 +335,76 @@ describe('Padded and blank TSV column names and sidecar keys', () => { assert.isEmpty(validate('onset\ttrial \t\n1.0\tgo\t\n', {})) }) + // Specification 3.2.10: a line other than the first must not be empty (reason e) and must have as many cells as + // the first (reason f); every offending line is reported, in line order, and the rows are not validated. + it('reports each empty line and wrong cell count by line, and does not validate the rows', () => { + const issues = validate('trial\tHED\ngo\tInvalidTagXYZ\n\nstop\nmaybe\tRed\tx\n\n') + assert.deepEqual(describeIssues(issues), [ + ['TABULAR_INVALID', 'emptyTsvLine'], + ['TABULAR_INVALID', 'tsvCellCount'], + ['TABULAR_INVALID', 'tsvCellCount'], + ['TABULAR_INVALID', 'emptyTsvLine'], + ]) + assert.deepEqual( + issues.map((issue) => issue.hedIssue.parameters.tsvLine), + ['3', '4', '5', '6'], + ) + assert.include(issues[0].issueMessage, 'Line 3 of the tabular file is empty') + assert.include( + issues[1].issueMessage, + 'Line 4 of the tabular file has a tab-separated cell count of 1, but the first line has a count of 2', + ) + }) + + it('accepts one line break at the end and a row of tabs with the right count', () => { + assert.isEmpty(validate('trial\tHED\ngo\tRed\n\t\n')) + }) + + it('reports header problems before row problems, and only them', () => { + assert.deepEqual(describeIssues(validate('trial\tHED\t\ngo\tRed\n\n')), [['TABULAR_INVALID', 'blankTsvColumn']]) + }) + + it('does not check the rows of a file with no HED', () => { + assert.isEmpty(validate('onset\ttrial\n1.0\n\n2.0\tgo\tx\n', {})) + }) + + it('reports a wrong cell count in old-format input but not in a Map', () => { + const oldFormat = { headers: ['trial', 'HED'], rows: [['trial', 'HED'], ['go']] } + assert.deepEqual(describeIssues(validate(oldFormat)), [['TABULAR_INVALID', 'tsvCellCount']]) + const file = new BidsTsvFile('events', { path: 'events.tsv' }, new Map([['HED', ['Red']]]), {}) + assert.deepEqual([file.emptyLines, file.badCellCounts], [[], []]) + }) + + // Specification 3.2.10: a file counts as having a HED column when a cell of its first non-empty line is HED once + // trimmed, so a broken first line does not hide it; the header is still line 1. + describe('the HED gate with a broken first line', () => { + it('checks a file whose empty first line is followed by a HED header', () => { + const file = new BidsTsvFile('events', { path: 'events.tsv' }, '\nresponse\tHED\na\tRed\n', {}) + assert.isTrue(file.hasHedData) + assert.deepEqual(describeIssues(file.validate(hedSchemas)), [['TABULAR_INVALID', 'blankTsvColumn']]) + }) + + it('checks a file whose two empty first lines are followed by a HED header', () => { + const file = new BidsTsvFile('events', { path: 'events.tsv' }, '\n\nresponse\tHED\na\tRed\n', {}) + assert.isTrue(file.hasHedData) + const issues = file.validate(hedSchemas) + assert.deepEqual(describeIssues(issues), [['TABULAR_INVALID', 'blankTsvColumn']]) + assert.strictEqual(issues[0].hedIssue.parameters.position, '1') + }) + + it('skips a file with an empty first line and no HED anywhere', () => { + const file = new BidsTsvFile('events', { path: 'events.tsv' }, '\nresponse\ttrial\na\tgo\n', {}) + assert.isFalse(file.hasHedData) + assert.isEmpty(file.validate(hedSchemas)) + }) + + it('does not open the gate for a blank column name alone', () => { + const file = new BidsTsvFile('events', { path: 'events.tsv' }, 'response\t\na\tgo\n', {}) + assert.isFalse(file.hasHedData) + assert.isEmpty(file.validate(hedSchemas)) + }) + }) + it('checks Map and old-format input the same way', () => { const map = new Map([ ['onset', ['1.0']], diff --git a/types/index.d.ts b/types/index.d.ts index 39a38b38..6dc24a99 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -95,6 +95,10 @@ export class BidsTsvFile { paddedColumns: string[] /** The 1-based positions of the blank column names (TABULAR_INVALID) */ blankColumns: number[] + /** The 1-based numbers of the empty lines other than the first (TABULAR_INVALID) */ + emptyLines: number[] + /** The lines other than the first whose number of cells differs from the first line's (TABULAR_INVALID) */ + badCellCounts: { line: number; cells: number; expected: number }[] /** HED strings in the "HED" column of the TSV data */ hedColumnHedStrings: string[] /** The pseudo-sidecar object representing the merged sidecar data */