From d430b663770dcad4d351096f80d55b1ade668c23 Mon Sep 17 00:00:00 2001 From: Kay Robbins <1189050+VisLab@users.noreply.github.com> Date: Mon, 23 Dec 2024 07:33:51 -0600 Subject: [PATCH] Revert "Completed implementation of HED validator" --- bids/schema.js | 2 +- bids/types/basic.js | 21 +- bids/types/dataset.js | 4 +- bids/types/issues.js | 2 +- bids/types/json.js | 230 +- bids/types/tsv.js | 171 +- bids/validator/sidecarValidator.js | 79 +- bids/validator/tsvValidator.js | 407 +- common/issues/data.js | 98 +- common/issues/issues.js | 10 +- converter/converter.js | 43 + converter/index.js | 8 + .../{reservedTags.json => specialTags.json} | 64 +- eventManager/columnSplicer.js | 211 - eventManager/definitionManager.js | 303 -- eventManager/parseUtils.js | 107 - eventManager/parsedHedColumnSplice.js | 40 - eventManager/parsedHedGroup.js | 307 -- eventManager/parsedHedString.js | 114 - eventManager/parsedHedSubstring.js | 57 - eventManager/parsedHedTag.js | 447 -- eventManager/parser.js | 170 - eventManager/splitter.js | 124 - eventManager/tagConverter.js | 183 - eventManager/tokenizer.js | 399 -- parser/columnSplicer.js | 9 +- parser/definitionManager.js | 313 -- parser/eventManager.js | 187 - parser/parseUtils.js | 107 - parser/parsedHedColumnSplice.js | 17 - parser/parsedHedGroup.js | 579 ++- parser/parsedHedString.js | 47 +- parser/parsedHedSubstring.js | 30 +- parser/parsedHedTag.js | 257 +- parser/parser.js | 121 +- parser/reservedChecker.js | 458 -- {eventManager => parser}/special.js | 399 +- parser/splitter.js | 63 +- parser/tagConverter.js | 6 +- parser/tokenizer.js | 10 +- schema/entries.js | 18 +- schema/parser.js | 2 +- schema/schemaMerger.js | 2 +- spec_tests/javascriptTests.json | 191 +- spec_tests/jsonTests.spec.js | 136 +- tests/bids.spec.js | 752 ++++ tests/bidsTests.spec.js | 70 +- tests/converter.spec.js | 899 ++++ tests/data/HED7.0.4.xml | 3719 ++++++++++++++++ tests/data/HED7.1.1.xml | 3950 +++++++++++++++++ tests/dataset.spec.js | 320 ++ tests/definitionManagerTests.spec.js | 89 - tests/event.spec.js | 1594 +++++++ tests/normalizerTests.spec.js | 66 - tests/schema.spec.js | 216 + tests/schemaBuildTests.spec.js | 3 +- tests/schemaSpecTests.spec.js | 1 - tests/splitterTests.spec.js | 30 +- tests/stringParser.spec.js | 431 ++ tests/stringParserTests.spec.js | 71 +- tests/tagParserTests.spec.js | 23 +- tests/testData/bids.spec.data.js | 782 ++++ tests/testData/bidsTests.data.js | 674 +-- tests/testData/definitionManagerTests.data.js | 156 - tests/testData/normalizerTests.data.js | 42 - tests/testData/splitterTests.data.js | 17 - tests/testData/stringParserTests.data.js | 347 +- tests/testUtilities.js | 17 +- tests/tokenizerTests.spec.js | 4 +- utils/array.js | 17 - utils/map.js | 2 +- validator/dataset.js | 42 +- validator/event/init.js | 12 +- validator/event/validator.js | 677 +-- 74 files changed, 15096 insertions(+), 6480 deletions(-) create mode 100644 converter/converter.js create mode 100644 converter/index.js rename data/json/{reservedTags.json => specialTags.json} (74%) delete mode 100644 eventManager/columnSplicer.js delete mode 100644 eventManager/definitionManager.js delete mode 100644 eventManager/parseUtils.js delete mode 100644 eventManager/parsedHedColumnSplice.js delete mode 100644 eventManager/parsedHedGroup.js delete mode 100644 eventManager/parsedHedString.js delete mode 100644 eventManager/parsedHedSubstring.js delete mode 100644 eventManager/parsedHedTag.js delete mode 100644 eventManager/parser.js delete mode 100644 eventManager/splitter.js delete mode 100644 eventManager/tagConverter.js delete mode 100644 eventManager/tokenizer.js delete mode 100644 parser/definitionManager.js delete mode 100644 parser/eventManager.js delete mode 100644 parser/parseUtils.js delete mode 100644 parser/reservedChecker.js rename {eventManager => parser}/special.js (51%) create mode 100644 tests/bids.spec.js create mode 100644 tests/converter.spec.js create mode 100644 tests/data/HED7.0.4.xml create mode 100644 tests/data/HED7.1.1.xml create mode 100644 tests/dataset.spec.js delete mode 100644 tests/definitionManagerTests.spec.js create mode 100644 tests/event.spec.js delete mode 100644 tests/normalizerTests.spec.js create mode 100644 tests/stringParser.spec.js create mode 100644 tests/testData/bids.spec.data.js delete mode 100644 tests/testData/definitionManagerTests.data.js delete mode 100644 tests/testData/normalizerTests.data.js diff --git a/bids/schema.js b/bids/schema.js index efe5c2c5..338e437a 100644 --- a/bids/schema.js +++ b/bids/schema.js @@ -2,7 +2,7 @@ import castArray from 'lodash/castArray' import semver from 'semver' import { buildSchemas } from '../schema/init' -import { IssueError } from '../common/issues/issues' +import { generateIssue, IssueError } from '../common/issues/issues' import { SchemaSpec, SchemasSpec } from '../schema/specs' const alphabeticRegExp = new RegExp('^[a-zA-Z]+$') diff --git a/bids/types/basic.js b/bids/types/basic.js index 662ae219..65e96241 100644 --- a/bids/types/basic.js +++ b/bids/types/basic.js @@ -1,5 +1,4 @@ import { BidsHedIssue } from './issues' -import { generateIssue } from '../../common/issues/issues' /** * A BIDS file. @@ -37,27 +36,15 @@ export class BidsFile { return false } - /** - * Validate this validator's tsv file - * - * @param {Schemas} schemas - * @returns {BidsIssue[]} Any issues found during validation of this TSV file. - */ - validate(schemas) { + validate(hedSchemas) { if (!this.hasHedData()) { return [] - } - if (!schemas) { - BidsHedIssue.fromHedIssue( - generateIssue('genericError', { - message: 'BIDS file HED validation requires a HED schema, but the schema received was null.', - }), - { path: this.file.file, relativePath: this.file.file }, - ) + } else if (hedSchemas === null) { + return null } try { - const validator = new this.validatorClass(this, schemas) + const validator = new this.validatorClass(this, hedSchemas) return validator.validate() } catch (internalError) { return BidsHedIssue.fromHedIssues(internalError, this.file) diff --git a/bids/types/dataset.js b/bids/types/dataset.js index 0a007144..109c1d81 100644 --- a/bids/types/dataset.js +++ b/bids/types/dataset.js @@ -1,3 +1,5 @@ +import { fallbackDatasetDescription } from './json' + export class BidsDataset { /** * The dataset's event file data. @@ -20,7 +22,7 @@ export class BidsDataset { */ datasetRootDirectory - constructor(eventData, sidecarData, datasetDescription, datasetRootDirectory = null) { + constructor(eventData, sidecarData, datasetDescription = fallbackDatasetDescription, datasetRootDirectory = null) { this.eventData = eventData this.sidecarData = sidecarData this.datasetDescription = datasetDescription diff --git a/bids/types/issues.js b/bids/types/issues.js index de7575e4..bdd23d7b 100644 --- a/bids/types/issues.js +++ b/bids/types/issues.js @@ -38,7 +38,7 @@ export class BidsIssue { * Determine if any of the passed issues are errors. * * @param {BidsIssue[]} issues A list of issues. - * @returns {boolean} Whether any of the passed issues are errors (rather than warnings). + * @return {boolean} Whether any of the passed issues are errors (rather than warnings). */ static anyAreErrors(issues) { return issues.some((issue) => issue.isError()) diff --git a/bids/types/json.js b/bids/types/json.js index de3f3f57..5ee71c38 100644 --- a/bids/types/json.js +++ b/bids/types/json.js @@ -6,7 +6,6 @@ import ParsedHedString from '../../parser/parsedHedString' import { BidsFile } from './basic' import BidsHedSidecarValidator from '../validator/sidecarValidator' import { IssueError } from '../../common/issues/issues' -import { DefinitionManager, Definition } from '../../parser/definitionManager' const ILLEGAL_SIDECAR_KEYS = new Set(['hed', 'n/a']) @@ -63,65 +62,37 @@ export class BidsSidecar extends BidsJsonFile { */ columnSpliceReferences - /** - * The object that manages definitions - * @type {DefinitionManager} - */ - definitions - /** * Constructor. * * @param {string} name The name of the sidecar file. * @param {Object} sidecarData The raw JSON data. * @param {Object} file The file object representing this file. - * @param {DefinitionManager } defManager - The external definitions to use */ - constructor(name, sidecarData = {}, file, defManager = null) { + constructor(name, sidecarData = {}, file) { super(name, sidecarData, file) this.columnSpliceMapping = new Map() this.columnSpliceReferences = new Set() - this._setDefinitions(defManager) this._filterHedStrings() this._categorizeHedStrings() } - _setDefinitions(defManager) { - if (defManager instanceof DefinitionManager) { - this.definitions = defManager - } else if (!defManager) { - this.definitions = new DefinitionManager() - } else { - IssueError.generateAndThrow('internalError', { - message: 'Improper format for defManager parameter -- must be null or DefinitionManager', - }) - } - } - - /** - * Create the sidecar key map from the JSON. - * @private - */ _filterHedStrings() { - this.sidecarKeys = new Map( - Object.entries(this.jsonData) - .map(([key, value]) => { - const trimmedKey = key.trim() - const lowerKey = trimmedKey.toLowerCase() - - if (ILLEGAL_SIDECAR_KEYS.has(lowerKey)) { - IssueError.generateAndThrow('illegalSidecarHedKey') - } - - if (sidecarValueHasHed(value)) { - return [trimmedKey, new BidsSidecarKey(trimmedKey, value.HED, this)] - } - - this._verifyKeyHasNoDeepHed(key, value) + const sidecarHedTags = Object.entries(this.jsonData) + .map(([sidecarKey, sidecarValue]) => { + const trimmedSidecarKey = sidecarKey.trim() + if (ILLEGAL_SIDECAR_KEYS.has(trimmedSidecarKey.toLowerCase())) { + IssueError.generateAndThrow('illegalSidecarHedKey') + } + if (sidecarValueHasHed(sidecarValue)) { + return [trimmedSidecarKey, new BidsSidecarKey(trimmedSidecarKey, sidecarValue.HED, this)] + } else { + this._verifyKeyHasNoDeepHed(sidecarKey, sidecarValue) return null - }) - .filter(Boolean), - ) + } + }) + .filter((x) => x !== null) + this.sidecarKeys = new Map(sidecarHedTags) } /** @@ -144,10 +115,6 @@ export class BidsSidecar extends BidsJsonFile { } } - /** - * Categorize the column strings into value strings and categorical strings - * @private - */ _categorizeHedStrings() { this.hedValueStrings = [] this.hedCategoricalStrings = [] @@ -177,15 +144,14 @@ export class BidsSidecar extends BidsJsonFile { * * The parsed strings are placed into {@link parsedHedData}. * - * @param {Schemas} hedSchemas - The HED schema collection. - * @param {boolean} fullCheck - If true, then it is assumed no splicing will occur and strings stand on their own. + * @param {Schemas} hedSchemas The HED schema collection. * @returns {Issue[]} Any issues found. */ - parseHedStrings(hedSchemas, fullCheck = false) { + parseHedStrings(hedSchemas) { this.parsedHedData = new Map() const issues = [] for (const [name, sidecarKey] of this.sidecarKeys.entries()) { - issues.push(...sidecarKey.parseHed(hedSchemas, fullCheck)) + issues.push(...sidecarKey.parseHed(hedSchemas)) if (sidecarKey.isValueKey) { this.parsedHedData.set(name, sidecarKey.parsedValueString) } else { @@ -202,15 +168,42 @@ export class BidsSidecar extends BidsJsonFile { * @private */ _generateSidecarColumnSpliceMap() { - this.columnSpliceMapping = new Map() - this.columnSpliceReferences = new Set() - for (const [sidecarKey, hedData] of this.parsedHedData) { - if (hedData instanceof ParsedHedString) { - this._parseValueSplice(sidecarKey, hedData) + if (hedData === null) { + // Skipped + } else if (hedData instanceof ParsedHedString) { + if (hedData.columnSplices.length === 0) { + continue + } + + const keyReferences = new Set() + + for (const columnSplice of hedData.columnSplices) { + keyReferences.add(columnSplice.originalTag) + this.columnSpliceReferences.add(columnSplice.originalTag) + } + + this.columnSpliceMapping.set(sidecarKey, keyReferences) } else if (hedData instanceof Map) { - this._parseCategorySplice(sidecarKey, hedData) - } else if (hedData) { + let keyReferences = null + + for (const valueString of hedData.values()) { + if (valueString === null || valueString.columnSplices.length === 0) { + continue + } + + keyReferences ??= new Set() + + for (const columnSplice of valueString.columnSplices) { + keyReferences.add(columnSplice.originalTag) + this.columnSpliceReferences.add(columnSplice.originalTag) + } + } + + if (keyReferences instanceof Set) { + this.columnSpliceMapping.set(sidecarKey, keyReferences) + } + } else { IssueError.generateAndThrow('internalConsistencyError', { message: 'Unexpected type found in sidecar parsedHedData map.', }) @@ -218,41 +211,6 @@ export class BidsSidecar extends BidsJsonFile { } } - _parseValueSplice(sidecarKey, hedData) { - if (hedData.columnSplices.length > 0) { - const keyReferences = this._processColumnSplices(new Set(), hedData.columnSplices) - this.columnSpliceMapping.set(sidecarKey, keyReferences) - } - } - - _parseCategorySplice(sidecarKey, hedData) { - let keyReferences = null - for (const valueString of hedData.values()) { - if (valueString?.columnSplices.length > 0) { - keyReferences = this._processColumnSplices(keyReferences, valueString.columnSplices) - } - } - if (keyReferences instanceof Set) { - this.columnSpliceMapping.set(sidecarKey, keyReferences) - } - } - - /** - * Add a list of columnSplices to a key map. - * @param {Set} keyReferences - * @param {ParsedHedColumnSplice[]} columnSplices - * @returns {*|Set} - * @private - */ - _processColumnSplices(keyReferences, columnSplices) { - keyReferences ??= new Set() - for (const columnSplice of columnSplices) { - keyReferences.add(columnSplice.originalTag) - this.columnSpliceReferences.add(columnSplice.originalTag) - } - return keyReferences - } - /** * The extracted HED strings. * @returns {string[]} @@ -260,6 +218,14 @@ export class BidsSidecar extends BidsJsonFile { get hedStrings() { return this.hedValueStrings.concat(this.hedCategoricalStrings) } + + /** + * An alias for {@link jsonData}. + * @returns {Object} + */ + get sidecarData() { + return this.jsonData + } } export class BidsSidecarKey { @@ -290,12 +256,10 @@ export class BidsSidecarKey { parsedValueString /** * Weak reference to the sidecar. - * @type {BidsSidecar} + * @type {WeakRef} */ sidecar - hasDefinitions - /** * Constructor. * @@ -305,8 +269,7 @@ export class BidsSidecarKey { */ constructor(key, data, sidecar) { this.name = key - this.hasDefinitions = false // May reset to true when definitions for this key are checked - this.sidecar = sidecar + this.sidecar = new WeakRef(sidecar) if (typeof data === 'string') { this.valueString = data } else if (!isPlainObject(data)) { @@ -320,39 +283,23 @@ export class BidsSidecarKey { * Parse the HED data for this key. * * @param {Schemas} hedSchemas The HED schema collection. - * @param {boolean} fullCheck - If true, then assume in final form and not up for potential splice. * @returns {Issue[]} Any issues found. */ - parseHed(hedSchemas, fullCheck) { + parseHed(hedSchemas) { if (this.isValueKey) { - return this._parseValueString(hedSchemas, fullCheck) + return this._parseValueString(hedSchemas) } - return this._parseCategory(hedSchemas, fullCheck) + return this._parseCategory(hedSchemas) } - /** - * Parse the value string in a sidecar - * @param {Schemas} hedSchemas - The HED schemas to use. - * @param {boolean} fullCheck - If true, then assume in final form and not up for potential splice. - * @returns {Issue[]} - * @private - * - * Note: value strings cannot contain definitions - */ - _parseValueString(hedSchemas, fullCheck) { - const [parsedString, parsingIssues] = parseHedString(this.valueString, hedSchemas, fullCheck, false, true) + _parseValueString(hedSchemas) { + const [parsedString, parsingIssues] = parseHedString(this.valueString, hedSchemas, false) + const flatIssues = Object.values(parsingIssues).flat() this.parsedValueString = parsedString - return parsingIssues + return flatIssues } - /** - * Parse the categorical values associated with this key. - * @param {Schemas} hedSchemas - The HED schemas used to check against. - * @param {boolean} fullCheck - If true, then assume in final form and not up for potential splice. - * @returns {Issue[]} - A list of issues if any - * @private - */ - _parseCategory(hedSchemas, fullCheck) { + _parseCategory(hedSchemas) { const issues = [] this.parsedCategoryMap = new Map() for (const [value, string] of Object.entries(this.categoryMap)) { @@ -362,34 +309,22 @@ export class BidsSidecarKey { } else if (typeof string !== 'string') { IssueError.generateAndThrow('illegalSidecarHedType', { key: value, - file: this.sidecar?.file?.relativePath, + file: this.sidecar.deref()?.file?.relativePath, }) } - const [parsedString, parsingIssues] = parseHedString(string, hedSchemas, fullCheck, true, true) + const [parsedString, parsingIssues] = parseHedString(string, hedSchemas, false) this.parsedCategoryMap.set(value, parsedString) - issues.push(...parsingIssues) - if (parsingIssues.length === 0) { - issues.push(...this._checkDefinitions(parsedString)) - } + issues.push(...Object.values(parsingIssues).flat()) } return issues } - _checkDefinitions(parsedString) { - const issues = [] - for (const group of parsedString.tagGroups) { - if (!group.isDefinitionGroup) { - continue - } - this.hasDefinitions = true - const [def, defIssues] = Definition.createDefinitionFromGroup(group) - if (defIssues.length > 0) { - issues.push(...defIssues) - } else { - issues.push(...this.sidecar.definitions.addDefinition(def)) - } - } - return issues + /** + * Whether this key is a categorical key. + * @returns {boolean} + */ + get isCategoricalKey() { + return Boolean(this.categoryMap) } /** @@ -400,3 +335,10 @@ export class BidsSidecarKey { return Boolean(this.valueString) } } + +/** + * Fallback default dataset_description.json file. + * @deprecated Will be removed in v4.0.0. + * @type {BidsJsonFile} + */ +export const fallbackDatasetDescription = new BidsJsonFile('./dataset_description.json', null) diff --git a/bids/types/tsv.js b/bids/types/tsv.js index 10e10848..59e5ae87 100644 --- a/bids/types/tsv.js +++ b/bids/types/tsv.js @@ -3,6 +3,7 @@ import isPlainObject from 'lodash/isPlainObject' import { BidsFile } from './basic' import { convertParsedTSVData, parseTSV } from '../tsvParser' import { BidsSidecar } from './json' +import ParsedHedString from '../../parser/parsedHedString' import BidsHedTsvValidator from '../validator/tsvValidator' import { IssueError } from '../../common/issues/issues' @@ -30,6 +31,11 @@ export class BidsTsvFile extends BidsFile { * @type {BidsSidecar} */ mergedSidecar + /** + * The extracted HED data for the merged pseudo-sidecar. + * @type {Map>} + */ + sidecarHedData /** * Constructor. @@ -41,9 +47,8 @@ export class BidsTsvFile extends BidsFile { * @param {object} file The file object representing this file. * @param {string[]} potentialSidecars The list of potential JSON sidecars. * @param {object} mergedDictionary The merged sidecar data. - * @param {DefinitionManager} defManager */ - constructor(name, tsvData, file, potentialSidecars = [], mergedDictionary = {}, defManager) { + constructor(name, tsvData, file, potentialSidecars = [], mergedDictionary = {}) { super(name, file, BidsHedTsvValidator) if (typeof tsvData === 'string') { @@ -57,7 +62,8 @@ export class BidsTsvFile extends BidsFile { } this.potentialSidecars = potentialSidecars - this.mergedSidecar = new BidsSidecar(name, mergedDictionary, this.file, defManager) + this.mergedSidecar = new BidsSidecar(name, mergedDictionary, this.file) + this.sidecarHedData = this.mergedSidecar.hedData this._parseHedColumn() } @@ -91,38 +97,85 @@ export class BidsTsvFile extends BidsFile { } } -export class BidsTsvElement { +/** + * A BIDS events.tsv file. + * + * @deprecated Use {@link BidsTsvFile}. Will be removed in version 4.0.0. + */ +export class BidsEventFile extends BidsTsvFile { /** - * The string representation of this row - * @type {string} + * Constructor. + * + * @param {string} name The name of the event TSV file. + * @param {string[]} potentialSidecars The list of potential JSON sidecars. + * @param {object} mergedDictionary The merged sidecar data. + * @param {{headers: string[], rows: string[][]}|string} tsvData This file's TSV data. + * @param {object} file The file object representing this file. */ - hedString + constructor(name, potentialSidecars, mergedDictionary, tsvData, file) { + super(name, tsvData, file, potentialSidecars, mergedDictionary) + } +} - parsedHedString +/** + * A BIDS TSV file other than an events.tsv file. + * + * @deprecated Use {@link BidsTsvFile}. Will be removed in version 4.0.0. + */ +export class BidsTabularFile extends BidsTsvFile { + /** + * Constructor. + * + * @param {string} name The name of the TSV file. + * @param {string[]} potentialSidecars The list of potential JSON sidecars. + * @param {object} mergedDictionary The merged sidecar data. + * @param {{headers: string[], rows: string[][]}|string} tsvData This file's TSV data. + * @param {object} file The file object representing this file. + */ + constructor(name, potentialSidecars, mergedDictionary, tsvData, file) { + super(name, tsvData, file, potentialSidecars, mergedDictionary) + } +} +/** + * A row in a BIDS TSV file. + */ +export class BidsTsvRow extends ParsedHedString { + /** + * The parsed string representing this row. + * @type {ParsedHedString} + */ + parsedString + /** + * The column-to-value mapping for this row. + * @type {Map} + */ + rowCells /** * The file this row belongs to. - * @type {Object} + * @type {BidsTsvFile} + */ + tsvFile + /** + * The line number in {@link BidsTsvRow.tsvFile} this line is located at. + * @type {number} */ - file - - onset - tsvLine + /** * Constructor. * - * @param {string} hedString The HED string representing this row + * @param {ParsedHedString} parsedString The parsed string representing this row. + * @param {Map} rowCells The column-to-value mapping for this row. * @param {BidsTsvFile} tsvFile The file this row belongs to. - * @param {number} onset - The onset for this element or undefined if none - * @param {string} tsvLine The line number(s) (string) corresponding to the lines in {@link tsvFile} this line is located at. - */ - constructor(hedString, tsvFile, onset, tsvLine) { - this.hedString = hedString - this.parsedHedString = null - this.file = tsvFile.file - this.fileName = tsvFile.name - this.onset = onset + * @param {number} tsvLine The line number in {@link tsvFile} this line is located at. + */ + constructor(parsedString, rowCells, tsvFile, tsvLine) { + super(parsedString.hedString, parsedString.parseTree) + this.parsedString = parsedString + this.context = parsedString.context + this.rowCells = rowCells + this.tsvFile = tsvFile this.tsvLine = tsvLine } @@ -132,35 +185,34 @@ export class BidsTsvElement { * @returns {string} */ toString() { - const onsetString = this.onset ? ` with onset=${this.onset.toString()}` : '' - return this.hedString + ` in TSV file "${this.fileName}" at line(s) ${this.tsvLine}` + onsetString + return super.toString() + ` in TSV file "${this.tsvFile.name}" at line ${this.tsvLine}` } -} -/** - * A row in a BIDS TSV file. - */ -export class BidsTsvRow extends BidsTsvElement { - rowCells /** - * Constructor. + * The onset of this row. * - * @param {string} hedString The parsed string representing this row. - * @param {Map} rowCells The column-to-value mapping for this row. - * @param {BidsTsvFile} tsvFile The file this row belongs to. - * @param {number} tsvLine The line number in {@link tsvFile} this line is located at. + * @return {number} The onset of this row. */ - constructor(hedString, tsvFile, tsvLine, rowCells) { - const onset = rowCells.has('onset') ? rowCells.get('onset') : undefined - super(hedString, tsvFile, onset, tsvLine.toString()) - this.rowCells = rowCells + get onset() { + const value = Number(this.rowCells.get('onset')) + if (Number.isNaN(value)) { + IssueError.generateAndThrow('internalError', { + message: 'Attempting to access the onset of a TSV row without one.', + }) + } + return value } } /** * An event in a BIDS TSV file. */ -export class BidsTsvEvent extends BidsTsvElement { +export class BidsTsvEvent extends ParsedHedString { + /** + * The file this row belongs to. + * @type {BidsTsvFile} + */ + tsvFile /** * The TSV rows making up this event. * @type {BidsTsvRow[]} @@ -174,33 +226,26 @@ export class BidsTsvEvent extends BidsTsvElement { * @param {BidsTsvRow[]} tsvRows The TSV rows making up this event. */ constructor(tsvFile, tsvRows) { - const hedString = tsvRows.map((tsvRow) => tsvRow.hedString).join(', ') - const tsvLine = tsvRows - .map((tsvRow) => tsvRow.tsvLine) - .flat() - .join(', ') - const onset = tsvRows[0].onset ? tsvRows[0].onset : undefined - super(hedString, tsvFile, onset, tsvLine) + super(tsvRows.map((tsvRow) => tsvRow.hedString).join(', '), tsvRows.map((tsvRow) => tsvRow.parseTree).flat()) + this.tsvFile = tsvFile this.tsvRows = tsvRows } -} -/** - * A BIDS events.tsv file. - * - * @deprecated Use {@link BidsTsvFile}. Will be removed in version 4.0.0. - */ -export class BidsEventFile extends BidsTsvFile { /** - * Constructor. + * The lines in the TSV file corresponding to this event. * - * @param {string} name The name of the event TSV file. - * @param {string[]} potentialSidecars The list of potential JSON sidecars. - * @param {object} mergedDictionary The merged sidecar data. - * @param {{headers: string[], rows: string[][]}|string} tsvData This file's TSV data. - * @param {object} file The file object representing this file. + * @return {string} The lines in the TSV file corresponding to this event. */ - constructor(name, potentialSidecars, mergedDictionary, tsvData, file) { - super(name, tsvData, file, potentialSidecars, mergedDictionary) + get tsvLines() { + return this.tsvRows.map((tsvRow) => tsvRow.tsvLine).join(', ') + } + + /** + * Override of {@link Object.prototype.toString}. + * + * @returns {string} + */ + toString() { + return super.toString() + ` in TSV file "${this.tsvFile.name}" at line(s) ${this.tsvLines}` } } diff --git a/bids/validator/sidecarValidator.js b/bids/validator/sidecarValidator.js index d01e7730..77472d68 100644 --- a/bids/validator/sidecarValidator.js +++ b/bids/validator/sidecarValidator.js @@ -1,9 +1,9 @@ import { BidsHedIssue } from '../types/issues' import ParsedHedString from '../../parser/parsedHedString' // IMPORTANT: This import cannot be shortened to '../../validator', as this creates a circular dependency until v4.0.0. -//import { validateHedString } from '../../validator/event/init' +import { validateHedString } from '../../validator/event/init' import { generateIssue, IssueError } from '../../common/issues/issues' -import { getCharacterCount } from '../../utils/string.js' + /** * Validator for HED data in BIDS JSON sidecars. */ @@ -28,7 +28,7 @@ export class BidsHedSidecarValidator { * Constructor. * * @param {BidsSidecar} sidecar The BIDS sidecar being validated. - * @param {Schemas} hedSchemas + * @param {Schemas} hedSchemas The HED schema collection being validated against. */ constructor(sidecar, hedSchemas) { this.sidecar = sidecar @@ -42,7 +42,6 @@ export class BidsHedSidecarValidator { * @returns {BidsIssue[]} Any issues found during validation of this sidecar file. */ validate() { - // Allow schema to be set a validation time -- this is checked by the superclass of BIDS file const sidecarParsingIssues = BidsHedIssue.fromHedIssues( this.sidecar.parseHedStrings(this.hedSchemas), this.sidecar.file, @@ -63,15 +62,25 @@ export class BidsHedSidecarValidator { _validateStrings() { const issues = [] - for (const [sidecarKeyName, hedData] of this.sidecar.parsedHedData) { + const categoricalOptions = { + checkForWarnings: true, + expectValuePlaceholderString: false, + definitionsAllowed: 'exclusive', + } + const valueOptions = { + checkForWarnings: true, + expectValuePlaceholderString: true, + definitionsAllowed: 'no', + } + + for (const [sidecarKey, hedData] of this.sidecar.parsedHedData) { if (hedData instanceof ParsedHedString) { // Value options have HED as string - issues.push(...this._checkDetails(sidecarKeyName, hedData, true)) + issues.push(...this._validateString(sidecarKey, hedData, valueOptions)) } else if (hedData instanceof Map) { // Categorical options have HED as a Map for (const valueString of hedData.values()) { - const placeholdersAllowed = this.sidecar.sidecarKeys.get(sidecarKeyName).hasDefinitions - issues.push(...this._checkDetails(sidecarKeyName, valueString, placeholdersAllowed)) + issues.push(...this._validateString(sidecarKey, valueString, categoricalOptions)) } } else { IssueError.generateAndThrow('internalConsistencyError', { @@ -79,51 +88,27 @@ export class BidsHedSidecarValidator { }) } } - return issues - } - _checkDetails(sidecarKeyName, hedString) { - const issues = this._checkDefs(sidecarKeyName, hedString, true) - issues.push(...this._checkPlaceholders(sidecarKeyName, hedString)) return issues } - _checkDefs(sidecarKeyName, sidecarString, placeholdersAllowed) { - let issues = this.sidecar.definitions.validateDefs(sidecarString, this.hedSchemas, placeholdersAllowed) - if (issues.length > 0) { - return BidsHedIssue.fromHedIssues(issues, this.sidecar.file, { sidecarKeyName: sidecarKeyName }) + /** + * Validate an individual string in this sidecar. + * + * @param {string} sidecarKey The sidecar key this string belongs to. + * @param {ParsedHedString} sidecarString The parsed sidecar HED string. + * @param {Object} options Options specific to this validation run to pass to {@link validateHedString}. + * @returns {BidsIssue[]} All issues found. + * @private + */ + _validateString(sidecarKey, sidecarString, options) { + // Parsing issues already pushed in validateSidecars() + if (sidecarString === null) { + return [] } - issues = this.sidecar.definitions.validateDefExpands(sidecarString, this.hedSchemas, placeholdersAllowed) - return BidsHedIssue.fromHedIssues(issues, this.sidecar.file, { sidecarKeyName: sidecarKeyName }) - } - _checkPlaceholders(sidecarKeyName, hedString) { - const numberPlaceholders = getCharacterCount(hedString.hedString, '#') - const sidecarKey = this.sidecar.sidecarKeys.get(sidecarKeyName) - if (!sidecarKey.valueString && !sidecarKey.hasDefinitions && numberPlaceholders > 0) { - return [ - BidsHedIssue.fromHedIssue( - generateIssue('invalidSidecarPlaceholder', { column: sidecarKeyName, string: hedString.hedString }), - this.sidecar.file, - ), - ] - } else if (sidecarKey.valueString && numberPlaceholders === 0) { - return [ - BidsHedIssue.fromHedIssue( - generateIssue('missingPlaceholder', { column: sidecarKeyName, string: hedString.hedString }), - this.sidecar.file, - ), - ] - } - if (sidecarKey.valueString && numberPlaceholders > 1) { - return [ - BidsHedIssue.fromHedIssue( - generateIssue('invalidSidecarPlaceholder', { column: sidecarKeyName, string: hedString.hedString }), - this.sidecar.file, - ), - ] - } - return [] + const [, hedIssues] = validateHedString(sidecarString, this.hedSchemas, options) + return BidsHedIssue.fromHedIssues(hedIssues, this.sidecar.file, { sidecarKey }) } /** diff --git a/bids/validator/tsvValidator.js b/bids/validator/tsvValidator.js index d6c40c19..65e840ea 100644 --- a/bids/validator/tsvValidator.js +++ b/bids/validator/tsvValidator.js @@ -1,11 +1,13 @@ +import BidsHedSidecarValidator from './sidecarValidator' import { BidsHedIssue, BidsIssue } from '../types/issues' -import { BidsTsvRow } from '../types/tsv' +import { BidsTsvEvent, BidsTsvRow } from '../types/tsv' import { parseHedString } from '../../parser/parser' +import ColumnSplicer from '../../parser/columnSplicer' import ParsedHedString from '../../parser/parsedHedString' import { generateIssue } from '../../common/issues/issues' -import { ReservedChecker } from '../../parser/reservedChecker' -import { getTagListString } from '../../parser/parseUtils' -import { EventManager } from '../../parser/eventManager' +import { validateHedDatasetWithContext } from '../../validator/dataset' +import { groupBy } from '../../utils/map' +import { validateHedString } from '../../validator/event/init' /** * Validator for HED data in BIDS TSV files. @@ -31,12 +33,11 @@ export class BidsHedTsvValidator { * Constructor. * * @param {BidsTsvFile} tsvFile The BIDS TSV file being validated. - * @param {Schemas} hedSchemas + * @param {Schemas} hedSchemas The HED schema collection being validated against. */ constructor(tsvFile, hedSchemas) { this.tsvFile = tsvFile - this.hedSchemas = hedSchemas // Will be set when the file is validated - this.special = ReservedChecker.getInstance() + this.hedSchemas = hedSchemas this.issues = [] } @@ -46,28 +47,31 @@ export class BidsHedTsvValidator { * @returns {BidsIssue[]} Any issues found during validation of this TSV file. */ validate() { - // Validate the BIDS sidecar if it exists. - if (this.tsvFile.mergedSidecar) { - const sidecarIssues = this.tsvFile.mergedSidecar.validate(this.hedSchemas) - this.issues.push(...sidecarIssues) - if (BidsIssue.anyAreErrors(sidecarIssues)) { - return this.issues - } + const parsingIssues = BidsHedIssue.fromHedIssues( + this.tsvFile.mergedSidecar.parseHedStrings(this.hedSchemas), + this.tsvFile.file, + ) + this.issues.push(...parsingIssues) + if (BidsIssue.anyAreErrors(parsingIssues)) { + return this.issues } - - // Valid the HED column by itself. + const curlyBraceIssues = new BidsHedSidecarValidator( + this.tsvFile.mergedSidecar, + this.hedSchemas, + ).validateCurlyBraces() const hedColumnIssues = this._validateHedColumn() - this.issues.push(...hedColumnIssues) + this.issues.push(...curlyBraceIssues, ...hedColumnIssues) if (BidsIssue.anyAreErrors(this.issues)) { return this.issues } - // Now do a full validation + const bidsHedTsvParser = new BidsHedTsvParser(this.tsvFile, this.hedSchemas) - const [bidsEvents, parsingIssues] = bidsHedTsvParser.parse() - this.issues.push(...parsingIssues) + const hedStrings = bidsHedTsvParser.parse() + this.issues.push(...bidsHedTsvParser.issues) if (!BidsIssue.anyAreErrors(this.issues)) { - this.issues.push(...this.validateDataset(bidsEvents)) + this.validateCombinedDataset(hedStrings) } + return this.issues } @@ -101,8 +105,16 @@ export class BidsHedTsvValidator { } const issues = [] - const [parsedString, parsingIssues] = parseHedString(hedString, this.hedSchemas, false, false, false) - issues.push(...BidsHedIssue.fromHedIssues(parsingIssues, this.tsvFile.file, { tsvLine: rowIndex })) + const options = { + checkForWarnings: true, + expectValuePlaceholderString: false, + definitionsAllowed: 'no', + } + + const [parsedString, parsingIssues] = parseHedString(hedString, this.hedSchemas, true) + issues.push( + ...BidsHedIssue.fromHedIssues(Object.values(parsingIssues).flat(), this.tsvFile.file, { tsvLine: rowIndex }), + ) if (parsedString === null) { return issues @@ -112,8 +124,8 @@ export class BidsHedTsvValidator { issues.push( BidsHedIssue.fromHedIssue( generateIssue('curlyBracesInHedColumn', { - string: parsedString.hedString, - tsvLine: rowIndex.toString(), + column: parsedString.columnSplices[0].format(), + tsvLine: rowIndex, }), this.tsvFile.file, ), @@ -121,99 +133,33 @@ export class BidsHedTsvValidator { return issues } - const defIssues = [ - ...this.tsvFile.mergedSidecar.definitions.validateDefs(parsedString, this.hedSchemas, false), - ...this.tsvFile.mergedSidecar.definitions.validateDefExpands(parsedString, this.hedSchemas, false), - ] - const convertedIssues = BidsHedIssue.fromHedIssues(defIssues, this.tsvFile.file, { tsvLine: rowIndex }) + const [, hedIssues] = validateHedString(parsedString, this.hedSchemas, options) + const convertedIssues = BidsHedIssue.fromHedIssues(hedIssues, this.tsvFile.file, { tsvLine: rowIndex }) issues.push(...convertedIssues) + return issues } /** * Validate the HED data in a combined event TSV file/sidecar BIDS data collection. * - * @param {BidsTsvElement[]} elements - The HED strings in the data collection. - * @returns {BidsHedIssue[]} - errors for dataset - */ - validateDataset(elements) { - const issues = this._checkNoTopTags(elements) - if (issues.length > 0) { - return issues - } - if (this.tsvFile.isTimelineFile) { - return this._validateTemporal(elements) - } - return this._checkNoTime(elements) - } - - /** - * Check the temporal relationships among events. - * @param {BidsTsvElement[]} elements - The elements representing the tsv file. - * @returns {BidsHedIssue[]} - Errors in temporal relationships among events - * @private - */ - _validateTemporal(elements) { - const eventManager = new EventManager() - const [eventList, temporalIssues] = eventManager.parseEvents(elements) - if (temporalIssues.length > 0) { - return temporalIssues - } - return eventManager.validate(eventList) - } - - /** - * Top group tag requirements may not be satisfied until all splices have been done. - * @param {BidsTsvElement[]} elements - The elements to be checked - * @returns {BidsHedIssue[]} - Issues from final check of top groups - * @private - */ - _checkNoTopTags(elements) { - const topGroupIssues = [] - for (const element of elements) { - const topTags = element.parsedHedString ? element.parsedHedString.topLevelTags : [] - const badTags = topTags.filter((tag) => ReservedChecker.hasTopLevelTagGroupAttribute(tag)) - if (badTags.length > 0) { - topGroupIssues.push( - BidsHedIssue.fromHedIssue( - generateIssue('invalidTopLevelTag', { tag: getTagListString(badTags), string: element.hedString }), - element.file, - { tsvLine: element.tsvLine }, - ), - ) - } - } - return topGroupIssues - } - - /** - * Verify that this non-temporal file does not contain any temporal tags. - * - * @param {BidsTsvElement[]} elements + * @param {ParsedHedString[]} hedStrings The HED strings in the data collection. */ - _checkNoTime(elements) { - const timeIssues = [] - for (const element of elements) { - if (element.parsedHedString.tags.some((tag) => this.special.temporalTags.has(tag.schemaTag.name))) { - timeIssues.push( - BidsHedIssue.fromHedIssue( - generateIssue('temporalTagInNonTemporalContext', { string: element.hedString, tsvLine: element.tsvLine }), - this.tsvFile.file, - ), - ) - } - } - return timeIssues + validateCombinedDataset(hedStrings) { + const [, hedIssues] = validateHedDatasetWithContext( + hedStrings, + this.tsvFile.mergedSidecar.hedStrings, + this.hedSchemas, + { + checkForWarnings: true, + validateDatasetLevel: this.tsvFile.isTimelineFile, + }, + ) + this.issues.push(...BidsHedIssue.fromHedIssues(hedIssues, this.tsvFile.file)) } } export class BidsHedTsvParser { - static nullSet = new Set([null, undefined, '', 'n/a']) - static braceRegEx = /\{([^{}]*?)\}/g - static parenthesesRegEx = /\(\s*[,\s]*(\(\s*[,\s]*\))*[,\s]*\)/g - static internalCommaRegEx = /,\s*,/g - static leadingCommaRegEx = /^\s*,+\s*/ - static trailingCommaRegEx = /\s*,+\s*$/ /** * The BIDS TSV file being parsed. * @type {BidsTsvFile} @@ -224,6 +170,11 @@ export class BidsHedTsvParser { * @type {Schemas} */ hedSchemas + /** + * The issues found during parsing. + * @type {BidsIssue[]} + */ + issues /** * Constructor. @@ -234,42 +185,23 @@ export class BidsHedTsvParser { constructor(tsvFile, hedSchemas) { this.tsvFile = tsvFile this.hedSchemas = hedSchemas + this.issues = [] } /** * Combine the BIDS sidecar HED data into a BIDS TSV file's HED data. * - * @returns {[BidsTsvElement[], BidsHedIssue[]]} The combined HED string collection for this BIDS TSV file. + * @returns {ParsedHedString[]} The combined HED string collection for this BIDS TSV file. */ parse() { const tsvHedRows = this._generateHedRows() - const tsvElements = this._parseHedRows(tsvHedRows) - const parsingIssues = this._parseElementStrings(tsvElements) - return [tsvElements, parsingIssues] - } + const hedStrings = this._parseHedRows(tsvHedRows) - /** - * Parse element HED strings - * @param { BidsTsvElement []} tsvElements - - * @returns {BidsHedIssue[]} - */ - _parseElementStrings(tsvElements) { - if (tsvElements.length === 0) { - return [] + if (this.tsvFile.isTimelineFile) { + return this._mergeEventRows(hedStrings) } - // Add the parsed HED strings to the elements and quite if there are serious errors - const cummulativeIssues = [] - for (const element of tsvElements) { - const [parsedHedString, parsingIssues] = parseHedString(element.hedString, this.hedSchemas, true, false, false) - element.parsedHedString = parsedHedString - if (parsingIssues.length > 0) { - cummulativeIssues.push( - ...BidsHedIssue.fromHedIssues(parsingIssues, this.tsvFile.file, { tsvLine: element.tsvLine }), - ) - } - } - return cummulativeIssues + return hedStrings } /** @@ -280,7 +212,7 @@ export class BidsHedTsvParser { */ _generateHedRows() { const tsvHedColumns = Array.from(this.tsvFile.parsedTsv.entries()).filter( - ([header]) => this.tsvFile.mergedSidecar.hedData.has(header) || header === 'HED' || header === 'onset', + ([header]) => this.tsvFile.sidecarHedData.has(header) || header === 'HED' || header === 'onset', ) const tsvHedRows = [] @@ -294,152 +226,163 @@ export class BidsHedTsvParser { } /** - * Parse the rows in the TSV file into HED strings. + * Parse the HED rows in the TSV file. * - * @param {Map[]} tsvHedRows - A list of single-row column-to-value mappings. - * @returns {BidsTsvRow[]} - A list of row-based parsed HED strings. + * @param {Map[]} tsvHedRows A list of single-row column-to-value mappings. + * @return {BidsTsvRow[]} A list of row-based parsed HED strings. * @private */ _parseHedRows(tsvHedRows) { - const hedRows = [] + const hedStrings = [] tsvHedRows.forEach((row, index) => { - const hedRow = this._parseHedRow(row, index + 2) - if (hedRow !== null) { - hedRows.push(hedRow) + const hedString = this._parseHedRow(row, index + 2) + if (hedString !== null) { + hedStrings.push(hedString) } }) - return hedRows + return hedStrings } /** - * Parse a row in a TSV file into a BIDS row. + * Merge rows with the same onset time into a single event string. * - * @param {Map} rowCells - The column-to-value mapping for a single row. - * @param {number} tsvLine - The index of this row in the TSV file. - * @returns {BidsTsvRow} - A parsed HED string. + * @param {BidsTsvRow[]} rowStrings A list of row-based parsed HED strings. + * @return {BidsTsvEvent[]} A list of event-based parsed HED strings. + * @private + */ + _mergeEventRows(rowStrings) { + const eventStrings = [] + const groupedTsvRows = groupBy(rowStrings, (rowString) => rowString.onset) + const sortedOnsetTimes = Array.from(groupedTsvRows.keys()).sort((a, b) => a - b) + for (const onset of sortedOnsetTimes) { + const onsetRows = groupedTsvRows.get(onset) + const onsetEventString = new BidsTsvEvent(this.tsvFile, onsetRows) + eventStrings.push(onsetEventString) + } + return eventStrings + } + + /** + * Parse a row in a TSV file. + * + * @param {Map} rowCells The column-to-value mapping for a single row. + * @param {number} tsvLine The index of this row in the TSV file. + * @return {BidsTsvRow} A parsed HED string. * @private */ _parseHedRow(rowCells, tsvLine) { const hedStringParts = [] - const columnMap = this._getColumnMapping(rowCells) - this.spliceValues(columnMap) - for (const [columnName, columnValue] of rowCells.entries()) { - // If a splice, it can't be used in an assembled HED string. - if ( - this.tsvFile.mergedSidecar.columnSpliceReferences.has(columnName) || - BidsHedTsvParser.nullSet.has(columnValue) - ) { - continue - } - if (columnMap.has(columnName) && !BidsHedTsvParser.nullSet.has(columnMap.get(columnName))) { - hedStringParts.push(columnMap.get(columnName)) + const hedStringPart = this._parseRowCell(columnName, columnValue, tsvLine) + if (hedStringPart !== null && !this.tsvFile.mergedSidecar.columnSpliceReferences.has(columnName)) { + hedStringParts.push(hedStringPart) } } + if (hedStringParts.length === 0) return null + const hedString = hedStringParts.join(',') - if (hedString === '' || hedString === 'n/a') { - return null - } - return new BidsTsvRow(hedString, this.tsvFile, tsvLine, rowCells) + + return this._parseHedRowString(rowCells, tsvLine, hedString) } /** - * Generate a mapping from tsv columns to strings (may have splices in the strings) + * Parse a row's generated HED string in a TSV file. * * @param {Map} rowCells The column-to-value mapping for a single row. - * @returns {Map} A mapping of column names to their corresponding parsed sidecar strings. + * @param {number} tsvLine The index of this row in the TSV file. + * @param {string} hedString The unparsed HED string for this row. + * @return {BidsTsvRow} A parsed HED string. * @private */ - _getColumnMapping(rowCells) { - const columnMap = new Map() - - if (rowCells.has('HED')) { - columnMap.set('HED', rowCells.get('HED')) - } + _parseHedRowString(rowCells, tsvLine, hedString) { + const columnSpliceMapping = this._generateColumnSpliceMapping(rowCells) - if (!this.tsvFile.mergedSidecar.hasHedData()) { - return columnMap + const [parsedString, parsingIssues] = parseHedString(hedString, this.hedSchemas) + const flatParsingIssues = Object.values(parsingIssues).flat() + if (flatParsingIssues.length > 0) { + this.issues.push(...BidsHedIssue.fromHedIssues(flatParsingIssues, this.tsvFile.file, { tsvLine })) + return null } - // Check for the columns with HED data in the sidecar - for (const [columnName, columnValues] of this.tsvFile.mergedSidecar.parsedHedData.entries()) { - if (!rowCells.has(columnName)) { - continue - } - const rowColumnValue = rowCells.get(columnName) - if (rowColumnValue === 'n/a' || rowColumnValue === '') { - columnMap.set(columnName, '') - continue - } - - if (columnValues instanceof ParsedHedString) { - const columnString = columnValues.hedString.replace('#', rowColumnValue) - columnMap.set(columnName, columnString) - } else if (columnValues instanceof Map) { - columnMap.set(columnName, columnValues.get(rowColumnValue).hedString) - } + const columnSplicer = new ColumnSplicer(parsedString, columnSpliceMapping, rowCells, this.hedSchemas) + const splicedParsedString = columnSplicer.splice() + const splicingIssues = columnSplicer.issues + if (splicingIssues.length > 0) { + this.issues.push(...BidsHedIssue.fromHedIssues(splicingIssues, this.tsvFile.file, { tsvLine })) + return null } - return columnMap + return new BidsTsvRow(splicedParsedString, rowCells, this.tsvFile, tsvLine) } /** - * Update the map to splice-in the values for columns that have splices. - * @param { Map } columnMap - Map of column name to HED string for a row. + * Generate the column splice mapping for a BIDS TSV file row. * - * Note: Updates the map in place. + * @param {Map} rowCells The column-to-value mapping for a single row. + * @return {Map} A mapping of column names to their corresponding parsed sidecar strings. + * @private */ - spliceValues(columnMap) { - // Only iterate over the column names that have splices - for (const column of this.tsvFile.mergedSidecar.columnSpliceMapping.keys()) { - if (!columnMap.has(column)) { + _generateColumnSpliceMapping(rowCells) { + const columnSpliceMapping = new Map() + + for (const [columnName, columnValue] of rowCells.entries()) { + if (columnValue === 'n/a' || columnValue === '') { + columnSpliceMapping.set(columnName, null) continue } - const unspliced = columnMap.get(column) - const result = this._replaceSplices(unspliced, columnMap) - columnMap.set(column, result) + + const sidecarEntry = this.tsvFile.mergedSidecar.parsedHedData.get(columnName) + + if (sidecarEntry instanceof ParsedHedString) { + columnSpliceMapping.set(columnName, sidecarEntry) + } else if (sidecarEntry instanceof Map) { + columnSpliceMapping.set(columnName, sidecarEntry.get(columnValue)) + } } - } - /** - * Replace a HED string containing slices with a resolved version for the column value in a row. - * - * @param {string} unspliced - A HED string possibly with unresolved splices. - * @param {Map} columnMap - The map of column name to HED string for a row. - * @returns {string} - The fully resolved HED string with no splices. - * @private - */ - _replaceSplices(unspliced, columnMap) { - const result = unspliced.replace(BidsHedTsvParser.braceRegEx, (match, content) => { - // Resolve the replacement value - const resolved = columnMap.has(content) ? columnMap.get(content) : '' - // Replace with resolved value or empty string if in nullSet - return BidsHedTsvParser.nullSet.has(resolved) ? '' : resolved - }) - return this._spliceCleanup(result) + return columnSpliceMapping } /** - * Remove empty tags or groups which occur because of an empty splice. - * @param {string} spliced - The result of splice removal -- which could have empty tags or groups. - * @returns {string} - The string with empty tags or groups removed. + * Parse a sidecar column cell in a TSV file. + * + * @param {string} columnName The name of the column/sidecar key. + * @param {string} cellValue The value of the cell. + * @param {number} tsvLine The index of this row in the TSV file. + * @return {string|null} A HED substring, or null if none was found. * @private */ - _spliceCleanup(spliced) { - let result = spliced - - // Remove extra internal empty parentheses due to empty splices - while (BidsHedTsvParser.parenthesesRegEx.test(result)) { - result = result.replace(BidsHedTsvParser.parenthesesRegEx, '') + _parseRowCell(columnName, cellValue, tsvLine) { + if (!cellValue || cellValue === 'n/a') { + return null } - // Remove leading commas - result = result.replace(BidsHedTsvParser.leadingCommaRegEx, '') - - // Remove trailing commas - result = result.replace(BidsHedTsvParser.trailingCommaRegEx, '') - - // Remove extra empty commas due to empty splices - return result.replace(BidsHedTsvParser.internalCommaRegEx, ',') + if (columnName === 'HED') { + return cellValue + } + const sidecarHedData = this.tsvFile.sidecarHedData.get(columnName) + if (!sidecarHedData) { + return null + } + if (typeof sidecarHedData === 'string') { + return sidecarHedData.replace('#', cellValue) + } else { + const sidecarHedString = sidecarHedData[cellValue] + if (sidecarHedString !== undefined) { + return sidecarHedString + } + } + this.issues.push( + BidsHedIssue.fromHedIssue( + generateIssue('sidecarKeyMissing', { + key: cellValue, + column: columnName, + file: this.tsvFile.file.relativePath, + }), + this.tsvFile.file, + { tsvLine }, + ), + ) + return null } } diff --git a/common/issues/data.js b/common/issues/data.js index 083250e3..ae17c817 100644 --- a/common/issues/data.js +++ b/common/issues/data.js @@ -30,7 +30,7 @@ export default { duplicateTag: { hedCode: 'TAG_EXPRESSION_REPEATED', level: 'error', - message: stringTemplate`Duplicate tags - "${'tags'} in "${'string'}".`, + message: stringTemplate`Duplicate tag - "${'tag'}".`, }, invalidCharacter: { hedCode: 'CHARACTER_INVALID', @@ -88,6 +88,11 @@ export default { level: 'warning', message: stringTemplate`Tag with prefix "${'tagPrefix'}" is required.`, }, + unitClassDefaultUsed: { + hedCode: 'UNITS_MISSING', + level: 'warning', + message: stringTemplate`No unit specified. Using "${'defaultUnit'}" as the default - "${'tag'}".`, + }, unitClassInvalidUnit: { hedCode: 'UNITS_INVALID', level: 'error', @@ -103,41 +108,21 @@ export default { level: 'warning', message: stringTemplate`Tag extension found - "${'tag'}".`, }, - invalidPlaceholderContext: { - hedCode: 'PLACEHOLDER_INVALID', - level: 'error', - message: stringTemplate`"${'string'}" has "#" placeholders, which are not allowed in this context.`, - }, - invalidSidecarPlaceholder: { - hedCode: 'PLACEHOLDER_INVALID', - level: 'error', - message: stringTemplate`"${'string'}" of column "${'column'}" has an invalid # placeholder.`, - }, // HED 3-specific validation issues invalidPlaceholder: { hedCode: 'PLACEHOLDER_INVALID', level: 'error', - message: stringTemplate`Invalid # placeholder - "${'tag'}".`, + message: stringTemplate`Invalid placeholder - "${'tag'}".`, }, missingPlaceholder: { hedCode: 'PLACEHOLDER_INVALID', level: 'error', - message: stringTemplate`HED value string "${'string'}" is missing a required # placeholder for column "${'column'}".`, - }, - extraPlaceholder: { - hedCode: 'PLACEHOLDER_INVALID', - level: 'error', - message: stringTemplate`HED value string "${'string'}" has too many placeholders in column "${'column'}".`, + message: stringTemplate`HED value string "${'string'}" is missing a required placeholder.`, }, invalidPlaceholderInDefinition: { hedCode: 'DEFINITION_INVALID', level: 'error', - message: stringTemplate`Invalid placeholder or missing placeholder in definition - "${'definition'}".`, - }, - invalidDefinition: { - hedCode: 'DEFINITION_INVALID', - level: 'error', - message: stringTemplate`Invalid definition - "${'definition'}".`, + message: stringTemplate`Invalid placeholder in definition - "${'definition'}".`, }, nestedDefinition: { hedCode: 'DEFINITION_INVALID', @@ -154,26 +139,11 @@ export default { level: 'error', message: stringTemplate`Def-expand tag found for definition name "${'definition'}" does not correspond to an existing definition.`, }, - defExpandContentsInvalid: { - hedCode: 'DEF_EXPAND_INVALID', - level: 'error', - message: stringTemplate`Def-expand contents "${'contents'}" disagree with evaluated definition "${'contentsDef'}".`, - }, duplicateDefinition: { hedCode: 'DEFINITION_INVALID', level: 'error', message: stringTemplate`Definition "${'definition'}" is declared multiple times. This instance's tag group is "${'tagGroup'}".`, }, - conflictingDefinitions: { - hedCode: 'DEFINITION_INVALID', - level: 'error', - message: stringTemplate`Definition "${'definition1'}" and "${'definition2'}' conflict.`, - }, - duplicateDefinitionNames: { - hedCode: 'DEFINITION_INVALID', - level: 'error', - message: stringTemplate`Definition "${'definition1'}" and "${'definition2'}" have same name but are not equivalent.`, - }, multipleTagGroupsInDefinition: { hedCode: 'DEFINITION_INVALID', level: 'error', @@ -187,7 +157,7 @@ export default { illegalDefinitionContext: { hedCode: 'DEFINITION_INVALID', level: 'error', - message: stringTemplate`Definition "${'definition'}" was found in string "${'string'}" in a context where definitions are not allowed.`, + message: stringTemplate`Definitions were found in string "${'string'}" in a context where definitions are not allowed.`, }, illegalInExclusiveContext: { hedCode: 'TAG_INVALID', @@ -204,45 +174,25 @@ export default { level: 'error', message: stringTemplate`${'tag'} found without an included inner top-level tag group. This instance's tag group is "${'tagGroup'}".`, }, - temporalWithWrongNumberDefs: { + temporalWithMultipleDefinitions: { hedCode: 'TEMPORAL_TAG_ERROR', level: 'error', - message: stringTemplate`${'tag'} found in tag group "${'tagGroup'}" with the wrong number of Def tags and Def-expand groups.`, + message: stringTemplate`${'tag'} found with multiple included definitions. This instance's tag group is "${'tagGroup'}".`, }, temporalWithoutDefinition: { hedCode: 'TEMPORAL_TAG_ERROR', level: 'error', - message: stringTemplate`${'tag'} found in tag group "${'tagGroup'}" without an included definition.`, + message: stringTemplate`${'tag'} found without an included definition. This instance's tag group is "${'tagGroup'}".`, }, extraTagsInTemporal: { hedCode: 'TEMPORAL_TAG_ERROR', level: 'error', - message: stringTemplate`Extra top-level tags or tag groups found in onset, inset, or offset group "${'tagGroup'}" with definition "${'definition'}".`, - }, - temporalTagInNonTemporalContext: { - hedCode: 'TEMPORAL_TAG_ERROR', - level: 'error', - message: stringTemplate`HED event string "${'string'}" has temporal tags on line(s) [${'tsvline'}] in a tsv file without an onset time.`, + message: stringTemplate`Extra non-definition top-level tags or tag groups found in onset or offset group with definition "${'definition'}".`, }, duplicateTemporal: { hedCode: 'TEMPORAL_TAG_ERROR', level: 'error', - message: stringTemplate`HED event string "${'string'}" has onset/offset/inset tags with duplicated definition "${'definition'}".`, - }, - multipleTemporalTags: { - hedCode: 'TEMPORAL_TAG_ERROR', - level: 'error', - message: stringTemplate`HED event string "${'string'}" has multiple temporal tags ${'tags'} in the same group.`, - }, - multipleRequiresDefTags: { - hedCode: 'TEMPORAL_TAG_ERROR', - level: 'error', - message: stringTemplate`HED event string "${'string'}" has multiple temporal tags ${'tags'} that require a definition in the same group.`, - }, - simultaneousDuplicateEvents: { - hedCode: 'TEMPORAL_TAG_ERROR', - level: 'error', - message: stringTemplate`Temporal tag group "${'tagGroup1'}" at ${'onset1'} line ${'tsvLine1'} is simultaneous with "${'tagGroup2'}" at ${'onset2'} line ${'tsvLine2'}.`, + message: stringTemplate`HED event string "${'string'}" has onset/offset tags with duplicated definition "${'definition'}".`, }, missingTagGroup: { hedCode: 'TAG_GROUP_ERROR', @@ -254,11 +204,6 @@ export default { level: 'error', message: stringTemplate`"${'tagGroup'}" has invalid group tags or invalid number of subgroups.`, }, - forbiddenSubgroupTags: { - hedCode: 'TAG_GROUP_ERROR', - level: 'error', - message: stringTemplate`Tag "${'tag'}" in "${'string'}" cannot have tags "${'tagList'}" in a subgroup.`, - }, invalidTopLevelTagGroupTag: { hedCode: 'TAG_GROUP_ERROR', level: 'error', @@ -274,11 +219,6 @@ export default { level: 'error', message: stringTemplate`Tags "${'tags'}" cannot be at the same level in group "${'string'}".`, }, - tooManyGroupTopTags: { - hedCode: 'TAG_GROUP_ERROR', - level: 'error', - message: stringTemplate`Group "${'string'}" has too many or too few tags at the top level.`, - }, multipleTopLevelTagGroupTags: { hedCode: 'TAG_GROUP_ERROR', level: 'error', @@ -287,17 +227,17 @@ export default { invalidNumberOfSubgroups: { hedCode: 'TAG_GROUP_ERROR', level: 'error', - message: stringTemplate`The tag "${'tag'} is in a "${'string'} with too many or too few subgroups.`, + message: stringTemplate`The tags "${'tags'} in "${'string'} require groups that do not agree or are not present in their group .`, }, invalidTopLevelTag: { hedCode: 'TAG_GROUP_ERROR', level: 'error', - message: stringTemplate`Tag(s) "${'tag'}" should be in a top group in "${'string'}".`, + message: stringTemplate`Tag "${'tag'}" is only allowed inside of a tag group.`, }, invalidGroupTag: { hedCode: 'TAG_GROUP_ERROR', level: 'error', - message: stringTemplate`Tag(s) "${'tag'}" should be in a group in "${'string'}".`, + message: stringTemplate`Tag "${'tag'}" should be in a group in "${'string'}" but is not.`, }, // Tag conversion issues invalidParentNode: { @@ -354,7 +294,7 @@ export default { curlyBracesInHedColumn: { hedCode: 'CHARACTER_INVALID', level: 'error', - message: stringTemplate`Curly brace expression "${'string'}" found in the HED column of a TSV file.`, + message: stringTemplate`Curly brace expression "${'column'}" found in the HED column of a TSV file.`, }, curlyBracesNotAllowed: { hedCode: 'CHARACTER_INVALID', diff --git a/common/issues/issues.js b/common/issues/issues.js index 7330a528..a54e1bf2 100644 --- a/common/issues/issues.js +++ b/common/issues/issues.js @@ -46,7 +46,14 @@ export class Issue { * @type {string} */ internalCode - + /** + * Also the internal error code. + * + * TODO: This is kept for backward compatibility until the next major version bump. + * @deprecated + * @type {string} + */ + code /** * The HED 3 error code. * @type {string} @@ -77,6 +84,7 @@ export class Issue { */ constructor(internalCode, hedCode, level, parameters) { this.internalCode = internalCode + this.code = internalCode this.hedCode = hedCode this.level = level this.parameters = parameters diff --git a/converter/converter.js b/converter/converter.js new file mode 100644 index 00000000..6c21c56f --- /dev/null +++ b/converter/converter.js @@ -0,0 +1,43 @@ +import { parseHedString } from '../parser/parser' + +/** + * Convert a HED string. + * + * @param {Schemas} hedSchemas The HED schema collection. + * @param {string} hedString The HED tag to convert. + * @param {boolean} long Whether the tags should be in long form. + * @returns {[string, Issue[]]} The converted string and any issues. + */ +const convertHedString = function (hedSchemas, hedString, long) { + const [parsedString, issues] = parseHedString(hedString, hedSchemas) + const flattenedIssues = Object.values(issues).flat() + if (flattenedIssues.some((issue) => issue.level === 'error')) { + return [hedString, flattenedIssues] + } + const convertedString = parsedString.format(long) + return [convertedString, flattenedIssues] +} + +/** + * Convert a HED string to long form. + * + * @param {Schemas} schemas The schema container object containing short-to-long mappings. + * @param {string} hedString The HED tag to convert. + * @returns {[string, Issue[]]} The long-form string and any issues. + * @deprecated Replaced with {@link ParsedHedString}. Will be removed in version 4.0.0 or earlier. + */ +export const convertHedStringToLong = function (schemas, hedString) { + return convertHedString(schemas, hedString, true) +} + +/** + * Convert a HED string to short form. + * + * @param {Schemas} schemas The schema container object containing short-to-long mappings. + * @param {string} hedString The HED tag to convert. + * @returns {[string, Issue[]]} The short-form string and any issues. + * @deprecated Replaced with {@link ParsedHedString}. Will be removed in version 4.0.0 or earlier. + */ +export const convertHedStringToShort = function (schemas, hedString) { + return convertHedString(schemas, hedString, false) +} diff --git a/converter/index.js b/converter/index.js new file mode 100644 index 00000000..f6278a35 --- /dev/null +++ b/converter/index.js @@ -0,0 +1,8 @@ +import { convertHedStringToLong, convertHedStringToShort } from './converter' + +export { convertHedStringToLong, convertHedStringToShort } + +export default { + convertHedStringToLong, + convertHedStringToShort, +} diff --git a/data/json/reservedTags.json b/data/json/specialTags.json similarity index 74% rename from data/json/reservedTags.json rename to data/json/specialTags.json index 614c68d5..2f8c8412 100644 --- a/data/json/reservedTags.json +++ b/data/json/specialTags.json @@ -8,14 +8,14 @@ "exclusive": false, "tagGroup": false, "topLevelTagGroup": false, - "maxNonDefSubgroups": null, - "minNonDefSubgroups": null, + "maxNonDefSubgroups": -1, + "minNonDefSubgroups": -1, "ERROR_CODE": "DEF_INVALID", "noSpliceInGroup": false, "forbiddenSubgroupTags": [], - "isTemporalTag": false, - "requiresDef": false, - "otherAllowedNonDefTags": null + "defTagRequired": false, + "otherAllowedTags": null, + "otherAllowedGroups": null }, "Def-expand": { "name": "Def-expand", @@ -30,10 +30,10 @@ "minNonDefSubgroups": 0, "ERROR_CODE": "DEF_EXPAND_INVALID", "noSpliceInGroup": true, - "forbiddenSubgroupTags": ["Def", "Def-expand"], - "isTemporalTag": false, - "requiresDef": false, - "otherAllowedNonDefTags": [] + "forbiddenSubgroupTags": ["Def", "Def-expand", "Definition"], + "defTagRequired": false, + "otherAllowedTags": [], + "otherAllowedGroups": [] }, "Definition": { "name": "Definition", @@ -48,10 +48,10 @@ "minNonDefSubgroups": 0, "ERROR_CODE": "DEFINITION_INVALID", "noSpliceInGroup": true, - "forbiddenSubgroupTags": ["Def", "Def-expand"], - "isTemporalTag": false, - "requiresDef": false, - "otherAllowedNonDefTags": [] + "forbiddenSubgroupTags": ["Def", "Def-expand", "Definition"], + "defTagRequired": false, + "otherAllowedTags": [], + "otherAllowedGroups": [] }, "Delay": { "name": "Delay", @@ -67,9 +67,9 @@ "ERROR_CODE": "TEMPORAL_TAG_ERROR", "noSpliceInGroup": false, "forbiddenSubgroupTags": [], - "isTemporalTag": true, - "requiresDef": false, - "otherAllowedNonDefTags": ["Duration", "Onset", "Offset", "Inset"] + "defTagRequired": false, + "otherAllowedTags": ["Duration", "Onset", "Offset", "Inset", "Def"], + "otherAllowedGroups": ["Def-expand"] }, "Duration": { "name": "Duration", @@ -85,9 +85,9 @@ "ERROR_CODE": "TEMPORAL_TAG_ERROR", "noSpliceInGroup": false, "forbiddenSubgroupTags": [], - "isTemporalTag": true, - "requiresDef": false, - "otherAllowedNonDefTags": ["Delay"] + "defTagRequired": false, + "otherAllowedTags": ["Delay"], + "otherAllowedGroups": [] }, "Event-context": { "name": "Event-context", @@ -103,9 +103,9 @@ "ERROR_CODE": "TAG_GROUP_ERROR", "noSpliceInGroup": true, "forbiddenSubgroupTags": [], - "isTemporalTag": false, - "requiresDef": false, - "otherAllowedNonDefTags": [] + "defTagRequired": false, + "otherAllowedTags": [], + "otherAllowedGroups": [] }, "Inset": { "name": "Inset", @@ -121,9 +121,9 @@ "ERROR_CODE": "TEMPORAL_TAG_ERROR", "noSpliceInGroup": false, "forbiddenSubgroupTags": [], - "isTemporalTag": true, - "requiresDef": true, - "otherAllowedNonDefTags": ["Delay"] + "defTagRequired": true, + "otherAllowedTags": ["Def", "Delay"], + "otherAllowedGroups": ["Def-expand"] }, "Offset": { "name": "Offset", @@ -134,14 +134,14 @@ "exclusive": false, "tagGroup": true, "topLevelTagGroup": true, - "maxNonDefSubgroups": 0, + "maxNonDefSubgroups": 1, "minNonDefSubgroups": 0, "ERROR_CODE": "TEMPORAL_TAG_ERROR", "noSpliceInGroup": false, "forbiddenSubgroupTags": [], - "isTemporalTag": true, - "requiresDef": true, - "otherAllowedNonDefTags": ["Def", "Delay"] + "defTagRequired": true, + "otherAllowedTags": ["Def", "Delay"], + "otherAllowedGroups": ["Def-expand"] }, "Onset": { "name": "Onset", @@ -157,8 +157,8 @@ "ERROR_CODE": "TEMPORAL_TAG_ERROR", "noSpliceInGroup": false, "forbiddenSubgroupTags": [], - "isTemporalTag": true, - "requiresDef": true, - "otherAllowedNonDefTags": ["Delay"] + "defTagRequired": true, + "otherAllowedTags": ["Def", "Delay"], + "otherAllowedGroups": ["Def-expand"] } } diff --git a/eventManager/columnSplicer.js b/eventManager/columnSplicer.js deleted file mode 100644 index d4744a7b..00000000 --- a/eventManager/columnSplicer.js +++ /dev/null @@ -1,211 +0,0 @@ -import ParsedHedString from './parsedHedString' -import ParsedHedColumnSplice from './parsedHedColumnSplice' -import ParsedHedGroup from './parsedHedGroup' -import { generateIssue } from '../common/issues/issues' -import { parseHedString } from './parser' - -export class ColumnSplicer { - /** - * The parsed HED string in which to make column splices. - * @type {ParsedHedString} - */ - parsedString - /** - * A mapping from column names to their replacement strings. - * @type {Map} - */ - columnReplacements - /** - * A mapping from column names to their values. - * @type {Map} - */ - columnValues - /** - * The active HED schema collection (passed to the {@link ParsedHedGroup} constructor). - * @type {Schemas} - */ - hedSchemas - /** - * Any issues found while splicing the columns. - * @type {Issue[]} - */ - issues - - /** - * Substitute replacement strings for column splice templates. - * - * @param {ParsedHedString} parsedString The parsed HED string in which to make column splices. - * @param {Map} columnReplacements A mapping from column names to their replacement strings. - * @param {Map} columnValues A mapping from column names to their values. - * @param {Schemas} hedSchemas The active HED schema collection (passed to the {@link ParsedHedGroup} constructor). - */ - constructor(parsedString, columnReplacements, columnValues, hedSchemas) { - this.parsedString = parsedString - this.columnReplacements = columnReplacements - this.columnValues = columnValues - this.hedSchemas = hedSchemas - this.issues = [] - } - - /** - * Substitute replacement strings for column splice templates. - * - * @returns {ParsedHedString} A new parsed HED string with the replacements made. - */ - splice() { - const originalData = this.parsedString.parseTree - const newData = this._spliceSubstrings(originalData) - return new ParsedHedString(this.parsedString.hedString, newData) - } - - /** - * Splice replacement strings into a list of substrings. - * - * @param {ParsedHedSubstring[]} substrings The parsed HED substrings in which to make column splices. - * @returns {ParsedHedSubstring[]} A new list of parsed HED substrings with the replacements made. - * @private - */ - _spliceSubstrings(substrings) { - const newData = [] - for (const substring of substrings) { - newData.push(...this._spliceSubstring(substring)) - } - return newData - } - - /** - * Splice replacement strings in place of a single substring. - * - * @param {ParsedHedSubstring} substring The parsed HED substring in which to make column splices. - * @returns {ParsedHedSubstring[]} A new list of parsed HED substrings with the replacements made. - * @private - */ - _spliceSubstring(substring) { - const newData = [] - if (substring instanceof ParsedHedColumnSplice) { - const substitution = this._spliceTemplate(substring) - if (substitution === null) { - return [] - } - newData.push(...substitution) - if (substitution.length === 0) { - newData.push(substring) - } - } else if (substring instanceof ParsedHedGroup) { - const substitution = this._spliceGroup(substring) - if (substitution !== null) { - newData.push(substitution) - } - } else { - newData.push(substring) - } - return newData - } - - /** - * Splice a replacement string in place of a column template. - * - * @param {ParsedHedColumnSplice} columnTemplate The parsed HED column splice template in which to make the column splice. - * @returns {ParsedHedSubstring[]|null} The spliced column substitution. - * @private - */ - _spliceTemplate(columnTemplate) { - const columnName = columnTemplate.originalTag - - // HED column handled specially - if (columnName === 'HED') { - return this._spliceHedColumnTemplate() - } - - // Not the HED column so treat as usual - const replacementString = this.columnReplacements.get(columnName) - - // Handle null or undefined replacement strings - if (replacementString === null) { - return null - } - if (replacementString === undefined) { - this.issues.push(generateIssue('undefinedCurlyBraces', { column: columnName })) - return [] - } - - // Handle recursive curly braces - if (replacementString.columnSplices.length > 0) { - this.issues.push(generateIssue('recursiveCurlyBraces', { column: columnName })) - return [] - } - - // Handle value templates with placeholder - const tagListHasPlaceholder = replacementString.tags.some((tag) => tag.originalTagName === '#') - if (tagListHasPlaceholder) { - return this._spliceValueTemplate(columnTemplate) - } - - // Default case - return replacementString.parseTree - } - - /** - * Splice a "HED" column value string in place of a column template. - * - * @returns {ParsedHedSubstring[]} The spliced column substitution. - * @private - */ - _spliceHedColumnTemplate() { - const columnName = 'HED' - const replacementString = this.columnValues.get(columnName) - const blankHedColumnValues = new Set([undefined, null, 'n/a', '']) - if (blankHedColumnValues.has(replacementString)) { - return null - } - - return this._reparseAndSpliceString(replacementString) - } - - /** - * Splice a value-taking replacement string in place of a column template. - * - * @param {ParsedHedColumnSplice} columnTemplate The parsed HED column splice template in which to make the column splice. - * @returns {ParsedHedSubstring[]} The spliced column substitution. - * @private - */ - _spliceValueTemplate(columnTemplate) { - const columnName = columnTemplate.originalTag - const replacementString = this.columnReplacements.get(columnName) - const replacedString = replacementString.hedString.replace('#', this.columnValues.get(columnName)) - return this._reparseAndSpliceString(replacedString) - } - - /** - * Re-parse a string to use in splicing. - * - * @param {string} replacementString A new string to parse. - * @returns {ParsedHedSubstring[]} The new string's parse tree. - * @private - */ - _reparseAndSpliceString(replacementString) { - const [newParsedString, parsingIssues] = parseHedString(replacementString, this.hedSchemas, true, false, false) - if (parsingIssues.length > 0) { - this.issues.push(...parsingIssues) - return [] - } - return newParsedString.parseTree - } - - /** - * Splice replacement strings into a parsed HED tag group. - * - * @param {ParsedHedGroup} group The parsed HED group in which to make column splices. - * @returns {ParsedHedGroup|null} A new parsed HED group with the replacements made. - * @private - */ - _spliceGroup(group) { - const newData = this._spliceSubstrings(group.tags) - if (newData.length === 0) { - return null - } - return new ParsedHedGroup(newData, this.parsedString.hedString, group.originalBounds) - } -} - -export default ColumnSplicer diff --git a/eventManager/definitionManager.js b/eventManager/definitionManager.js deleted file mode 100644 index 39d3a887..00000000 --- a/eventManager/definitionManager.js +++ /dev/null @@ -1,303 +0,0 @@ -import { generateIssue, IssueError } from '../common/issues/issues' -import { parseHedString } from './parser' -//import { filterNonEqualDuplicates } from './parseUtils' -import { filterByTagName } from './parseUtils' - -export class Definition { - /** - * The name of the definition. - * @type {string} - */ - name - - /** - * The name of the definition. - * @type {ParsedHedTag} - */ - defTag - - /** - * The parsed HED group representing the definition - * @type {ParsedHedGroup} - */ - defGroup - - /** - * The definition contents group - * @type {ParsedHedGroup} - */ - defContents - - placeholder - - /** - * A single definition - * - * @param {ParsedHedGroup} definitionGroup - the parsedHedGroup representing the definition. - */ - constructor(definitionGroup) { - this.defGroup = definitionGroup - this._initializeDefinition(definitionGroup) - } - - _initializeDefinition(definitionGroup) { - if (definitionGroup.topTags?.length !== 1 || definitionGroup.topGroups?.length > 1) { - IssueError.generateAndThrow('invalidDefinition', { definition: definitionGroup.originalTag }) - } - this.defTag = definitionGroup.topTags[0] - this.name = this.defTag._value - this.placeholder = this.defTag._splitValue - this.defContents = this.defGroup.topGroups.length > 0 ? this.defGroup.topGroups[0] : null - } - - /** - * Return the evaluated definition contents and any issues. - * @param {ParsedHedTag} tag - The parsed HEd tag whose details should be checked. - * @param {Schemas} hedSchema - The HED schemas used to validate against. - * @param {boolean} placeholderAllowed - If true then placeholder is allowed in the def tag. - * @returns {[string | null, Issue[]]} - The evaluated normalized definition string and any issues in the evaluation, - */ - evaluateDefinition(tag, hedSchema, placeholderAllowed) { - // Check that the level of the value of tag agrees with the definition - if (!!this.defTag._splitValue !== !!tag._splitValue) { - const errorType = tag.schemaTag.name === 'Def' ? 'missingDefinitionForDef' : 'missingDefinitionForDefExpand' - return [null, [generateIssue(errorType, { definition: tag._value })]] - } - // Check that the evaluated definition contents okay (if two-level value) - if (!this.defContents) { - return ['', []] - } - if (!this.defTag._splitValue || (placeholderAllowed && tag._splitValue === '#')) { - return [this.defContents.normalized, []] - } - const evalString = this.defContents.originalTag.replace('#', tag._splitValue) - const [normalizedValue, issues] = parseHedString(evalString, hedSchema, true, false, false) - if (issues.length > 0) { - return [null, issues] - } - return [normalizedValue.normalized, []] - } - - /** - * Return true if this definition is the same as the other - * @param other - * @returns {boolean} - */ - - equivalent(other) { - if (this.name !== other.name || this.defTag._splitValue !== other.defTag._splitValue) { - return false - } else if (this.defContents?.normalized !== other.defContents?.normalized) { - return false - } - return true - } - - _checkDefinitionPlaceholderCount() { - const placeholderCount = this.defContents ? this.defContents.originalTag.split('#').length - 1 : 0 - return !((placeholderCount !== 1 && this.placeholder) || (placeholderCount !== 0 && !this.placeholder)) - } - - /** - * Create a list of Definition objects from a list of strings - * - * @param {string} hedString - A list of string definitions - * @param {Schemas} hedSchemas - The HED schemas to use in creation - * @returns { [Definition[], Issue[]]} - The Definition list and any issues found - */ - static createDefinition(hedString, hedSchemas) { - const [parsedString, issues] = parseHedString(hedString, hedSchemas, true, true, true) - if (issues.length > 0) { - return [null, issues] - } - if (parsedString.topLevelTags.length !== 0 || parsedString.tagGroups.length > 1) { - return [null, [generateIssue('invalidDefinition', { definition: hedString })]] - } - return Definition.createDefinitionFromGroup(parsedString.tagGroups[0]) - } - - static createDefinitionFromGroup(group) { - const def = new Definition(group) - if (def._checkDefinitionPlaceholderCount()) { - return [def, []] - } - return [null, [generateIssue('invalidPlaceholderInDefinition', { definition: def.defGroup.originalTag })]] - } -} - -export class DefinitionManager { - /** - * @type { Map} -definitions for this manager - */ - definitions - - constructor() { - this.definitions = new Map() - } - - /** - * Add the non-null items to this manager - * @param {Definition[]} defs - The list of items to be added - */ - addDefinitions(defs) { - const issues = [] - for (const def of defs) { - issues.push(...this.addDefinition(def)) - } - return issues - } - - /** - * Add a Definition object to this manager - * @param {Definition} definition - The definition to be added. - * @returns {Issue[]} - */ - addDefinition(definition) { - const lowerName = definition.name.toLowerCase() - const existingDefinition = this.definitions.get(lowerName) - if (existingDefinition && !existingDefinition.equivalent(definition)) { - return [ - generateIssue('conflictingDefinitions', { - definition1: definition.defTag.originalTag, - definition2: existingDefinition.defGroup.originalTag, - }), - ] - } - if (!existingDefinition) { - this.definitions.set(lowerName, definition) - } - return [] - } - - /** - * Check the Def tags in a HED string for missing or incorrectly used Def tags. - * @param {ParsedHedString} hedString - A parsed HED string to be checked. - * @param {Schemas} hedSchemas - Schemas to validate against. - * @param {boolean} placeholderAllowed - If true then placeholder is allowed in the def tag. - * @returns {Issue []} - If there is no matching definition or definition applied incorrectly. - */ - validateDefs(hedString, hedSchemas, placeholderAllowed) { - const defTags = filterByTagName(hedString.tags, 'Def') - const issues = [] - for (const tag of defTags) { - const defIssues = this.evaluateTag(tag, hedSchemas, placeholderAllowed)[1] - if (defIssues.length > 0) { - issues.push(...defIssues) - } - } - return issues - } - - /** - * Check the Def tags in a HED string for missing or incorrectly used Def-expand tags. - * @param {ParsedHedString} hedString - A parsed HED string to be checked. - * @param {Schemas} hedSchemas - Schemas to validate against. - * @param {boolean} placeholderAllowed - If true then placeholder is allowed in the def tag. - * @returns {Issue []} - If there is no matching definition or definition applied incorrectly. - */ - validateDefExpands(hedString, hedSchemas, placeholderAllowed) { - //Def-expand tags should be rare, so don't look if there aren't any Def-expand tags - const defExpandTags = filterByTagName(hedString.tags, 'Def-expand') - if (defExpandTags.length === 0) { - return [] - } - const issues = [] - for (const topGroup of hedString.tagGroups) { - issues.push(...this._checkDefExpandGroup(topGroup, hedSchemas, placeholderAllowed)) - } - return issues - } - - /** - * Evaluate the definition based on a parsed HED tag - * @param {ParsedHedTag} tag - The tag to evaluate against the definitions. - * @param {Schemas} hedSchemas - The schemas to be used to assist in the evaluation. - * @param {boolean} placeholderAllowed - If true then placeholder is allowed in the def tag. - * @returns {[string, Issue[]]} - The evaluated definition contents with this tag and any issues. - * - * Note: If the tag is not a Def or Def-expand, this returns null for the string and [] for the issues. - */ - evaluateTag(tag, hedSchemas, placeholderAllowed) { - const [definition, missingIssues] = this.findDefinition(tag) - if (missingIssues.length > 0) { - return [null, missingIssues] - } else if (definition) { - return definition.evaluateDefinition(tag, hedSchemas, placeholderAllowed) - } - return [null, []] - } - - /** - * Recursively check for Def-expand groups in this group. - * @param {ParsedHedGroup} topGroup - a top group in a HED string to be evaluated for Def-expand groups. - * @param {Schemas} hedSchemas - The HED schemas to used in the check. - * @param {boolean} placeholderAllowed - If true then placeholder is allowed in the def tag. - * @returns {Issue[]} - * @private - */ - _checkDefExpandGroup(topGroup, hedSchemas, placeholderAllowed) { - const issues = [] - for (const group of topGroup.subParsedGroupIterator('Def-expand')) { - const defExpandTags = group.getSpecial('Def-expand') - if (defExpandTags.length === 0) { - continue - } - // There should be only one Def-expand in this group as special requirements have been checked at parsing time. - const [normalizedValue, normalizedIssues] = this.evaluateTag(defExpandTags[0], hedSchemas, placeholderAllowed) - issues.push(...normalizedIssues) - if (normalizedIssues.length > 0) { - continue - } - if (group.topGroups.length === 0 && normalizedValue !== '') { - issues.push(generateIssue('defExpandContentsInvalid', { contents: '', defContents: normalizedValue })) - } else if (group.topGroups.length > 0 && group.topGroups[0].normalized !== normalizedValue) { - issues.push( - generateIssue('defExpandContentsInvalid', { - contents: group.topGroups[0].normalized, - defContents: normalizedValue, - }), - ) - } - } - return issues - } - - /** - * Find the definition associated with a tag, if any - * @param {ParsedHedTag} tag - The parsed HEd tag to be checked. - * @returns {[Definition | null, Issue []]} - If there is no matching definition. - */ - findDefinition(tag) { - if (tag.schemaTag._name !== 'Def' && tag.schemaTag.name !== 'Def-expand') { - return [null, []] - } - const name = tag._value.toLowerCase() - const existingDefinition = this.definitions.get(name) - const errorType = tag.schemaTag.name === 'Def' ? 'missingDefinitionForDef' : 'missingDefinitionForDefExpand' - if (!existingDefinition) { - return [null, [generateIssue(errorType, { definition: name })]] - } - if (!!existingDefinition.defTag._splitValue !== !!tag._splitValue) { - return [null, [generateIssue(errorType, { definition: name })]] - } - return [existingDefinition, []] - } - - /** - * Create a list of Definition objects from a list of strings. - * - * @param {string[]} defStrings - A list of string definitions. - * @param {Schemas} hedSchemas - The HED schemas to use in creation. - * @returns { [Definition[], Issue[]]} - The Definition list and any issues found. - */ - static createDefinitions(defStrings, hedSchemas) { - const defList = [] - const issues = [] - for (const defString of defStrings) { - const [nextDef, defIssues] = Definition.createDefinition(defString, hedSchemas) - defList.push(nextDef) - issues.push(...defIssues) - } - return [defList, issues] - } -} diff --git a/eventManager/parseUtils.js b/eventManager/parseUtils.js deleted file mode 100644 index ea9e4a0d..00000000 --- a/eventManager/parseUtils.js +++ /dev/null @@ -1,107 +0,0 @@ -import ParsedHedTag from './parsedHedTag' - -/** - * Extract the items of a specified subtype from a list of ParsedHedSubstring - * @param {ParsedHedSubstring[]} items - to be filtered by class type - * @param {Class} classType - class type to filter by - * @returns {*|*[]} - */ - -export function filterByClass(items, classType) { - return items && items.length ? items.filter((item) => item instanceof classType) : [] -} - -/** - * Extract the ParsedHedTag tags with a specified tag name - * @param {ParsedHedTag[]} tags - to be filtered by name - * @param {string} tagName - name of the tag to filter by - * @returns {ParsedHedTag[]} - */ - -export function filterByTagName(tags, tagName) { - if (!tags) { - return [] - } - return tags.filter((tag) => tag instanceof ParsedHedTag && tag.schemaTag?.name === tagName) -} - -/** - * Extract the ParsedHedTag tags with a specified tag name. - * @param {Map} tagMap - The Map of parsed HED tags for extraction (must be defined). - * @param {string[]} tagNames - The names to use as keys for the filter. - * @returns {ParsedHedTag[]} - A list of temporal tags. - */ -export function filterTagMapByNames(tagMap, tagNames) { - if (!tagNames || tagMap.size === 0) { - return [] - } - - const keys = [...tagNames].filter((name) => tagMap.has(name)) - if (keys.length === 0) { - return [] - } - - return keys.flatMap((key) => tagMap.get(key)) -} - -/*/!** - * Extract the ParsedHedTag tags that have a name from a specified list of names - * @param {ParsedHedTag[]} tags - to be filtered by name - * @param {[string]} tagList - List of tag names to filter by. - * @returns {ParsedHedTag[]} - A list of tags whose - *!/ - -export function filterByTagNames(tags, tagList) { - if (!tags || !tagList) { - return [] - } - return tags.filter((tag) => tagList.includes(tag.schemaTag.name)) -}*/ - -/** - * Convert a list of ParsedHedTag objects into a comma-separated string of their string representations. - * @param {ParsedHedTag []} tagList - The HED tags whose string representations should be put in a comma-separated list. - * @returns {string} A comma separated list of original tag names for tags in tagList. - */ -export function getTagListString(tagList) { - return tagList.map((tag) => tag.toString()).join(', ') -} - -/** - * Create a map of the ParsedHedTags by type - * @param { ParsedHedTag[] } tagList - * @param {Set} tagNames - * @returns {Map} - */ -export function categorizeTagsByName(tagList, tagNames = null) { - // Initialize the map with keys from tagNames and an "other" key - const resultMap = new Map() - - // Iterate through A and categorize - tagList.forEach((tag) => { - if (!tagNames || tagNames.has(tag.schemaTag.name)) { - const tagList = resultMap.get(tag.schemaTag.name) || [] - tagList.push(tag) - resultMap.set(tag.schemaTag.name, tagList) // Add to matching key list - } - }) - return resultMap -} - -/** - * Return a list of duplicate strings - * @param { string[] } itemList - A list of strings to look for duplicates in - * @returns {string []} - a list of unique duplicate strings (multiple copies not repeated - */ -export function getDuplicates(itemList) { - const checkSet = new Set() - const dupSet = new Set() - for (const item of itemList) { - if (!checkSet.has(item)) { - checkSet.add(item) - } else { - dupSet.add(item) - } - } - return [...dupSet] -} diff --git a/eventManager/parsedHedColumnSplice.js b/eventManager/parsedHedColumnSplice.js deleted file mode 100644 index ef76a726..00000000 --- a/eventManager/parsedHedColumnSplice.js +++ /dev/null @@ -1,40 +0,0 @@ -import ParsedHedSubstring from './parsedHedSubstring' - -/** - * A template for an inline column splice in a {@link ParsedHedString} or {@link ParsedHedGroup}. - */ -export class ParsedHedColumnSplice extends ParsedHedSubstring { - /** - * Constructor. - * - * @param {string} columnName The token for this tag. - * @param {number[]} bounds The collection of HED schemas. - */ - constructor(columnName, bounds) { - super(columnName, bounds) // Sets originalTag and originalBounds - this._normalized = this.format(false) // Sets various forms of the tag. - } - - get normalized() { - { - return this._normalized - } - } - - /** - * Nicely format this column splice template. - * - * @param {boolean} long Whether the tags should be in long form. - * @returns {string} - */ - // eslint-disable-next-line no-unused-vars - format(long = true) { - return '{' + this.originalTag + '}' - } - - equivalent(other) { - return other instanceof ParsedHedColumnSplice && this.originalTag === other.originalTag - } -} - -export default ParsedHedColumnSplice diff --git a/eventManager/parsedHedGroup.js b/eventManager/parsedHedGroup.js deleted file mode 100644 index b0367ac5..00000000 --- a/eventManager/parsedHedGroup.js +++ /dev/null @@ -1,307 +0,0 @@ -import differenceWith from 'lodash/differenceWith' - -import { IssueError } from '../common/issues/issues' -import ParsedHedSubstring from './parsedHedSubstring' -import ParsedHedTag from './parsedHedTag' -import ParsedHedColumnSplice from './parsedHedColumnSplice' -import { SpecialChecker } from './special' -import { - filterByClass, - categorizeTagsByName, - getDuplicates, - filterByTagName, - filterTagMapByNames, - getTagListString, -} from './parseUtils' - -/** - * A parsed HED tag group. - */ -export default class ParsedHedGroup extends ParsedHedSubstring { - /** - * The parsed HED tags or parsedHedGroups or parsedColumnSplices in the HED tag group at the top level - * @type {ParsedHedSubstring[]} - */ - tags - - topTags - - topGroups - - topSplices - - allTags - /** - * Any HED tags with special handling. This only covers top-level tags in the group - * @type {Map} - */ - specialTags - /** - * Whether this HED tag group has child groups with a Def-expand tag. - * @type {boolean} - */ - hasDefExpandChildren - - /** - * The top-level child subgroups containing Def-expand tags. - * @type {ParsedHedGroup[]} - */ - defExpandChildren - - isDefExpandGroup - - isDefinitionGroup - - defCount - - requiresDefTag - - /** - * Constructor. - * @param {ParsedHedSubstring[]} parsedHedTags The parsed HED tags, groups or column splices in the HED tag group. - * @param {string} hedString The original HED string. - * @param {number[]} originalBounds The bounds of the HED tag in the original HED string. - */ - constructor(parsedHedTags, hedString, originalBounds) { - const originalTag = hedString.substring(originalBounds[0], originalBounds[1]) - super(originalTag, originalBounds) - this.tags = parsedHedTags - this.topGroups = filterByClass(parsedHedTags, ParsedHedGroup) - this.topTags = filterByClass(parsedHedTags, ParsedHedTag) - this.topSplices = filterByClass(parsedHedTags, ParsedHedColumnSplice) - this.allTags = this._getAllTags() - this._normalized = undefined - this._initializeGroups() - } - - _getAllTags() { - const subgroupTags = this.topGroups.flatMap((tagGroup) => tagGroup.allTags) - return this.topTags.concat(subgroupTags) - } - - _initializeGroups() { - const special = SpecialChecker.getInstance() - this.specialTags = categorizeTagsByName(this.topTags, special.specialNames) - this.isDefExpandGroup = this.specialTags.has('Def-expand') - this.isDefinitionGroup = this.specialTags.has('Definition') - this.defExpandChildren = this._filterSubgroupsByTagName('Def-expand') - this.hasDefExpandChildren = this.defExpandChildren.length !== 0 - this.defCount = this.getSpecial('Def').length + this.defExpandChildren.length - this.requiresDefTag = this._getRequiresDefTag(special.requiresDefTags) - } - - /** - * Filter top subgroups that include a special at the top-level of the group - * - * @param {string} tagName - The schemaTag name to filter by. - * @returns {Array} - Array of subgroups containing the specified tag. - */ - _filterSubgroupsByTagName(tagName) { - return Array.from(this.topLevelGroupIterator()).filter((subgroup) => subgroup.specialTags.has(tagName)) - } - - /** - * Return the unique requiresDef tag associated with this group (if any). - * @param {string[]} tagNames - The list of requiresDef tag names to use (based on the special tag requirements). - * @returns {ParsedHedTag | null} - The parsed requiresDef tag (if any) or null. - * @throws {IssueError} - If there are too many or too few defs or too many requiresDef tags in this group. - * @private - */ - _getRequiresDefTag(tagNames) { - const requiresDefTags = filterTagMapByNames(this.specialTags, tagNames) - if (requiresDefTags.length > 1) { - IssueError.generateAndThrow('multipleRequiresDefTags', { - tags: getTagListString(requiresDefTags), - string: this.originalTag, - }) - } - if (requiresDefTags.length === 0) { - return null - } - if (this.defCount > 1) { - return [ - IssueError.generateAndThrow('temporalWithWrongNumberDefs', { - tag: requiresDefTags[0].originalTag, - tagGroup: this.originalTag, - }), - ] - } - if (this.topSplices.length === 0 && this.defCount === 0) { - return [ - IssueError.generateAndThrow('temporalWithWrongNumberDefs', { - tag: requiresDefTags[0].originalTag, - tagGroup: this.originalTag, - }), - ] - } - return requiresDefTags[0] - } - - /** - * Nicely format this tag group. - * - * @param {boolean} long Whether the tags should be in long form. - * @returns {string} - */ - format(long = true) { - return '(' + this.tags.map((substring) => substring.format(long)).join(', ') + ')' - } - - getSpecial(tagName) { - return this.specialTags.get(tagName) ?? [] - } - - isSpecialGroup(tagName) { - return this.specialTags.has(tagName) - } - - /** - * Whether this HED tag group is an onset, offset, or inset group. - * @returns {boolean} - */ - get isTemporalGroup() { - return this.isSpecialGroup('Onset') || this.isSpecialGroup('Offset') || this.isSpecialGroup('Inset') - } - - /** - * Whether this HED tag group is an onset, offset, or inset group. - * @returns {string} - */ - get temporalGroupName() { - if (this.isSpecialGroup('Onset')) { - return 'Onset' - } else if (this.isSpecialGroup('Offset')) { - return 'Offset' - } else if (this.isSpecialGroup('Inset')) { - return 'Inset' - } else { - return undefined - } - } - - equivalent(other) { - if (!(other instanceof ParsedHedGroup)) { - return false - } - const equivalence = (ours, theirs) => ours.equivalent(theirs) - return ( - differenceWith(this.tags, other.tags, equivalence).length === 0 && - differenceWith(other.tags, this.tags, equivalence).length === 0 - ) - } - - /** - * The deeply nested array of parsed tags. - * @returns {ParsedHedTag[]} - */ - nestedGroups() { - const currentGroup = [] - for (const innerTag of this.tags) { - if (innerTag instanceof ParsedHedTag) { - currentGroup.push(innerTag) - } else if (innerTag instanceof ParsedHedGroup) { - currentGroup.push(innerTag.nestedGroups()) - } - } - return currentGroup - } - - /** - * Return a normalized string representation - * @returns {string} - */ - get normalized() { - if (this._normalized) { - return this._normalized - } - // Recursively normalize each item in the group - const normalizedItems = this.tags.map((item) => item.normalized) - - // Sort normalized items to ensure order independence - const sortedNormalizedItems = normalizedItems.sort() - - const duplicates = getDuplicates(sortedNormalizedItems) - if (duplicates.length > 0) { - IssueError.generateAndThrow('duplicateTag', { - tags: '[' + duplicates.join('],[') + ']', - string: this.originalTag, - }) - } - this._normalized = '(' + sortedNormalizedItems.join(',') + ')' - // Return the normalized group as a string - return `(${sortedNormalizedItems.join(',')})` // Using curly braces to indicate unordered group - } - - /** - * Iterator over the full HED groups and subgroups in this HED tag group. - * - * @yields {ParsedHedTag[]} The subgroups of this tag group. - */ - *subGroupArrayIterator() { - for (const innerTag of this.tags) { - if (innerTag instanceof ParsedHedGroup) { - yield* innerTag.subGroupArrayIterator() - } - } - yield this.tags - } - - /** - * Iterator over the ParsedHedGroup objects in this HED tag group. - * @param {string | null} tagName - The name of the tag whose groups are to be iterated over or null if all tags. - * @yields {ParsedHedGroup} - This object and the ParsedHedGroup objects belonging to this tag group. - */ - *subParsedGroupIterator(tagName = null) { - if (!tagName || filterByTagName(this.topTags, tagName)) { - yield this - } - for (const innerTag of this.tags) { - if (innerTag instanceof ParsedHedGroup) { - yield* innerTag.subParsedGroupIterator(tagName) - } - } - } - - /** - * Iterator over the parsed HED tags in this HED tag group. - * - * @yields {ParsedHedTag} This tag group's HED tags. - */ - *tagIterator() { - for (const innerTag of this.tags) { - if (innerTag instanceof ParsedHedTag) { - yield innerTag - } else if (innerTag instanceof ParsedHedGroup) { - yield* innerTag.tagIterator() - } - } - } - - /** - * Iterator over the parsed HED column splices in this HED tag group. - * - * @yields {ParsedHedColumnSplice} This tag group's HED column splices. - */ - *columnSpliceIterator() { - for (const innerTag of this.tags) { - if (innerTag instanceof ParsedHedColumnSplice) { - yield innerTag - } else if (innerTag instanceof ParsedHedGroup) { - yield* innerTag.columnSpliceIterator() - } - } - } - - /** - * Iterator over the top-level parsed HED groups in this HED tag group. - * - * @yields {ParsedHedTag} This tag group's top-level HED groups. - */ - *topLevelGroupIterator() { - for (const innerTag of this.tags) { - if (innerTag instanceof ParsedHedGroup) { - yield innerTag - } - } - } -} diff --git a/eventManager/parsedHedString.js b/eventManager/parsedHedString.js deleted file mode 100644 index 3e4a5f7a..00000000 --- a/eventManager/parsedHedString.js +++ /dev/null @@ -1,114 +0,0 @@ -import ParsedHedTag from './parsedHedTag' -import ParsedHedGroup from './parsedHedGroup' -import ParsedHedColumnSplice from './parsedHedColumnSplice' -import { filterByClass, getDuplicates } from './parseUtils' -import { IssueError } from '../common/issues/issues' - -/** - * A parsed HED string. - */ -export class ParsedHedString { - /** - * The original HED string. - * @type {string} - */ - hedString - /** - * The parsed substring data in unfiltered form. - * @type {ParsedHedSubstring[]} - */ - parseTree - /** - * The tag groups in the string (top-level). - * @type {ParsedHedGroup[]} - */ - tagGroups - /** - * All the top-level tags in the string. - * @type {ParsedHedTag[]} - */ - topLevelTags - /** - * All the tags in the string at all levels - * @type {ParsedHedTag[]} - */ - tags - /** - * All the column splices in the string at all levels. - * @type {ParsedHedColumnSplice[]} - */ - columnSplices - /** - * The top-level tag groups in the string, split into arrays. - * @type {ParsedHedTag[][]} - */ - topLevelGroupTags - /** - * The top-level definition tag groups in the string. - * @type {ParsedHedGroup[]} - */ - definitions - - /** - * Constructor. - * @param {string} hedString The original HED string. - * @param {ParsedHedSubstring[]} parsedTags The nested list of parsed HED tags and groups. - */ - constructor(hedString, parsedTags) { - this.hedString = hedString - this.parseTree = parsedTags - this.tagGroups = filterByClass(parsedTags, ParsedHedGroup) - this.topLevelTags = filterByClass(parsedTags, ParsedHedTag) - - const subgroupTags = this.tagGroups.flatMap((tagGroup) => Array.from(tagGroup.tagIterator())) - this.tags = this.topLevelTags.concat(subgroupTags) - - const topLevelColumnSplices = filterByClass(parsedTags, ParsedHedColumnSplice) - const subgroupColumnSplices = this.tagGroups.flatMap((tagGroup) => Array.from(tagGroup.columnSpliceIterator())) - this.columnSplices = topLevelColumnSplices.concat(subgroupColumnSplices) - - //this.topLevelGroupTags = this.tagGroups.map((tagGroup) => filterByClass(tagGroup.tags, ParsedHedTag)) - this.topLevelGroupTags = this.tagGroups.flatMap((tagGroup) => filterByClass(tagGroup.tags, ParsedHedTag)) - this.definitions = this.tagGroups.filter((group) => group.isDefinitionGroup) - this.normalized = this._getNormalized() - } - - /** - * Nicely format this HED string. (Doesn't allow column splices). - * - * @param {boolean} long Whether the tags should be in long form. - * @returns {string} - */ - format(long = true) { - return this.parseTree.map((substring) => substring.format(long)).join(', ') - } - - /** - * Return a normalized string representation - * @returns {string} - */ - _getNormalized() { - // This is an implicit recursion as the items have the same call. - const normalizedItems = this.parseTree.map((item) => item.normalized) - - // Sort normalized items to ensure order independence - const sortedNormalizedItems = normalizedItems.sort() - const duplicates = getDuplicates(sortedNormalizedItems) - if (duplicates.length > 0) { - IssueError.generateAndThrow('duplicateTag', { tags: '[' + duplicates.join('],[') + ']', string: this.hedString }) - } - // Return the normalized group as a string - return `${sortedNormalizedItems.join(',')}` // Using curly braces to indicate unordered group - } - - /** - * Override of {@link Object.prototype.toString}. - * - * @returns {string} - */ - toString() { - return this.hedString - } -} - -export default ParsedHedString diff --git a/eventManager/parsedHedSubstring.js b/eventManager/parsedHedSubstring.js deleted file mode 100644 index 27ea6a4b..00000000 --- a/eventManager/parsedHedSubstring.js +++ /dev/null @@ -1,57 +0,0 @@ -/** - * A parsed HED substring. - */ -export class ParsedHedSubstring { - /** - * The original pre-parsed version of the HED tag. - * @type {string} - */ - originalTag - /** - * The bounds of the HED tag in the original HED string. - * @type {int[]} - */ - originalBounds - - /** - * Constructor. - * @param {string} originalTag The original HED tag. - * @param {number[]} originalBounds The bounds of the HED tag in the original HED string. - */ - constructor(originalTag, originalBounds) { - this.originalTag = originalTag - this.originalBounds = originalBounds - } - - /** - * Nicely format this substring. This is left blank for the subclasses to override. - * - * This is left blank for the subclasses to override. - * - * @param {boolean} long - Whether the tags should be in long form. - * @returns {string} - * @abstract - */ - format(long = true) {} - - /** - * Get the normalized version of the object. - * - * @returns {string} - * @abstract - */ - get normalized() { - return '' - } - - /** - * Override of {@link Object.prototype.toString}. - * - * @returns {string} The original form of this HED substring. - */ - toString() { - return this.originalTag - } -} - -export default ParsedHedSubstring diff --git a/eventManager/parsedHedTag.js b/eventManager/parsedHedTag.js deleted file mode 100644 index 140e582a..00000000 --- a/eventManager/parsedHedTag.js +++ /dev/null @@ -1,447 +0,0 @@ -import { IssueError } from '../common/issues/issues' -import ParsedHedSubstring from './parsedHedSubstring' -import { SchemaValueTag } from '../schema/entries' -import TagConverter from './tagConverter' -import { SpecialChecker } from './special' - -const allowedRegEx = /^[^{}\,]*$/ - -/** - * A parsed HED tag. - */ -export default class ParsedHedTag extends ParsedHedSubstring { - /** - * The formatted canonical version of the HED tag. - * @type {string} - */ - formattedTag - /** - * The canonical form of the HED tag. - * @type {string} - */ - canonicalTag - /** - * The HED schema this tag belongs to. - * @type {Schema} - */ - schema - /** - * The schema's representation of this tag. - * - * @type {SchemaTag} - * @private - */ - _schemaTag - - /** - * The remaining part of the tag after the portion actually in the schema. - * - * @type {string} - * @private - */ - _remainder - - /** - * The value if any - * - * @type {string} - * @private - */ - _value - - /** - * If definition this is the second value if - * - * @type {string} - * @private - */ - _splitValue - - /** - * The units if any - * - * @type {string} - * @private - */ - _units - - /** - * Constructor. - * - * @param {TagSpec} tagSpec The token for this tag. - * @param {Schemas} hedSchemas The collection of HED schemas. - * @param {string} hedString The original HED string. - * @throws {IssueError} If tag conversion or parsing fails. - */ - constructor(tagSpec, hedSchemas, hedString) { - super(tagSpec.tag, tagSpec.bounds) // Sets originalTag and originalBounds - this._convertTag(hedSchemas, hedString, tagSpec) - this._normalized = this.format(false) // Sets various forms of the tag. - this._validUnits = null - } - - /** - * Convert this tag to its various forms - * - * @param {Schemas} hedSchemas The collection of HED schemas. - * @param {string} hedString The original HED string. - * @param {TagSpec} tagSpec The token for this tag. - * @throws {IssueError} If tag conversion or parsing fails. - */ - _convertTag(hedSchemas, hedString, tagSpec) { - const schemaName = tagSpec.library - this.schema = hedSchemas.getSchema(schemaName) - if (this.schema === undefined) { - if (schemaName !== '') { - IssueError.generateAndThrow('unmatchedLibrarySchema', { - tag: this.originalTag, - library: schemaName, - }) - } else { - IssueError.generateAndThrow('unmatchedBaseSchema', { - tag: this.originalTag, - }) - } - } - - const [schemaTag, remainder] = new TagConverter(tagSpec, hedSchemas).convert() - this._schemaTag = schemaTag - this._remainder = remainder - this.canonicalTag = this._schemaTag.longExtend(remainder) - this.formattedTag = this.canonicalTag.toLowerCase() - this._handleRemainder(schemaTag, remainder) - } - - /** - * Handle the remainder portion for value tag (converter handles others) - * - * @param {SchemaTag} schemaTag - The part of the tag that is in the schema - * @param {string} remainder - the leftover part - * @throws {IssueError} If parsing the remainder section fails. - */ - _handleRemainder(schemaTag, remainder) { - if (!(schemaTag instanceof SchemaValueTag)) { - return - } - // Check that there is a value if required - const special = SpecialChecker.getInstance() - if ( - (schemaTag.hasAttributeName('requireChild') || special.requireValueTags.has(schemaTag.name)) && - remainder === '' - ) { - IssueError.generateAndThrow('valueRequired', { tag: this.originalTag }) - } - // Check if this could have a two-level value - const [value, rest] = this._getSplitValue(remainder, special) - this._splitValue = rest - - // Resolve the units and check - const [actualUnit, actualUnitString, actualValueString] = this._separateUnits(schemaTag, value) - this._units = actualUnitString - this._value = actualValueString - - if (actualUnit === null && actualUnitString !== null) { - IssueError.generateAndThrow('unitClassInvalidUnit', { tag: this.originalTag }) - } - if (!this.checkValue(actualValueString)) { - IssueError.generateAndThrow('invalidValue', { tag: this.originalTag }) - } - } - - /** - * Separate the remainder of the tag into three parts: - * - * @param {SchemaTag} schemaTag - The part of the tag that is in the schema - * @param {string} remainder - the leftover part - * @returns {[SchemaUnit, string, string]} - The actual Unit, the unit string and the value string. - * @throws {IssueError} If parsing the remainder section fails. - */ - _separateUnits(schemaTag, remainder) { - const unitClasses = schemaTag.unitClasses - let actualUnit = null - let actualUnitString = null - let actualValueString = remainder // If no unit class, the remainder is the value - for (let i = 0; i < unitClasses.length; i++) { - ;[actualUnit, actualUnitString, actualValueString] = unitClasses[i].extractUnit(remainder) - if (actualUnit !== null) { - break // found the unit - } - } - return [actualUnit, actualUnitString, actualValueString] - } - - /** - * Handle special three-level tags - * @param {string} remainder - the remainder of the tag string after schema tag - * @param {SpecialChecker} special - the special checker for checking the special tag properties - */ - _getSplitValue(remainder, special) { - if (!special.allowTwoLevelValueTags.has(this.schemaTag.name)) { - return [remainder, null] - } - const [first, ...rest] = remainder.split('/') - return [first, rest.join('/')] - } - - /** - * Nicely format this tag. - * - * @param {boolean} long Whether the tags should be in long form. - * @returns {string} The nicely formatted version of this tag. - */ - format(long = true) { - let tagName - if (long) { - tagName = this._schemaTag?.longExtend(this._remainder) - } else { - tagName = this._schemaTag?.extend(this._remainder) - } - if (tagName === undefined) { - tagName = this.originalTag - } - if (this.schema?.prefix) { - return this.schema.prefix + ':' + tagName - } else { - return tagName - } - } - - get normalized() { - return this._normalized - } - - /** - * Override of {@link Object.prototype.toString}. - * - * @returns {string} The original form of this HED tag. - */ - toString() { - if (this.schema?.prefix) { - return this.schema.prefix + ':' + this.originalTag - } else { - return this.originalTag - } - } - - /** - * Determine whether this tag has a given attribute. - * - * @param {string} attribute An attribute name. - * @returns {boolean} Whether this tag has the named attribute. - */ - hasAttribute(attribute) { - return this.schema?.tagHasAttribute(this.formattedTag, attribute) - } - - /** - * Determine whether this tag's parent tag has a given attribute. - * - * @param {string} attribute An attribute name. - * @returns {boolean} Whether this tag's parent tag has the named attribute. - */ - parentHasAttribute(attribute) { - return this.schema?.tagHasAttribute(this.parentFormattedTag, attribute) - } - - /** - * Get the last part of a HED tag. - * - * @param {string} tagString A HED tag. - * @returns {string} The last part of the tag using the given separator. - */ - static getTagName(tagString) { - const lastSlashIndex = tagString.lastIndexOf('/') - if (lastSlashIndex === -1) { - return tagString - } else { - return tagString.substring(lastSlashIndex + 1) - } - } - - /** - * The trailing portion of {@link originalTag}. - * - * @returns {string} The "name" portion of the original tag. - */ - get originalTagName() { - return ParsedHedTag.getTagName(this.originalTag) - } - - /** - * Get the HED tag prefix (up to the last slash). - * - * @param {string} tagString A HED tag. - * @returns {string} The portion of the tag up to the last slash. - */ - static getParentTag(tagString) { - const lastSlashIndex = tagString.lastIndexOf('/') - if (lastSlashIndex === -1) { - return tagString - } else { - return tagString.substring(0, lastSlashIndex) - } - } - - /** - * The parent portion of {@link formattedTag}. - * - * @returns {string} The "parent" portion of the formatted tag. - */ - get parentFormattedTag() { - return ParsedHedTag.getParentTag(this.formattedTag) - } - - /** - * Iterate through a tag's ancestor tag strings. - * - * @param {string} tagString A tag string. - * @yields {string} The tag's ancestor tags. - */ - static *ancestorIterator(tagString) { - while (tagString.lastIndexOf('/') >= 0) { - yield tagString - tagString = ParsedHedTag.getParentTag(tagString) - } - yield tagString - } - /* - - /!** - * Determine whether this tag is a descendant of another tag. - * - * @param {ParsedHedTag|string} parent The possible parent tag. - * @returns {boolean} Whether {@link parent} is the parent tag of this tag. - *!/ - isDescendantOf(parent) { - if (parent instanceof ParsedHedTag) { - if (this.schema !== parent.schema) { - return false - } - parent = parent.formattedTag - } - for (const ancestor of ParsedHedTag.ancestorIterator(this.formattedTag)) { - if (ancestor === parent) { - return true - } - } - return false - } -*/ - - /** - * Determine if this HED tag is equivalent to another HED tag. - * - * Note: HED tags are deemed equivalent if they have the same schema and normalized tag string. - * - * @param {ParsedHedTag} other - A HED tag to compare with this one. - * @returns {boolean} Whether {@link other} True, if other is equivalent to this HED tag. - */ - equivalent(other) { - return other instanceof ParsedHedTag && this.formattedTag === other.formattedTag && this.schema === other.schema - } - - /** - * Get the schema tag object for this tag. - * - * @returns {SchemaTag} The schema tag object for this tag. - */ - get schemaTag() { - if (this._schemaTag instanceof SchemaValueTag) { - return this._schemaTag.parent - } else { - return this._schemaTag - } - } - - /** - * Get the schema tag object for this tag's value-taking form. - * - * @returns {SchemaValueTag} The schema tag object for this tag's value-taking form. - */ - get takesValueTag() { - if (this._schemaTag instanceof SchemaValueTag) { - return this._schemaTag - } - return undefined - } - - /** - * Checks if this HED tag has the {@code takesValue} attribute. - * - * @returns {boolean} Whether this HED tag has the {@code takesValue} attribute. - */ - get takesValue() { - return this.takesValueTag !== undefined - } - - /** - * Checks if this HED tag has the {@code unitClass} attribute. - * - * @returns {boolean} Whether this HED tag has the {@code unitClass} attribute. - */ - get hasUnitClass() { - if (!this.takesValueTag) { - return false - } - return this.takesValueTag.hasUnitClasses - } - - /** - * Get the unit classes for this HED tag. - * - * @returns {SchemaUnitClass[]} The unit classes for this HED tag. - */ - get unitClasses() { - if (this.hasUnitClass) { - return this.takesValueTag.unitClasses - } - return [] - } - - /** - * Get the legal units for this HED tag. - * - * @returns {Set} The legal units for this HED tag. - */ - get validUnits() { - if (this._validUnits) { - return this._validUnits - } - const tagUnitClasses = this.unitClasses - this._validUnits = new Set() - for (const unitClass of tagUnitClasses) { - const unitClassUnits = this.schema?.entries.unitClasses.getEntry(unitClass.name).units - for (const unit of unitClassUnits.values()) { - this._validUnits.add(unit) - } - } - return this._validUnits - } - - /** - * Check if value is a valid value for this tag. - * - * @param {string} value - The value to be checked. - * @returns {boolean} The result of check -- false if not a valid value. - */ - checkValue(value) { - if (!this.takesValue) { - return false - } - if (value === '#') { - // Placeholders work - return true - } - const valueAttributeNames = this._schemaTag.valueAttributeNames - const valueClassNames = valueAttributeNames?.get('valueClass') - if (!valueClassNames) { - // No specified value classes - return allowedRegEx.test(value) - } - const entryManager = this.schema.entries.valueClasses - for (let i = 0; i < valueClassNames.length; i++) { - if (entryManager.getEntry(valueClassNames[i]).validateValue(value)) return true - } - return false - } -} diff --git a/eventManager/parser.js b/eventManager/parser.js deleted file mode 100644 index b2d1e3c5..00000000 --- a/eventManager/parser.js +++ /dev/null @@ -1,170 +0,0 @@ -import { mergeParsingIssues } from '../utils/hedData' -import ParsedHedString from './parsedHedString' -import HedStringSplitter from './splitter' -import { generateIssue } from '../common/issues/issues' -import { SpecialChecker } from './special' -import { getTagListString } from './parseUtils' - -/** - * A parser for HED strings. - */ -class HedStringParser { - /** - * The HED string being parsed. - * @type {string|ParsedHedString} - */ - hedString - /** - * The collection of HED schemas. - * @type {Schemas} - */ - hedSchemas - - definitionsAllowed - - placeholdersAllowed - - /** - * Constructor. - * - * @param {string|ParsedHedString} hedString The HED string to be parsed. - * @param {Schemas} hedSchemas The collection of HED schemas. - * @param {boolean} definitionsAllowed - True if definitions are allowed - * @param {boolean} placeholdersAllowed - True if placeholders are allowed - */ - constructor(hedString, hedSchemas, definitionsAllowed, placeholdersAllowed) { - this.hedString = hedString - this.hedSchemas = hedSchemas - this.definitionsAllowed = definitionsAllowed - this.placeholdersAllowed = placeholdersAllowed - } - - /** - * Parse a full HED string. - * @param {boolean} fullCheck whether the string is in final form and can be fully parsed - - * @returns {[ParsedHedString|null, Issue[]]} The parsed HED string and any parsing issues. - */ - parseHedString(fullCheck) { - if (this.hedString === null || this.hedString === undefined) { - return [null, [generateIssue('invalidTagString', {})]] - } - // if (!this.hedString) { - // return [null, []] - // } - const placeholderIssues = this._getPlaceholderCountIssues() - if (placeholderIssues.length > 0) { - return [null, placeholderIssues] - } - if (this.hedString instanceof ParsedHedString) { - return [this.hedString, []] - } - if (!this.hedSchemas) { - return [null, [generateIssue('missingSchemaSpecification', {})]] - } - - // This assumes that splitter errors are only errors and not warnings - const [parsedTags, parsingIssues] = new HedStringSplitter(this.hedString, this.hedSchemas).splitHedString() - if (parsedTags === null || parsingIssues.length > 0) { - return [null, parsingIssues] - } - - const parsedString = new ParsedHedString(this.hedString, parsedTags) - - // This checks whether there are any definitions in the string - const simpleDefinitionIssues = this._checkDefinitionContext(parsedString) - if (simpleDefinitionIssues.length > 0) { - return [null, simpleDefinitionIssues] - } - const checkIssues = SpecialChecker.getInstance().checkHedString(parsedString, fullCheck) - if (checkIssues.length > 0) { - return [null, checkIssues] - } - return [parsedString, []] - } - - _checkDefinitionContext(parsedString) { - if (this.definitionsAllowed || !parsedString) { - return [] - } - const definitionTags = parsedString.tags.filter((tag) => tag.schemaTag.name === 'Definition') - if (definitionTags.length > 0) { - return [ - generateIssue('illegalDefinitionContext', { - definition: getTagListString(definitionTags), - string: parsedString.hedString, - }), - ] - } - return [] - } - - _getPlaceholderCountIssues() { - if (this.placeholdersAllowed) { - return [] - } - const checkString = this.hedString instanceof ParsedHedString ? this.hedString.hedString : this.hedString - if (checkString.split('#').length > 1) { - return [generateIssue('invalidPlaceholderContext', { string: checkString })] - } - return [] - } - - /** - * Parse a list of HED strings. - * - * @param {string[]|ParsedHedString[]} hedStrings A list of HED strings. - * @param {Schemas} hedSchemas The collection of HED schemas. - * @param {boolean} fullCheck whether the strings are in final form and can be fully parsed - * @param {boolean} definitionsAllowed - True if definitions are allowed - * @param {boolean} placeholdersAllowed - True if placeholders are allowed - * @returns {[ParsedHedString[], Issue[]]} The parsed HED strings and any issues found. - */ - static parseHedStrings(hedStrings, hedSchemas, fullCheck, definitionsAllowed, placeholdersAllowed) { - if (!hedSchemas) { - return [null, [generateIssue('missingSchemaSpecification', {})]] - } - const parsedStrings = [] - const cumulativeIssues = [] - for (const hedString of hedStrings) { - const [parsedString, currentIssues] = new HedStringParser( - hedString, - hedSchemas, - definitionsAllowed, - placeholdersAllowed, - ).parseHedString(fullCheck) - parsedStrings.push(parsedString) - cumulativeIssues.push(...currentIssues) - } - - return [parsedStrings, cumulativeIssues] - } -} - -/** - * Parse a HED string. - * - * @param {string|ParsedHedString} hedString A (possibly already parsed) HED string. - * @param {Schemas} hedSchemas - The collection of HED schemas. - * @param {boolean} fullCheck - If the string is in final form -- can be fully parsed - * @param {boolean} definitionsAllowed - True if definitions are allowed - * @param {boolean} placeholdersAllowed - True if placeholders are allowed - * @returns {[ParsedHedString, Issue[]]} - The parsed HED string and any issues found. - */ -export function parseHedString(hedString, hedSchemas, fullCheck, definitionsAllowed, placeholdersAllowed) { - return new HedStringParser(hedString, hedSchemas, definitionsAllowed, placeholdersAllowed).parseHedString(fullCheck) -} - -/** - * Parse a list of HED strings. - * - * @param {string[]|ParsedHedString[]} hedStrings A list of HED strings. - * @param {Schemas} hedSchemas - The collection of HED schemas. - * @param {boolean} fullCheck - If the strings is in final form -- can be fully parsed - * @param {boolean} definitionsAllowed - True if definitions are allowed - * @param {boolean} placeholdersAllowed - True if placeholders are allowed - * @returns {[ParsedHedString[], Issue[]]} The parsed HED strings and any issues found. - */ -export function parseHedStrings(hedStrings, hedSchemas, fullCheck, definitionsAllowed, placeholdersAllowed) { - return HedStringParser.parseHedStrings(hedStrings, hedSchemas, fullCheck, definitionsAllowed, placeholdersAllowed) -} diff --git a/eventManager/splitter.js b/eventManager/splitter.js deleted file mode 100644 index ea8ecbdd..00000000 --- a/eventManager/splitter.js +++ /dev/null @@ -1,124 +0,0 @@ -import ParsedHedTag from './parsedHedTag' -import ParsedHedColumnSplice from './parsedHedColumnSplice' -import ParsedHedGroup from './parsedHedGroup' -import { recursiveMap } from '../utils/array' -import { HedStringTokenizer, ColumnSpliceSpec, TagSpec } from './tokenizer' -import { generateIssue, IssueError } from '../common/issues/issues' -import { SpecialChecker } from './special' - -export default class HedStringSplitter { - /** - * The HED string being split. - * @type {string} - */ - hedString - /** - * The collection of HED schemas. - * @type {Schemas} - */ - hedSchemas - - issues - - /** - * Constructor. - * - * @param {string} hedString The HED string to be split and parsed. - * @param {Schemas} hedSchemas The collection of HED schemas. - */ - constructor(hedString, hedSchemas) { - this.hedString = hedString - this.hedSchemas = hedSchemas - this.special = SpecialChecker.getInstance() - this.issues = [] - } - - /** - * Split and parse a HED string into tags and groups. - * - * @returns {[ParsedHedSubstring[], Issue[]]} The parsed HED string data and any issues found. - */ - splitHedString() { - if (this.hedString === null || this.hedString === undefined || typeof this.hedString !== 'string') { - return [null, [generateIssue('invalidTagString', {})]] - } - if (this.hedString.length === 0) { - return [[], []] - } - const [tagSpecs, groupBounds, issues] = new HedStringTokenizer(this.hedString).tokenize() - if (issues.length > 0) { - return [null, issues] - } - const [parsedTags, parsingIssues] = this._createParsedTags(tagSpecs, groupBounds) - return [parsedTags, parsingIssues] - } - - /** - * Create parsed HED tags and groups from specifications. - * - * @param {TagSpec[]} tagSpecs The tag specifications. - * @param {GroupSpec} groupSpecs The group specifications. - * @returns {[ParsedHedSubstring[], Issue[]]} The parsed HED tags and any issues. - */ - _createParsedTags(tagSpecs, groupSpecs) { - // Create tags from specifications - this.issues = [] - const parsedTags = recursiveMap((tagSpec) => this._createParsedTag(tagSpec), tagSpecs) - - // Create groups from the parsed tags - const parsedTagsWithGroups = this._createParsedGroups(parsedTags, groupSpecs.children) - return [parsedTagsWithGroups, this.issues] - } - - _createParsedTag(tagSpec) { - if (tagSpec instanceof TagSpec) { - try { - return new ParsedHedTag(tagSpec, this.hedSchemas, this.hedString) - } catch (issueError) { - this.issues.push(this._handleIssueError(issueError)) - return null - } - } else if (tagSpec instanceof ColumnSpliceSpec) { - return new ParsedHedColumnSplice(tagSpec.columnName, tagSpec.bounds) - } - } - - /** - * Handle any issue encountered during tag parsing. - * - * @param {Error|IssueError} issueError The error encountered. - */ - _handleIssueError(issueError) { - if (issueError instanceof IssueError) { - return issueError.issue - } else if (issueError instanceof Error) { - return generateIssue('internalError', { message: issueError.message }) - } - } - - /** - * Create parsed HED groups from parsed tags and group specifications. - * - * @param {ParsedHedTag[]} tags The parsed HED tags. - * @param {GroupSpec[]} groupSpecs The group specifications. - * @returns {ParsedHedGroup[]} The parsed HED groups. - */ - _createParsedGroups(tags, groupSpecs) { - const tagGroups = [] - let index = 0 - - for (const tag of tags) { - if (Array.isArray(tag)) { - const groupSpec = groupSpecs[index] - tagGroups.push( - new ParsedHedGroup(this._createParsedGroups(tag, groupSpec.children), this.hedString, groupSpec.bounds), - ) - index++ - } else if (tag !== null) { - tagGroups.push(tag) - } - } - - return tagGroups - } -} diff --git a/eventManager/tagConverter.js b/eventManager/tagConverter.js deleted file mode 100644 index 1724b9d9..00000000 --- a/eventManager/tagConverter.js +++ /dev/null @@ -1,183 +0,0 @@ -import { IssueError } from '../common/issues/issues' -import { getTagSlashIndices } from '../utils/hedStrings' -import { SpecialChecker } from './special' - -/** - * Converter from a tag specification to a schema-based tag object. - */ -export default class TagConverter { - /** - * A parsed tag token. - * @type {TagSpec} - */ - tagSpec - /** - * The tag string to convert. - * @type {string} - */ - tagString - /** - * The tag string split by slashes. - * @type {string[]} - */ - tagLevels - /** - * The indices of the tag string's slashes. - * @type {number[]} - */ - tagSlashes - /** - * A HED schema collection. - * @type {Schemas} - */ - hedSchemas - /** - * The entry manager for the tags in the active schema. - * @type {SchemaTagManager} - */ - tagMapping - /** - * The converted tag in the schema. - * @type {SchemaTag} - */ - schemaTag - /** - * The remainder (e.g. value, extension) of the tag string. - * @type {string} - */ - remainder - - /** - * Constructor. - * - * @param {TagSpec} tagSpec The tag specification to convert. - * @param {Schemas} hedSchemas The HED schema collection. - */ - constructor(tagSpec, hedSchemas) { - this.hedSchemas = hedSchemas - this.tagMapping = hedSchemas.getSchema(tagSpec.library).entries.tags - - this.tagSpec = tagSpec - this.tagString = tagSpec.tag - this.tagLevels = this.tagString.split('/') - this.tagSlashes = getTagSlashIndices(this.tagString) - this.remainder = undefined - this.special = SpecialChecker.getInstance() - } - - /** - * Retrieve the {@link SchemaTag} object for a tag specification. - * - * @returns {[SchemaTag, string]} The schema's corresponding tag object and the remainder of the tag string. - * @throws {IssueError} If tag conversion. - */ - convert() { - let parentTag = undefined - for (let tagLevelIndex = 0; tagLevelIndex < this.tagLevels.length; tagLevelIndex++) { - if (parentTag?.valueTag) { - // It is a value tag - this._setSchemaTag(parentTag.valueTag, tagLevelIndex) - return [this.schemaTag, this.remainder] - } - const childTag = this._validateChildTag(parentTag, tagLevelIndex) - if (childTag === undefined) { - // It is an extended tag and the rest is undefined - this._setSchemaTag(parentTag, tagLevelIndex) - } - parentTag = childTag - } - this._setSchemaTag(parentTag, this.tagLevels.length + 1) // Fix the ending - return [this.schemaTag, this.remainder] - } - - _validateChildTag(parentTag, tagLevelIndex) { - const childTag = this._getSchemaTag(tagLevelIndex) - if (childTag === undefined) { - // This is an extended tag - if (tagLevelIndex === 0) { - // Top level tags can't be extensions - IssueError.generateAndThrow('invalidTag', { tag: this.tagString }) - } - if ( - parentTag !== undefined && - (!parentTag.hasAttributeName('extensionAllowed') || this.special.noExtensionTags.has(parentTag.name)) - ) { - IssueError.generateAndThrow('invalidExtension', { - tag: this.tagLevels[tagLevelIndex], - parentTag: this.tagLevels.slice(0, tagLevelIndex).join('/'), - }) - } - this._checkExtensions(tagLevelIndex) - return childTag - } - - if (tagLevelIndex > 0 && (childTag.parent === undefined || childTag.parent !== parentTag)) { - IssueError.generateAndThrow('invalidParentNode', { - tag: this.tagLevels[tagLevelIndex], - parentTag: this.tagLevels.slice(0, tagLevelIndex).join('/'), - }) - } - - return childTag - } - - _checkExtensions(tagLevelIndex) { - // A non-tag has been detected --- from here on must be non-tags. - this._checkNameClass(tagLevelIndex) // This is an extension - for (let index = tagLevelIndex + 1; index < this.tagLevels.length; index++) { - const child = this._getSchemaTag(index) - if (child !== undefined) { - // A schema tag showed up after a non-schema tag - IssueError.generateAndThrow('invalidParentNode', { - tag: child.name, - parentTag: this.tagLevels.slice(0, index).join('/'), - }) - } - this._checkNameClass(index) - } - } - - /* /!** - * Handle the case where it does not allow an extension or if it requires a child and doesn't have one. - * @param {int} tagLevelIndex index of the tag - * @throws {IssueError} If the tag has an extension that is not allowed. - *!/ - _checkExtensionRequirements(tagLevelIndex) { - // Check allow extension or requires a child - const schemaTag = this.getSchemaTag(tagLevelIndex - 1) - const remainder = this.tagLevels.slice(tagLevelIndex).join('/') - if (this.special.noExtension.includes(schemaTag.name) && remainder !== '') { - IssueError.generateAndThrow('invalidExtension',{tag: remainder, parentTag: schemaTag.name}) - } - if (remainder === '' && schemaTag.hasAttributeName('requireChild')) ( - IssueError.generateAndThrow('childRequired', {tag:schemaTag.name}) - ) - }*/ - - _getSchemaTag(tagLevelIndex) { - const tagLevel = this.tagLevels[tagLevelIndex].toLowerCase() - return this.tagMapping.getEntry(tagLevel) - } - - _setSchemaTag(schemaTag, remainderStartLevelIndex) { - if (this.schemaTag !== undefined) { - return - } - this.schemaTag = schemaTag - this.remainder = this.tagLevels.slice(remainderStartLevelIndex).join('/') - if (this.schemaTag?.hasAttributeName('requireChild') && !this.remainder) { - IssueError.generateAndThrow('childRequired', { tag: this.tagString }) - } - } - - _checkNameClass(index) { - // Check whether the tagLevel is a valid name class - const valueClasses = this.hedSchemas.getSchema(this.tagSpec.library).entries.valueClasses - if (!valueClasses._definitions.get('nameClass').validateValue(this.tagLevels[index])) { - IssueError.generateAndThrow('invalidExtension', { - tag: this.tagLevels[index], - parentTag: this.tagLevels.slice(0, index).join('/'), - }) - } - } -} diff --git a/eventManager/tokenizer.js b/eventManager/tokenizer.js deleted file mode 100644 index 571b8913..00000000 --- a/eventManager/tokenizer.js +++ /dev/null @@ -1,399 +0,0 @@ -import { unicodeName } from 'unicode-name' - -import { generateIssue } from '../common/issues/issues' - -const CHARACTERS = { - BLANK: ' ', - OPENING_GROUP: '(', - CLOSING_GROUP: ')', - OPENING_COLUMN: '{', - CLOSING_COLUMN: '}', - COMMA: ',', - COLON: ':', - SLASH: '/', - PLACEHOLDER: '#', -} - -function getTrimmedBounds(originalString) { - const start = originalString.search(/\S/) - - if (start === -1) { - // The string contains only whitespace - return null - } - const end = originalString.search(/\S\s*$/) - return [start, end + 1] -} - -const invalidCharacters = new Set(['[', ']', '~', '"']) -// Add control codes to invalidCharacters -for (let i = 0x00; i <= 0x1f; i++) { - invalidCharacters.add(String.fromCodePoint(i)) -} -for (let i = 0x7f; i <= 0x9f; i++) { - invalidCharacters.add(String.fromCodePoint(i)) -} - -/** - * A specification for a tokenized substring. - */ -export class SubstringSpec { - /** - * The starting and ending bounds of the substring. - * @type {number[]} - */ - bounds - - constructor(start, end) { - this.bounds = [start, end] - } -} - -/** - * A specification for a tokenized tag. - */ -export class TagSpec extends SubstringSpec { - /** - * The tag this spec represents. - * @type {string} - */ - tag - /** - * The schema prefix for this tag, if any. - * @type {string} - */ - library - - constructor(tag, start, end, librarySchema) { - super(start, end) - - this.tag = tag.trim() - this.library = librarySchema - } -} - -/** - * A specification for a tokenized tag group. - */ -export class GroupSpec extends SubstringSpec { - /** - * The child group specifications. - * @type {GroupSpec[]} - */ - children - - constructor(start, end, children) { - super(start, end) - - this.children = children - } -} - -/** - * A specification for a tokenized column splice template. - */ -export class ColumnSpliceSpec extends SubstringSpec { - /** - * The column name this spec refers to. - * @type {string} - */ - columnName - - constructor(name, start, end) { - super(start, end) - - this.columnName = name.trim() - } -} - -class TokenizerState { - constructor() { - this.currentToken = '' // Characters in the token currently being parsed - this.groupDepth = 0 - this.startingIndex = 0 // Starting index of this token - this.lastDelimiter = [undefined, -1] // Type and position of the last delimiter - this.librarySchema = '' - this.lastSlash = -1 // Position of the last slash in current token - this.currentGroupStack = [[]] - this.parenthesesStack = [] - } -} - -/** - * Class for tokenizing HED strings. - */ -export class HedStringTokenizer { - constructor(hedString) { - this.hedString = hedString - this.issues = [] - this.state = null - } - - /** - * Split the HED string into delimiters and tags. - * - * @returns {[TagSpec[], GroupSpec, Issue[]]} The tag specifications, group bounds, and any issues found. - */ - tokenize() { - this.initializeTokenizer() - // Empty strings cannot be tokenized - if (this.hedString.trim().length === 0) { - this.pushIssue('emptyTagFound', 0) - return [[], null, this.issues] - } - for (let i = 0; i < this.hedString.length; i++) { - const character = this.hedString.charAt(i) - this.handleCharacter(i, character) - if (this.issues.length > 0) { - return [[], null, this.issues] - } - } - this.finalizeTokenizer() - if (this.issues.length > 0) { - return [[], null, this.issues] - } else { - return [this.state.currentGroupStack.pop(), this.state.parenthesesStack.pop(), []] - } - } - - resetToken(i) { - this.state.startingIndex = i + 1 - this.state.currentToken = '' - this.state.librarySchema = '' - this.state.lastSlash = '-1' - } - - finalizeTokenizer() { - if (this.state.lastDelimiter[0] === CHARACTERS.OPENING_COLUMN) { - // Extra opening brace - this.pushIssue('unclosedCurlyBrace', this.state.lastDelimiter[1]) - } else if (this.state.lastDelimiter[0] === CHARACTERS.OPENING_GROUP) { - // Extra opening parenthesis - this.pushIssue('unclosedParentheses', this.state.lastDelimiter[1]) - } else if ( - this.state.lastDelimiter[0] === CHARACTERS.COMMA && - this.hedString.slice(this.state.lastDelimiter[1] + 1).trim().length === 0 - ) { - this.pushIssue('emptyTagFound', this.state.lastDelimiter[1]) // Extra comma - } else if (this.state.lastSlash >= 0 && this.hedString.slice(this.state.lastSlash + 1).trim().length === 0) { - this.pushIssue('extraSlash', this.state.lastSlash) // Extra slash - } - if ( - this.state.currentToken.trim().length > 0 && - ![undefined, CHARACTERS.COMMA].includes(this.state.lastDelimiter[0]) - ) { - // Missing comma - this.pushIssue('commaMissing', this.state.lastDelimiter[1] + 1) - } else { - if (this.state.currentToken.trim().length > 0) { - this.pushTag(this.hedString.length) - } - this.unwindGroupStack() - } - } - - initializeTokenizer() { - this.issues = [] - this.state = new TokenizerState() - this.state.parenthesesStack = [new GroupSpec(0, this.hedString.length, [])] - } - - handleCharacter(i, character) { - const characterHandler = { - [CHARACTERS.OPENING_GROUP]: () => this.handleOpeningGroup(i), - [CHARACTERS.CLOSING_GROUP]: () => this.handleClosingGroup(i), - [CHARACTERS.OPENING_COLUMN]: () => this.handleOpeningColumn(i), - [CHARACTERS.CLOSING_COLUMN]: () => this.handleClosingColumn(i), - [CHARACTERS.COMMA]: () => this.handleComma(i), - [CHARACTERS.COLON]: () => this.handleColon(i), - [CHARACTERS.SLASH]: () => this.handleSlash(i), - }[character] // Selects the character handler based on the value of character - - if (characterHandler) { - characterHandler() - } else if (invalidCharacters.has(character)) { - this.pushInvalidCharacterIssue(character, i) - } else { - this.state.currentToken += character - } - } - - handleComma(i) { - const trimmed = this.hedString.slice(this.state.lastDelimiter[1] + 1, i).trim() - if ( - [CHARACTERS.OPENING_GROUP, CHARACTERS.COMMA, undefined].includes(this.state.lastDelimiter[0]) && - trimmed.length === 0 - ) { - this.pushIssue('emptyTagFound', i) // Empty tag Ex: ",x" or "(, x" or "y, ,x" - } else if (this.state.lastDelimiter[0] === CHARACTERS.OPENING_COLUMN) { - this.pushIssue('unclosedCurlyBrace', this.state.lastDelimiter[1]) // Unclosed curly brace Ex: "{ x," - } - if ( - [CHARACTERS.CLOSING_GROUP, CHARACTERS.CLOSING_COLUMN].includes(this.state.lastDelimiter[0]) && - trimmed.length > 0 - ) { - // A tag followed a group or column with no comma Ex: (x) yz - this.pushIssue('invalidTag', i, trimmed) - } else if (trimmed.length > 0) { - this.pushTag(i) // Tag has just finished - } else { - this.resetToken(i) // After a group or column - } - this.state.lastDelimiter = [CHARACTERS.COMMA, i] - } - - handleSlash(i) { - if (this.state.currentToken.trim().length === 0) { - // Slash at beginning of tag. - this.pushIssue('extraSlash', i) // Slash at beginning of tag. - } else if (this.state.lastSlash >= 0 && this.hedString.slice(this.state.lastSlash + 1, i).trim().length === 0) { - this.pushIssue('extraSlash', i) // Slashes with only blanks between - } else if (i > 0 && this.hedString.charAt(i - 1) === CHARACTERS.BLANK) { - this.pushIssue('extraBlank', i - 1) // Blank before slash such as slash in value - } else if (i < this.hedString.length - 1 && this.hedString.charAt(i + 1) === CHARACTERS.BLANK) { - this.pushIssue('extraBlank', i + 1) //Blank after a slash - } else if (this.hedString.slice(i).trim().length === 0) { - this.pushIssue('extraSlash', this.state.startingIndex) // Extra slash at the end - } else { - this.state.currentToken += CHARACTERS.SLASH - this.state.lastSlash = i - } - } - - handleOpeningGroup(i) { - if (this.state.lastDelimiter[0] === CHARACTERS.OPENING_COLUMN) { - this.pushIssue('unclosedCurlyBrace', this.state.lastDelimiter[1]) // After open curly brace Ex: "{ (" - } else if (this.state.lastDelimiter[0] === CHARACTERS.CLOSING_COLUMN) { - this.pushIssue('commaMissing', this.state.lastDelimiter[1]) // After close curly brace Ex: "} (" - } else if (this.state.lastDelimiter[0] === CHARACTERS.CLOSING_GROUP) { - this.pushIssue('commaMissing', this.state.lastDelimiter[1] + 1) // After close group Ex: ") (" - } else if (this.state.currentToken.trim().length > 0) { - this.pushInvalidTag('commaMissing', i, this.state.currentToken.trim()) // After tag Ex: "x (" - } else { - this.state.currentGroupStack.push([]) - this.state.parenthesesStack.push(new GroupSpec(i, undefined, [])) - this.resetToken(i) - this.state.groupDepth++ - this.state.lastDelimiter = [CHARACTERS.OPENING_GROUP, i] - } - } - - handleClosingGroup(i) { - if (this.state.groupDepth <= 0) { - this.pushIssue('unopenedParenthesis', i) // No corresponding opening group - } else if (this.state.lastDelimiter[0] === CHARACTERS.OPENING_COLUMN) { - this.pushIssue('unclosedCurlyBrace', this.state.lastDelimiter[1]) // After open curly brace Ex: "{ )" - } else { - if ([CHARACTERS.OPENING_GROUP, CHARACTERS.COMMA].includes(this.state.lastDelimiter[0])) { - // Should be a tag here - this.pushTag(i) - } - this.closeGroup(i) // Close the group by updating its bounds and moving it to the parent group. - this.state.lastDelimiter = [CHARACTERS.CLOSING_GROUP, i] - } - } - - handleOpeningColumn(i) { - if (this.state.currentToken.trim().length > 0) { - this.pushInvalidCharacterIssue(CHARACTERS.OPENING_COLUMN, i) // Middle of a token Ex: "x {" - } else if (this.state.lastDelimiter[0] === CHARACTERS.OPENING_COLUMN) { - this.pushIssue('nestedCurlyBrace', i) // After open curly brace Ex: "{x{" - } else { - this.state.lastDelimiter = [CHARACTERS.OPENING_COLUMN, i] - } - } - - handleClosingColumn(i) { - if (this.state.lastDelimiter[0] !== CHARACTERS.OPENING_COLUMN) { - this.pushIssue('unopenedCurlyBrace', i) // No matching open brace Ex: " x}" - } else if (!this.state.currentToken.trim()) { - this.pushIssue('emptyCurlyBrace', i) // Column slice cannot be empty Ex: "{ }" - } else { - // Close column by updating bounds and moving it to the parent group, push a column splice on the stack. - this.state.currentGroupStack[this.state.groupDepth].push( - new ColumnSpliceSpec(this.state.currentToken.trim(), this.state.lastDelimiter[1], i), - ) - this.resetToken(i) - this.state.lastDelimiter = [CHARACTERS.CLOSING_COLUMN, i] - } - } - - handleColon(i) { - if (this.state.librarySchema || this.state.currentToken.trim().includes(CHARACTERS.BLANK)) { - this.state.currentToken += CHARACTERS.COLON // If colon has not been seen, it is a library. Ignore other colons. - } else if (/[^A-Za-z]/.test(this.state.currentToken.trim())) { - this.pushIssue('invalidTagPrefix', i) // Prefix not alphabetic Ex: "1a:xxx" - } else { - const lib = this.state.currentToken.trimStart() - this.resetToken(i) - this.state.librarySchema = lib - } - } - - unwindGroupStack() { - while (this.state.groupDepth > 0) { - this.pushIssue( - 'unclosedParenthesis', - this.state.parenthesesStack[this.state.parenthesesStack.length - 1].bounds[0], - ) - this.closeGroup(this.hedString.length) - } - } - - pushTag(i) { - if (this.state.currentToken.trim().length === 0) { - this.pushIssue('emptyTagFound', i) - } else if (this.checkForBadPlaceholderIssues(i)) { - this.pushInvalidTag('invalidPlaceholder', i, this.state.currentToken) - } else { - const bounds = getTrimmedBounds(this.state.currentToken) - this.state.currentGroupStack[this.state.groupDepth].push( - new TagSpec( - this.state.currentToken.trim(), - this.state.startingIndex + bounds[0], - this.state.startingIndex + bounds[1], - this.state.librarySchema, - ), - ) - this.resetToken(i) - } - } - - checkForBadPlaceholderIssues() { - const tokenSplit = this.state.currentToken.split(CHARACTERS.PLACEHOLDER) - if (tokenSplit.length === 1) { - // No placeholders to worry about for this tag - return false - } - return ( - tokenSplit.length > 2 || - !tokenSplit[0].endsWith(CHARACTERS.SLASH) || // A placeholder must be after a slash - (tokenSplit[1].trim().length > 0 && tokenSplit[1][0] !== CHARACTERS.BLANK) - ) - } - - closeGroup(i) { - const groupSpec = this.state.parenthesesStack.pop() - groupSpec.bounds[1] = i + 1 - if (this.hedString.slice(groupSpec.bounds[0] + 1, i).trim().length === 0) { - this.pushIssue('emptyTagFound', i) //The group is empty - } - this.state.parenthesesStack[this.state.groupDepth - 1].children.push(groupSpec) - this.state.currentGroupStack[this.state.groupDepth - 1].push(this.state.currentGroupStack.pop()) - this.state.groupDepth-- - } - - pushIssue(issueCode, index) { - this.issues.push(generateIssue(issueCode, { index, string: this.hedString })) - } - - pushInvalidTag(issueCode, index, tag) { - this.issues.push(generateIssue(issueCode, { index, tag: tag, string: this.hedString })) - } - - pushInvalidCharacterIssue(character, index) { - this.issues.push( - generateIssue('invalidCharacter', { character: unicodeName(character), index, string: this.hedString }), - ) - } -} diff --git a/parser/columnSplicer.js b/parser/columnSplicer.js index d4744a7b..d1ea2606 100644 --- a/parser/columnSplicer.js +++ b/parser/columnSplicer.js @@ -184,9 +184,10 @@ export class ColumnSplicer { * @private */ _reparseAndSpliceString(replacementString) { - const [newParsedString, parsingIssues] = parseHedString(replacementString, this.hedSchemas, true, false, false) - if (parsingIssues.length > 0) { - this.issues.push(...parsingIssues) + const [newParsedString, parsingIssues] = parseHedString(replacementString, this.hedSchemas) + const flatParsingIssues = Object.values(parsingIssues).flat() + if (flatParsingIssues.length > 0) { + this.issues.push(...flatParsingIssues) return [] } return newParsedString.parseTree @@ -204,7 +205,7 @@ export class ColumnSplicer { if (newData.length === 0) { return null } - return new ParsedHedGroup(newData, this.parsedString.hedString, group.originalBounds) + return new ParsedHedGroup(newData, this.hedSchemas, this.parsedString.hedString, group.originalBounds) } } diff --git a/parser/definitionManager.js b/parser/definitionManager.js deleted file mode 100644 index ce2328a4..00000000 --- a/parser/definitionManager.js +++ /dev/null @@ -1,313 +0,0 @@ -import { generateIssue, IssueError } from '../common/issues/issues' -import { parseHedString } from './parser' -//import { filterNonEqualDuplicates } from './parseUtils' -import { filterByTagName } from './parseUtils' - -export class Definition { - /** - * The name of the definition. - * @type {string} - */ - name - - /** - * The name of the definition. - * @type {ParsedHedTag} - */ - defTag - - /** - * The parsed HED group representing the definition - * @type {ParsedHedGroup} - */ - defGroup - - /** - * The definition contents group - * @type {ParsedHedGroup} - */ - defContents - - placeholder - - /** - * A single definition - * - * @param {ParsedHedGroup} definitionGroup - the parsedHedGroup representing the definition. - */ - constructor(definitionGroup) { - this.defGroup = definitionGroup - this._initializeDefinition(definitionGroup) - } - - _initializeDefinition(definitionGroup) { - if (definitionGroup.topTags?.length !== 1 || definitionGroup.topGroups?.length > 1) { - IssueError.generateAndThrow('invalidDefinition', { definition: definitionGroup.originalTag }) - } - this.defTag = definitionGroup.topTags[0] - this.name = this.defTag._value - this.placeholder = this.defTag._splitValue - this.defContents = this.defGroup.topGroups.length > 0 ? this.defGroup.topGroups[0] : null - } - - /** - * Return the evaluated definition contents and any issues. - * @param {ParsedHedTag} tag - The parsed HEd tag whose details should be checked. - * @param {Schemas} hedSchema - The HED schemas used to validate against. - * @param {boolean} placeholderAllowed - If true then placeholder is allowed in the def tag. - * @returns {[string | null, Issue[]]} - The evaluated normalized definition string and any issues in the evaluation, - */ - evaluateDefinition(tag, hedSchema, placeholderAllowed) { - // Check that the level of the value of tag agrees with the definition - if (!!this.defTag._splitValue !== !!tag._splitValue) { - const errorType = tag.schemaTag.name === 'Def' ? 'missingDefinitionForDef' : 'missingDefinitionForDefExpand' - return [null, [generateIssue(errorType, { definition: tag._value })]] - } - // Check that the evaluated definition contents okay (if two-level value) - if (!this.defContents) { - return ['', []] - } - if (!this.defTag._splitValue || (placeholderAllowed && tag._splitValue === '#')) { - return [this.defContents.normalized, []] - } - const evalString = this.defContents.originalTag.replace('#', tag._splitValue) - const [normalizedValue, issues] = parseHedString(evalString, hedSchema, true, false, false) - if (issues.length > 0) { - return [null, issues] - } - return [normalizedValue.normalized, []] - } - - /** - * Return true if this definition is the same as the other. - * @param {Definition} other - Another definition to compare with this one. - * @returns {boolean} - True if the definitions are equivalent - */ - equivalent(other) { - if (this.name !== other.name || this.defTag._splitValue !== other.defTag._splitValue) { - return false - } else if (this.defContents?.normalized !== other.defContents?.normalized) { - return false - } - return true - } - - /** - * Verify that the placeholder count is correct in the definition. - * @returns {boolean} - True if the placeholder count is as expected. - * @private - */ - _checkDefinitionPlaceholderCount() { - const placeholderCount = this.defContents ? this.defContents.originalTag.split('#').length - 1 : 0 - return !((placeholderCount !== 1 && this.placeholder) || (placeholderCount !== 0 && !this.placeholder)) - } - - /** - * Create a list of Definition objects from a list of strings. - * - * @param {string} hedString - A list of string definitions. - * @param {Schemas} hedSchemas - The HED schemas to use in creation. - * @returns { [Definition, Issue[]]} - The Definition list and any issues found. - */ - static createDefinition(hedString, hedSchemas) { - const [parsedString, issues] = parseHedString(hedString, hedSchemas, true, true, true) - if (issues.length > 0) { - return [null, issues] - } - if (parsedString.topLevelTags.length !== 0 || parsedString.tagGroups.length > 1) { - return [null, [generateIssue('invalidDefinition', { definition: hedString })]] - } - return Definition.createDefinitionFromGroup(parsedString.tagGroups[0]) - } - - /** - * Create a definition from a ParsedHedGroup. - * @param {ParsedHedGroup} group - The group to create a definition from. - * @returns { [Definition, Issue[]]} - The definition and any issues. (The definition will be null if issues.) - */ - static createDefinitionFromGroup(group) { - const def = new Definition(group) - if (def._checkDefinitionPlaceholderCount()) { - return [def, []] - } - return [null, [generateIssue('invalidPlaceholderInDefinition', { definition: def.defGroup.originalTag })]] - } -} - -export class DefinitionManager { - /** - * @type { Map} - Definitions for this manager. - */ - definitions - - constructor() { - this.definitions = new Map() - } - - /** - * Add the non-null definitions to this manager. - * @param {Definition[]} defs - The list of definitions to add to this manager. - * @returns {Issue[]} - Issues encountered in adding the definition. - */ - addDefinitions(defs) { - const issues = [] - for (const def of defs) { - issues.push(...this.addDefinition(def)) - } - return issues - } - - /** - * Add a Definition object to this manager - * @param {Definition} definition - The definition to be added. - * @returns {Issue[]} - */ - addDefinition(definition) { - const lowerName = definition.name.toLowerCase() - const existingDefinition = this.definitions.get(lowerName) - if (existingDefinition && !existingDefinition.equivalent(definition)) { - return [ - generateIssue('conflictingDefinitions', { - definition1: definition.defTag.originalTag, - definition2: existingDefinition.defGroup.originalTag, - }), - ] - } - if (!existingDefinition) { - this.definitions.set(lowerName, definition) - } - return [] - } - - /** - * Check the Def tags in a HED string for missing or incorrectly used Def tags. - * @param {ParsedHedString} hedString - A parsed HED string to be checked. - * @param {Schemas} hedSchemas - Schemas to validate against. - * @param {boolean} placeholderAllowed - If true then placeholder is allowed in the def tag. - * @returns {Issue []} - If there is no matching definition or definition applied incorrectly. - */ - validateDefs(hedString, hedSchemas, placeholderAllowed) { - const defTags = filterByTagName(hedString.tags, 'Def') - const issues = [] - for (const tag of defTags) { - const defIssues = this.evaluateTag(tag, hedSchemas, placeholderAllowed)[1] - if (defIssues.length > 0) { - issues.push(...defIssues) - } - } - return issues - } - - /** - * Check the Def tags in a HED string for missing or incorrectly used Def-expand tags. - * @param {ParsedHedString} hedString - A parsed HED string to be checked. - * @param {Schemas} hedSchemas - Schemas to validate against. - * @param {boolean} placeholderAllowed - If true then placeholder is allowed in the def tag. - * @returns {Issue []} - If there is no matching definition or definition applied incorrectly. - */ - validateDefExpands(hedString, hedSchemas, placeholderAllowed) { - //Def-expand tags should be rare, so don't look if there aren't any Def-expand tags - const defExpandTags = filterByTagName(hedString.tags, 'Def-expand') - if (defExpandTags.length === 0) { - return [] - } - const issues = [] - for (const topGroup of hedString.tagGroups) { - issues.push(...this._checkDefExpandGroup(topGroup, hedSchemas, placeholderAllowed)) - } - return issues - } - - /** - * Evaluate the definition based on a parsed HED tag - * @param {ParsedHedTag} tag - The tag to evaluate against the definitions. - * @param {Schemas} hedSchemas - The schemas to be used to assist in the evaluation. - * @param {boolean} placeholderAllowed - If true then placeholder is allowed in the def tag. - * @returns {[string, Issue[]]} - The evaluated definition contents with this tag and any issues. - * - * Note: If the tag is not a Def or Def-expand, this returns null for the string and [] for the issues. - */ - evaluateTag(tag, hedSchemas, placeholderAllowed) { - const [definition, missingIssues] = this.findDefinition(tag) - if (missingIssues.length > 0) { - return [null, missingIssues] - } else if (definition) { - return definition.evaluateDefinition(tag, hedSchemas, placeholderAllowed) - } - return [null, []] - } - - /** - * Recursively check for Def-expand groups in this group. - * @param {ParsedHedGroup} topGroup - a top group in a HED string to be evaluated for Def-expand groups. - * @param {Schemas} hedSchemas - The HED schemas to used in the check. - * @param {boolean} placeholderAllowed - If true then placeholder is allowed in the def tag. - * @returns {Issue[]} - * @private - */ - _checkDefExpandGroup(topGroup, hedSchemas, placeholderAllowed) { - const issues = [] - for (const group of topGroup.subParsedGroupIterator('Def-expand')) { - const defExpandTags = group.getSpecial('Def-expand') - if (defExpandTags.length === 0) { - continue - } - // There should be only one Def-expand in this group as special requirements have been checked at parsing time. - const [normalizedValue, normalizedIssues] = this.evaluateTag(defExpandTags[0], hedSchemas, placeholderAllowed) - issues.push(...normalizedIssues) - if (normalizedIssues.length > 0) { - continue - } - if (group.topGroups.length === 0 && normalizedValue !== '') { - issues.push(generateIssue('defExpandContentsInvalid', { contents: '', defContents: normalizedValue })) - } else if (group.topGroups.length > 0 && group.topGroups[0].normalized !== normalizedValue) { - issues.push( - generateIssue('defExpandContentsInvalid', { - contents: group.topGroups[0].normalized, - defContents: normalizedValue, - }), - ) - } - } - return issues - } - - /** - * Find the definition associated with a tag, if any - * @param {ParsedHedTag} tag - The parsed HEd tag to be checked. - * @returns {[Definition | null, Issue []]} - If there is no matching definition. - */ - findDefinition(tag) { - if (tag.schemaTag._name !== 'Def' && tag.schemaTag.name !== 'Def-expand') { - return [null, []] - } - const name = tag._value.toLowerCase() - const existingDefinition = this.definitions.get(name) - const errorType = tag.schemaTag.name === 'Def' ? 'missingDefinitionForDef' : 'missingDefinitionForDefExpand' - if (!existingDefinition) { - return [null, [generateIssue(errorType, { definition: name })]] - } - if (!!existingDefinition.defTag._splitValue !== !!tag._splitValue) { - return [null, [generateIssue(errorType, { definition: name })]] - } - return [existingDefinition, []] - } - - /** - * Create a list of Definition objects from a list of strings. - * - * @param {string[]} defStrings - A list of string definitions. - * @param {Schemas} hedSchemas - The HED schemas to use in creation. - * @returns { [Definition[], Issue[]]} - The Definition list and any issues found. - */ - static createDefinitions(defStrings, hedSchemas) { - const defList = [] - const issues = [] - for (const defString of defStrings) { - const [nextDef, defIssues] = Definition.createDefinition(defString, hedSchemas) - defList.push(nextDef) - issues.push(...defIssues) - } - return [defList, issues] - } -} diff --git a/parser/eventManager.js b/parser/eventManager.js deleted file mode 100644 index 0c4c33ac..00000000 --- a/parser/eventManager.js +++ /dev/null @@ -1,187 +0,0 @@ -import { generateIssue } from '../common/issues/issues' -import { BidsHedIssue } from '../bids' - -export class Event { - /** - * The name of the definition. - * @type {number} - */ - onset - - /** - * The parsed HED group representing the definition - * @type {string} - */ - defName - - type - group - element - - /** - * The parsed HED group representing the definition - * @type {ParsedHedGroup} - */ - - constructor(defName, eventType, onset, group, element) { - this.defName = defName - this.type = eventType - this.onset = onset - this.group = group - this.file = element.file - this.tsvLine = element.tsvLine - } - - /** - * Create an event from a ParsedHedGroup. - * @param {ParsedHedGroup} group - A group to extract an event from a temporal group, if it is a group. - * @param {BidsTsvElement} element - The element in which this group appears. - * @returns {[Event, BidsHedIssue[]]} - The event extracted from the group - */ - static createEvent(group, element) { - if (!group.requiresDefTag) { - return [null, []] - } - let onset = Number(element.onset) - if (!Number.isFinite(onset)) { - return [ - null, - [ - BidsHedIssue.fromHedIssue( - generateIssue('temporalTagInNonTemporalContext', { string: element.hedString }), - element.file, - { tsvLine: element.tsvLine }, - ), - ], - ] - } - onset = onset + Event.extractDelay(group) - const eventType = group.requiresDefTag.schemaTag.name - let defName = null - const defTags = group.defTags - if (defTags.length === 1) { - defName = defTags[0]._remainder.toLowerCase() - } else { - return [ - null, - [ - BidsHedIssue.fromHedIssue( - generateIssue('temporalWithWrongNumberDefs', { tagGroup: group.originalTag, tag: eventType }), - element.file, - { tsvLine: element.tsvLine }, - ), - ], - ] - } - const event = new Event(defName, eventType, onset, group, element) - return [event, []] - } - - static extractDelay(group) { - if (!group.specialTags.has('Delay')) { - return 0 - } - const tags = group.specialTags.get('Delay') - const delay = Number(tags[0]._value) - return Number.isFinite(delay) ? delay : 0 - } -} - -export class EventManager { - static TOLERANCE = 1e-7 - constructor() {} - - /** - * Create a list of temporal events from BIDS elements. - * @param {BidsTsvElement[]} elements - The elements representing the contents of a tsv file. - * @returns {[Event[], BidsHedIssue[]]} - */ - parseEvents(elements) { - const eventList = [] - for (const element of elements) { - if (!element.parsedHedString) { - continue - } - - for (const group of element.parsedHedString.tagGroups) { - const [event, eventIssues] = Event.createEvent(group, element) - if (eventIssues.length > 0) { - return [null, eventIssues] - } - if (event) { - eventList.push(event) - } - } - } - eventList.sort((a, b) => a.onset - b.onset) - return [eventList, []] - } - - validate(eventList) { - const currentMap = new Map() - for (const event of eventList) { - if (!currentMap.has(event.defName)) { - if (event.type === 'Offset' || event.type === 'Inset') { - return [ - BidsHedIssue.fromHedIssue( - generateIssue('inactiveOnset', { tag: event.type, definition: event.defName }), - event.file, - { tsvLine: event.tsvLine }, - ), - ] - } - currentMap.set(event.defName, event) - continue - } - const issues = this._resolveConflicts(currentMap, event) - if (issues.length > 0) { - return issues - } - } - return [] - } - - _resolveConflicts(currentMap, event) { - const currentEvent = currentMap.get(event.defName) - // Make sure that these events are not at the same time - if (Math.abs(currentEvent.onset - event.onset) < EventManager.TOLERANCE) { - return [ - BidsHedIssue.fromHedIssue( - generateIssue('simultaneousDuplicateEvents', { - tagGroup1: event.group.originalTag, - onset1: event.onset.toString(), - tsvLine1: event.tsvLine, - tagGroup2: currentEvent.group.originalTag, - onset2: currentEvent.onset.toString(), - tsvLine2: currentEvent.tsvLine, - }), - event.file, - ), - ] - } - - if (event.type === 'Onset') { - currentMap.set(event.defName, event) - } else if (event.type === 'Inset' && currentEvent.type !== 'Offset') { - currentMap.set(event.defName, event) - } else if (event.type === 'Offset' && currentEvent.type !== 'Offset') { - currentMap.set(event.defName, event) - // } else { - // return [ - // BidsHedIssue.fromHedIssue( - // generateIssue('simultaneousDuplicateEvents', { - // tagGroup1: event.group.originalTag, - // onset1: event.onset.toString(), - // tsvLine1: event.element.tsvLine, - // tagGroup2: currentEvent.group.originalTag, - // onset2: currentEvent.onset.toString(), - // tsvLine2: currentEvent.element.tsvLine, - // }), - // event.tsvFile, - // ), - // ] - } - - return [] - } -} diff --git a/parser/parseUtils.js b/parser/parseUtils.js deleted file mode 100644 index ea9e4a0d..00000000 --- a/parser/parseUtils.js +++ /dev/null @@ -1,107 +0,0 @@ -import ParsedHedTag from './parsedHedTag' - -/** - * Extract the items of a specified subtype from a list of ParsedHedSubstring - * @param {ParsedHedSubstring[]} items - to be filtered by class type - * @param {Class} classType - class type to filter by - * @returns {*|*[]} - */ - -export function filterByClass(items, classType) { - return items && items.length ? items.filter((item) => item instanceof classType) : [] -} - -/** - * Extract the ParsedHedTag tags with a specified tag name - * @param {ParsedHedTag[]} tags - to be filtered by name - * @param {string} tagName - name of the tag to filter by - * @returns {ParsedHedTag[]} - */ - -export function filterByTagName(tags, tagName) { - if (!tags) { - return [] - } - return tags.filter((tag) => tag instanceof ParsedHedTag && tag.schemaTag?.name === tagName) -} - -/** - * Extract the ParsedHedTag tags with a specified tag name. - * @param {Map} tagMap - The Map of parsed HED tags for extraction (must be defined). - * @param {string[]} tagNames - The names to use as keys for the filter. - * @returns {ParsedHedTag[]} - A list of temporal tags. - */ -export function filterTagMapByNames(tagMap, tagNames) { - if (!tagNames || tagMap.size === 0) { - return [] - } - - const keys = [...tagNames].filter((name) => tagMap.has(name)) - if (keys.length === 0) { - return [] - } - - return keys.flatMap((key) => tagMap.get(key)) -} - -/*/!** - * Extract the ParsedHedTag tags that have a name from a specified list of names - * @param {ParsedHedTag[]} tags - to be filtered by name - * @param {[string]} tagList - List of tag names to filter by. - * @returns {ParsedHedTag[]} - A list of tags whose - *!/ - -export function filterByTagNames(tags, tagList) { - if (!tags || !tagList) { - return [] - } - return tags.filter((tag) => tagList.includes(tag.schemaTag.name)) -}*/ - -/** - * Convert a list of ParsedHedTag objects into a comma-separated string of their string representations. - * @param {ParsedHedTag []} tagList - The HED tags whose string representations should be put in a comma-separated list. - * @returns {string} A comma separated list of original tag names for tags in tagList. - */ -export function getTagListString(tagList) { - return tagList.map((tag) => tag.toString()).join(', ') -} - -/** - * Create a map of the ParsedHedTags by type - * @param { ParsedHedTag[] } tagList - * @param {Set} tagNames - * @returns {Map} - */ -export function categorizeTagsByName(tagList, tagNames = null) { - // Initialize the map with keys from tagNames and an "other" key - const resultMap = new Map() - - // Iterate through A and categorize - tagList.forEach((tag) => { - if (!tagNames || tagNames.has(tag.schemaTag.name)) { - const tagList = resultMap.get(tag.schemaTag.name) || [] - tagList.push(tag) - resultMap.set(tag.schemaTag.name, tagList) // Add to matching key list - } - }) - return resultMap -} - -/** - * Return a list of duplicate strings - * @param { string[] } itemList - A list of strings to look for duplicates in - * @returns {string []} - a list of unique duplicate strings (multiple copies not repeated - */ -export function getDuplicates(itemList) { - const checkSet = new Set() - const dupSet = new Set() - for (const item of itemList) { - if (!checkSet.has(item)) { - checkSet.add(item) - } else { - dupSet.add(item) - } - } - return [...dupSet] -} diff --git a/parser/parsedHedColumnSplice.js b/parser/parsedHedColumnSplice.js index ef76a726..c67381c4 100644 --- a/parser/parsedHedColumnSplice.js +++ b/parser/parsedHedColumnSplice.js @@ -4,23 +4,6 @@ import ParsedHedSubstring from './parsedHedSubstring' * A template for an inline column splice in a {@link ParsedHedString} or {@link ParsedHedGroup}. */ export class ParsedHedColumnSplice extends ParsedHedSubstring { - /** - * Constructor. - * - * @param {string} columnName The token for this tag. - * @param {number[]} bounds The collection of HED schemas. - */ - constructor(columnName, bounds) { - super(columnName, bounds) // Sets originalTag and originalBounds - this._normalized = this.format(false) // Sets various forms of the tag. - } - - get normalized() { - { - return this._normalized - } - } - /** * Nicely format this column splice template. * diff --git a/parser/parsedHedGroup.js b/parser/parsedHedGroup.js index 35590c55..f5092af0 100644 --- a/parser/parsedHedGroup.js +++ b/parser/parsedHedGroup.js @@ -1,36 +1,32 @@ import differenceWith from 'lodash/differenceWith' import { IssueError } from '../common/issues/issues' +import { getTagName } from '../utils/hedStrings' import ParsedHedSubstring from './parsedHedSubstring' import ParsedHedTag from './parsedHedTag' import ParsedHedColumnSplice from './parsedHedColumnSplice' -import { ReservedChecker } from './reservedChecker' -import { - filterByClass, - categorizeTagsByName, - getDuplicates, - filterByTagName, - filterTagMapByNames, - getTagListString, -} from './parseUtils' /** * A parsed HED tag group. */ export default class ParsedHedGroup extends ParsedHedSubstring { + static SPECIAL_SHORT_TAGS = new Set([ + 'Definition', + 'Def', + 'Def-expand', + 'Onset', + 'Offset', + 'Inset', + 'Delay', + 'Duration', + 'Event-context', + ]) + /** * The parsed HED tags or parsedHedGroups or parsedColumnSplices in the HED tag group at the top level * @type {ParsedHedSubstring[]} */ tags - - topTags - - topGroups - - topSplices - - allTags /** * Any HED tags with special handling. This only covers top-level tags in the group * @type {Map} @@ -41,96 +37,60 @@ export default class ParsedHedGroup extends ParsedHedSubstring { * @type {boolean} */ hasDefExpandChildren - /** * The top-level child subgroups containing Def-expand tags. * @type {ParsedHedGroup[]} */ defExpandChildren - isDefExpandGroup - - isDefinitionGroup - - defCount - - requiresDefTag - /** * Constructor. * @param {ParsedHedSubstring[]} parsedHedTags The parsed HED tags, groups or column splices in the HED tag group. + * @param {Schemas} hedSchemas The collection of HED schemas. * @param {string} hedString The original HED string. * @param {number[]} originalBounds The bounds of the HED tag in the original HED string. */ - constructor(parsedHedTags, hedString, originalBounds) { - const originalTag = hedString.substring(originalBounds[0], originalBounds[1]) + constructor(parsedHedTags, hedSchemas, hedString, originalBounds) { + const originalTag = hedString.substring(...originalBounds) super(originalTag, originalBounds) this.tags = parsedHedTags - this.topGroups = filterByClass(parsedHedTags, ParsedHedGroup) - this.topTags = filterByClass(parsedHedTags, ParsedHedTag) - this.topSplices = filterByClass(parsedHedTags, ParsedHedColumnSplice) - this.allTags = this._getAllTags() - this._normalized = undefined - this._initializeGroups() - } - - _getAllTags() { - const subgroupTags = this.topGroups.flatMap((tagGroup) => tagGroup.allTags) - return this.topTags.concat(subgroupTags) + this._findSpecialGroups(hedSchemas) } - _initializeGroups() { - const special = ReservedChecker.getInstance() - this.specialTags = categorizeTagsByName(this.topTags, special.specialNames) - this.isDefExpandGroup = this.specialTags.has('Def-expand') - this.isDefinitionGroup = this.specialTags.has('Definition') - this.defExpandChildren = this._filterSubgroupsByTagName('Def-expand') + _findSpecialGroups(hedSchemas) { + this.specialTags = new Map() + for (const shortTag of ParsedHedGroup.SPECIAL_SHORT_TAGS) { + const tags = ParsedHedGroup.findGroupTags(this, hedSchemas, shortTag) + if (tags !== undefined) { + this.specialTags.set(shortTag, tags) + } + } + this.defExpandChildren = Array.from(this.topLevelGroupIterator()).filter((subgroup) => subgroup.isDefExpandGroup) this.hasDefExpandChildren = this.defExpandChildren.length !== 0 - this.defCount = this.getSpecial('Def').length + this.defExpandChildren.length - this.requiresDefTag = this._getRequiresDefTag(special.requiresDefTags) } /** - * Filter top subgroups that include a special at the top-level of the group + * Determine a parsed HED tag group's special tags. * - * @param {string} tagName - The schemaTag name to filter by. - * @returns {Array} - Array of subgroups containing the specified tag. - */ - _filterSubgroupsByTagName(tagName) { - return Array.from(this.topLevelGroupIterator()).filter((subgroup) => subgroup.specialTags.has(tagName)) - } - - /** - * Return the unique requiresDef tag associated with this group (if any). - * @param {string[]} tagNames - The list of requiresDef tag names to use (based on the special tag requirements). - * @returns {ParsedHedTag | null} - The parsed requiresDef tag (if any) or null. - * @throws {IssueError} - If there are too many or too few defs or too many requiresDef tags in this group. - * @private + * @param {ParsedHedGroup} group The parsed HED tag group. + * @param {Schemas} hedSchemas The collection of HED schemas. + * @param {string} shortTag The short tag to search for. + * @returns {null|ParsedHedTag[]} The tag(s) matching the short tag. */ - _getRequiresDefTag(tagNames) { - const requiresDefTags = filterTagMapByNames(this.specialTags, tagNames) - if (requiresDefTags.length > 1) { - IssueError.generateAndThrow('multipleRequiresDefTags', { - tags: getTagListString(requiresDefTags), - string: this.originalTag, - }) - } - if (requiresDefTags.length === 0) { - return null - } - if (this.defCount > 1) { - IssueError.generateAndThrow('temporalWithWrongNumberDefs', { - tag: requiresDefTags[0].originalTag, - tagGroup: this.originalTag, - }) - } - if (this.topSplices.length === 0 && this.defCount === 0) { - IssueError.generateAndThrow('temporalWithWrongNumberDefs', { - tag: requiresDefTags[0].originalTag, - tagGroup: this.originalTag, - }) + static findGroupTags(group, hedSchemas, shortTag) { + const tags = group.tags.filter((tag) => { + if (!(tag instanceof ParsedHedTag)) { + return false + } + const schemaTag = tag.schemaTag + return schemaTag.name === shortTag + }) + switch (tags.length) { + case 0: + return undefined + default: + return tags } - return requiresDefTags[0] } /** @@ -143,12 +103,76 @@ export default class ParsedHedGroup extends ParsedHedSubstring { return '(' + this.tags.map((substring) => substring.format(long)).join(', ') + ')' } - getSpecial(tagName) { - return this.specialTags.get(tagName) ?? [] + /** + * The {@code Definition} tags associated with this HED tag group. + * @returns {ParsedHedTag[]} + */ + get definitionTags() { + return this.specialTags.get('Definition') + } + + /** + * The {@code Def} tags associated with this HED tag group. + * @returns {ParsedHedTag[]} + */ + get defTags() { + return this.specialTags.get('Def') } - isSpecialGroup(tagName) { - return this.specialTags.has(tagName) + /** + * The {@code Def-expand} tags associated with this HED tag group. + * @returns {ParsedHedTag[]} + */ + get defExpandTags() { + return this.specialTags.get('Def-expand') + } + + /** + * Whether this HED tag group is a definition group. + * @returns {boolean} + */ + get isDefinitionGroup() { + return this.specialTags.has('Definition') + } + + /** + * Whether this HED tag group has a {@code Def} tag. + * @returns {boolean} + */ + get isDefGroup() { + return this.specialTags.has('Def') + } + + /** + * Whether this HED tag group has a {@code Def-expand} tag. + * @returns {boolean} + */ + get isDefExpandGroup() { + return this.specialTags.has('Def-expand') + } + + /** + * Whether this HED tag group is an onset group. + * @returns {boolean} + */ + get isOnsetGroup() { + return this.specialTags.has('Onset') + } + + /** + * Whether this HED tag group is an offset group. + * @returns {boolean} + */ + get isOffsetGroup() { + return this.specialTags.has('Offset') + } + + /** + * Whether this HED tag group is an inset group. + * @returns {boolean} + */ + get isInsetGroup() { + return this.specialTags.has('Inset') } /** @@ -156,15 +180,336 @@ export default class ParsedHedGroup extends ParsedHedSubstring { * @returns {boolean} */ get isTemporalGroup() { - return this.isSpecialGroup('Onset') || this.isSpecialGroup('Offset') || this.isSpecialGroup('Inset') + return this.isOnsetGroup || this.isOffsetGroup || this.isInsetGroup } - get defTags() { - const tags = this.getSpecial('Def') - for (const group of this.defExpandChildren) { - tags.push(...group.getSpecial('Def-expand')) + /** + * Whether this HED tag group is an onset, offset, or inset group. + * @returns {string} + */ + get temporalGroupName() { + if (this.isOnsetGroup) { + return 'Onset' + } else if (this.isOffsetGroup) { + return 'Offset' + } else if (this.isInsetGroup) { + return 'Inset' + } else { + return undefined } - return tags + } + + /** + * Find what should be the sole definition tag, or throw an error if more than one is found. + * + * @returns {ParsedHedTag} This group's definition tag. + */ + get definitionTag() { + return this.getSingleDefinitionTag('definitionTag', 'Definition') + } + + /** + * Find what should be the sole {@code Def-expand} tag, or throw an error if more than one is found. + * + * @returns {ParsedHedTag} This group's {@code Def-expand} tag. + */ + get defExpandTag() { + return this.getSingleDefinitionTag('defExpandTag', 'Def-expand') + } + + getSingleDefinitionTag(fieldName, parentTag) { + return this._memoize(fieldName, () => { + switch (this.specialTags.get(parentTag).length) { + case 0: + return undefined + case 1: + return this.specialTags.get(parentTag)[0] + default: + throw new Error(`Single ${parentTag} tag asserted, but multiple ${parentTag} tags found.`) + } + }) + } + + /** + * A list of all tags in the group + * @returns {ParsedHedTag[]} + */ + get allTags() { + return this._memoize('allTags', () => { + const tagGroups = this.tags.filter((obj) => obj instanceof ParsedHedGroup) + const subgroupTags = tagGroups.flatMap((tagGroup) => tagGroup.allTags) + return this.topTags.concat(subgroupTags) + }) + } + + /** + * A list of all tags in the subgroups + * @returns {ParsedHedTag[]} + */ + get allSubgroupTags() { + return this._memoize('allSubgroupTags', () => { + const tagGroups = this.tags.filter((obj) => obj instanceof ParsedHedGroup) + return tagGroups.flatMap((tagGroup) => tagGroup.allTags) + }) + } + + get topColumnSplices() { + return this._memoize('topColumnSplices', () => { + return this.tags.filter((tagOrGroup) => tagOrGroup instanceof ParsedHedColumnSplice) + }) + } + + get topTags() { + return this._memoize('topTags', () => { + return this.tags.filter((tagOrGroup) => tagOrGroup instanceof ParsedHedTag) + }) + } + + get topGroups() { + return this._memoize('topGroups', () => { + return this.tags.filter((tagOrGroup) => tagOrGroup instanceof ParsedHedGroup) + }) + } + + /* get specialTagList() { + return this._memoize('specialTagList', () => { + const special = new SpecialChecker() + return this.allTags.filter((obj) => special.specialNames.includes(obj.schemaTag.name)) + }) + }*/ + + /* get hasForbiddenSubgroupTags() { + return this._memoize('hasForbiddenSubgroupTags', () => { + return this.allTags.some((obj) => new SpecialChecker().hasForbiddenSubgroupTags.includes(obj.schemaTag.name)) + }) + }*/ + + /* /!** + * A list of all column splices at all levels + * + * @returns {ParsedHedColumnSplice[]} The "name" portion of the canonical tag. + *!/ + get allColumnSplices() { + return this._memoize('allColumnSplices', () => { + return Array.from(this.columnSpliceIterator()) + }) + }*/ + + /** + * Determine the name of this group's definition. + */ + static findDefinitionName(canonicalTag, definitionBase) { + const tag = canonicalTag + let value = getTagName(tag) + let previousValue + for (const level of ParsedHedTag.ancestorIterator(tag)) { + if (value.toLowerCase() === definitionBase.toLowerCase()) { + return previousValue + } + previousValue = value + value = getTagName(level) + } + throw Error( + `Completed iteration through ${definitionBase.toLowerCase()} tag without finding ${definitionBase} level.`, + ) + } + + /** + * Determine the name of this group's definition. + * @returns {string|null} + */ + get definitionName() { + return this.getSingleDefinitionName('definitionName', 'Definition') + } + + /** + * Determine the name of this group's definition. + * @returns {string|null} + */ + get defExpandName() { + return this.getSingleDefinitionName('defExpandName', 'Def-expand') + } + + getSingleDefinitionName(fieldName, parentTag) { + return this._memoize(fieldName, () => { + if (!this.specialTags.has(parentTag)) { + return null + } + return ParsedHedGroup.findDefinitionName( + this.getSingleDefinitionTag(fieldName, parentTag).canonicalTag, + parentTag, + ) + }) + } + + /** + * Determine the value of this group's definition. + * @returns {string|null} + */ + get definitionValue() { + return this.getSingleDefinitionValue('definitionValue', 'Definition') + } + + /** + * Determine the value of this group's definition. + * @returns {string|null} + */ + get defExpandValue() { + return this.getSingleDefinitionValue('defExpandValue', 'Def-expand') + } + + getSingleDefinitionValue(fieldName, parentTag) { + return this._memoize(fieldName, () => { + if (!this.specialTags.has(parentTag)) { + return null + } + return ParsedHedGroup.getDefinitionTagValue(this.getSingleDefinitionTag(fieldName, parentTag), parentTag) + }) + } + + /** + * Determine the name and value of this group's definition. + * @returns {string|null} + */ + get definitionNameAndValue() { + return this.getSingleDefinitionNameAndValue('definition', 'Definition') + } + + /** + * Determine the name and value of this group's definition. + * @returns {string|null} + */ + get defExpandNameAndValue() { + return this.getSingleDefinitionNameAndValue('defExpand', 'Def-expand') + } + + getSingleDefinitionNameAndValue(fieldName, parentTag) { + return this._memoize(fieldName + 'NameAndValue', () => { + if (!this.specialTags.has(parentTag)) { + return null + } else if (this.getSingleDefinitionValue(fieldName + 'Value', parentTag)) { + return ( + this.getSingleDefinitionName(fieldName + 'Name', parentTag) + + '/' + + this.getSingleDefinitionValue(fieldName + 'Value', parentTag) + ) + } else { + return this.getSingleDefinitionName(fieldName + 'Name', parentTag) + } + }) + } + + /** + * Determine the name(s) of this group's definition. + * @returns {string|string[]|null} + */ + get defName() { + return this._memoize('defName', () => { + if (!this.isDefGroup && !this.hasDefExpandChildren) { + return null + } else if (!this.isTemporalGroup) { + return [].concat( + this.defExpandChildren.map((defExpandChild) => defExpandChild.defExpandName), + this.defTags.map((defTag) => ParsedHedGroup.findDefinitionName(defTag.canonicalTag, 'Def')), + ) + } else if (this.defCount > 1) { + IssueError.generateAndThrow('temporalWithMultipleDefinitions', { + tagGroup: this.originalTag, + tag: this.temporalGroupName, + }) + } else if (this.hasDefExpandChildren) { + return this.defExpandChildren[0].defExpandName + } + return ParsedHedGroup.findDefinitionName(this.defTags[0].canonicalTag, 'Def') + }) + } + + /** + * Determine the name of this group's definition. + * @returns {string|null} + */ + get defValue() { + return this._memoize('defValue', () => { + if (!this.isDefGroup && !this.hasDefExpandChildren) { + return null + } else if (!this.isTemporalGroup) { + return [].concat( + this.defExpandChildren.map((defExpandChild) => defExpandChild.defExpandValue), + this.defTags.map((defTag) => ParsedHedGroup.getDefinitionTagValue(defTag, 'Def')), + ) + } else if (this.defCount > 1) { + IssueError.generateAndThrow('temporalWithMultipleDefinitions', { + tagGroup: this.originalTag, + tag: this.temporalGroupName, + }) + } else if (this.hasDefExpandChildren) { + return this.defExpandChildren[0].defExpandValue + } + return ParsedHedGroup.getDefinitionTagValue(this.defTags[0], 'Def') + }) + } + + /** + * Determine the name and value of this group's {@code Def} or {@code Def-expand}. + * @returns {string|null} + */ + get defNameAndValue() { + return this._memoize('defNameAndValue', () => { + if (!this.isDefGroup && !this.hasDefExpandChildren) { + return null + } else if (this.defValue) { + return this.defName + '/' + this.defValue + } else { + return this.defName + } + }) + } + + /** + * Extract the value from a definition tag. + * + * @param {ParsedHedTag} tag A definition-type tag. + * @param {string} parentTag The expected parent of the tag. + * @returns {string} The parameterized value of the definition, or an empty string if no value was found. + */ + static getDefinitionTagValue(tag, parentTag) { + if (getTagName(tag.parentCanonicalTag) === parentTag) { + return '' + } else { + return tag.originalTagName + } + } + + /** + * Determine the value of this group's definition. + * @returns {ParsedHedGroup|null} + */ + get definitionGroup() { + return this._memoize('definitionGroup', () => { + if (!this.isDefinitionGroup) { + return null + } + for (const subgroup of this.tags) { + if (subgroup instanceof ParsedHedGroup) { + return subgroup + } + } + return null + }) + } + + /** + * Determine the number of {@code Def} and {@code Def-expand} tag/tag groups included in this group. + * @returns {number} The number of first-level definition reference tags and tag groups in this group. + */ + get defCount() { + return this._memoize('defCount', () => { + if (this.isDefGroup) { + return this.defTags.length + this.defExpandChildren.length + } else { + return this.defExpandChildren.length + } + }) } equivalent(other) { @@ -183,41 +528,15 @@ export default class ParsedHedGroup extends ParsedHedSubstring { * @returns {ParsedHedTag[]} */ nestedGroups() { - const groups = [] + const currentGroup = [] for (const innerTag of this.tags) { if (innerTag instanceof ParsedHedTag) { - groups.push(innerTag) + currentGroup.push(innerTag) } else if (innerTag instanceof ParsedHedGroup) { - groups.push(innerTag.nestedGroups()) + currentGroup.push(innerTag.nestedGroups()) } } - return groups - } - - /** - * Return a normalized string representation - * @returns {string} - */ - get normalized() { - if (this._normalized) { - return this._normalized - } - // Recursively normalize each item in the group - const normalizedItems = this.tags.map((item) => item.normalized) - - // Sort normalized items to ensure order independence - const sortedNormalizedItems = normalizedItems.sort() - - const duplicates = getDuplicates(sortedNormalizedItems) - if (duplicates.length > 0) { - IssueError.generateAndThrow('duplicateTag', { - tags: '[' + duplicates.join('],[') + ']', - string: this.originalTag, - }) - } - this._normalized = '(' + sortedNormalizedItems.join(',') + ')' - // Return the normalized group as a string - return `(${sortedNormalizedItems.join(',')})` // Using curly braces to indicate unordered group + return currentGroup } /** @@ -236,16 +555,14 @@ export default class ParsedHedGroup extends ParsedHedSubstring { /** * Iterator over the ParsedHedGroup objects in this HED tag group. - * @param {string | null} tagName - The name of the tag whose groups are to be iterated over or null if all tags. - * @yields {ParsedHedGroup} - This object and the ParsedHedGroup objects belonging to this tag group. + * + * @yields {ParsedHedGroup} This object and the ParsedHedGroup objects belonging to this tag group. */ - *subParsedGroupIterator(tagName = null) { - if (!tagName || filterByTagName(this.topTags, tagName)) { - yield this - } + *subParsedGroupIterator() { + yield this for (const innerTag of this.tags) { if (innerTag instanceof ParsedHedGroup) { - yield* innerTag.subParsedGroupIterator(tagName) + yield* innerTag.subParsedGroupIterator() } } } diff --git a/parser/parsedHedString.js b/parser/parsedHedString.js index 3e4a5f7a..5c756eb1 100644 --- a/parser/parsedHedString.js +++ b/parser/parsedHedString.js @@ -1,8 +1,6 @@ import ParsedHedTag from './parsedHedTag' import ParsedHedGroup from './parsedHedGroup' import ParsedHedColumnSplice from './parsedHedColumnSplice' -import { filterByClass, getDuplicates } from './parseUtils' -import { IssueError } from '../common/issues/issues' /** * A parsed HED string. @@ -47,7 +45,12 @@ export class ParsedHedString { * The top-level definition tag groups in the string. * @type {ParsedHedGroup[]} */ - definitions + definitionGroups + /** + * The context in which this string was defined. Applicable definitions. + * @type {Map} + */ + context /** * Constructor. @@ -57,20 +60,24 @@ export class ParsedHedString { constructor(hedString, parsedTags) { this.hedString = hedString this.parseTree = parsedTags - this.tagGroups = filterByClass(parsedTags, ParsedHedGroup) - this.topLevelTags = filterByClass(parsedTags, ParsedHedTag) + this.tagGroups = parsedTags.filter((tagOrGroup) => tagOrGroup instanceof ParsedHedGroup) + this.topLevelTags = parsedTags.filter((tagOrGroup) => tagOrGroup instanceof ParsedHedTag) + const topLevelColumnSplices = parsedTags.filter((tagOrGroup) => tagOrGroup instanceof ParsedHedColumnSplice) const subgroupTags = this.tagGroups.flatMap((tagGroup) => Array.from(tagGroup.tagIterator())) this.tags = this.topLevelTags.concat(subgroupTags) - const topLevelColumnSplices = filterByClass(parsedTags, ParsedHedColumnSplice) const subgroupColumnSplices = this.tagGroups.flatMap((tagGroup) => Array.from(tagGroup.columnSpliceIterator())) this.columnSplices = topLevelColumnSplices.concat(subgroupColumnSplices) - //this.topLevelGroupTags = this.tagGroups.map((tagGroup) => filterByClass(tagGroup.tags, ParsedHedTag)) - this.topLevelGroupTags = this.tagGroups.flatMap((tagGroup) => filterByClass(tagGroup.tags, ParsedHedTag)) - this.definitions = this.tagGroups.filter((group) => group.isDefinitionGroup) - this.normalized = this._getNormalized() + this.topLevelGroupTags = this.tagGroups.map((tagGroup) => + tagGroup.tags.filter((tagOrGroup) => tagOrGroup instanceof ParsedHedTag), + ) + this.definitionGroups = this.tagGroups.filter((group) => { + return group.isDefinitionGroup + }) + + this.context = new Map() } /** @@ -83,22 +90,10 @@ export class ParsedHedString { return this.parseTree.map((substring) => substring.format(long)).join(', ') } - /** - * Return a normalized string representation - * @returns {string} - */ - _getNormalized() { - // This is an implicit recursion as the items have the same call. - const normalizedItems = this.parseTree.map((item) => item.normalized) - - // Sort normalized items to ensure order independence - const sortedNormalizedItems = normalizedItems.sort() - const duplicates = getDuplicates(sortedNormalizedItems) - if (duplicates.length > 0) { - IssueError.generateAndThrow('duplicateTag', { tags: '[' + duplicates.join('],[') + ']', string: this.hedString }) - } - // Return the normalized group as a string - return `${sortedNormalizedItems.join(',')}` // Using curly braces to indicate unordered group + get definitions() { + return this.definitionGroups.map((group) => { + return [group.definitionName, group] + }) } /** diff --git a/parser/parsedHedSubstring.js b/parser/parsedHedSubstring.js index 27ea6a4b..491e7b4e 100644 --- a/parser/parsedHedSubstring.js +++ b/parser/parsedHedSubstring.js @@ -1,7 +1,9 @@ +import Memoizer from '../utils/memoizer' + /** * A parsed HED substring. */ -export class ParsedHedSubstring { +export class ParsedHedSubstring extends Memoizer { /** * The original pre-parsed version of the HED tag. * @type {string} @@ -19,30 +21,36 @@ export class ParsedHedSubstring { * @param {number[]} originalBounds The bounds of the HED tag in the original HED string. */ constructor(originalTag, originalBounds) { + super() + this.originalTag = originalTag this.originalBounds = originalBounds } /** - * Nicely format this substring. This is left blank for the subclasses to override. + * Determine whether this tag is a descendant of another tag. * - * This is left blank for the subclasses to override. + * This is a default implementation. Subclasses should override as appropriate. * - * @param {boolean} long - Whether the tags should be in long form. - * @returns {string} - * @abstract + * @param {ParsedHedTag|string} parent The possible parent tag. + * @return {boolean} Whether {@code parent} is the parent tag of this tag. */ - format(long = true) {} + // eslint-disable-next-line no-unused-vars + isDescendantOf(parent) { + return false + } /** - * Get the normalized version of the object. + * Nicely format this substring. + * + * This is left blank for the subclasses to override. * + * @param {boolean} long Whether the tags should be in long form. * @returns {string} * @abstract */ - get normalized() { - return '' - } + // eslint-disable-next-line no-unused-vars + format(long = true) {} /** * Override of {@link Object.prototype.toString}. diff --git a/parser/parsedHedTag.js b/parser/parsedHedTag.js index c4f3ad20..4ca4260a 100644 --- a/parser/parsedHedTag.js +++ b/parser/parsedHedTag.js @@ -2,7 +2,7 @@ import { IssueError } from '../common/issues/issues' import ParsedHedSubstring from './parsedHedSubstring' import { SchemaValueTag } from '../schema/entries' import TagConverter from './tagConverter' -import { ReservedChecker } from './reservedChecker' +import { SpecialChecker } from './special' const allowedRegEx = /^[^{}\,]*$/ @@ -33,6 +33,14 @@ export default class ParsedHedTag extends ParsedHedSubstring { */ _schemaTag + /** + * The extension part if it has an extension rather than a value + * + * @type {string} + * @private + */ + _extension + /** * The remaining part of the tag after the portion actually in the schema. * @@ -75,9 +83,7 @@ export default class ParsedHedTag extends ParsedHedSubstring { */ constructor(tagSpec, hedSchemas, hedString) { super(tagSpec.tag, tagSpec.bounds) // Sets originalTag and originalBounds - this._convertTag(hedSchemas, hedString, tagSpec) - this._normalized = this.format(false) // Sets various forms of the tag. - this._validUnits = null + this._convertTag(hedSchemas, hedString, tagSpec) // Sets various forms of the tag. } /** @@ -115,7 +121,7 @@ export default class ParsedHedTag extends ParsedHedSubstring { /** * Handle the remainder portion for value tag (converter handles others) * - * @param {SchemaTag} schemaTag - The part of the tag that is in the schema + * @param (SchemaTag) schemaTag - the part of the tag that is in the schema * @param {string} remainder - the leftover part * @throws {IssueError} If parsing the remainder section fails. */ @@ -124,9 +130,9 @@ export default class ParsedHedTag extends ParsedHedSubstring { return } // Check that there is a value if required - const special = ReservedChecker.getInstance() + const special = SpecialChecker.getInstance() if ( - (schemaTag.hasAttributeName('requireChild') || special.requireValueTags.has(schemaTag.name)) && + (schemaTag.hasAttributeName('requireChild') || special.requireValueTags.includes(schemaTag.name)) && remainder === '' ) { IssueError.generateAndThrow('valueRequired', { tag: this.originalTag }) @@ -137,7 +143,7 @@ export default class ParsedHedTag extends ParsedHedSubstring { // Resolve the units and check const [actualUnit, actualUnitString, actualValueString] = this._separateUnits(schemaTag, value) - this._units = actualUnitString + this._units = actualUnit this._value = actualValueString if (actualUnit === null && actualUnitString !== null) { @@ -148,14 +154,6 @@ export default class ParsedHedTag extends ParsedHedSubstring { } } - /** - * Separate the remainder of the tag into three parts: - * - * @param {SchemaTag} schemaTag - The part of the tag that is in the schema - * @param {string} remainder - the leftover part - * @returns {[SchemaUnit, string, string]} - The actual Unit, the unit string and the value string. - * @throws {IssueError} If parsing the remainder section fails. - */ _separateUnits(schemaTag, remainder) { const unitClasses = schemaTag.unitClasses let actualUnit = null @@ -173,14 +171,15 @@ export default class ParsedHedTag extends ParsedHedSubstring { /** * Handle special three-level tags * @param {string} remainder - the remainder of the tag string after schema tag - * @param {ReservedChecker} special - the special checker for checking the special tag properties + * @param {SpecialChecker} special - the special checker for checking the special tag properties */ _getSplitValue(remainder, special) { - if (!special.allowTwoLevelValueTags.has(this.schemaTag.name)) { + if (!special.allowTwoLevelValueTags.includes(this.schemaTag.name)) { return [remainder, null] } - const [first, ...rest] = remainder.split('/') - return [first, rest.join('/')] + const split = remainder.split('/', 2) + const rest = split.length > 1 ? split[1] : null + return [split[0], rest] } /** @@ -206,10 +205,6 @@ export default class ParsedHedTag extends ParsedHedSubstring { } } - get normalized() { - return this._normalized - } - /** * Override of {@link Object.prototype.toString}. * @@ -233,6 +228,16 @@ export default class ParsedHedTag extends ParsedHedSubstring { return this.schema?.tagHasAttribute(this.formattedTag, attribute) } + /** + * Determine whether this tag's parent tag has a given attribute. + * + * @param {string} attribute An attribute name. + * @returns {boolean} Whether this tag's parent tag has the named attribute. + */ + parentHasAttribute(attribute) { + return this.schema?.tagHasAttribute(this.parentFormattedTag, attribute) + } + /** * Get the last part of a HED tag. * @@ -248,13 +253,39 @@ export default class ParsedHedTag extends ParsedHedSubstring { } } + /* + /!** + * The trailing portion of {@link canonicalTag}. + * + * @returns {string} The "name" portion of the canonical tag. + *!/ + get canonicalTagName() { + return this._memoize('canonicalTagName', () => { + return ParsedHedTag.getTagName(this.canonicalTag) + }) + }*/ + + /* + /!** + * The trailing portion of {@link formattedTag}. + * + * @returns {string} The "name" portion of the formatted tag. + *!/ + get formattedTagName() { + return this._memoize('formattedTagName', () => { + return ParsedHedTag.getTagName(this.formattedTag) + }) + }*/ + /** * The trailing portion of {@link originalTag}. * * @returns {string} The "name" portion of the original tag. */ get originalTagName() { - return ParsedHedTag.getTagName(this.originalTag) + return this._memoize('originalTagName', () => { + return ParsedHedTag.getTagName(this.originalTag) + }) } /** @@ -272,11 +303,45 @@ export default class ParsedHedTag extends ParsedHedSubstring { } } + /** + * The parent portion of {@link canonicalTag}. + * + * @returns {string} The "parent" portion of the canonical tag. + */ + get parentCanonicalTag() { + return this._memoize('parentCanonicalTag', () => { + return ParsedHedTag.getParentTag(this.canonicalTag) + }) + } + + /** + * The parent portion of {@link formattedTag}. + * + * @returns {string} The "parent" portion of the formatted tag. + */ + get parentFormattedTag() { + return this._memoize('parentFormattedTag', () => { + return ParsedHedTag.getParentTag(this.formattedTag) + }) + } + + /* + /!** + * The parent portion of {@link originalTag}. + * + * @returns {string} The "parent" portion of the original tag. + *!/ + get parentOriginalTag() { + return this._memoize('parentOriginalTag', () => { + return ParsedHedTag.getParentTag(this.originalTag) + }) + }*/ + /** * Iterate through a tag's ancestor tag strings. * - * @param {string} tagString - A tag string. - * @yields {string} - The tag's ancestor tags. + * @param {string} tagString A tag string. + * @yields {string} The tag's ancestor tags. */ static *ancestorIterator(tagString) { while (tagString.lastIndexOf('/') >= 0) { @@ -285,14 +350,13 @@ export default class ParsedHedTag extends ParsedHedSubstring { } yield tagString } - /* - /!** + /** * Determine whether this tag is a descendant of another tag. * * @param {ParsedHedTag|string} parent The possible parent tag. * @returns {boolean} Whether {@link parent} is the parent tag of this tag. - *!/ + */ isDescendantOf(parent) { if (parent instanceof ParsedHedTag) { if (this.schema !== parent.schema) { @@ -307,31 +371,64 @@ export default class ParsedHedTag extends ParsedHedSubstring { } return false } -*/ + + /* + /!** + * Check if any level of this HED tag allows extensions. + * + * @returns {boolean} Whether any level of this HED tag allows extensions. + *!/ + get allowsExtensions() { + return this._memoize('allowsExtensions', () => { + if (this.originalTagName === '#') { + return false + } + const extensionAllowedAttribute = 'extensionAllowed' + if (this.hasAttribute(extensionAllowedAttribute)) { + return true + } + return getTagLevels(this.formattedTag).some((tagSubstring) => + this.schema?.tagHasAttribute(tagSubstring, extensionAllowedAttribute), + ) + }) + }*/ /** * Determine if this HED tag is equivalent to another HED tag. * - * Note: HED tags are deemed equivalent if they have the same schema and normalized tag string. + * HED tags are deemed equivalent if they have the same schema and formatted tag string. * - * @param {ParsedHedTag} other - A HED tag to compare with this one. - * @returns {boolean} Whether {@link other} True, if other is equivalent to this HED tag. + * @param {ParsedHedTag} other A HED tag. + * @returns {boolean} Whether {@link other} is equivalent to this HED tag. */ equivalent(other) { return other instanceof ParsedHedTag && this.formattedTag === other.formattedTag && this.schema === other.schema } + /** + * Determine if this HED tag is in the linked schema. + * + * @returns {boolean} Whether this HED tag is in the linked schema. + */ + get existsInSchema() { + return this._memoize('existsInSchema', () => { + return this.schema?.entries?.tags?.hasLongNameEntry(this.formattedTag) + }) + } + /** * Get the schema tag object for this tag. * * @returns {SchemaTag} The schema tag object for this tag. */ get schemaTag() { - if (this._schemaTag instanceof SchemaValueTag) { - return this._schemaTag.parent - } else { - return this._schemaTag - } + return this._memoize('takesValueTag', () => { + if (this._schemaTag instanceof SchemaValueTag) { + return this._schemaTag.parent + } else { + return this._schemaTag + } + }) } /** @@ -340,10 +437,13 @@ export default class ParsedHedTag extends ParsedHedSubstring { * @returns {SchemaValueTag} The schema tag object for this tag's value-taking form. */ get takesValueTag() { - if (this._schemaTag instanceof SchemaValueTag) { - return this._schemaTag - } - return undefined + return this._memoize('takesValueTag', () => { + if (this._schemaTag instanceof SchemaValueTag) { + return this._schemaTag + } else { + return undefined + } + }) } /** @@ -352,7 +452,9 @@ export default class ParsedHedTag extends ParsedHedSubstring { * @returns {boolean} Whether this HED tag has the {@code takesValue} attribute. */ get takesValue() { - return this.takesValueTag !== undefined + return this._memoize('takesValue', () => { + return this.takesValueTag !== undefined + }) } /** @@ -361,10 +463,12 @@ export default class ParsedHedTag extends ParsedHedSubstring { * @returns {boolean} Whether this HED tag has the {@code unitClass} attribute. */ get hasUnitClass() { - if (!this.takesValueTag) { - return false - } - return this.takesValueTag.hasUnitClasses + return this._memoize('hasUnitClass', () => { + if (!this.takesValueTag) { + return false + } + return this.takesValueTag.hasUnitClasses + }) } /** @@ -373,10 +477,33 @@ export default class ParsedHedTag extends ParsedHedSubstring { * @returns {SchemaUnitClass[]} The unit classes for this HED tag. */ get unitClasses() { - if (this.hasUnitClass) { - return this.takesValueTag.unitClasses - } - return [] + return this._memoize('unitClasses', () => { + if (this.hasUnitClass) { + return this.takesValueTag.unitClasses + } else { + return [] + } + }) + } + + /** + * Get the default unit for this HED tag. + * + * @returns {string} The default unit for this HED tag. + */ + get defaultUnit() { + return this._memoize('defaultUnit', () => { + const defaultUnitsForUnitClassAttribute = 'defaultUnits' + if (!this.hasUnitClass) { + return '' + } + const tagDefaultUnit = this.takesValueTag.getNamedAttributeValue(defaultUnitsForUnitClassAttribute) + if (tagDefaultUnit) { + return tagDefaultUnit + } + const firstUnitClass = this.unitClasses[0] + return firstUnitClass.getNamedAttributeValue(defaultUnitsForUnitClassAttribute) + }) } /** @@ -385,26 +512,26 @@ export default class ParsedHedTag extends ParsedHedSubstring { * @returns {Set} The legal units for this HED tag. */ get validUnits() { - if (this._validUnits) { - return this._validUnits - } - const tagUnitClasses = this.unitClasses - this._validUnits = new Set() - for (const unitClass of tagUnitClasses) { - const unitClassUnits = this.schema?.entries.unitClasses.getEntry(unitClass.name).units - for (const unit of unitClassUnits.values()) { - this._validUnits.add(unit) + return this._memoize('validUnits', () => { + const tagUnitClasses = this.unitClasses + const units = new Set() + for (const unitClass of tagUnitClasses) { + const unitClassUnits = this.schema?.entries.unitClasses.getEntry(unitClass.name).units + for (const unit of unitClassUnits.values()) { + units.add(unit) + } } - } - return this._validUnits + return units + }) } /** * Check if value is a valid value for this tag. * - * @param {string} value - The value to be checked. - * @returns {boolean} The result of check -- false if not a valid value. + * @param {string} The value to be checked + * @returns {boolean} The result of check -- false if not a valid value */ + checkValue(value) { if (!this.takesValue) { return false diff --git a/parser/parser.js b/parser/parser.js index c80a3174..6c468b77 100644 --- a/parser/parser.js +++ b/parser/parser.js @@ -1,8 +1,8 @@ +import { mergeParsingIssues } from '../utils/hedData' import ParsedHedString from './parsedHedString' import HedStringSplitter from './splitter' import { generateIssue } from '../common/issues/issues' -import { ReservedChecker } from './reservedChecker' -import { getTagListString } from './parseUtils' +import { SpecialChecker } from './special' /** * A parser for HED strings. @@ -19,94 +19,42 @@ class HedStringParser { */ hedSchemas - definitionsAllowed - - placeholdersAllowed - /** * Constructor. * * @param {string|ParsedHedString} hedString The HED string to be parsed. * @param {Schemas} hedSchemas The collection of HED schemas. - * @param {boolean} definitionsAllowed - True if definitions are allowed - * @param {boolean} placeholdersAllowed - True if placeholders are allowed */ - constructor(hedString, hedSchemas, definitionsAllowed, placeholdersAllowed) { + constructor(hedString, hedSchemas) { this.hedString = hedString this.hedSchemas = hedSchemas - this.definitionsAllowed = definitionsAllowed - this.placeholdersAllowed = placeholdersAllowed } /** * Parse a full HED string. - * @param {boolean} fullCheck whether the string is in final form and can be fully parsed - - * @returns {[ParsedHedString|null, Issue[]]} The parsed HED string and any parsing issues. + * @type {boolean} fullCheck whether the string is in final form and can be fully parsed + * @returns {[ParsedHedString|null, Object]} The parsed HED string and any parsing issues. */ parseHedString(fullCheck) { - if (this.hedString === null || this.hedString === undefined) { - return [null, [generateIssue('invalidTagString', {})]] - } - // if (!this.hedString) { - // return [null, []] - // } - const placeholderIssues = this._getPlaceholderCountIssues() - if (placeholderIssues.length > 0) { - return [null, placeholderIssues] - } if (this.hedString instanceof ParsedHedString) { - return [this.hedString, []] + return [this.hedString, {}] } if (!this.hedSchemas) { - return [null, [generateIssue('missingSchemaSpecification', {})]] + return [null, { syntaxIssues: [generateIssue('missingSchemaSpecification', {})] }] } - // This assumes that splitter errors are only errors and not warnings const [parsedTags, parsingIssues] = new HedStringSplitter(this.hedString, this.hedSchemas).splitHedString() - if (parsedTags === null || parsingIssues.length > 0) { + if (parsedTags === null) { return [null, parsingIssues] } - const parsedString = new ParsedHedString(this.hedString, parsedTags) - - // This checks whether there are any definitions in the string - const simpleDefinitionIssues = this._checkDefinitionContext(parsedString) - if (simpleDefinitionIssues.length > 0) { - return [null, simpleDefinitionIssues] - } - const checkIssues = ReservedChecker.getInstance().checkHedString(parsedString, fullCheck) + const checkIssues = SpecialChecker.getInstance().checkHedString(parsedString, fullCheck) + mergeParsingIssues(parsingIssues, { syntaxIssues: checkIssues }) if (checkIssues.length > 0) { - return [null, checkIssues] - } - return [parsedString, []] - } - - _checkDefinitionContext(parsedString) { - if (this.definitionsAllowed || !parsedString) { - return [] - } - const definitionTags = parsedString.tags.filter((tag) => tag.schemaTag.name === 'Definition') - if (definitionTags.length > 0) { - return [ - generateIssue('illegalDefinitionContext', { - definition: getTagListString(definitionTags), - string: parsedString.hedString, - }), - ] - } - return [] - } - - _getPlaceholderCountIssues() { - if (this.placeholdersAllowed) { - return [] - } - const checkString = this.hedString instanceof ParsedHedString ? this.hedString.hedString : this.hedString - if (checkString.split('#').length > 1) { - return [generateIssue('invalidPlaceholderContext', { string: checkString })] + return [null, parsingIssues] } - return [] + //mergeParsingIssues(parsingIssues, {syntax: checkIssues}) + return [parsedString, parsingIssues] } /** @@ -115,25 +63,18 @@ class HedStringParser { * @param {string[]|ParsedHedString[]} hedStrings A list of HED strings. * @param {Schemas} hedSchemas The collection of HED schemas. * @param {boolean} fullCheck whether the strings are in final form and can be fully parsed - * @param {boolean} definitionsAllowed - True if definitions are allowed - * @param {boolean} placeholdersAllowed - True if placeholders are allowed - * @returns {[ParsedHedString[], Issue[]]} The parsed HED strings and any issues found. + * @returns {[ParsedHedString[], Object]} The parsed HED strings and any issues found. */ - static parseHedStrings(hedStrings, hedSchemas, fullCheck, definitionsAllowed, placeholdersAllowed) { + static parseHedStrings(hedStrings, hedSchemas, fullCheck) { if (!hedSchemas) { - return [null, [generateIssue('missingSchemaSpecification', {})]] + return [null, { syntaxIssues: [generateIssue('missingSchemaSpecification', {})] }] } const parsedStrings = [] - const cumulativeIssues = [] + const cumulativeIssues = {} for (const hedString of hedStrings) { - const [parsedString, currentIssues] = new HedStringParser( - hedString, - hedSchemas, - definitionsAllowed, - placeholdersAllowed, - ).parseHedString(fullCheck) + const [parsedString, currentIssues] = new HedStringParser(hedString, hedSchemas).parseHedString(fullCheck) parsedStrings.push(parsedString) - cumulativeIssues.push(...currentIssues) + mergeParsingIssues(cumulativeIssues, currentIssues) } return [parsedStrings, cumulativeIssues] @@ -144,26 +85,22 @@ class HedStringParser { * Parse a HED string. * * @param {string|ParsedHedString} hedString A (possibly already parsed) HED string. - * @param {Schemas} hedSchemas - The collection of HED schemas. - * @param {boolean} fullCheck - If the string is in final form -- can be fully parsed - * @param {boolean} definitionsAllowed - True if definitions are allowed - * @param {boolean} placeholdersAllowed - True if placeholders are allowed - * @returns {[ParsedHedString, Issue[]]} - The parsed HED string and any issues found. + * @param {Schemas} hedSchemas The collection of HED schemas. + * @param {boolean} fullCheck If the string is in final form -- can be fully parsed + * @returns {[ParsedHedString, Object]} The parsed HED string and any issues found. */ -export function parseHedString(hedString, hedSchemas, fullCheck, definitionsAllowed, placeholdersAllowed) { - return new HedStringParser(hedString, hedSchemas, definitionsAllowed, placeholdersAllowed).parseHedString(fullCheck) +export function parseHedString(hedString, hedSchemas, fullCheck) { + return new HedStringParser(hedString, hedSchemas).parseHedString(fullCheck) } /** * Parse a list of HED strings. * * @param {string[]|ParsedHedString[]} hedStrings A list of HED strings. - * @param {Schemas} hedSchemas - The collection of HED schemas. - * @param {boolean} fullCheck - If the strings is in final form -- can be fully parsed - * @param {boolean} definitionsAllowed - True if definitions are allowed - * @param {boolean} placeholdersAllowed - True if placeholders are allowed - * @returns {[ParsedHedString[], Issue[]]} The parsed HED strings and any issues found. + * @param {Schemas} hedSchemas The collection of HED schemas. + * @param {boolean} fullCheck If the strings is in final form -- can be fully parsed + * @returns {[ParsedHedString[], Object]} The parsed HED strings and any issues found. */ -export function parseHedStrings(hedStrings, hedSchemas, fullCheck, definitionsAllowed, placeholdersAllowed) { - return HedStringParser.parseHedStrings(hedStrings, hedSchemas, fullCheck, definitionsAllowed, placeholdersAllowed) +export function parseHedStrings(hedStrings, hedSchemas, fullCheck) { + return HedStringParser.parseHedStrings(hedStrings, hedSchemas, fullCheck) } diff --git a/parser/reservedChecker.js b/parser/reservedChecker.js deleted file mode 100644 index 606aa084..00000000 --- a/parser/reservedChecker.js +++ /dev/null @@ -1,458 +0,0 @@ -import specialTags from '../data/json/reservedTags.json' -import { generateIssue } from '../common/issues/issues' -import { filterTagMapByNames, getTagListString } from './parseUtils' - -export class ReservedChecker { - static instance = null - static reservedMap = new Map(Object.entries(specialTags)) - - constructor() { - if (ReservedChecker.instance) { - throw new Error('Use ReservedChecker.getInstance() to get an instance of this class.') - } - - this._initializeSpecialTags() - } - - // Static method to control access to the singleton instance - static getInstance() { - if (!ReservedChecker.instance) { - ReservedChecker.instance = new ReservedChecker() - } - return ReservedChecker.instance - } - - _initializeSpecialTags() { - this.specialNames = new Set(ReservedChecker.reservedMap.keys()) - this.requireValueTags = ReservedChecker._getSpecialTagsByProperty('requireValue') - this.noExtensionTags = ReservedChecker._getSpecialTagsByProperty('noExtension') - this.allowTwoLevelValueTags = ReservedChecker._getSpecialTagsByProperty('allowTwoLevelValue') - this.topGroupTags = ReservedChecker._getSpecialTagsByProperty('topLevelTagGroup') - this.requiresDefTags = ReservedChecker._getSpecialTagsByProperty('requiresDef') - this.groupTags = ReservedChecker._getSpecialTagsByProperty('tagGroup') - this.exclusiveTags = ReservedChecker._getSpecialTagsByProperty('exclusive') - this.temporalTags = ReservedChecker._getSpecialTagsByProperty('isTemporalTag') - this.noSpliceInGroup = ReservedChecker._getSpecialTagsByProperty('noSpliceInGroup') - this.hasForbiddenSubgroupTags = new Set( - [...ReservedChecker.reservedMap.values()] - .filter((value) => value.forbiddenSubgroupTags.length > 0) - .map((value) => value.name), - ) - } - - static _getSpecialTagsByProperty(property) { - return new Set( - [...ReservedChecker.reservedMap.values()].filter((value) => value[property] === true).map((value) => value.name), - ) - } - - /** - * Perform syntactical checks on the provided HED string to detect violations. - * - * @param {ParsedHedString} hedString - The HED string to be checked. - * @param {boolean} fullCheck - If true, assumes that all splices have been resolved. - * @returns {Issue[]} An array of issues if violations are found otherwise, an empty array. - */ - checkHedString(hedString, fullCheck) { - const checks = [ - () => this.spliceCheck(hedString, fullCheck), - () => this.checkUnique(hedString), - () => this.checkTagGroupLevels(hedString, fullCheck), - () => this.checkExclusive(hedString), - () => this.checkNoSpliceInGroupTags(hedString), - () => this.checkTopGroupRequirements(hedString, fullCheck), - () => this.checkForbiddenGroups(hedString), - () => this.checkNonTopGroups(hedString, fullCheck), - ] - for (const check of checks) { - const issues = check() - if (issues.length > 0) { - return issues - } - } - return [] - } - - /** - * Check whether column splices are allowed - * - * @param {ParsedHedString} hedString - The HED string to check for splice conflicts. - * @param {boolean} fullCheck - If true, then column splices should have been resolved. - * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. - */ - spliceCheck(hedString, fullCheck) { - if (hedString.columnSplices.length === 0) { - // No column splices in string so skip test - return [] - } - - // If doing a full-check, column splices should be resolved - if (fullCheck || hedString.tags.some((tag) => this.exclusiveTags.has(tag.schemaTag._name))) { - return [generateIssue('curlyBracesNotAllowed', { string: hedString.hedString })] - } - return [] - } - - /** - * Check for tags with the unique attribute. - * - * @param {ParsedHedString} hedString - The HED string to be checked for tags with the unique attribute. - * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. - */ - checkUnique(hedString) { - const uniqueTags = hedString.tags.filter((tag) => tag.hasAttribute('unique')) - const uniqueNames = new Set() - for (const tag of uniqueTags) { - if (uniqueNames.has(tag.schemaTag._name)) { - return [generateIssue('multipleUniqueTags', { tag: tag.originalTag, string: hedString.hedString })] - } - uniqueNames.add(tag.schemaTag._name) - } - return [] - } - - /** - * Check whether tags are not in groups -- or top-level groups as required - * - * @param {ParsedHedString} hedString - The HED string to be checked for special tag syntax. - * @param {boolean} fullCheck - If true, can assume that no column splices are around. - * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. - */ - checkTagGroupLevels(hedString, fullCheck) { - const issues = [] - const topGroupTags = hedString.topLevelGroupTags - hedString.tags.forEach((tag) => { - // Check for top-level violations because tag is deep - if (ReservedChecker.hasTopLevelTagGroupAttribute(tag)) { - //Tag is in a top-level tag group - if (topGroupTags.includes(tag)) { - return - } - - // Check the top-level tag requirements - if (!hedString.topLevelTags.includes(tag) || (fullCheck && hedString.topLevelTags.includes(tag))) { - issues.push( - generateIssue('invalidTopLevelTagGroupTag', { tag: tag.originalTag, string: hedString.hedString }), - ) - return - } - } - - // In final form --- if not in a group (not just a top group) but has the group tag attribute - if (fullCheck && hedString.topLevelTags.includes(tag) && ReservedChecker.hasGroupAttribute(tag)) { - issues.push(generateIssue('missingTagGroup', { tag: tag.originalTag, string: hedString.hedString })) - } - }) - return issues - } - - /** - * Check the exclusive property (so far only for Definitions) - only groups of same kind allowed in string - * - * @param hedString {ParsedHedString} - the HED string to be checked. - * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. - * - * Notes: Can only be in a top group and with other top groups of the same kind - */ - checkExclusive(hedString) { - const exclusiveTags = hedString.tags.filter((tag) => this.exclusiveTags.has(tag.schemaTag._name)) - if (exclusiveTags.length === 0) { - return [] - } - - // Exclusive tags don't allow splices and must be in groups - if (hedString.topLevelTags.length > 0 || hedString.columnSplices.length > 0) { - return [ - generateIssue('illegalInExclusiveContext', { - tag: exclusiveTags[0].originalTag, - string: hedString.hedString, - }), - ] - } - - // Make sure that all the objects in exclusiveTags have same schema tag name - not an issue currently - const badList = exclusiveTags.filter((tag) => tag.schemaTag._name !== exclusiveTags[0].schemaTag._name) - if (badList.length > 0) { - return [generateIssue('illegalExclusiveContext', { tag: badList[0].originalTag, string: hedString.hedString })] - } - return [] - } - - /** - * Check that no splices appear with tags that don't allow spaced - * - * @param {ParsedHedString} hedString - the HED string to be checked for disallowed spices - * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. - */ - checkNoSpliceInGroupTags(hedString) { - const topNoSpliceTags = hedString.topLevelTags.filter((tag) => this.noSpliceInGroup.has(tag.schemaTag._name)) - if (topNoSpliceTags.length > 0) { - return [generateIssue('missingTagGroup', { tag: topNoSpliceTags[0].originalTag, string: hedString.hedString })] - } - const allSpliceTags = hedString.tags.filter((tag) => this.noSpliceInGroup.has(tag.schemaTag._name)) - if (allSpliceTags.length > 0 && hedString.columnSplices.length > 0) { - return [generateIssue('curlyBracesNotAllowed', { string: hedString.hedString })] - } - return [] - } - - /** - * Check the group conditions of the special tags. The top-level has already been verified. - * - * @param {ParsedHedString} hedString - The HED string to check for group conflicts. - * @param {boolean} fullCheck - If true, no splices so the HED string is complete - * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. - * - * Notes: These include the number of groups and top tag compatibility in the group - */ - checkTopGroupRequirements(hedString, fullCheck) { - const issues = [] - for (const group of hedString.tagGroups) { - const specialTags = [...group.specialTags.values()].flat() - for (const specialTag of specialTags) { - const nextIssues = this._checkGroupRequirements(group, specialTag, fullCheck) - issues.push(...nextIssues) - // If an error is found in this group -- there is no point looking for more. - if (nextIssues.length > 0) { - break - } - } - } - return issues - } - - /** - * Check the group tag requirements of a special Tag - * @param {ParsedHedGroup} group - the group to check for tag requirements - * @param {ParsedHedTag} specialTag - a top-level special tag in group - * @param { boolean} fullCheck - if True, assume this is the final version and all tags must be present - * @returns {Issue[]} - */ - _checkGroupRequirements(group, specialTag, fullCheck) { - const specialRequirements = ReservedChecker.reservedMap.get(specialTag.schemaTag.name) - const issues = this._checkAllowedTags(group, specialTag, specialRequirements.otherAllowedNonDefTags) - if (issues.length > 0) { - return issues - } - issues.push(...this._checkAllowedGroups(group, specialTag, specialRequirements, fullCheck)) - return issues - } - - /** - * Verify that the tags in the group are allowed with the special tag - * - * @param {ParsedHedGroup} group - The enclosing tag group - * @param {ParsedHedTag} specialTag - The special tag whose tag requirements are to be checked - * @param { string[]} otherAllowed - The list of tags that are allowed with this tag - * @returns {Issue[]|[]} - * @private - */ - _checkAllowedTags(group, specialTag, otherAllowed) { - if (otherAllowed === null || otherAllowed === undefined) { - return [] - } - const otherTopTags = group.topTags.filter((tag) => tag !== specialTag) - if (otherTopTags.length === 0) { - return [] - } - const encountered = new Set() - for (const tag of otherTopTags) { - if (encountered.has(tag.schemaTag.name)) { - return [generateIssue('tooManyGroupTopTags', { string: group.originalTag })] - } - encountered.add(tag.schemaTag.name) - if (tag.schemaTag.name === 'Def' && group.requiresDefTag !== null) { - continue - } - // This tag is not allowed with the special tag - if (!otherAllowed.includes(tag.schemaTag.name)) { - return [ - generateIssue('invalidGroupTopTags', { tags: getTagListString(group.topTags), string: group.originalTag }), - ] - } - } - return [] - } - - /** - * Verify the group conditions - * - * @param {ParsedHedGroup} group - The enclosing tag group - * @param {ParsedHedTag} specialTag - The special tag whose tag requirements are to be checked - * @param { Object } requirements - The requirements for this special tag. - * @param {boolean} fullCheck - If true, all splices have been resolved and everything should be there - * @returns {Issue[]} - * @private - */ - _checkAllowedGroups(group, specialTag, requirements, fullCheck) { - // Group checks are not applicable to this special tag - if (!requirements.tagGroup) { - return [] - } - let subgroupCount = group.topGroups.length - if (group.hasDefExpandChildren && group.requiresDefTag !== null) { - subgroupCount = subgroupCount - 1 - } - - // Check maximum limit - const maxLimit = requirements.maxNonDefSubgroups != null ? requirements.maxNonDefSubgroups : Infinity - if (subgroupCount > maxLimit) { - return [generateIssue('invalidNumberOfSubgroups', { tag: specialTag.originalTag, string: group.originalTag })] - } - - // Check if you can't do more because of splices - if (!fullCheck && group.topSplices.length > 0) { - return [] - } - - // Check that it has the minimum number of subgroups - const minLimit = requirements.minNonDefSubgroups != null ? requirements.minNonDefSubgroups : -Infinity - if (fullCheck && subgroupCount < minLimit) { - return [generateIssue('invalidNumberOfSubgroups', { tag: specialTag.originalTag, string: group.originalTag })] - } - return [] - } - - checkNonTopGroups(hedString, fullCheck) { - if (!hedString.tags.some((tag) => this.groupTags.has(tag.schemaTag._name) && !this.topGroupTags.has(tag))) { - return [] - } - const issues = [] - for (const topGroup of hedString.tagGroups) { - for (const group of topGroup.subParsedGroupIterator()) { - const theseIssues = this._checkGroup(group, fullCheck) - issues.push(...theseIssues) - if (theseIssues.length > 0) { - break - } - } - } - return issues - } - - _checkGroup(group, fullCheck) { - const specialTags = filterTagMapByNames(group.specialTags, this.groupTags) - const notTopGroupTags = specialTags.filter((tag) => !this.topGroupTags.has(tag.schemaTag.name)) - for (const tag of notTopGroupTags) { - const issues = this._checkGroupRequirements(group, tag, fullCheck) - if (issues.length > 0) { - return issues - } - } - return [] - } - - /** - * Check if there are conflicting subgroup tags. - * - * @param {ParsedHedString} hedString - the HED string to be checked. - * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. - */ - checkForbiddenGroups(hedString) { - // Only do this if there are any tags in the string with forbidden tags. - const hasForbidden = hedString.tags.filter((tag) => this.hasForbiddenSubgroupTags.has(tag.schemaTag.name)) - if (hasForbidden.length === 0) { - return [] - } - let forbiddenCount = hasForbidden.length - for (const tag of hedString.topLevelTags) { - if (!this.hasForbiddenSubgroupTags.has(tag.schemaTag.name)) { - continue - } - // This tag has - const forbidden = ReservedChecker.reservedMap.get(tag.schemaTag.name).forbiddenSubgroupTags - for (const group of hedString.tagGroups) { - if (group.allTags.some((tag) => forbidden.has(tag.schemaTag.name))) { - return [ - generateIssue('forbiddenSubgroupTags', { - tag: tag.originalTag, - string: hedString.hedString, - tagList: getTagListString(forbidden), - }), - ] - } - } - forbiddenCount-- - if (forbiddenCount === 0) { - return [] - } - } - const issues = [] - for (const group of hedString.tagGroups) { - // Only check the group if there are tags with forbidden subgroup tags - if (group.allTags.some((tag) => this.hasForbiddenSubgroupTags.has(tag.schemaTag.name))) { - issues.push(...this._checkForbiddenGroup(group)) - } - } - return issues - } - - /** - * Check a group completely for forbidden tag conflicts such as a Def in a Definition group. - * - * @param {ParsedHedGroup} group - HED group to check for forbidden group conflicts. - * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. - * - * Note: Returns in a given group as soon as it finds a conflict - */ - _checkForbiddenGroup(group) { - for (const subGroup of group.subParsedGroupIterator()) { - // if this group does not have top tags with forbidden subgroups -- must go deeper - const forbiddenTags = subGroup.topTags.filter((tag) => this.hasForbiddenSubgroupTags.has(tag.schemaTag._name)) - if (forbiddenTags.length === 0) { - continue - } - - // Check the tags in - for (const tag of forbiddenTags) { - const otherTags = subGroup.allTags.filter((otherTag) => otherTag !== tag) - const badTags = otherTags.filter((otherTag) => - ReservedChecker.reservedMap.get(tag.schemaTag.name)?.forbiddenSubgroupTags.includes(otherTag.schemaTag.name), - ) - - if (badTags?.length > 0) { - return [ - generateIssue('invalidGroupTags', { - tags: getTagListString(badTags), - string: subGroup.originalTag, - }), - ] - } - } - } - return [] - } - - /** - * Indicate whether a tag should be a top-level tag. - * - * @param {ParsedHedTag} tag - HED tag to check for top-level requirements. - * @returns {boolean} If true, the tag is required to be at the top level. - * - * Note: This check both the special requirements and the 'topLevelTagGroup' attribute in the schema. - * - */ - static hasTopLevelTagGroupAttribute(tag) { - return ( - tag.hasAttribute('topLevelTagGroup') || - (ReservedChecker.reservedMap.has(tag.schemaTag.name) && - ReservedChecker.reservedMap.get(tag.schemaTag.name).topLevelTagGroup) - ) - } - - /** - * Return a boolean indicating whether a tag is required to be in a tag group. - * - * @param {ParsedHedTag} tag - The HED tag to be checked. - * @returns {boolean} If true, this indicates that tag must be in a tag group. - * - * Note: This checks both special and schema tag requirements. - */ - static hasGroupAttribute(tag) { - return ( - tag.hasAttribute('tagGroup') || - (ReservedChecker.reservedMap.has(tag.schemaTag.name) && - ReservedChecker.reservedMap.get(tag.schemaTag.name).tagGroup) - ) - } -} diff --git a/eventManager/special.js b/parser/special.js similarity index 51% rename from eventManager/special.js rename to parser/special.js index 79b10982..bd345726 100644 --- a/eventManager/special.js +++ b/parser/special.js @@ -1,6 +1,5 @@ -import specialTags from '../data/json/reservedTags.json' +import specialTags from '../data/json/specialTags.json' import { generateIssue } from '../common/issues/issues' -import { filterTagMapByNames, getTagListString } from './parseUtils' export class SpecialChecker { static instance = null @@ -8,7 +7,7 @@ export class SpecialChecker { constructor() { if (SpecialChecker.instance) { - throw new Error('Use ReservedChecker.getInstance() to get an instance of this class.') + throw new Error('Use SpecialChecker.getInstance() to get an instance of this class.') } this._initializeSpecialTags() @@ -23,13 +22,12 @@ export class SpecialChecker { } _initializeSpecialTags() { - this.specialNames = new Set(SpecialChecker.specialMap.keys()) + this.specialNames = [...SpecialChecker.specialMap.keys()] this.requireValueTags = SpecialChecker._getSpecialTagsByProperty('requireValue') this.noExtensionTags = SpecialChecker._getSpecialTagsByProperty('noExtension') this.allowTwoLevelValueTags = SpecialChecker._getSpecialTagsByProperty('allowTwoLevelValue') - this.topGroupTags = SpecialChecker._getSpecialTagsByProperty('topLevelTagGroup') - this.requiresDefTags = SpecialChecker._getSpecialTagsByProperty('requiresDef') - this.groupTags = SpecialChecker._getSpecialTagsByProperty('tagGroup') + this.specialGroupTags = SpecialChecker._getSpecialTagsByProperty('tagGroup') + this.specialTopGroupTags = SpecialChecker._getSpecialTagsByProperty('topLevelTagGroup') this.exclusiveTags = SpecialChecker._getSpecialTagsByProperty('exclusive') this.noSpliceInGroup = SpecialChecker._getSpecialTagsByProperty('noSpliceInGroup') this.hasForbiddenSubgroupTags = new Set( @@ -40,9 +38,9 @@ export class SpecialChecker { } static _getSpecialTagsByProperty(property) { - return new Set( - [...SpecialChecker.specialMap.values()].filter((value) => value[property] === true).map((value) => value.name), - ) + return [...SpecialChecker.specialMap.values()] + .filter((value) => value[property] === true) + .map((value) => value.name) } /** @@ -55,14 +53,14 @@ export class SpecialChecker { checkHedString(hedString, fullCheck) { const checks = [ () => this.spliceCheck(hedString, fullCheck), - () => this.checkUnique(hedString), () => this.checkTagGroupLevels(hedString, fullCheck), + () => this.checkUnique(hedString), () => this.checkExclusive(hedString), + () => this.checkSpecialTopGroups(hedString), () => this.checkNoSpliceInGroupTags(hedString), - () => this.checkTopGroupRequirements(hedString, fullCheck), () => this.checkForbiddenGroups(hedString), - () => this.checkNonTopGroups(hedString, fullCheck), ] + for (const check of checks) { const issues = check() if (issues.length > 0) { @@ -73,11 +71,11 @@ export class SpecialChecker { } /** - * Check whether column splices are allowed + * Check whether column splices are allowed * - * @param {ParsedHedString} hedString - The HED string to check for splice conflicts. - * @param {boolean} fullCheck - If true, then column splices should have been resolved. - * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. + * @param {ParsedHedString} hedString - The HED string to check for splice conflicts. + * @param {boolean} fullCheck - If true, then column splices should have been resolved. + * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. */ spliceCheck(hedString, fullCheck) { if (hedString.columnSplices.length === 0) { @@ -86,30 +84,12 @@ export class SpecialChecker { } // If doing a full-check, column splices should be resolved - if (fullCheck || hedString.tags.some((tag) => this.exclusiveTags.has(tag.schemaTag._name))) { + if (fullCheck || hedString.tags.some((tag) => this.exclusiveTags.includes(tag.schemaTag._name))) { return [generateIssue('curlyBracesNotAllowed', { string: hedString.hedString })] } return [] } - /** - * Check for tags with the unique attribute. - * - * @param {ParsedHedString} hedString - The HED string to be checked for tags with the unique attribute. - * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. - */ - checkUnique(hedString) { - const uniqueTags = hedString.tags.filter((tag) => tag.hasAttribute('unique')) - const uniqueNames = new Set() - for (const tag of uniqueTags) { - if (uniqueNames.has(tag.schemaTag._name)) { - return [generateIssue('multipleUniqueTags', { tag: tag.originalTag, string: hedString.hedString })] - } - uniqueNames.add(tag.schemaTag._name) - } - return [] - } - /** * Check whether tags are not in groups -- or top-level groups as required * @@ -119,10 +99,10 @@ export class SpecialChecker { */ checkTagGroupLevels(hedString, fullCheck) { const issues = [] - const topGroupTags = hedString.topLevelGroupTags + const topGroupTags = hedString.topLevelGroupTags.flat() hedString.tags.forEach((tag) => { // Check for top-level violations because tag is deep - if (SpecialChecker.hasTopLevelTagGroupAttribute(tag)) { + if (this.hasTopLevelTagGroupAttribute(tag)) { //Tag is in a top-level tag group if (topGroupTags.includes(tag)) { return @@ -138,7 +118,7 @@ export class SpecialChecker { } // In final form --- if not in a group (not just a top group) but has the group tag attribute - if (fullCheck && hedString.topLevelTags.includes(tag) && SpecialChecker.hasGroupAttribute(tag)) { + if (fullCheck && hedString.topLevelTags.includes(tag) && this.hasGroupAttribute(tag)) { issues.push(generateIssue('missingTagGroup', { tag: tag.originalTag, string: hedString.hedString })) } }) @@ -154,13 +134,13 @@ export class SpecialChecker { * Notes: Can only be in a top group and with other top groups of the same kind */ checkExclusive(hedString) { - const exclusiveTags = hedString.tags.filter((tag) => this.exclusiveTags.has(tag.schemaTag._name)) + const exclusiveTags = hedString.tags.filter((tag) => this.exclusiveTags.includes(tag.schemaTag._name)) if (exclusiveTags.length === 0) { return [] } // Exclusive tags don't allow splices and must be in groups - if (hedString.topLevelTags.length > 0 || hedString.columnSplices.length > 0) { + if (hedString.topLevelTags.length > 0) { return [ generateIssue('illegalInExclusiveContext', { tag: exclusiveTags[0].originalTag, @@ -177,170 +157,133 @@ export class SpecialChecker { return [] } - /** - * Check that no splices appear with tags that don't allow spaced - * - * @param {ParsedHedString} hedString - the HED string to be checked for disallowed spices - * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. - */ - checkNoSpliceInGroupTags(hedString) { - const topNoSpliceTags = hedString.topLevelTags.filter((tag) => this.noSpliceInGroup.has(tag.schemaTag._name)) - if (topNoSpliceTags.length > 0) { - return [generateIssue('missingTagGroup', { tag: topNoSpliceTags[0].originalTag, string: hedString.hedString })] - } - const allSpliceTags = hedString.tags.filter((tag) => this.noSpliceInGroup.has(tag.schemaTag._name)) - if (allSpliceTags.length > 0 && hedString.columnSplices.length > 0) { - return [generateIssue('curlyBracesNotAllowed', { string: hedString.hedString })] - } - return [] - } - /** * Check the group conditions of the special tags. The top-level has already been verified. * * @param {ParsedHedString} hedString - The HED string to check for group conflicts. - * @param {boolean} fullCheck - If true, no splices so the HED string is complete * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. * * Notes: These include the number of groups and top tag compatibility in the group */ - checkTopGroupRequirements(hedString, fullCheck) { + checkSpecialTopGroups(hedString) { const issues = [] for (const group of hedString.tagGroups) { - const specialTags = [...group.specialTags.values()].flat() - for (const specialTag of specialTags) { - const nextIssues = this._checkGroupRequirements(group, specialTag, fullCheck) - issues.push(...nextIssues) - // If an error is found in this group -- there is no point looking for more. - if (nextIssues.length > 0) { - break - } - } - } - return issues - } - - /** - * Check the group tag requirements of a special Tag - * @param {ParsedHedGroup} group - the group to check for tag requirements - * @param {ParsedHedTag} specialTag - a top-level special tag in group - * @param { boolean} fullCheck - if True, assume this is the final version and all tags must be present - * @returns {Issue[]} - */ - _checkGroupRequirements(group, specialTag, fullCheck) { - const specialRequirements = SpecialChecker.specialMap.get(specialTag.schemaTag.name) - const issues = this._checkAllowedTags(group, specialTag, specialRequirements.otherAllowedNonDefTags) - if (issues.length > 0) { - return issues + const nextIssues = this._checkSpecialTopGroup(group) + issues.push(...nextIssues) } - issues.push(...this._checkAllowedGroups(group, specialTag, specialRequirements, fullCheck)) return issues } /** - * Verify that the tags in the group are allowed with the special tag + * Check special group requirements for top-level tags in a parsed HED group. * - * @param {ParsedHedGroup} group - The enclosing tag group - * @param {ParsedHedTag} specialTag - The special tag whose tag requirements are to be checked - * @param { string[]} otherAllowed - The list of tags that are allowed with this tag - * @returns {Issue[]|[]} - * @private + * This method verifies whether the given group complies with specific requirements + * for top-level tags, focusing on groups with special properties such as + * 'Def' or 'Def-expand'. It ensures that group combinations are valid and that + * restrictions on subgroup relationships are respected. + * + * @param {ParsedHedGroup} group - The parsed HED group containing tags to be validated. + * @returns {Issue[]} An array of `Issue` objects if violations are found; otherwise, an empty array. + * + * Note: This is a top-group check only */ - _checkAllowedTags(group, specialTag, otherAllowed) { - if (otherAllowed === null || otherAllowed === undefined) { + _checkSpecialTopGroup(group) { + const specialTags = group.topTags.filter((tag) => this.specialTopGroupTags.includes(tag.schemaTag.name)) + + // If there are no special tags, there are no issues to check + if (specialTags.length === 0) { return [] } - const otherTopTags = group.topTags.filter((tag) => tag !== specialTag) - if (otherTopTags.length === 0) { + + // Ensure that groups with special tags can only contain other special tags or a Def tag. + if (specialTags.length > group.topTags.length) { + // ASSUME that groups with special tags can only have other tags which are special or a Def //TODO fix when map is removed from specialTags + return [generateIssue('invalidTagGroup', { tagGroup: group.originalTag })] + } + + // Validate Def-expand groups: must have only one tag and limited subgroups. + if (group.isDefExpandGroup && group.topTags.length === 1 && group.topGroups.length <= 1) { return [] } - const encountered = new Set() - for (const tag of otherTopTags) { - if (encountered.has(tag.schemaTag.name)) { - return [generateIssue('tooManyGroupTopTags', { string: group.originalTag })] - } - encountered.add(tag.schemaTag.name) - if (tag.schemaTag.name === 'Def' && group.requiresDefTag !== null) { - continue - } - // This tag is not allowed with the special tag - if (!otherAllowed.includes(tag.schemaTag.name)) { - return [ - generateIssue('invalidGroupTopTags', { tags: getTagListString(group.topTags), string: group.originalTag }), - ] - } + + // Check if group is an invalid Def-expand group or has too many Def-expand subgroups. + if (group.isDefExpandGroup || (group.hasDefExpandChildren && group.defExpandChildren.length > 1)) { + return [generateIssue('invalidTagGroup', { tagGroup: group.originalTag })] } - return [] + + // Ensure a group does not contain both a Def tag and a Def-expand group, which is disallowed. + if (group.hasDefExpandChildren && group.topTags.some((tag) => tag.schemaTag.name === 'Def')) { + return [generateIssue('invalidTagGroup', { tagGroup: group.originalTag })] + } + + // Delegate to check special tags in the group for further validation. + return this._checkSpecialTagsInGroup(group, specialTags) } /** - * Verify the group conditions + * Check the compatibility of special tags within a group after the initial guard conditions have been handled. + * + * This function verifies whether special tags in the provided group meet the required constraints. + * Specifically, it checks if tags are allowed based on group properties, if there are any duplicate names, + * and if certain required tags or group conditions are missing. * - * @param {ParsedHedGroup} group - The enclosing tag group - * @param {ParsedHedTag} specialTag - The special tag whose tag requirements are to be checked - * @param { Object } requirements - The requirements for this special tag. - * @param {boolean} fullCheck - If true, all splices have been resolved and everything should be there - * @returns {Issue[]} + * @param {ParsedHedGroup} - The HED group object containing tags to be validated. + * @param {ParsedHedTag[]} specialTags - Tags within the group that have special properties requiring validation. + * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. * @private */ - _checkAllowedGroups(group, specialTag, requirements, fullCheck) { - // Group checks are not applicable to this special tag - if (!requirements.tagGroup) { - return [] - } - let subgroupCount = group.topGroups.length - if (group.hasDefExpandChildren && group.requiresDefTag !== null) { - subgroupCount = subgroupCount - 1 - } + _checkSpecialTagsInGroup(group, specialTags) { + const hasDef = group.topTags.some((tag) => tag.schemaTag.name === 'Def') || group.hasDefExpandChildren - // Check maximum limit - const maxLimit = requirements.maxNonDefSubgroups != null ? requirements.maxNonDefSubgroups : Infinity - if (subgroupCount > maxLimit) { - return [generateIssue('invalidNumberOfSubgroups', { tag: specialTag.originalTag, string: group.originalTag })] - } + for (const specialTag of specialTags) { + const specialRequirements = SpecialChecker.specialMap.get(specialTag.schemaTag.name) + const otherTags = group.topTags.filter((tag) => tag !== specialTag) - // Check if you can't do more because of splices - if (!fullCheck && group.topSplices.length > 0) { - return [] - } + // Check for disallowed tags in the group + const disallowedTags = this._getDisallowedTags(otherTags, specialRequirements) - // Check that it has the minimum number of subgroups - const minLimit = requirements.minNonDefSubgroups != null ? requirements.minNonDefSubgroups : -Infinity - if (fullCheck && subgroupCount < minLimit) { - return [generateIssue('invalidNumberOfSubgroups', { tag: specialTag.originalTag, string: group.originalTag })] - } - return [] - } + // Make sure that there are not any duplicates in the allowed tags + if (disallowedTags.length > 0 || this.hasDuplicateNames(otherTags)) { + return [generateIssue('invalidTagGroup', { tagGroup: group.originalTag })] + } - checkNonTopGroups(hedString, fullCheck) { - if (!hedString.tags.some((tag) => this.groupTags.has(tag.schemaTag._name) && !this.topGroupTags.has(tag))) { - return [] - } - const issues = [] - for (const topGroup of hedString.tagGroups) { - for (const group of topGroup.subParsedGroupIterator()) { - const theseIssues = this._checkGroup(group, fullCheck) - issues.push(...theseIssues) - if (theseIssues.length > 0) { - break - } + // Check if required definition tag is missing + if (!hasDef && specialRequirements.defTagRequired && group.topColumnSplices.length === 0) { + return [ + generateIssue('temporalWithoutDefinition', { + tag: specialTag.schemaTag.name, + tagGroup: group.originalTag, + }), + ] } - } - return issues - } - _checkGroup(group, fullCheck) { - const specialTags = filterTagMapByNames(group.specialTags, this.groupTags) - const notTopGroupTags = specialTags.filter((tag) => !this.topGroupTags.has(tag.schemaTag.name)) - for (const tag of notTopGroupTags) { - const issues = this._checkGroupRequirements(group, tag, fullCheck) - if (issues.length > 0) { - return issues + // Check for the maximum number of subgroups allowed + const defExpandCount = specialRequirements.defTagRequired && group.hasDefExpandChildren ? 1 : 0 + const maxRequired = (specialRequirements.maxNonDefSubgroups ?? Infinity) + defExpandCount + if (group.topGroups.length > maxRequired) { + return [generateIssue('invalidTagGroup', { tagGroup: group.originalTag })] + } + + // Check if no column splices so the minimum number of groups can be verified + if (group.topColumnSplices.length === 0 && group.topGroups.length < specialRequirements.minNonDefSubgroups) { + return [generateIssue('invalidTagGroup', { tagGroup: group.originalTag })] } } return [] } + /** + * Get tags that are not allowed in the current group. + * + * @param {ParsedHedTag[]} tags - The HED tags to be evaluated. + * @param {Object} specialTagRequirements - The requirements for the special tag. + * @returns {ParsedHedTag[]} An array of tags that are not allowed. + * @private + */ + _getDisallowedTags(tags, specialTagRequirements) { + return tags.filter((tag) => !specialTagRequirements.otherAllowedTags?.includes(tag.schemaTag.name)) + } + /** * Check if there are conflicting subgroup tags. * @@ -348,39 +291,11 @@ export class SpecialChecker { * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. */ checkForbiddenGroups(hedString) { - // Only do this if there are any tags in the string with forbidden tags. - const hasForbidden = hedString.tags.filter((tag) => this.hasForbiddenSubgroupTags.has(tag.schemaTag.name)) - if (hasForbidden.length === 0) { - return [] - } - let forbiddenCount = hasForbidden.length - for (const tag of hedString.topLevelTags) { - if (!this.hasForbiddenSubgroupTags.has(tag.schemaTag.name)) { - continue - } - // This tag has - const forbidden = this.specialMap.get(tag.schemaTag.name).forbiddenSubgroupTags - for (const group of hedString.tagGroups) { - if (group.allTags.some((tag) => forbidden.has(tag.schemaTag.name))) { - return [ - generateIssue('forbiddenSubgroupTags', { - tag: tag.originalTag, - string: hedString.hedString, - tagList: getTagListString(forbidden), - }), - ] - } - } - forbiddenCount-- - if (forbiddenCount === 0) { - return [] - } - } const issues = [] for (const group of hedString.tagGroups) { // Only check the group if there are tags with forbidden subgroup tags if (group.allTags.some((tag) => this.hasForbiddenSubgroupTags.has(tag.schemaTag.name))) { - issues.push(...this._checkForbiddenGroup(group)) + issues.push(...this.checkForbiddenGroup(group)) } } return issues @@ -394,7 +309,7 @@ export class SpecialChecker { * * Note: Returns in a given group as soon as it finds a conflict */ - _checkForbiddenGroup(group) { + checkForbiddenGroup(group) { for (const subGroup of group.subParsedGroupIterator()) { // if this group does not have top tags with forbidden subgroups -- must go deeper const forbiddenTags = subGroup.topTags.filter((tag) => this.hasForbiddenSubgroupTags.has(tag.schemaTag._name)) @@ -412,7 +327,7 @@ export class SpecialChecker { if (badTags?.length > 0) { return [ generateIssue('invalidGroupTags', { - tags: getTagListString(badTags), + tags: this.getTagListString(badTags), string: subGroup.originalTag, }), ] @@ -422,6 +337,92 @@ export class SpecialChecker { return [] } + /** + * Check for special tags that have no splice in group that no forbidden tags are in their subgroup. + * + * @param {ParsedHedString} hedString - the HED string to be checked. + * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. + * + * Notes: Currently these are Definition and Def-expand + */ + + /** + * Check that no splice in group tags are not at the top level. + * + * @param {ParsedHedString} hedString - the HED string to be checked for splices at the top level. + * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. + */ + checkNoSpliceInGroupTags(hedString) { + const spliceTags = hedString.topLevelTags.filter((tag) => this.noSpliceInGroup.includes(tag.schemaTag._name)) + if (spliceTags.length > 0) { + return [generateIssue('missingTagGroup', { tag: spliceTags[0].originalTag, string: hedString.hedString })] + } + return [] + } + + /** + * Check for tags with the unique attribute. + * + * @param {ParsedHedString} hedString - The HED string to be checked for tags with the unique attribute. + * @returns {Issue[]} An array of `Issue` objects if there are violations; otherwise, an empty array. + */ + checkUnique(hedString) { + const uniqueTags = hedString.tags.filter((tag) => tag.hasAttribute('unique')) + if (uniqueTags.length > 1) { + const uniqueNames = new Set() + for (const tag of uniqueTags) { + if (uniqueNames.has(tag.schemaTag._name)) { + return [generateIssue('multipleUniqueTags', { tag: tag.originalTag, string: hedString.hedString })] + } + uniqueNames.add(tag.schemaTag._name) + } + } + return [] + } + + /** + * Return the value of a special attribute if special tag, otherwise undefined. + * + * @param {ParsedHedTag} tag - The HED tag to be checked for the attribute + * @param {string} attributeName - The name of the special attribute to check for. + * + * @returns value of the property or undefined. + */ + getSpecialAttributeForTag(tag, attributeName) { + return this.specialMap.get(tag.schemaTag.name)?.[attributeName] + } + + /** + * Return true if a list of tags has any duplicate names. + * + * @param {list} - A list of ParsedHedTag objects to be checked. + * @returns {boolean} If true, indicates that there tags with duplicate names in the list. + * + */ + hasDuplicateNames(list) { + const seen = new Set() + for (const obj of list) { + if (seen.has(obj.schemaTag.name)) { + return true + } + seen.add(obj.schemaTag.name) + } + return false + } + + /** + * Return a string of original tag names for error messages. + * @param {ParsedHedTag} tagList - The HED tags whose string representations should be put in a comma-separated list. + * @returns {string} A comma separated list of original tag names for tags in tagList. + */ + getTagListString(tagList) { + return tagList.map((tag) => tag.toString()).join(', ') + } + + _hasExclusiveTags(hedString) { + return hedString.tags.some((tag) => this.exclusiveTags.includes(tag.schemaTag._name)) + } + /** * Indicate whether a tag should be a top-level tag. * @@ -431,7 +432,7 @@ export class SpecialChecker { * Note: This check both the special requirements and the 'topLevelTagGroup' attribute in the schema. * */ - static hasTopLevelTagGroupAttribute(tag) { + hasTopLevelTagGroupAttribute(tag) { return ( tag.hasAttribute('topLevelTagGroup') || (SpecialChecker.specialMap.has(tag.schemaTag.name) && @@ -447,7 +448,7 @@ export class SpecialChecker { * * Note: This checks both special and schema tag requirements. */ - static hasGroupAttribute(tag) { + hasGroupAttribute(tag) { return ( tag.hasAttribute('tagGroup') || (SpecialChecker.specialMap.has(tag.schemaTag.name) && SpecialChecker.specialMap.get(tag.schemaTag.name).tagGroup) diff --git a/parser/splitter.js b/parser/splitter.js index b88a0945..c2f514bb 100644 --- a/parser/splitter.js +++ b/parser/splitter.js @@ -2,9 +2,10 @@ import ParsedHedTag from './parsedHedTag' import ParsedHedColumnSplice from './parsedHedColumnSplice' import ParsedHedGroup from './parsedHedGroup' import { recursiveMap } from '../utils/array' +import { mergeParsingIssues } from '../utils/hedData' import { HedStringTokenizer, ColumnSpliceSpec, TagSpec } from './tokenizer' import { generateIssue, IssueError } from '../common/issues/issues' -import { ReservedChecker } from './reservedChecker' +import { SpecialChecker } from './special' export default class HedStringSplitter { /** @@ -17,8 +18,16 @@ export default class HedStringSplitter { * @type {Schemas} */ hedSchemas - - issues + /** + * Any issues found during tag conversion. + * @type {Issue[]} + */ + conversionIssues + /** + * Any syntax issues found. + * @type {Issue[]} + */ + syntaxIssues /** * Constructor. @@ -29,28 +38,32 @@ export default class HedStringSplitter { constructor(hedString, hedSchemas) { this.hedString = hedString this.hedSchemas = hedSchemas - this.special = ReservedChecker.getInstance() - this.issues = [] + this.conversionIssues = [] + this.syntaxIssues = [] + this.special = SpecialChecker.getInstance() } /** * Split and parse a HED string into tags and groups. * - * @returns {[ParsedHedSubstring[], Issue[]]} The parsed HED string data and any issues found. + * @returns {[ParsedHedSubstring[], Object]} The parsed HED string data and any issues found. */ splitHedString() { if (this.hedString === null || this.hedString === undefined || typeof this.hedString !== 'string') { - return [null, [generateIssue('invalidTagString', {})]] + return [null, { syntax: [generateIssue('invalidTagString', {})] }] } if (this.hedString.length === 0) { - return [[], []] + return [[], {}] } - const [tagSpecs, groupBounds, issues] = new HedStringTokenizer(this.hedString).tokenize() - if (issues.length > 0) { - return [null, issues] + const [tagSpecs, groupBounds, tokenizingIssues] = new HedStringTokenizer(this.hedString).tokenize() + if (tokenizingIssues.syntax.length > 0) { + return [null, tokenizingIssues] } + const [parsedTags, parsingIssues] = this._createParsedTags(tagSpecs, groupBounds) - return [parsedTags, parsingIssues] + mergeParsingIssues(tokenizingIssues, parsingIssues) + + return [parsedTags, tokenizingIssues] } /** @@ -58,24 +71,31 @@ export default class HedStringSplitter { * * @param {TagSpec[]} tagSpecs The tag specifications. * @param {GroupSpec} groupSpecs The group specifications. - * @returns {[ParsedHedSubstring[], Issue[]]} The parsed HED tags and any issues. + * @returns {[ParsedHedSubstring[], Object]} The parsed HED tags and any issues. */ _createParsedTags(tagSpecs, groupSpecs) { // Create tags from specifications - this.issues = [] const parsedTags = recursiveMap((tagSpec) => this._createParsedTag(tagSpec), tagSpecs) // Create groups from the parsed tags const parsedTagsWithGroups = this._createParsedGroups(parsedTags, groupSpecs.children) - return [parsedTagsWithGroups, this.issues] + + const issues = { syntax: this.syntaxIssues, conversion: this.conversionIssues } + return [parsedTagsWithGroups, issues] } + /** + * Create a parsed tag object based on the tag specification. + * + * @param {TagSpec|ColumnSpliceSpec} tagSpec The tag or column splice specification. + * @returns {ParsedHedTag|ParsedHedColumnSplice|null} The parsed HED tag or column splice. + */ _createParsedTag(tagSpec) { if (tagSpec instanceof TagSpec) { try { return new ParsedHedTag(tagSpec, this.hedSchemas, this.hedString) } catch (issueError) { - this.issues.push(this._handleIssueError(issueError)) + this._handleIssueError(issueError) return null } } else if (tagSpec instanceof ColumnSpliceSpec) { @@ -90,9 +110,9 @@ export default class HedStringSplitter { */ _handleIssueError(issueError) { if (issueError instanceof IssueError) { - return issueError.issue + this.conversionIssues.push(issueError.issue) } else if (issueError instanceof Error) { - return generateIssue('internalError', { message: issueError.message }) + this.conversionIssues.push(generateIssue('internalError', { message: issueError.message })) } } @@ -111,7 +131,12 @@ export default class HedStringSplitter { if (Array.isArray(tag)) { const groupSpec = groupSpecs[index] tagGroups.push( - new ParsedHedGroup(this._createParsedGroups(tag, groupSpec.children), this.hedString, groupSpec.bounds), + new ParsedHedGroup( + this._createParsedGroups(tag, groupSpec.children), + this.hedSchemas, + this.hedString, + groupSpec.bounds, + ), ) index++ } else if (tag !== null) { diff --git a/parser/tagConverter.js b/parser/tagConverter.js index 837a6ac4..d56fa6d9 100644 --- a/parser/tagConverter.js +++ b/parser/tagConverter.js @@ -1,6 +1,6 @@ import { IssueError } from '../common/issues/issues' import { getTagSlashIndices } from '../utils/hedStrings' -import { ReservedChecker } from './reservedChecker' +import { SpecialChecker } from './special' /** * Converter from a tag specification to a schema-based tag object. @@ -62,7 +62,7 @@ export default class TagConverter { this.tagLevels = this.tagString.split('/') this.tagSlashes = getTagSlashIndices(this.tagString) this.remainder = undefined - this.special = ReservedChecker.getInstance() + this.special = SpecialChecker.getInstance() } /** @@ -100,7 +100,7 @@ export default class TagConverter { } if ( parentTag !== undefined && - (!parentTag.hasAttributeName('extensionAllowed') || this.special.noExtensionTags.has(parentTag.name)) + (!parentTag.hasAttributeName('extensionAllowed') || this.special.noExtensionTags.includes(parentTag.name)) ) { IssueError.generateAndThrow('invalidExtension', { tag: this.tagLevels[tagLevelIndex], diff --git a/parser/tokenizer.js b/parser/tokenizer.js index 571b8913..dcdfb4fb 100644 --- a/parser/tokenizer.js +++ b/parser/tokenizer.js @@ -132,27 +132,27 @@ export class HedStringTokenizer { /** * Split the HED string into delimiters and tags. * - * @returns {[TagSpec[], GroupSpec, Issue[]]} The tag specifications, group bounds, and any issues found. + * @returns {[TagSpec[], GroupSpec, Object]} The tag specifications, group bounds, and any issues found. */ tokenize() { this.initializeTokenizer() // Empty strings cannot be tokenized if (this.hedString.trim().length === 0) { this.pushIssue('emptyTagFound', 0) - return [[], null, this.issues] + return [[], null, { syntax: this.issues }] } for (let i = 0; i < this.hedString.length; i++) { const character = this.hedString.charAt(i) this.handleCharacter(i, character) if (this.issues.length > 0) { - return [[], null, this.issues] + return [[], null, { syntax: this.issues }] } } this.finalizeTokenizer() if (this.issues.length > 0) { - return [[], null, this.issues] + return [[], null, { syntax: this.issues }] } else { - return [this.state.currentGroupStack.pop(), this.state.parenthesesStack.pop(), []] + return [this.state.currentGroupStack.pop(), this.state.parenthesesStack.pop(), { syntax: [] }] } } diff --git a/schema/entries.js b/schema/entries.js index 0312c180..5d82a98c 100644 --- a/schema/entries.js +++ b/schema/entries.js @@ -159,8 +159,8 @@ export class SchemaEntryManager extends Memoizer { /** * Get the entry with the given name. * - * @param {string} name - The name of the entry to retrieve. - * @returns {T} - The entry with that name. + * @param {string} name The name of the entry to retrieve. + * @return {T} The entry with that name. */ getEntry(name) { return this._definitions.get(name) @@ -169,8 +169,8 @@ export class SchemaEntryManager extends Memoizer { /** * Get a collection of entries with the given boolean attribute. * - * @param {string} booleanAttributeName - The name of boolean attribute to filter on. - * @returns {Map} - A collection of entries with that attribute. + * @param {string} booleanAttributeName The name of boolean attribute to filter on. + * @return {Map} A collection of entries with that attribute. */ getEntriesWithBooleanAttribute(booleanAttributeName) { return this._memoize(booleanAttributeName, () => { @@ -240,8 +240,8 @@ export class SchemaTagManager extends SchemaEntryManager { /** * Determine whether the tag with the given name exists. * - * @param {string} longName - The long name of the tag. - * @returns {boolean} -True if the tag exists. + * @param {string} longName The long name of the tag. + * @return {boolean} Whether the tag exists. */ hasLongNameEntry(longName) { return this._definitionsByLongName.has(longName) @@ -250,8 +250,8 @@ export class SchemaTagManager extends SchemaEntryManager { /** * Get the tag with the given name. * - * @param {string} longName - The long name of the tag to retrieve. - * @returns {SchemaTag} - The tag with that name. + * @param {string} longName The long name of the tag to retrieve. + * @return {SchemaTag} The tag with that name. */ getLongNameEntry(longName) { return this._definitionsByLongName.get(longName) @@ -664,7 +664,7 @@ export class SchemaUnitClass extends SchemaEntryWithAttributes { /** * Extracts the Unit class and remainder - * @returns {SchemaUnit, string, string} unit class, unit string, and value string + * @returns {Unit, string, string} unit class, unit string, and value string */ extractUnit(value) { let actualUnit = null // The Unit class of the value diff --git a/schema/parser.js b/schema/parser.js index 9c94421e..a5993a2e 100644 --- a/schema/parser.js +++ b/schema/parser.js @@ -24,7 +24,7 @@ import { } from './entries' import { IssueError } from '../common/issues/issues' -//const specialTags = require('../data/json/reservedTags.json') +//const specialTags = require('../data/json/specialTags.json') import classRegex from '../data/json/class_regex.json' diff --git a/schema/schemaMerger.js b/schema/schemaMerger.js index 38001db1..c9c2405d 100644 --- a/schema/schemaMerger.js +++ b/schema/schemaMerger.js @@ -74,7 +74,7 @@ export default class PartneredSchemaMerger { /** * The destination schema's tag collection. * - * @returns {SchemaTagManager} + * @return {SchemaTagManager} */ get destinationTags() { return this.destination.entries.tags diff --git a/spec_tests/javascriptTests.json b/spec_tests/javascriptTests.json index c27dc03b..41aa6ac0 100644 --- a/spec_tests/javascriptTests.json +++ b/spec_tests/javascriptTests.json @@ -2,7 +2,7 @@ { "error_code": "CHARACTER_INVALID", "alt_codes": ["TAG_INVALID", "UNITS_INVALID", "VALUE_INVALID"], - "name": "character-invalid-non-printing-appears", + "name": "character-invalid-non-printing appears", "description": "The HED string contains a UTF-8 character.", "warning": false, "schema": "8.3.0", @@ -1181,7 +1181,7 @@ "definitions": ["(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))", "(Definition/MyColor, (Label/Pie))"], "tests": { "string_tests": { - "fails": ["(Def-expand/MyColor2, (Label/Pie))"], + "fails": ["(Def-expand/Acc2/4.5, (Acceleration/4.5 m-per-s^2, Red))", "(Def-expand/MyColor2, (Label/Pie))"], "passes": ["(Def-expand/Acc/4.5, (Acceleration/4.5 m-per-s^2, Red))"] }, "sidecar_tests": { @@ -1229,7 +1229,7 @@ "event_code": { "HED": { "face": "(Def-expand/Acc2/4.5, (Acceleration/4.5 m-per-s^2, Red))", - "ball": "(Def-expand/MyColor2, (Label/Pie))" + "ball": "(Def-expand/MyColor2, (Label/Pie)))" } } }, @@ -1263,7 +1263,7 @@ }, { "error_code": "DEF_EXPAND_INVALID", - "alt_codes": ["VALUE_INVALID"], + "alt_codes": [], "name": "def-expand-invalid-missing-placeholder", "description": "A `Def-expand` is missing an expected placeholder value or has an unexpected placeholder value.", "warning": false, @@ -1271,8 +1271,15 @@ "definitions": ["(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))", "(Definition/MyColor, (Label/Pie))"], "tests": { "string_tests": { - "fails": ["(Def-expand/Acc, (Acceleration, Red))", "(Def-expand/MyColor/Blue, (Label/Pie))"], - "passes": ["(Def-expand/Acc/4.5, (Acceleration/4.5 m-per-s^2, Red))"] + "fails": [ + "(Def-expand/Acc, (Acceleration, Red))", + "(Def-expand/Acc/4.5/3, (Acceleration, Red))", + "(Def-expand/MyColor/Blue, (Label/Pie))" + ], + "passes": [ + "(Def-expand/Acc/4.5, (Acceleration/4.5 m-per-s^2, Red))", + "(Def-expand/Acc/4.5, (Acceleration/4.5 m-per-s^2, Red))" + ] }, "sidecar_tests": { "fails": [ @@ -1280,7 +1287,8 @@ "event_code": { "HED": { "face": "(Def-expand/Acc, (Acceleration, Red))", - "ball": "(Def-expand/Acc/4.5, (Acceleration, Red))" + "ball": "(Def-expand/Acc/4.5/3, (Acceleration, Red))", + "circle": "(Def-expand/MyColor2/4, (Label/Pie))" } } } @@ -1289,7 +1297,8 @@ { "event_code": { "HED": { - "face": "(Def-expand/Acc/4.5, (Acceleration/4.5 m-per-s^2, Red))" + "face": "(Def-expand/Acc/4.5, (Acceleration/4.5 m-per-s^2, Red))", + "ball": "(Def-expand/Acc/4.5, (Acceleration/4.5 m-per-s^2, Red))" } } } @@ -1300,13 +1309,15 @@ [ ["onset", "duration", "HED"], [4.5, 0, "(Def-expand/Acc, (Acceleration, Red))"], - [5.5, 0, "(Def-expand/Acc/4.5, (Acceleration, Red))"] + [5.5, 0, "(Def-expand/Acc/4.5/3, (Acceleration, Red))"], + [6.5, 0, "(Def-expand/MyColor2/4, (Label/Pie))"] ] ], "passes": [ [ ["onset", "duration", "HED"], - [4.5, 0, "(Def-expand/Acc/4.5, (Acceleration/4.5 m-per-s^2, Red))"] + [4.5, 0, "(Def-expand/Acc/4.5, (Acceleration/4.5 m-per-s^2, Red))"], + [5.5, 0, "(Def-expand/Acc/4.5, (Acceleration/4.5 m-per-s^2, Red))"] ] ] }, @@ -1317,14 +1328,17 @@ "event_code": { "HED": { "face": "(Def-expand/Acc, (Acceleration, Red))", - "ball": "(Def-expand/Acc/4.5, (Acceleration, Red))" + "ball": "(Def-expand/Acc/4.5/3, (Acceleration, Red))", + "circle": "(Def-expand/MyColor2/4, (Label/Pie))" } } }, "events": [ - ["onset", "duration", "event_code"], - [4.5, 0, "face"], - [5.5, 0, "ball"] + ["onset", "duration", "event_code", "HED"], + [4.5, 0, "face", " (Def-expand/Acc, (Acceleration, Red))"], + [5.5, 0, "ball", "(Def-expand/Acc/4.5/3, (Acceleration, Red))"], + [5.6, 0, "circle", "(Def-expand/Acc/4.5/3, (Acceleration, Red))"], + [6.5, 0, "n/a", "(Def-expand/MyColor2/4, (Label/Pie))"] ] } ], @@ -1512,7 +1526,7 @@ "HED": { "face": "(Def-expand/Acc/4.5, (Acceleration/6, Red))", "ball": "(Def-expand/Acc/4.5, (Acceleration/4.5 m-per-s^2, Blue))", - "square": "(Def-expand/MyColor, (Label/Cake))" + "square": "(Def-expand/MyColor, (Label/Cake)))" } } }, @@ -1562,7 +1576,7 @@ { "event_code": { "HED": { - "face": "Orange", + "face": "(Def-expand/Acc/4.5, Red)", "ball": "(Def-expand/Acc/4.5)" } } @@ -1583,6 +1597,7 @@ "fails": [ [ ["onset", "duration", "HED"], + [4.5, 0, "(Def-expand/Acc/4.5, Red)"], [5.4, 0, "(Def-expand/Acc/5.4 )"] ] ], @@ -1606,7 +1621,7 @@ }, "events": [ ["onset", "duration", "event_code", "HED"], - [4.5, 0, "ball", "(Def-expand/Acc/4.5)"], + [4.5, 0, "ball", "Def/Acc, (Def-expand/Acc/4.5)"], [5.4, 0, "n/a", "Green"], [6.4, 0, "face", "n/a"] ] @@ -1796,8 +1811,7 @@ "sidecar": { "event_code": { "HED": { - "face": "Def/Acc/4.5", - "ball": "Item" + "face": "Def/Acc/4.5" } } }, @@ -1902,8 +1916,8 @@ }, { "error_code": "DEF_INVALID", - "alt_codes": ["VALUE_INVALID", "UNITS_INVALID"], - "name": "def-invalid-bad-placeholder-value", + "alt_codes": [], + "name": "def-invalid-bad-placeholder-units", "description": "A `Def` has a placeholder value of incorrect format or units for definition.", "warning": false, "schema": "8.3.0", @@ -2573,6 +2587,60 @@ } } }, + { + "error_code": "REQUIRED_TAG_MISSING", + "alt_codes": [], + "name": "required-tag-missing", + "description": "An event-level annotation does not have a tag corresponding to a node with the `required` schema attribute. (Note this is deprecated so no tests.)", + "warning": false, + "schema": "8.3.0", + "definitions": ["(Definition/Acc/#, (Acceleration/# m-per-s^2 Red))", "(Definition/MyColor, (Label/Pie))"], + "tests": { + "string_tests": { + "fails": [], + "passes": [] + }, + "sidecar_tests": { + "fails": [], + "passes": [] + }, + "event_tests": { + "fails": [], + "passes": [] + }, + "combo_tests": { + "fails": [], + "passes": [] + } + } + }, + { + "error_code": "REQUIRED_TAG_MISSING", + "alt_codes": [], + "name": "required-tag-in-definition", + "description": "A required tag appears in a definition", + "warning": false, + "schema": "8.3.0", + "definitions": ["(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))", "(Definition/MyColor, (Label/Pie))"], + "tests": { + "string_tests": { + "fails": [], + "passes": [] + }, + "sidecar_tests": { + "fails": [], + "passes": [] + }, + "event_tests": { + "fails": [], + "passes": [] + }, + "combo_tests": { + "fails": [], + "passes": [] + } + } + }, { "error_code": "SIDECAR_BRACES_INVALID", "alt_codes": ["CHARACTER_INVALID", "SIDECAR_INVALID"], @@ -2935,7 +3003,7 @@ [4.5, 0, 3.4, "face", "Blue"], [5.0, 0, 6.8, "ball", "Green, Def/MyColor"], [5.2, 0, "n/a", "face", ""], - [5.5, 0, "7,3", "face", "n/a"] + [5.5, 0, "any", "face", "n/a"] ] } ] @@ -4036,7 +4104,7 @@ }, { "error_code": "TAG_GROUP_ERROR", - "alt_codes": ["TEMPORAL_TAG_ERROR", "TAG_INVALID", "DEFINITION_INVALID"], + "alt_codes": ["TEMPORAL_TAG_ERROR", "TAG_INVALID"], "name": "tag-group-error-missing", "description": "A tag has tagGroup or topLevelTagGroup attribute, but is not enclosed in parentheses.", "warning": false, @@ -4191,7 +4259,7 @@ "event_code": { "HED": { "face": "Acceleration/5 m-per-s^2", - "ball": "(Def/Acc/3.0, Onset)" + "ball": "(Def/Acc/3.0 m-per-s^2, Onset)" } } }, @@ -4237,11 +4305,11 @@ "event_code": { "HED": { "face": "Acceleration/5.0", - "ball": "Red" + "ball": "Onset" } }, "val_col": { - "HED": "Time-interval/# s, (Duration/5.0 s, ({event_code}), Duration/6.0)" + "HED": "Time-interval/# s, (Duration/5.0 s, {event_code})" } } ], @@ -4270,7 +4338,7 @@ "passes": [ [ ["onset", "duration", "HED"], - [4.5, 0, "Red, (Event-context, (Def/MyColor))"], + [4.5, 0, "Red, (Event-context, Def/MyColor)"], [5.0, 0, "Green"] ] ] @@ -4305,7 +4373,7 @@ }, "events": [ ["onset", "duration", "event_code", "HED"], - [4.5, 0, "face", "Blue, (Event-context, (Label/Red))"], + [4.5, 0, "face", "Blue, (Event-context, Label/Red)"], [5.0, 0, "ball", "Green, Def/MyColor"] ] } @@ -4793,7 +4861,16 @@ "passes": ["(Onset, Def/Acc/5.4)"] }, "sidecar_tests": { - "fails": [], + "fails": [ + { + "event_code": { + "HED": { + "face": "Onset, Red", + "ball": "Offset, Def/Acc/5.4" + } + } + } + ], "passes": [ { "event_code": { @@ -4809,7 +4886,8 @@ [ ["onset", "duration", "HED"], [4.5, 0, "Onset, Red"], - [5.0, 0, "Onset, Def/MyColor"] + [4.8, 0, "(Onset, MyColor)"], + [5.0, 0, "Offset, MyColor"] ] ], "passes": [ @@ -4828,13 +4906,15 @@ "event_code": { "HED": { "face": "Onset, Red", - "ball": "(Onset, Def/Acc/5.4)" + "ball": "Offset, Def/Acc/5.4" } } }, "events": [ ["onset", "duration", "event_code", "HED"], - [4.5, 0, "ball", "Onset, Def/MyColor"] + [4.5, 0, "ball", "Onset, Red"], + [4.8, 0, "n/a", "(Onset/MyColor)"], + [5.0, 0, "face", "Offset, MyColor"] ] } ], @@ -5129,8 +5209,8 @@ "definitions": ["(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))", "(Definition/MyColor, (Label/Pie))"], "tests": { "string_tests": { - "fails": [], - "passes": [] + "fails": ["(Offset, Def/MyColor, Red)", "((Def-expand/MyColor, (Label/Pie)), Offset, (Red))"], + "passes": ["(Offset, Def/MyColor)"] }, "sidecar_tests": { "fails": [ @@ -5799,7 +5879,7 @@ "definitions": ["(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))", "(Definition/MyColor, (Label/Pie))"], "tests": { "string_tests": { - "fails": ["(Delay, Red)", "(Delay, Def/Acc/5.4)"], + "fails": ["Delay, Red", "Delay, Def/Acc/5.4"], "passes": ["(Delay/5 s, (Def/Acc/5.4))"] }, "sidecar_tests": { @@ -5808,7 +5888,15 @@ "event_code": { "HED": { "face": "Blue, Red", - "ball": "(Delay/5.0 s, Def/Acc/5.4)" + "ball": "Delay/5.0 s, Def/Acc/5.4" + } + } + }, + { + "event_code": { + "HED": { + "face": "Blue, Red", + "ball": "Delay/5.0 s, (Def/Acc/5.4)" } } } @@ -5858,6 +5946,20 @@ [4.8, 0, "n/a", "Blue"], [5.0, 0, "face", "Green"] ] + }, + { + "sidecar": { + "event_code": { + "HED": { + "face": "Red", + "ball": "Delay/5.0 s, (Def/Acc/5.4)" + } + } + }, + "events": [ + ["onset", "duration", "event_code", "HED"], + [4.5, 0, "ball", "Red"] + ] } ], "passes": [ @@ -5883,13 +5985,13 @@ "error_code": "TEMPORAL_TAG_ERROR", "alt_codes": ["TAG_GROUP_ERROR"], "name": "temporal-tag-error-nested-group-delay", - "description": "A delay appears in a group not in the top level.", + "description": "An Onset or Offset tag appears in a nested tag group (not a top-level tag group).", "schema": "8.2.0", "definitions": ["(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))", "(Definition/MyColor, (Label/Pie))"], "tests": { "string_tests": { - "fails": ["((Delay/5.0 s, Delay/7.0 s, (Def/MyColor)), Red)"], - "passes": ["(Onset, Delay/6 s, Def/MyColor), Red"] + "fails": ["((Delay/5.0 s, Delay/7.0 s (Def/MyColor)), Red)"], + "passes": ["(Onset, Delay/6 s, (Def/MyColor)), Red"] }, "sidecar_tests": { "fails": [ @@ -6154,8 +6256,11 @@ "definitions": ["(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))", "(Definition/MyColor, (Label/Pie))"], "tests": { "string_tests": { - "fails": [], - "passes": [] + "fails": [ + "(Delay/5.0 s, Offset, Def/MyColor, Red)", + "(Delay/5.0 s, (Def-expand/MyColor, (Label/Pie)), Offset, (Red))" + ], + "passes": ["(Delay/5.0 s, Offset, Def/MyColor)"] }, "sidecar_tests": { "fails": [ @@ -6221,7 +6326,7 @@ [4.5, 0, "face", "n/a"], [4.8, 0, "square", "n/a"], [4.9, 0, "ball", "Green"], - [5.5, 0, "n/a", "(Delay/5.0 s, (Def-expand/MyColor, (Label/Pie)), Offset, Blue)", "Orange"] + [5.5, 0, "(Delay/5.0 s, (Def-expand/MyColor, (Label/Pie)), Offset, Blue)", "Orange"] ] } ], @@ -6242,7 +6347,7 @@ [4.5, 0, "face", "n/a"], [4.8, 0, "square", "n/a"], [4.9, 0, "ball", "Green"], - [10.0, 0, "n/a", "(Delay/5.0 s, (Def-expand/MyColor, (Label/Pie)), Offset)"] + [5.5, 0, "(Delay/5.0 s, (Def-expand/MyColor, (Label/Pie)), Offset, Blue)", "Orange"] ] } ] diff --git a/spec_tests/jsonTests.spec.js b/spec_tests/jsonTests.spec.js index 81b4fd7c..0471466e 100644 --- a/spec_tests/jsonTests.spec.js +++ b/spec_tests/jsonTests.spec.js @@ -8,26 +8,17 @@ import { SchemaSpec, SchemasSpec } from '../schema/specs' import path from 'path' import { BidsSidecar, BidsTsvFile } from '../bids' import { generateIssue, IssueError } from '../common/issues/issues' -import { DefinitionManager } from '../parser/definitionManager' -import parseTSV from '../bids/tsvParser' -import { shouldRun } from '../tests/testUtilities' const fs = require('fs') -const skipMap = new Map() -const runAll = true -//const runMap = new Map([['DEF_EXPAND_INVALID', ['def-expand-invalid-missing-placeholder']]]) -//const runMap = new Map([['TAG_GROUP_ERROR', ['tag-group-error-missing']]]) -const runMap = new Map([['TAG_GROUP_ERROR', ['tag-group-error-missing']]]) -const runOnly = new Set() - const skippedErrors = { - VERSION_DEPRECATED: 'not handling in the spec tests.', - ELEMENT_DEPRECATED: 'not handling tag deprecated in the spec tests.', - STYLE_WARNING: 'not handling style warnings at this time', - 'invalid-character-name-value-class-deprecated': 'not handling deprecated in the spec tests.', + VERSION_DEPRECATED: 'Not handling in the spec tests', + ELEMENT_DEPRECATED: 'Not handling in this round. This is a warning', + STYLE_WARNING: 'Not handling style warnings at this time', + 'invalid-character-name-value-class-deprecated': 'We will let this pass regardless of schema version.', } const readFileSync = fs.readFileSync const test_file_name = 'javascriptTests.json' +//const test_file_name = 'temp3.json' function comboListToStrings(items) { const comboItems = [] @@ -41,6 +32,10 @@ function comboListToStrings(items) { return comboItems } +function getMergedSidecar(side1, side2) { + return Object.assign({}, JSON.parse(side1), side2) +} + function loadTestData() { const testFile = path.join(__dirname, test_file_name) return JSON.parse(readFileSync(testFile, 'utf8')) @@ -106,9 +101,9 @@ describe('HED validation using JSON tests', () => { '$error_code $name : $description', ({ error_code, alt_codes, name, schema, definitions, warning, tests }) => { let hedSchema - let defList + let defs let expectedErrors - let noErrors + const noErrors = new Set() const failedSidecars = stringifyList(tests.sidecar_tests.fails) const passedSidecars = stringifyList(tests.sidecar_tests.passes) @@ -138,39 +133,33 @@ describe('HED validation using JSON tests', () => { } const comboValidator = function (side, events, expectedErrors) { - const status = expectedErrors.size === 0 ? 'Expect pass' : 'Expect fail' + const status = expectedErrors.size === 0 ? 'Expect fail' : 'Expect pass' const header = `\n[${error_code} ${name}](${status})\tCOMBO\t"${side}"\n"${events}"` - let issues + const mergedSide = getMergedSidecar(side, defs) + let sidecarIssues = [] + try { + const bidsSide = new BidsSidecar(`sidecar`, mergedSide, { relativePath: 'combo test sidecar' }) + sidecarIssues = bidsSide.validate(hedSchema) + } catch (e) { + sidecarIssues = [convertIssue(e)] + } + let eventsIssues = [] try { - const defManager = new DefinitionManager() - defManager.addDefinitions(defList) - const parsedTsv = parseTSV(events) - assert.instanceOf(parsedTsv, Map, `${events} cannot be parsed`) - const bidsTsv = new BidsTsvFile( - `events`, - parsedTsv, - { relativePath: 'combo test tsv' }, - [], - JSON.parse(side), - defManager, - ) - issues = bidsTsv.validate(hedSchema) + const bidsTsv = new BidsTsvFile(`events`, events, { relativePath: 'combo test tsv' }, [side], mergedSide) + eventsIssues = bidsTsv.validate(hedSchema) } catch (e) { - issues = [convertIssue(e)] + eventsIssues = [convertIssue(e)] } - assertErrors(expectedErrors, issues, header) + const allIssues = [...sidecarIssues, ...eventsIssues] + assertErrors(expectedErrors, allIssues, header) } const eventsValidator = function (events, expectedErrors) { - const status = expectedErrors.size === 0 ? 'Expect pass' : 'Expect fail' + const status = expectedErrors.size === 0 ? 'Expect fail' : 'Expect pass' const header = `\n[${error_code} ${name}](${status})\tEvents:\n"${events}"` - let eventsIssues + let eventsIssues = [] try { - const defManager = new DefinitionManager() - defManager.addDefinitions(defList) - const parsedTsv = parseTSV(events) - assert.instanceOf(parsedTsv, Map, `${events} cannot be parsed`) - const bidsTsv = new BidsTsvFile(`events`, parsedTsv, { relativePath: 'events test' }, [], {}, defManager) + const bidsTsv = new BidsTsvFile(`events`, events, { relativePath: 'events test' }, [], defs) eventsIssues = bidsTsv.validate(hedSchema) } catch (e) { eventsIssues = [convertIssue(e)] @@ -179,31 +168,26 @@ describe('HED validation using JSON tests', () => { } const sideValidator = function (side, expectedErrors) { - const status = expectedErrors.size === 0 ? 'Expect pass' : 'Expect fail' + const status = expectedErrors.size === 0 ? 'Expect fail' : 'Expect pass' const header = `\n[${error_code} ${name}](${status})\tSIDECAR "${side}"` - let issues + const side1 = getMergedSidecar(side, defs) + let sidecarIssues = [] try { - const defManager = new DefinitionManager() - defManager.addDefinitions(defList) - const bidsSide = new BidsSidecar(`sidecar`, JSON.parse(side), { relativePath: 'sidecar test' }, defManager) - issues = bidsSide.validate(hedSchema) + const bidsSide = new BidsSidecar(`sidecar`, side1, { relativePath: 'sidecar test' }) + sidecarIssues = bidsSide.validate(hedSchema) } catch (e) { - issues = [convertIssue(e)] + sidecarIssues = [convertIssue(e)] } - assertErrors(expectedErrors, issues, header) + assertErrors(expectedErrors, sidecarIssues, header) } const stringValidator = function (str, expectedErrors) { const status = expectedErrors.size === 0 ? 'Expect pass' : 'Expect fail' const header = `\n[${error_code} ${name}](${status})\tSTRING: "${str}"` - const hTsv = `onset\tHED\n5.4\t${str}\n` - let stringIssues + const hTsv = `HED\n${str}\n` + let stringIssues = [] try { - const defManager = new DefinitionManager() - defManager.addDefinitions(defList) - const parsedTsv = parseTSV(hTsv) - assert.instanceOf(parsedTsv, Map, `${str} cannot be parsed`) - const bidsTsv = new BidsTsvFile(`string`, parsedTsv, { relativePath: 'string test tsv' }, [], {}, defManager) + const bidsTsv = new BidsTsvFile(`events`, hTsv, { relativePath: 'string test tsv' }, [], defs) stringIssues = bidsTsv.validate(hedSchema) } catch (e) { stringIssues = [convertIssue(e)] @@ -227,77 +211,67 @@ describe('HED validation using JSON tests', () => { beforeAll(async () => { hedSchema = schemaMap.get(schema) - let defIssues - ;[defList, defIssues] = DefinitionManager.createDefinitions(definitions, hedSchema) - assert.equal(defIssues.length, 0, `${name}: input definitions "${definitions}" have errors "${defIssues}"`) + if (definitions.length === 0) { + defs = {} + } else { + defs = { definitions: { HED: { defList: definitions.join(',') } } } + } expectedErrors = new Set(alt_codes) expectedErrors.add(error_code) - noErrors = new Set() }) afterAll(() => {}) - // If debugging a single test - if (!shouldRun(error_code, name, runAll, runMap, skipMap)) { - // eslint-disable-next-line no-console - console.log(`----Skipping JSON Spec tests ${error_code} [${name}]}`) - return - } - // Run tests except for the ones explicitly skipped or because they are warnings - if (warning) { - test.skip(`Skipping tests ${error_code} [${name}] skipped because warning not error`, () => {}) - } else if (error_code in skippedErrors) { - test.skip(`Skipping tests ${error_code} [${name}] skipped because ${skippedErrors[error_code]}`, () => {}) - } else if (name in skippedErrors) { - test.skip(`Skipping tests ${error_code} [${name}] skipped because ${skippedErrors[name]}`, () => {}) + if (error_code in skippedErrors || name in skippedErrors || warning) { + test.skip(`Skipping tests ${error_code} skipped because ${skippedErrors['error_code']}`, () => {}) } else { test('it should have HED schema defined', () => { expect(hedSchema).toBeDefined() }) - if (tests.string_tests.passes.length > 0 && (runOnly.size === 0 || runOnly.has('stringPass'))) { + if (tests.string_tests.passes.length > 0) { test.each(tests.string_tests.passes)('Valid string: %s', (str) => { - stringValidator(str, new Set()) + stringValidator(str, noErrors) }) } - if (tests.string_tests.fails.length > 0 && (runOnly.size === 0 || runOnly.has('stringFail'))) { + if (tests.string_tests.fails.length > 0) { test.each(tests.string_tests.fails)('Invalid string: %s', (str) => { stringValidator(str, expectedErrors) }) } - if (passedSidecars.length > 0 && (runOnly.size === 0 || runOnly.has('sidecarPass'))) { + if (passedSidecars.length > 0) { test.each(passedSidecars)(`Valid sidecar: %s`, (side) => { sideValidator(side, noErrors) }) } - if (failedSidecars.length > 0 && (runOnly.size === 0 || runOnly.has('sidecarFail'))) { + if (failedSidecars.length > 0) { test.each(failedSidecars)(`Invalid sidecar: %s`, (side) => { sideValidator(side, expectedErrors) }) } - if (passedEvents.length > 0 && (runOnly.size === 0 || runOnly.has('eventsPass'))) { + if (passedEvents.length > 0) { test.each(passedEvents)(`Valid events: %s`, (events) => { eventsValidator(events, noErrors) }) } - if (failedEvents.length > 0 && (runOnly.size === 0 || runOnly.has('eventsFail'))) { + if (failedEvents.length > 0) { test.each(failedEvents)(`Invalid events: %s`, (events) => { eventsValidator(events, expectedErrors) }) } - if (passedCombos.length > 0 && (runOnly.size === 0 || runOnly.has('combosPass'))) { + if (passedCombos.length > 0) { test.each(passedCombos)(`Valid combo: [%s] [%s]`, (side, events) => { comboValidator(side, events, noErrors) }) } - if (failedCombos.length > 0 && (runOnly.size === 0 || runOnly.has('combosFail'))) { + if (failedCombos.length > 0) { test.each(failedCombos)(`Invalid combo: [%s] [%s]`, (side, events) => { comboValidator(side, events, expectedErrors) }) diff --git a/tests/bids.spec.js b/tests/bids.spec.js new file mode 100644 index 00000000..acfec715 --- /dev/null +++ b/tests/bids.spec.js @@ -0,0 +1,752 @@ +import chai from 'chai' +const assert = chai.assert +import { beforeAll, describe, it } from '@jest/globals' +import cloneDeep from 'lodash/cloneDeep' + +import { generateIssue } from '../common/issues/issues' +import { SchemaSpec, SchemasSpec } from '../schema/specs' +import { buildBidsSchemas, parseSchemasSpec } from '../bids/schema' +import { BidsDataset, BidsHedIssue, BidsIssue, validateBidsDataset } from '../bids' +import { bidsDatasetDescriptions, bidsSidecars, bidsTsvFiles } from './testData/bids.spec.data' +import { parseHedString } from '../parser/parser' +import { BidsHedTsvParser } from '../bids/validator/tsvValidator' + +describe('BIDS datasets', () => { + /** + * @type {SchemasSpec} + */ + let specs + + beforeAll(() => { + const spec1 = new SchemaSpec('', '8.3.0') + specs = new SchemasSpec().addSchemaSpec(spec1) + }) + + /** + * Validate the test datasets. + * @param {Object} testDatasets The datasets to test with. + * @param {Object} expectedIssues The expected issues. + * @param {SchemasSpec} versionSpec The schema version to test with. + * @returns {Promise} + */ + const validator = (testDatasets, expectedIssues, versionSpec) => { + return Promise.all( + Object.entries(testDatasets).map(async ([datasetName, dataset]) => { + assert.property(expectedIssues, datasetName, datasetName + ' is not in expectedIssues') + const issues = await validateBidsDataset(dataset, versionSpec) + assert.sameDeepMembers(issues, expectedIssues[datasetName], datasetName) + }), + ) + } + + /** + * Validate the test datasets. + * @param {Object} testDatasets The datasets to test with. + * @param {Object} expectedIssues The expected issues. + * @param {SchemasSpec} versionSpecs The schema version to test with. + * @returns {Promise} + */ + const validatorWithSpecs = (testDatasets, expectedIssues, versionSpecs) => { + return Promise.all( + Object.entries(testDatasets).map(async ([datasetName, dataset]) => { + assert.property(expectedIssues, datasetName, datasetName + ' is not in expectedIssues') + let specs = versionSpecs + if (versionSpecs) { + assert.property(versionSpecs, datasetName, datasetName + ' is not in versionSpecs') + specs = versionSpecs[datasetName] + } + const issues = await validateBidsDataset(dataset, specs) + assert.sameDeepMembers(issues, expectedIssues[datasetName], datasetName) + }), + ) + } + // + // describe('Sidecar-only datasets', () => { + // it('should validate non-placeholder HED strings in BIDS sidecars', () => { + // const goodDatasets = bidsSidecars[0] + // const testDatasets = { + // single: new BidsDataset([], [bidsSidecars[0][0]]), + // all_good: new BidsDataset([], goodDatasets), + // warning_and_good: new BidsDataset([], goodDatasets.concat([bidsSidecars[1][0]])), + // error_and_good: new BidsDataset([], goodDatasets.concat([bidsSidecars[1][1]])), + // } + // const expectedIssues = { + // single: [], + // all_good: [], + // warning_and_good: [ + // BidsHedIssue.fromHedIssue( + // generateIssue('extension', { tag: 'Train/Maglev', sidecarKey: 'transport' }), + // bidsSidecars[1][0].file, + // ), + // ], + // error_and_good: [ + // BidsHedIssue.fromHedIssue(generateIssue('invalidTag', { tag: 'Confused' }), bidsSidecars[1][1].file), + // ], + // } + // validator(testDatasets, expectedIssues, specs) + // }, 10000) + // + // it('should validate placeholders in BIDS sidecars', () => { + // const placeholderDatasets = bidsSidecars[2] + // const testDatasets = { + // placeholders: new BidsDataset([], placeholderDatasets), + // } + // const expectedIssues = { + // placeholders: [ + // BidsHedIssue.fromHedIssue( + // generateIssue('invalidPlaceholderInDefinition', { + // definition: 'InvalidDefinitionGroup', + // sidecarKey: 'invalid_definition_group', + // }), + // placeholderDatasets[2].file, + // ), + // BidsHedIssue.fromHedIssue( + // generateIssue('invalidPlaceholderInDefinition', { + // definition: 'InvalidDefinitionTag', + // sidecarKey: 'invalid_definition_tag', + // }), + // placeholderDatasets[3].file, + // ), + // BidsHedIssue.fromHedIssue( + // generateIssue('invalidPlaceholderInDefinition', { + // definition: 'MultiplePlaceholdersInGroupDefinition', + // sidecarKey: 'multiple_placeholders_in_group', + // }), + // placeholderDatasets[4].file, + // ), + // BidsHedIssue.fromHedIssue( + // generateIssue('invalidPlaceholder', { tag: 'Label/#', sidecarKey: 'multiple_value_tags' }), + // placeholderDatasets[5].file, + // ), + // BidsHedIssue.fromHedIssue( + // generateIssue('invalidPlaceholder', { tag: 'Description/#', sidecarKey: 'multiple_value_tags' }), + // placeholderDatasets[5].file, + // ), + // BidsHedIssue.fromHedIssue( + // generateIssue('missingPlaceholder', { string: 'Sad', sidecarKey: 'no_value_tags' }), + // placeholderDatasets[6].file, + // ), + // BidsHedIssue.fromHedIssue( + // generateIssue('invalidPlaceholder', { tag: 'RGB-green/#', sidecarKey: 'value_in_categorical' }), + // placeholderDatasets[7].file, + // ), + // ], + // } + // return validator(testDatasets, expectedIssues, specs) + // }, 10000) + // }) + // + // describe('TSV-only datasets', () => { + // it('should validate HED strings in BIDS event files', () => { + // const goodDatasets = bidsTsvFiles[0] + // const badDatasets = bidsTsvFiles[1] + // const testDatasets = { + // all_good: new BidsDataset(goodDatasets, []), + // all_bad: new BidsDataset(badDatasets, []), + // } + // const legalSpeedUnits = ['m-per-s', 'kph', 'mph'] + // const speedIssue = generateIssue('unitClassInvalidUnit', { + // tag: 'Speed/300 miles', + // unitClassUnits: legalSpeedUnits.sort().join(','), + // }) + // const maglevError = generateIssue('invalidTag', { tag: 'Maglev' }) + // const maglevWarning = generateIssue('extension', { tag: 'Train/Maglev' }) + // const expectedIssues = { + // all_good: [], + // all_bad: [ + // BidsHedIssue.fromHedIssue(cloneDeep(speedIssue), badDatasets[0].file, { tsvLine: 2 }), + // BidsHedIssue.fromHedIssue(cloneDeep(maglevWarning), badDatasets[1].file, { tsvLine: 2 }), + // BidsHedIssue.fromHedIssue(cloneDeep(speedIssue), badDatasets[2].file, { tsvLine: 3 }), + // BidsHedIssue.fromHedIssue(cloneDeep(maglevError), badDatasets[3].file, { tsvLine: 2 }), + // BidsHedIssue.fromHedIssue(cloneDeep(speedIssue), badDatasets[3].file, { tsvLine: 3 }), + // BidsHedIssue.fromHedIssue(cloneDeep(maglevWarning), badDatasets[4].file, { tsvLine: 2 }), + // BidsHedIssue.fromHedIssue(cloneDeep(speedIssue), badDatasets[4].file, { tsvLine: 3 }), + // ], + // } + // return validator(testDatasets, expectedIssues, specs) + // }, 10000) + // }) + + // TODO: Find out why this doesn't work + describe.skip('Combined datasets', () => { + it('should validate BIDS event files combined with JSON sidecar data', () => { + const goodDatasets = bidsTsvFiles[2] + const badDatasets = bidsTsvFiles[3] + const testDatasets = { + /* all_good: new BidsDataset(goodDatasets, []),*/ + all_bad: new BidsDataset(badDatasets, []), + } + const expectedIssues = { + all_good: [], + all_bad: [ + BidsHedIssue.fromHedIssue(generateIssue('invalidTag', { tag: 'Confused' }), badDatasets[0].file), + // TODO: Catch warning in sidecar validation + /* BidsHedIssue.fromHedIssue( + generateIssue('extension', { tag: 'Train/Maglev' }), + badDatasets[1].file, + ), */ + BidsHedIssue.fromHedIssue( + generateIssue('duplicateTag', { + tag: 'Boat', + }), + badDatasets[2].file, + { tsvLine: 5 }, + ), + BidsHedIssue.fromHedIssue( + generateIssue('duplicateTag', { + tag: 'Boat', + }), + badDatasets[2].file, + { tsvLine: 5 }, + ), + BidsHedIssue.fromHedIssue( + generateIssue('invalidValue', { + tag: 'Duration/ferry s', + }), + badDatasets[3].file, + { tsvLine: 2 }, + ), + BidsHedIssue.fromHedIssue( + generateIssue('duplicateTag', { + tag: 'Age/30', + }), + badDatasets[3].file, + { tsvLine: 2 }, + ), + BidsHedIssue.fromHedIssue( + generateIssue('duplicateTag', { + tag: 'Age/30', + }), + badDatasets[3].file, + { tsvLine: 2 }, + ), + BidsHedIssue.fromHedIssue( + generateIssue('invalidTopLevelTagGroupTag', { + tag: 'Duration/ferry s', + }), + badDatasets[3].file, + { tsvLine: 2 }, + ), + BidsHedIssue.fromHedIssue( + generateIssue('sidecarKeyMissing', { + key: 'purple', + column: 'color', + file: '/sub04/sub04_task-test_run-5_events.tsv', + }), + badDatasets[4].file, + { tsvLine: 2 }, + ), + ], + } + return validator(testDatasets, expectedIssues, specs) + }, 10000) + }) + + describe('HED 3 library schema tests', () => { + let goodEvents + let goodDatasetDescriptions, badDatasetDescriptions + + beforeAll(() => { + goodEvents = bidsTsvFiles[5] + goodDatasetDescriptions = bidsDatasetDescriptions[0] + badDatasetDescriptions = bidsDatasetDescriptions[1] + }) + + describe('HED 3 library schema good tests', () => { + it('should validate HED 3 in BIDS event with json and a dataset description and no version spec', () => { + const testDatasets = { + basestd_with_std_no_defs: new BidsDataset([goodEvents[2]], [], goodDatasetDescriptions[0]), + basestd_with_std_and_libtestlib_nodefs: new BidsDataset([goodEvents[2]], [], goodDatasetDescriptions[1]), + basestd_with_std_and_two_libtestlibs_nodefs: new BidsDataset([goodEvents[2]], [], goodDatasetDescriptions[3]), + libtestlib_with_basestd_and_libtestlib_nodefs: new BidsDataset( + [goodEvents[1]], + [], + goodDatasetDescriptions[1], + ), + libtestlib_with_basestd_and_two_libtestlibs_nodefs: new BidsDataset( + [goodEvents[1]], + [], + goodDatasetDescriptions[3], + ), + libtestlib_with_two_libtestlibs_nodefs: new BidsDataset([goodEvents[1]], [], goodDatasetDescriptions[4]), + basestd_libtestlib_with_basestd_and_libtestlib_defs: new BidsDataset( + [goodEvents[0]], + [], + goodDatasetDescriptions[1], + ), + basestd_libtestlib_with_basestd_and_two_libtestlib_defs: new BidsDataset( + [goodEvents[0]], + [], + goodDatasetDescriptions[3], + ), + basescore_with_basescore_no_defs: new BidsDataset([goodEvents[3]], [], goodDatasetDescriptions[5]), + libscore_with_libscore_nodefs: new BidsDataset([goodEvents[4]], [], goodDatasetDescriptions[6]), + basetestlib_with_basetestlib_with_defs: new BidsDataset([goodEvents[5]], [], goodDatasetDescriptions[7]), + libtestlib_with_basestd_and_libtestlib_with_defs: new BidsDataset( + [goodEvents[6]], + [], + goodDatasetDescriptions[1], + ), + libtestlib_with_libtestlib_with_defs: new BidsDataset([goodEvents[6]], [], goodDatasetDescriptions[2]), + libtestlib_with_basestd_and_two_libtestlib_with_defs: new BidsDataset( + [goodEvents[6]], + [], + goodDatasetDescriptions[3], + ), + libtestlib_with_two_libtestlib_with_defs: new BidsDataset([goodEvents[6]], [], goodDatasetDescriptions[4]), + } + const expectedIssues = { + basestd_with_std_no_defs: [], + basestd_with_std_and_libtestlib_nodefs: [], + basestd_with_std_and_two_libtestlibs_nodefs: [], + libtestlib_with_basestd_and_libtestlib_nodefs: [], + libtestlib_with_basestd_and_two_libtestlibs_nodefs: [], + libtestlib_with_two_libtestlibs_nodefs: [], + basestd_libtestlib_with_basestd_and_libtestlib_defs: [], + basestd_libtestlib_with_basestd_and_two_libtestlib_defs: [], + basescore_with_basescore_no_defs: [], + libscore_with_libscore_nodefs: [], + basetestlib_with_basetestlib_with_defs: [], + libtestlib_with_basestd_and_libtestlib_with_defs: [], + libtestlib_with_libtestlib_with_defs: [], + libtestlib_with_basestd_and_two_libtestlib_with_defs: [], + libtestlib_with_two_libtestlib_with_defs: [], + } + return validator(testDatasets, expectedIssues, null) + }, 10000) + }) + + describe('HED 3 library schema bad tests', () => { + it('should not validate when library schema specifications are invalid', () => { + const testDatasets = { + unknown_library: new BidsDataset([goodEvents[2]], [], badDatasetDescriptions[0]), + leading_colon: new BidsDataset([goodEvents[2]], [], badDatasetDescriptions[1]), + bad_nickname: new BidsDataset([goodEvents[2]], [], badDatasetDescriptions[2]), + multipleColons1: new BidsDataset([goodEvents[2]], [], badDatasetDescriptions[3]), + multipleColons2: new BidsDataset([goodEvents[2]], [], badDatasetDescriptions[4]), + noLibraryName: new BidsDataset([goodEvents[2]], [], badDatasetDescriptions[5]), + badVersion1: new BidsDataset([goodEvents[2]], [], badDatasetDescriptions[6]), + badVersion2: new BidsDataset([goodEvents[2]], [], badDatasetDescriptions[7]), + badRemote1: new BidsDataset([goodEvents[2]], [], badDatasetDescriptions[8]), + badRemote2: new BidsDataset([goodEvents[2]], [], badDatasetDescriptions[9]), + noHedVersion: new BidsDataset([goodEvents[2]], [], badDatasetDescriptions[10]), + } + + const expectedIssues = { + unknown_library: [ + BidsHedIssue.fromHedIssue( + generateIssue('remoteSchemaLoadFailed', { + spec: JSON.stringify(new SchemaSpec('ts', '1.0.2', 'badlib')), + error: + 'Server responded to https://raw.githubusercontent.com/hed-standard/hed-schemas/refs/heads/main/library_schemas/badlib/hedxml/HED_badlib_1.0.2.xml with status code 404: Not Found', + }), + badDatasetDescriptions[0].file, + ), + ], + leading_colon: [ + BidsHedIssue.fromHedIssue( + generateIssue('invalidSchemaNickname', { nickname: '', spec: ':testlib_1.0.2' }), + badDatasetDescriptions[1].file, + ), + ], + bad_nickname: [ + BidsHedIssue.fromHedIssue( + generateIssue('invalidSchemaNickname', { nickname: 't-s', spec: 't-s:testlib_1.0.2' }), + badDatasetDescriptions[2].file, + ), + ], + multipleColons1: [ + BidsHedIssue.fromHedIssue( + generateIssue('invalidSchemaSpecification', { spec: 'ts::testlib_1.0.2' }), + badDatasetDescriptions[3].file, + ), + ], + multipleColons2: [ + BidsHedIssue.fromHedIssue( + generateIssue('invalidSchemaSpecification', { spec: ':ts:testlib_1.0.2' }), + badDatasetDescriptions[4].file, + ), + ], + noLibraryName: [ + BidsHedIssue.fromHedIssue( + generateIssue('invalidSchemaSpecification', { spec: 'ts:_1.0.2' }), + badDatasetDescriptions[5].file, + ), + ], + badVersion1: [ + BidsHedIssue.fromHedIssue( + generateIssue('invalidSchemaSpecification', { spec: 'ts:testlib1.0.2' }), + badDatasetDescriptions[6].file, + ), + ], + badVersion2: [ + BidsHedIssue.fromHedIssue( + generateIssue('invalidSchemaSpecification', { spec: 'ts:testlib_1.a.2' }), + badDatasetDescriptions[7].file, + ), + ], + badRemote1: [ + BidsHedIssue.fromHedIssue( + generateIssue('remoteSchemaLoadFailed', { + spec: JSON.stringify(new SchemaSpec('ts', '1.800.2', 'testlib')), + error: + 'Server responded to https://raw.githubusercontent.com/hed-standard/hed-schemas/refs/heads/main/library_schemas/testlib/hedxml/HED_testlib_1.800.2.xml with status code 404: Not Found', + }), + badDatasetDescriptions[8].file, + ), + ], + badRemote2: [ + BidsHedIssue.fromHedIssue( + generateIssue('remoteSchemaLoadFailed', { + spec: JSON.stringify(new SchemaSpec('', '8.828.0', '')), + error: + 'Server responded to https://raw.githubusercontent.com/hed-standard/hed-schemas/refs/heads/main/standard_schema/hedxml/HED8.828.0.xml with status code 404: Not Found', + }), + badDatasetDescriptions[9].file, + ), + ], + noHedVersion: [ + BidsHedIssue.fromHedIssue(generateIssue('missingSchemaSpecification', {}), badDatasetDescriptions[10].file), + ], + } + return validator(testDatasets, expectedIssues, null) + }, 10000) + }) + + describe('HED 3 library schema with version spec', () => { + it('should validate HED 3 in BIDS event files sidecars and libraries using version spec', () => { + const specs0 = parseSchemasSpec(['8.1.0']) + const specs1 = parseSchemasSpec(['8.1.0', 'ts:testlib_1.0.2']) + const specs2 = parseSchemasSpec(['ts:testlib_1.0.2']) + const specs3 = parseSchemasSpec(['8.1.0', 'ts:testlib_1.0.2', 'bg:testlib_1.0.2']) + const specs4 = parseSchemasSpec(['ts:testlib_1.0.2', 'bg:testlib_1.0.2']) + const testDatasets1 = { + library_and_defs_base_ignored: new BidsDataset([goodEvents[0]], [], goodDatasetDescriptions[1]), + library_and_defs_no_base: new BidsDataset([goodEvents[0]], [], goodDatasetDescriptions[3]), + library_only_with_extra_base: new BidsDataset([goodEvents[1]], [], goodDatasetDescriptions[1]), + library_only: new BidsDataset([goodEvents[1]], [], goodDatasetDescriptions[1]), + just_base2: new BidsDataset([goodEvents[2]], [], goodDatasetDescriptions[0]), + library_not_needed1: new BidsDataset([goodEvents[2]], [], goodDatasetDescriptions[1]), + library_not_needed2: new BidsDataset([goodEvents[2]], [], goodDatasetDescriptions[3]), + library_and_base_with_extra_schema: new BidsDataset([goodEvents[0]], [], goodDatasetDescriptions[1]), + } + const expectedIssues1 = { + library_and_defs_base_ignored: [], + library_and_defs_no_base: [], + library_only_with_extra_base: [], + library_only: [], + just_base2: [], + library_not_needed1: [], + library_not_needed2: [], + library_and_base_with_extra_schema: [], + } + const schemaSpecs = { + library_and_defs_base_ignored: specs1, + library_and_defs_no_base: specs3, + library_only_with_extra_base: specs1, + library_only: specs1, + just_base2: specs0, + library_not_needed1: specs1, + library_not_needed2: specs3, + library_and_base_with_extra_schema: specs1, + } + return validatorWithSpecs(testDatasets1, expectedIssues1, schemaSpecs) + }, 10000) + }) + }) + + describe('Definition context', () => { + it('should validate the BIDS context of HED definitions', () => { + const badTsvDatasets = bidsTsvFiles[6] + const defSidecars = bidsSidecars[5] + const testDatasets = { + bad_tsv: new BidsDataset(badTsvDatasets, []), + sidecars: new BidsDataset([], defSidecars), + } + const expectedIssues = { + bad_tsv: [ + BidsHedIssue.fromHedIssue( + generateIssue('illegalDefinitionContext', { + string: '(Definition/myDef, (Label/Red, Green))', + tsvLine: 2, + }), + badTsvDatasets[0].file, + ), + ], + sidecars: [ + BidsHedIssue.fromHedIssue( + generateIssue('illegalDefinitionContext', { + string: bidsSidecars[5][2].hedData.get('event_code'), + sidecarKey: 'event_code', + }), + defSidecars[2].file, + ), + BidsHedIssue.fromHedIssue( + generateIssue('missingPlaceholder', { + string: bidsSidecars[5][2].hedData.get('event_code'), + sidecarKey: 'event_code', + }), + defSidecars[2].file, + ), + BidsHedIssue.fromHedIssue( + generateIssue('illegalInExclusiveContext', { + string: 'Red, Blue, (Definition/myDef, (Label/Red, Blue))', + tag: 'Definition/myDef', + }), + defSidecars[3].file, + ), + /* TODO: Fix cross-string exclusive context tests. + BidsHedIssue.fromHedIssue( + generateIssue('illegalDefinitionInExclusiveContext', { string: 'Def/Acc/5.4 m-per-s^2' }), + defSidecars[3].file, + ), + BidsHedIssue.fromHedIssue( + generateIssue('illegalDefinitionInExclusiveContext', { string: 'Def/Acc/4.5 m-per-s^2' }), + defSidecars[4].file, + ), */ + ], + } + return validator(testDatasets, expectedIssues, specs) + }, 10000) + }) + + // TODO: transfer to new format as the error codes have changed. + describe('Curly braces', () => { + it.skip('(REVISIT)should validate the use of HED curly braces in BIDS data', () => { + const standaloneTsvFiles = bidsTsvFiles[7] + const standaloneSidecars = bidsSidecars[6] + const combinedDatasets = bidsTsvFiles[8] + const hedColumnDatasets = bidsTsvFiles[9] + const syntaxSidecars = bidsSidecars[8].slice(0, 1) + const testDatasets = { + tsv: new BidsDataset(standaloneTsvFiles, []), + sidecars: new BidsDataset([], standaloneSidecars), + combined: new BidsDataset(combinedDatasets, []), + hedColumn: new BidsDataset(hedColumnDatasets, []), + syntax: new BidsDataset([], syntaxSidecars), + } + const expectedIssues = { + tsv: [ + BidsHedIssue.fromHedIssue( + generateIssue('curlyBracesInHedColumn', { column: '{response_time}' }), + standaloneTsvFiles[1].file, + { tsvLine: 2 }, + ), + ], + sidecars: [ + BidsHedIssue.fromHedIssue( + generateIssue('curlyBracesNotAllowed', { + string: '(Definition/Acc/#, {event_code}, (Acceleration/#, Red))', + }), + standaloneSidecars[1].file, + ), + BidsHedIssue.fromHedIssue( + generateIssue('curlyBracesNotAllowed', { + string: '(Definition/MyColor, (Label/Pie, {response_time}))', + }), + standaloneSidecars[1].file, + ), + BidsHedIssue.fromHedIssue( + generateIssue('undefinedCurlyBraces', { + column: 'event_code', + }), + standaloneSidecars[1].file, + ), + BidsHedIssue.fromHedIssue( + generateIssue('undefinedCurlyBraces', { + column: 'response_time', + }), + standaloneSidecars[1].file, + ), + BidsHedIssue.fromHedIssue( + generateIssue('recursiveCurlyBracesWithKey', { + column: 'response_time', + referrer: 'event_code', + }), + standaloneSidecars[6].file, + ), + BidsHedIssue.fromHedIssue( + generateIssue('recursiveCurlyBracesWithKey', { + column: 'event_code', + referrer: 'response_time', + }), + standaloneSidecars[6].file, + ), + BidsHedIssue.fromHedIssue( + generateIssue('recursiveCurlyBracesWithKey', { + column: 'event_type', + referrer: 'event_code', + }), + standaloneSidecars[7].file, + ), + BidsHedIssue.fromHedIssue( + generateIssue('recursiveCurlyBracesWithKey', { + column: 'event_code', + referrer: 'event_type', + }), + standaloneSidecars[7].file, + ), + BidsHedIssue.fromHedIssue( + generateIssue('recursiveCurlyBracesWithKey', { + column: 'response_time', + referrer: 'response_time', + }), + standaloneSidecars[8].file, + ), + BidsHedIssue.fromHedIssue( + generateIssue('unclosedCurlyBrace', { + index: 15, + string: standaloneSidecars[9].hedData.get('event_code').ball, + }), + standaloneSidecars[9].file, + ), + BidsHedIssue.fromHedIssue( + generateIssue('nestedCurlyBrace', { + index: 1, + string: standaloneSidecars[9].hedData.get('event_code2').ball, + }), + standaloneSidecars[9].file, + ), + BidsHedIssue.fromHedIssue( + generateIssue('unopenedCurlyBrace', { + index: 15, + string: standaloneSidecars[9].hedData.get('event_code3').ball, + }), + standaloneSidecars[9].file, + ), + BidsHedIssue.fromHedIssue( + generateIssue('emptyCurlyBrace', { + index: 1, + string: standaloneSidecars[9].hedData.get('event_code4').ball, + }), + standaloneSidecars[9].file, + ), + ], + combined: [ + BidsHedIssue.fromHedIssue( + generateIssue('undefinedCurlyBraces', { + column: 'response_time', + }), + combinedDatasets[0].file, + ), + BidsHedIssue.fromHedIssue( + generateIssue('undefinedCurlyBraces', { + column: 'response_time', + }), + combinedDatasets[1].file, + ), + BidsHedIssue.fromHedIssue( + generateIssue('duplicateTag', { + tag: 'Label/1', + }), + combinedDatasets[2].file, + { tsvLine: 3 }, + ), + BidsHedIssue.fromHedIssue( + generateIssue('duplicateTag', { + tag: 'Label/1', + }), + combinedDatasets[2].file, + { tsvLine: 3 }, + ), + ], + hedColumn: [ + BidsHedIssue.fromHedIssue( + generateIssue('curlyBracesInHedColumn', { column: '{response_time}' }), + hedColumnDatasets[0].file, + { tsvLine: 2 }, + ), + ], + syntax: [ + BidsHedIssue.fromHedIssue( + generateIssue('invalidCharacter', { + character: 'LEFT CURLY BRACKET', + index: 9, + string: '(Def/Acc/{response_time})', + }), + syntaxSidecars[0].file, + ), + ], + } + return validator(testDatasets, expectedIssues, specs) + }, 10000) + + it('should splice strings by replacing placeholders and deleting "n/a" values', async () => { + const tsvFiles = bidsTsvFiles[10] + const expectedStrings = [ + 'Label/1, (Def/Acc/3.5), (Item-count/2, Label/1)', + '(Def/Acc/3.5)', + '(Def/Acc/3.5), (Green, Def/MyColor)', + 'Label/1, (Def/Acc/3.5)', + '(Def/Acc/3.5)', + '(Red, Blue), (Green, (Yellow))', + ] + const dataset = new BidsDataset(tsvFiles, []) + const hedSchemas = await buildBidsSchemas(dataset, specs) + const parsedExpectedStrings = [] + for (const expectedString of expectedStrings) { + const [parsedExpectedString, parsingIssues] = parseHedString(expectedString, hedSchemas) + assert.isEmpty(Object.values(parsingIssues).flat(), `String "${expectedString}" failed to parse`) + parsedExpectedStrings.push(parsedExpectedString) + } + const tsvHedStrings = [] + for (const tsvFile of tsvFiles) { + tsvFile.mergedSidecar.parseHedStrings(hedSchemas) + const tsvValidator = new BidsHedTsvParser(tsvFile, hedSchemas) + const tsvHed = tsvValidator.parse() + assert.isEmpty(tsvValidator.issues, 'TSV file failed to parse') + tsvHedStrings.push(...tsvHed) + } + const formatMap = (hedString) => hedString.format() + assert.deepStrictEqual( + tsvHedStrings.map(formatMap), + parsedExpectedStrings.map(formatMap), + 'Mismatch in parsed strings', + ) + }, 10000) + }) + + describe('HED 3 partnered schema tests', () => { + let goodEvent + let goodDatasetDescriptions, badDatasetDescriptions + + beforeAll(() => { + goodEvent = bidsTsvFiles[11][0] + goodDatasetDescriptions = bidsDatasetDescriptions[0] + badDatasetDescriptions = bidsDatasetDescriptions[1] + }) + + it('should validate HED 3 in BIDS event TSV files with JSON sidecar data using tags from merged partnered schemas', () => { + const testDatasets = { + validPartneredTestlib: new BidsDataset([goodEvent], [], goodDatasetDescriptions[8]), + validPartneredTestlibWithStandard: new BidsDataset([goodEvent], [], goodDatasetDescriptions[9]), + invalidPartneredTestlib1: new BidsDataset([goodEvent], [], badDatasetDescriptions[11]), + invalidPartneredTestlib2: new BidsDataset([goodEvent], [], badDatasetDescriptions[12]), + invalidPartneredTestlibWithStandard: new BidsDataset([goodEvent], [], badDatasetDescriptions[13]), + } + const expectedIssues = { + validPartneredTestlib: [], + validPartneredTestlibWithStandard: [], + invalidPartneredTestlib1: [ + BidsHedIssue.fromHedIssue( + generateIssue('lazyPartneredSchemasShareTag', { tag: 'A-nonextension' }), + badDatasetDescriptions[11].file, + ), + ], + invalidPartneredTestlib2: [ + BidsHedIssue.fromHedIssue( + generateIssue('lazyPartneredSchemasShareTag', { tag: 'Piano-sound' }), + badDatasetDescriptions[12].file, + ), + ], + invalidPartneredTestlibWithStandard: [ + BidsHedIssue.fromHedIssue( + generateIssue('differentWithStandard', { first: '8.1.0', second: '8.2.0' }), + badDatasetDescriptions[13].file, + ), + ], + } + return validator(testDatasets, expectedIssues, null) + }, 10000) + }) +}) diff --git a/tests/bidsTests.spec.js b/tests/bidsTests.spec.js index fb5a5316..1a66e452 100644 --- a/tests/bidsTests.spec.js +++ b/tests/bidsTests.spec.js @@ -4,25 +4,32 @@ import { beforeAll, describe, afterAll } from '@jest/globals' import path from 'path' import { buildSchemas } from '../schema/init' import { SchemaSpec, SchemasSpec } from '../schema/specs' -import { BidsSidecar, BidsTsvFile } from '../bids' +import { BidsHedTsvValidator, BidsSidecar, BidsTsvFile } from '../bids' import parseTSV from '../bids/tsvParser' + import { bidsTestData } from './testData/bidsTests.data' import { shouldRun } from './testUtilities' -import { DefinitionManager } from '../parser/definitionManager' // Ability to select individual tests to run //const skipMap = new Map([['definition-tests', ['invalid-missing-definition-for-def', 'invalid-nested-definition']]]) const skipMap = new Map() -const runAll = false -const runMap = new Map([['temporal-tests', ['delayed-onset-with-offset-before-with-sidecar']]]) +const runAll = true +const runMap = new Map([['definition-tests', ['valid-definition-no-placeholder']]]) describe('BIDS validation', () => { - const schemaMap = new Map([['8.3.0', undefined]]) + const schemaMap = new Map([ + ['8.2.0', undefined], + ['8.3.0', undefined], + ]) beforeAll(async () => { + const spec2 = new SchemaSpec('', '8.2.0', '', path.join(__dirname, '../tests/data/HED8.2.0.xml')) + const specs2 = new SchemasSpec().addSchemaSpec(spec2) + const schemas2 = await buildSchemas(specs2) const spec3 = new SchemaSpec('', '8.3.0', '', path.join(__dirname, '../tests/data/HED8.3.0.xml')) const specs3 = new SchemasSpec().addSchemaSpec(spec3) const schemas3 = await buildSchemas(specs3) + schemaMap.set('8.2.0', schemas2) schemaMap.set('8.3.0', schemas3) }) @@ -41,59 +48,35 @@ describe('BIDS validation', () => { const thisSchema = schemaMap.get(test.schemaVersion) assert.isDefined(thisSchema, `${header}:${test.schemaVersion} is not available in test ${test.name}`) - //Make sure the test definitions are okay before proceeding - const [defList, defIssues] = DefinitionManager.createDefinitions(test.definitions, thisSchema) - assert.equal(defIssues.length, 0, `${header}: input definitions "${test.definitions}" have errors "${defIssues}"`) - const defManager1 = new DefinitionManager() - const defAddIssues = defManager1.addDefinitions(defList) - assert.equal( - defAddIssues.length, - 0, - `${header}: input definitions "${test.definitions}" have conflicts "${defAddIssues}"`, - ) - - // Validate the sidecar by itself + //Validate the sidecar by itself const sidecarName = test.testname + '.json' - const bidsSidecar = new BidsSidecar( - 'thisOne', - test.sidecar, - { relativePath: sidecarName, path: sidecarName }, - defManager1, - ) + const bidsSidecar = new BidsSidecar('thisOne', test.sidecar, { relativePath: sidecarName, path: sidecarName }) assert.instanceOf(bidsSidecar, BidsSidecar, 'Test') const sidecarIssues = bidsSidecar.validate(thisSchema) assertErrors(test, 'Sidecar only', test.sidecarErrors, sidecarIssues) - // Validate the events file with no sidecar + // Parse the events file const eventName = test.testname + '.tsv' const parsedTsv = parseTSV(test.eventsString) assert.instanceOf(parsedTsv, Map, `${eventName} cannot be parsed`) - const defManager2 = new DefinitionManager() - defManager2.addDefinitions(defList) - const bidsTsv = new BidsTsvFile( - test.testname, - parsedTsv, - { relativePath: eventName, path: eventName }, - [], - {}, - defManager2, - ) - const noSideIssues = bidsTsv.validate(thisSchema) - assertErrors(test, 'Events', test.tsvErrors, noSideIssues) - // Validate the events file with the sidecar (use the definitions from the sidecar) - const defManager3 = new DefinitionManager() - defManager3.addDefinitions(defList) + // Validate the events file with no sidecar + const bidsTsv = new BidsTsvFile(test.testname, parsedTsv, { relativePath: eventName, path: eventName }, [], {}) + const validatorNoSide = new BidsHedTsvValidator(bidsTsv, thisSchema) + validatorNoSide.validate() + assertErrors(test, 'Events', test.tsvErrors, validatorNoSide.issues) + + // Validate the events file with the sidecar const bidsTsvSide = new BidsTsvFile( test.testname, parsedTsv, { relativePath: eventName, path: eventName }, [], test.sidecar, - defManager3, ) - const withSideIssues = bidsTsvSide.validate(thisSchema) - assertErrors(test, 'Events+side', test.comboErrors, withSideIssues) + const validatorWithSide = new BidsHedTsvValidator(bidsTsvSide, thisSchema) + validatorWithSide.validate() + assertErrors(test, 'Events+side', test.comboErrors, validatorWithSide.issues) } if (tests && tests.length > 0) { @@ -101,8 +84,7 @@ describe('BIDS validation', () => { if (shouldRun(name, test.testname, runAll, runMap, skipMap)) { validate(test) } else { - // eslint-disable-next-line no-console - //console.log(`----Skipping bidsTest ${name}: ${test.testname}`) + console.log(`----Skipping bidsTest ${name}: ${test.testname}`) } }) } diff --git a/tests/converter.spec.js b/tests/converter.spec.js new file mode 100644 index 00000000..88f7fb36 --- /dev/null +++ b/tests/converter.spec.js @@ -0,0 +1,899 @@ +import chai from 'chai' +const assert = chai.assert +import { beforeAll, describe, it } from '@jest/globals' + +import * as converter from '../converter/converter' +import { generateIssue } from '../common/issues/issues' +import { SchemaSpec, SchemasSpec } from '../schema/specs' +import { buildSchemas } from '../schema/init' + +describe('HED string conversion', () => { + const hedSchemaFile = 'tests/data/HED8.0.0.xml' + let hedSchemas + + beforeAll(async () => { + const spec1 = new SchemaSpec('', '8.0.0', '', hedSchemaFile) + const specs = new SchemasSpec().addSchemaSpec(spec1) + hedSchemas = await buildSchemas(specs) + }) + + describe('HED tags', () => { + /** + * Base validation function. + * + * @param {Object} testStrings The test strings. + * @param {Object} expectedResults The expected results. + * @param {Object} expectedIssues The expected issues. + * @param {function (Schema, string): [string, Issue[]]} testFunction The test function. + * @returns {Promise} + */ + const validatorBase = async function (testStrings, expectedResults, expectedIssues, testFunction) { + for (const [testStringKey, testString] of Object.entries(testStrings)) { + const [testResult, issues] = testFunction(hedSchemas, testString) + assert.strictEqual(testResult, expectedResults[testStringKey], testString) + assert.sameDeepMembers(issues, expectedIssues[testStringKey], testString) + } + } + + describe('Long-to-short', () => { + const validator = function (testStrings, expectedResults, expectedIssues) { + return validatorBase(testStrings, expectedResults, expectedIssues, converter.convertHedStringToShort) + } + + // TODO: Remove this test - Test now implemented as valid-long-to-short in conversionTestsData + it.skip('(REMOVE) should convert basic HED tags to short form', () => { + const testStrings = { + singleLevel: 'Event', + twoLevel: 'Event/Sensory-event', + fullLong: 'Item/Object/Geometric-object', + partialShort: 'Object/Geometric-object', + alreadyShort: 'Geometric-object', + } + const expectedResults = { + singleLevel: 'Event', + twoLevel: 'Sensory-event', + fullLong: 'Geometric-object', + partialShort: 'Geometric-object', + alreadyShort: 'Geometric-object', + } + const expectedIssues = { + singleLevel: [], + twoLevel: [], + fullLong: [], + partialShort: [], + alreadyShort: [], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + // TODO: Remove this test - Test now implemented as valid-long-to-short in conversionTestsData - note not values + it.skip('(REMOVE) should convert HED tags with values to short form', () => { + const testStrings = { + uniqueValue: 'Item/Sound/Environmental-sound/Unique Value', + multiLevel: 'Item/Sound/Environmental-sound/Long Unique Value With/Slash Marks', + partialPath: 'Sound/Environmental-sound/Unique Value', + } + const expectedResults = { + uniqueValue: 'Environmental-sound/Unique Value', + multiLevel: 'Environmental-sound/Long Unique Value With/Slash Marks', + partialPath: 'Environmental-sound/Unique Value', + } + const expectedIssues = { + uniqueValue: [], + multiLevel: [], + partialPath: [], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + // TODO: Remove this test - Test now implemented as valid-long-to-short in conversionTestsData + it.skip('(REMOVE) should raise an issue if a "value" is an already valid node', () => { + const testStrings = { + singleLevel: 'Item/Sound/Environmental-sound/Event', + multiLevel: 'Item/Sound/Environmental-sound/Event/Sensory-event', + mixed: 'Item/Sound/Event/Sensory-event/Environmental-sound', + } + const expectedResults = { + singleLevel: 'Item/Sound/Environmental-sound/Event', + multiLevel: 'Item/Sound/Environmental-sound/Event/Sensory-event', + mixed: 'Item/Sound/Event/Sensory-event/Environmental-sound', + } + const expectedIssues = { + singleLevel: [generateIssue('invalidParentNode', { tag: 'Event', parentTag: 'Event' })], + multiLevel: [generateIssue('invalidParentNode', { tag: 'Event', parentTag: 'Event' })], + mixed: [generateIssue('invalidParentNode', { tag: 'Event', parentTag: 'Event' })], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + //TODO: This is a repeat of the tags with values (which are tags with extensions) Remove this. + it.skip('(REMOVE)should convert HED tags with extensions to short form', () => { + const testStrings = { + singleLevel: 'Item/Object/extended lvl1', + multiLevel: 'Item/Object/extended lvl1/Extension2', + partialPath: 'Object/Man-made-object/Vehicle/Boat/Yacht', + } + const expectedResults = { + singleLevel: 'Object/extended lvl1', + multiLevel: 'Object/extended lvl1/Extension2', + partialPath: 'Boat/Yacht', + } + const expectedIssues = { + singleLevel: [], + multiLevel: [], + partialPath: [], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + //TODO: Remove this -- shouldn't parse when invalid -- the rest is tested for + it.skip('(REMOVE) should raise an issue if an "extension" is already a valid node', () => { + const testStrings = { + validThenInvalid: 'Item/Object/valid extension followed by invalid/Event', + singleLevel: 'Item/Object/Visual-presentation', + singleLevelAlreadyShort: 'Object/Visual-presentation', + twoLevels: 'Item/Object/Visual-presentation/Event', + duplicate: 'Item/Object/Geometric-object/Item/Object/Geometric-object', + } + const expectedResults = { + validThenInvalid: 'Item/Object/valid extension followed by invalid/Event', + singleLevel: 'Item/Object/Visual-presentation', + singleLevelAlreadyShort: 'Object/Visual-presentation', + twoLevels: 'Item/Object/Visual-presentation/Event', + duplicate: 'Item/Object/Geometric-object/Item/Object/Geometric-object', + } + const expectedIssues = { + validThenInvalid: [generateIssue('invalidParentNode', { tag: 'Event', parentTag: 'Event' })], + singleLevel: [ + generateIssue('invalidParentNode', { + tag: 'Visual-presentation', + parentTag: 'Property/Sensory-property/Sensory-presentation/Visual-presentation', + }), + ], + singleLevelAlreadyShort: [ + generateIssue('invalidParentNode', { + tag: 'Visual-presentation', + parentTag: 'Property/Sensory-property/Sensory-presentation/Visual-presentation', + }), + ], + twoLevels: [ + generateIssue('invalidParentNode', { + tag: 'Visual-presentation', + parentTag: 'Property/Sensory-property/Sensory-presentation/Visual-presentation', + }), + ], + duplicate: [generateIssue('invalidParentNode', { tag: 'Item', parentTag: 'Item' })], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + //TODO: Remove -- these cases are handled by stringParserTests + it.skip('(REMOVE)should raise an issue if an invalid node is found', () => { + const testStrings = { + invalidParentWithExistingGrandchild: 'InvalidItem/Object/Visual-presentation', + invalidChildWithExistingGrandchild: 'Event/InvalidEvent/Geometric-object', + invalidParentWithExistingChild: 'InvalidEvent/Geometric-object', + invalidSingle: 'InvalidEvent', + invalidWithExtension: 'InvalidEvent/InvalidExtension', + } + const expectedResults = { + invalidParentWithExistingGrandchild: 'InvalidItem/Object/Visual-presentation', + invalidChildWithExistingGrandchild: 'Event/InvalidEvent/Geometric-object', + invalidParentWithExistingChild: 'InvalidEvent/Geometric-object', + invalidSingle: 'InvalidEvent', + invalidWithExtension: 'InvalidEvent/InvalidExtension', + } + const expectedIssues = { + invalidParentWithExistingGrandchild: [ + generateIssue('invalidTag', { tag: testStrings.invalidParentWithExistingGrandchild }), + ], + invalidChildWithExistingGrandchild: [ + generateIssue('invalidExtension', { tag: 'InvalidEvent', parentTag: 'Event' }), + ], + invalidParentWithExistingChild: [ + generateIssue('invalidTag', { tag: testStrings.invalidParentWithExistingChild }), + ], + invalidSingle: [generateIssue('invalidTag', { tag: testStrings.invalidSingle })], + invalidWithExtension: [generateIssue('invalidTag', { tag: testStrings.invalidWithExtension })], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + //TODO: Remove --- these cases are handled by tagParserTests + it.skip('(REMODE)should validate whether a node actually allows extensions', () => { + const testStrings = { + validTakesValue: 'Property/Agent-property/Agent-trait/Age/15', + cascadeExtension: 'Property/Agent-property/Agent-state/Agent-emotional-state/Awed/Cascade Extension', + invalidExtension: 'Event/Agent-action/Good/Time', + } + const expectedResults = { + validTakesValue: 'Age/15', + cascadeExtension: 'Awed/Cascade Extension', + invalidExtension: 'Event/Agent-action/Good/Time', + } + const expectedIssues = { + validTakesValue: [], + cascadeExtension: [], + invalidExtension: [generateIssue('invalidExtension', { tag: 'Good', parentTag: 'Event/Agent-action' })], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + // TODO: This test is handled by tokenizer and should be removed. + it.skip('(REMOVE)should handle leading and trailing spaces correctly', () => { + const testStrings = { + leadingSpace: ' Item/Sound/Environmental-sound/Unique Value', + trailingSpace: 'Item/Sound/Environmental-sound/Unique Value ', + } + const expectedResults = { + leadingSpace: 'Environmental-sound/Unique Value', + trailingSpace: 'Environmental-sound/Unique Value', + } + const expectedIssues = { + leadingSpace: [], + trailingSpace: [], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + // TODO: Remove as these are now errors + it.skip('(REMOVE)should strip leading and trailing slashes', () => { + const testStrings = { + leadingSingle: '/Event', + leadingExtension: '/Event/Extension', + leadingMultiLevel: '/Item/Object/Man-made-object/Vehicle/Train', + leadingMultiLevelExtension: '/Item/Object/Man-made-object/Vehicle/Train/Maglev', + trailingSingle: 'Event/', + trailingExtension: 'Event/Extension/', + trailingMultiLevel: 'Item/Object/Man-made-object/Vehicle/Train/', + trailingMultiLevelExtension: 'Item/Object/Man-made-object/Vehicle/Train/Maglev/', + bothSingle: '/Event/', + bothExtension: '/Event/Extension/', + bothMultiLevel: '/Item/Object/Man-made-object/Vehicle/Train/', + bothMultiLevelExtension: '/Item/Object/Man-made-object/Vehicle/Train/Maglev/', + } + const expectedResults = { + leadingSingle: 'Event', + leadingExtension: 'Event/Extension', + leadingMultiLevel: 'Train', + leadingMultiLevelExtension: 'Train/Maglev', + trailingSingle: 'Event', + trailingExtension: 'Event/Extension', + trailingMultiLevel: 'Train', + trailingMultiLevelExtension: 'Train/Maglev', + bothSingle: 'Event', + bothExtension: 'Event/Extension', + bothMultiLevel: 'Train', + bothMultiLevelExtension: 'Train/Maglev', + } + const expectedIssues = { + leadingSingle: [], + leadingExtension: [], + leadingMultiLevel: [], + leadingMultiLevelExtension: [], + trailingSingle: [], + trailingExtension: [], + trailingMultiLevel: [], + trailingMultiLevelExtension: [], + bothSingle: [], + bothExtension: [], + bothMultiLevel: [], + bothMultiLevelExtension: [], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + }) + + describe('Short-to-long', () => { + const validator = function (testStrings, expectedResults, expectedIssues) { + return validatorBase(testStrings, expectedResults, expectedIssues, converter.convertHedStringToLong) + } + //TODO: Remove as it is now part of tag parsing + it.skip('(REMOVE)should convert basic HED tags to long form', () => { + const testStrings = { + singleLevel: 'Event', + twoLevel: 'Sensory-event', + alreadyLong: 'Item/Object/Geometric-object', + partialLong: 'Object/Geometric-object', + fullShort: 'Geometric-object', + } + const expectedResults = { + singleLevel: 'Event', + twoLevel: 'Event/Sensory-event', + alreadyLong: 'Item/Object/Geometric-object', + partialLong: 'Item/Object/Geometric-object', + fullShort: 'Item/Object/Geometric-object', + } + const expectedIssues = { + singleLevel: [], + twoLevel: [], + alreadyLong: [], + partialLong: [], + fullShort: [], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + //TODO: Remove as it is now part of tag parsing + it.skip('(REMOVE)should convert HED tags with values to long form', () => { + const testStrings = { + uniqueValue: 'Environmental-sound/Unique Value', + multiLevel: 'Environmental-sound/Long Unique Value With/Slash Marks', + partialPath: 'Sound/Environmental-sound/Unique Value', + } + const expectedResults = { + uniqueValue: 'Item/Sound/Environmental-sound/Unique Value', + multiLevel: 'Item/Sound/Environmental-sound/Long Unique Value With/Slash Marks', + partialPath: 'Item/Sound/Environmental-sound/Unique Value', + } + const expectedIssues = { + uniqueValue: [], + multiLevel: [], + partialPath: [], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + //TODO: Remove as it is now part of tag parsing + it.skip('(REMOVE)should convert HED tags with extensions to long form', () => { + const testStrings = { + singleLevel: 'Object/extended lvl1', + multiLevel: 'Object/extended lvl1/Extension2', + partialPath: 'Vehicle/Boat/Yacht', + } + const expectedResults = { + singleLevel: 'Item/Object/extended lvl1', + multiLevel: 'Item/Object/extended lvl1/Extension2', + partialPath: 'Item/Object/Man-made-object/Vehicle/Boat/Yacht', + } + const expectedIssues = { + singleLevel: [], + multiLevel: [], + partialPath: [], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + //TODO: Remove as it is now part of tag parsing + it.skip('(REMOVE)should raise an issue if an "extension" is already a valid node', () => { + const testStrings = { + validThenInvalid: 'Object/valid extension followed by invalid/Event', + singleLevel: 'Object/Visual-presentation', + singleLevelAlreadyLong: 'Item/Object/Visual-presentation', + twoLevels: 'Object/Visual-presentation/Event', + partialDuplicate: 'Geometric-object/Item/Object/Geometric-object', + } + const expectedResults = { + validThenInvalid: 'Object/valid extension followed by invalid/Event', + singleLevel: 'Object/Visual-presentation', + singleLevelAlreadyLong: 'Item/Object/Visual-presentation', + twoLevels: 'Object/Visual-presentation/Event', + partialDuplicate: 'Geometric-object/Item/Object/Geometric-object', + } + const expectedIssues = { + validThenInvalid: [generateIssue('invalidParentNode', { tag: 'Event', parentTag: 'Event' })], + singleLevel: [ + generateIssue('invalidParentNode', { + tag: 'Visual-presentation', + parentTag: 'Property/Sensory-property/Sensory-presentation/Visual-presentation', + }), + ], + singleLevelAlreadyLong: [ + generateIssue('invalidParentNode', { + tag: 'Visual-presentation', + parentTag: 'Property/Sensory-property/Sensory-presentation/Visual-presentation', + }), + ], + twoLevels: [ + generateIssue('invalidParentNode', { + tag: 'Visual-presentation', + parentTag: 'Property/Sensory-property/Sensory-presentation/Visual-presentation', + }), + ], + partialDuplicate: [generateIssue('invalidParentNode', { tag: 'Item', parentTag: 'Item' })], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + //TODO: Remove as it is now part of tag parsing + it.skip('(REMOVE) should raise an issue if an invalid node is found', () => { + const testStrings = { + single: 'InvalidEvent', + invalidChild: 'InvalidEvent/InvalidExtension', + validChild: 'InvalidEvent/Event', + } + const expectedResults = { + single: 'InvalidEvent', + invalidChild: 'InvalidEvent/InvalidExtension', + validChild: 'InvalidEvent/Event', + } + const expectedIssues = { + single: [generateIssue('invalidTag', { tag: testStrings.single })], + invalidChild: [generateIssue('invalidTag', { tag: testStrings.invalidChild })], + validChild: [generateIssue('invalidTag', { tag: testStrings.validChild })], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + //TODO: Remove as it is now part of tag parsing + it.skip('(REMOVE)should validate whether a node actually allows extensions', () => { + const testStrings = { + validTakesValue: 'Age/15', + cascadeExtension: 'Awed/Cascade Extension', + invalidExtension: 'Agent-action/Good/Time', + } + const expectedResults = { + validTakesValue: 'Property/Agent-property/Agent-trait/Age/15', + cascadeExtension: 'Property/Agent-property/Agent-state/Agent-emotional-state/Awed/Cascade Extension', + invalidExtension: 'Agent-action/Good/Time', + } + const expectedIssues = { + validTakesValue: [], + cascadeExtension: [], + invalidExtension: [generateIssue('invalidExtension', { tag: 'Good', parentTag: 'Event/Agent-action' })], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + //TODO: Remove as it is now part of tag parsing + it.skip('(REMOVE)should handle leading and trailing spaces correctly', () => { + const testStrings = { + leadingSpace: ' Environmental-sound/Unique Value', + trailingSpace: 'Environmental-sound/Unique Value ', + } + const expectedResults = { + leadingSpace: 'Item/Sound/Environmental-sound/Unique Value', + trailingSpace: 'Item/Sound/Environmental-sound/Unique Value', + } + const expectedIssues = { + leadingSpace: [], + trailingSpace: [], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + // TODO: These are taken care of in the tokenizer + it.skip('(REMOVE)should strip leading and trailing slashes', () => { + const testStrings = { + leadingSingle: '/Event', + leadingExtension: '/Event/Extension', + leadingMultiLevel: '/Vehicle/Train', + leadingMultiLevelExtension: '/Vehicle/Train/Maglev', + trailingSingle: 'Event/', + trailingExtension: 'Event/Extension/', + trailingMultiLevel: 'Vehicle/Train/', + trailingMultiLevelExtension: 'Vehicle/Train/Maglev/', + bothSingle: '/Event/', + bothExtension: '/Event/Extension/', + bothMultiLevel: '/Vehicle/Train/', + bothMultiLevelExtension: '/Vehicle/Train/Maglev/', + } + const expectedResults = { + leadingSingle: 'Event', + leadingExtension: 'Event/Extension', + leadingMultiLevel: 'Item/Object/Man-made-object/Vehicle/Train', + leadingMultiLevelExtension: 'Item/Object/Man-made-object/Vehicle/Train/Maglev', + trailingSingle: 'Event', + trailingExtension: 'Event/Extension', + trailingMultiLevel: 'Item/Object/Man-made-object/Vehicle/Train', + trailingMultiLevelExtension: 'Item/Object/Man-made-object/Vehicle/Train/Maglev', + bothSingle: 'Event', + bothExtension: 'Event/Extension', + bothMultiLevel: 'Item/Object/Man-made-object/Vehicle/Train', + bothMultiLevelExtension: 'Item/Object/Man-made-object/Vehicle/Train/Maglev', + } + const expectedIssues = { + leadingSingle: [], + leadingExtension: [], + leadingMultiLevel: [], + leadingMultiLevelExtension: [], + trailingSingle: [], + trailingExtension: [], + trailingMultiLevel: [], + trailingMultiLevelExtension: [], + bothSingle: [], + bothExtension: [], + bothMultiLevel: [], + bothMultiLevelExtension: [], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + // TODO: Remove as this is handled by tag parser + it.skip('(REMOVE) should properly handle node names in value-taking strings', () => { + const testStrings = { + valueTaking: 'Label/Red', + nonValueTaking: 'Train/Car', + definitionName: 'Definition/Blue', + definitionNameWithPlaceholder: 'Definition/BlueCircle/#', + definitionNameWithNodeValue: 'Definition/BlueSquare/SteelBlue', + definitionNodeNameWithValue: 'Definition/Blue/Cobalt', + } + const expectedResults = { + valueTaking: 'Property/Informational-property/Label/Red', + nonValueTaking: 'Train/Car', + definitionName: 'Property/Organizational-property/Definition/Blue', + definitionNameWithPlaceholder: 'Property/Organizational-property/Definition/BlueCircle/#', + definitionNameWithNodeValue: 'Property/Organizational-property/Definition/BlueSquare/SteelBlue', + definitionNodeNameWithValue: 'Property/Organizational-property/Definition/Blue/Cobalt', + } + const expectedIssues = { + valueTaking: [], + nonValueTaking: [ + generateIssue('invalidParentNode', { tag: 'Car', parentTag: 'Item/Object/Man-made-object/Vehicle/Car' }), + ], + definitionName: [], // To be caught in validation. + definitionNameWithPlaceholder: [], + definitionNameWithNodeValue: [], + definitionNodeNameWithValue: [], // To be caught in validation. + } + return validator(testStrings, expectedResults, expectedIssues) + }) + }) + }) + + describe('HED strings', () => { + /** + * Base validation function. + * + * @param {Object} testStrings The test strings. + * @param {Object} expectedResults The expected results. + * @param {Object} expectedIssues The expected issues. + * @param {function (Schemas, string): [string, Issue[]]} testFunction The test function. + * @returns {Promise} + */ + const validatorBase = async function (testStrings, expectedResults, expectedIssues, testFunction) { + for (const [testStringKey, testString] of Object.entries(testStrings)) { + const [testResult, issues] = testFunction(hedSchemas, testString) + assert.strictEqual(testResult, expectedResults[testStringKey], testString) + assert.sameDeepMembers(issues, expectedIssues[testStringKey], testString) + } + } + + describe.skip('Long-to-short', () => { + const validator = function (testStrings, expectedResults, expectedIssues) { + return validatorBase(testStrings, expectedResults, expectedIssues, converter.convertHedStringToShort) + } + + //TODO: Remove as this is covered in string parser. + it.skip('(REMOVE) should properly convert HED strings to short form', () => { + const testStrings = { + singleLevel: 'Event', + multiLevel: 'Event/Sensory-event', + twoSingle: 'Event, Property', + oneExtension: 'Item/Extension', + threeMulti: + 'Event/Sensory-event, Item/Object/Man-made-object/Vehicle/Train, Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red/0.5', + simpleGroup: + '(Item/Object/Man-made-object/Vehicle/Train, Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red/0.5)', + groupAndTag: + '(Item/Object/Man-made-object/Vehicle/Train, Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red/0.5), Item/Object/Man-made-object/Vehicle/Car', + } + const expectedResults = { + singleLevel: 'Event', + multiLevel: 'Sensory-event', + twoSingle: 'Event, Property', + oneExtension: 'Item/Extension', + threeMulti: 'Sensory-event, Train, RGB-red/0.5', + simpleGroup: '(Train, RGB-red/0.5)', + groupAndTag: '(Train, RGB-red/0.5), Car', + } + const expectedIssues = { + singleLevel: [], + multiLevel: [], + twoSingle: [], + oneExtension: [], + threeMulti: [], + simpleGroup: [], + groupAndTag: [], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + it('should raise an issue if an invalid node is found', () => { + const single = 'InvalidEvent' + const double = 'InvalidEvent/InvalidExtension' + const testStrings = { + single: single, + double: double, + both: single + ', ' + double, + singleWithTwoValid: 'Property, ' + single + ', Event', + doubleWithValid: double + ', Item/Object/Man-made-object/Vehicle/Car/Minivan', + } + const expectedResults = { + single: single, + double: double, + both: single + ', ' + double, + singleWithTwoValid: 'Property, ' + single + ', Event', + doubleWithValid: double + ', Item/Object/Man-made-object/Vehicle/Car/Minivan', + } + const expectedIssues = { + single: [generateIssue('invalidTag', { tag: single })], + double: [generateIssue('invalidTag', { tag: double })], + both: [generateIssue('invalidTag', { tag: single }), generateIssue('invalidTag', { tag: double })], + singleWithTwoValid: [generateIssue('invalidTag', { tag: single })], + doubleWithValid: [generateIssue('invalidTag', { tag: double })], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + it.skip('(REMOVE) should ignore leading and trailing spaces', () => { + const testStrings = { + leadingSpace: ' Item/Sound/Environmental-sound/Unique Value', + trailingSpace: 'Item/Sound/Environmental-sound/Unique Value ', + bothSpace: ' Item/Sound/Environmental-sound/Unique Value ', + leadingSpaceTwo: ' Item/Sound/Environmental-sound/Unique Value, Event', + trailingSpaceTwo: 'Event, Item/Sound/Environmental-sound/Unique Value ', + bothSpaceTwo: ' Event, Item/Sound/Environmental-sound/Unique Value ', + } + const expectedResults = { + leadingSpace: 'Environmental-sound/Unique Value', + trailingSpace: 'Environmental-sound/Unique Value', + bothSpace: 'Environmental-sound/Unique Value', + leadingSpaceTwo: 'Environmental-sound/Unique Value, Event', + trailingSpaceTwo: 'Event, Environmental-sound/Unique Value', + bothSpaceTwo: 'Event, Environmental-sound/Unique Value', + } + const expectedIssues = { + leadingSpace: [], + trailingSpace: [], + bothSpace: [], + leadingSpaceTwo: [], + trailingSpaceTwo: [], + bothSpaceTwo: [], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + it('should detect bad leading and trailing slashes', () => { + const testStrings = { + leadingSingle: '/Event', + leadingMultiLevel: '/Item/Object/Man-made-object/Vehicle/Train', + trailingSingle: 'Event/', + trailingMultiLevel: 'Item/Object/Man-made-object/Vehicle/Train/', + bothSingle: '/Event/', + bothMultiLevel: '/Item/Object/Man-made-object/Vehicle/Train/', + twoMixedOuter: '/Event,Item/Object/Man-made-object/Vehicle/Train/', + //twoMixedInner: 'Event/,/Item/Object/Man-made-object/Vehicle/Train', + twoMixedBoth: '/Event/,/Item/Object/Man-made-object/Vehicle/Train/', + twoMixedBothGroup: '(/Event/,/Item/Object/Man-made-object/Vehicle/)', + } + const expectedResults = testStrings + const expectedIssues = { + leadingSingle: [generateIssue('extraSlash', { index: 0, string: testStrings.leadingSingle })], + leadingMultiLevel: [generateIssue('extraSlash', { index: 0, string: testStrings.leadingMultiLevel })], + trailingSingle: [generateIssue('extraSlash', { index: 5, string: testStrings.trailingSingle })], + trailingMultiLevel: [generateIssue('extraSlash', { index: 41, string: testStrings.trailingMultiLevel })], + bothSingle: [generateIssue('extraSlash', { index: 0, string: testStrings.bothSingle })], + bothMultiLevel: [generateIssue('extraSlash', { index: 0, string: testStrings.bothMultiLevel })], + twoMixedOuter: [generateIssue('extraSlash', { index: 0, string: testStrings.twoMixedOuter })], + // twoMixedInner: [ + // generateIssue('extraSlash', { index: 7, string: testStrings.twoMixedOuter })], + // // generateIssue('invalidTag', { tag: '/Item/Object/Man-made-object/Vehicle/Train' }), + // // ], + twoMixedBoth: [generateIssue('extraSlash', { index: 0, string: testStrings.twoMixedBoth })], + twoMixedBothGroup: [generateIssue('extraSlash', { index: 1, string: testStrings.twoMixedBothGroup })], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + it('should replace extra spaces and slashes with single slashes', () => { + const testStrings = { + twoLevelDoubleSlash: 'Item//Extension', + threeLevelDoubleSlash: 'Item//Object//Geometric-object', + tripleSlashes: 'Item///Object///Geometric-object', + mixedSingleAndDoubleSlashes: 'Item///Object/Geometric-object', + singleSlashWithSpace: 'Item/ Extension', + doubleSlashSurroundingSpace: 'Item/ /Extension', + doubleSlashThenSpace: 'Item// Extension', + sosPattern: 'Item/// ///Extension', + alternatingSlashSpace: 'Item/ / Object/ / Geometric-object', + leadingDoubleSlash: '//Item/Extension', + trailingDoubleSlash: 'Item/Extension//', + leadingDoubleSlashWithSpace: '/ /Item/Extension', + trailingDoubleSlashWithSpace: 'Item/Extension/ /', + } + const expectedResults = testStrings + const expectedIssues = { + twoLevelDoubleSlash: [generateIssue('extraSlash', { index: 5, string: testStrings.twoLevelDoubleSlash })], + threeLevelDoubleSlash: [generateIssue('extraSlash', { index: 5, string: testStrings.threeLevelDoubleSlash })], + tripleSlashes: [generateIssue('extraSlash', { index: 5, string: testStrings.tripleSlashes })], + mixedSingleAndDoubleSlashes: [ + generateIssue('extraSlash', { index: 5, string: testStrings.mixedSingleAndDoubleSlashes }), + ], + singleSlashWithSpace: [generateIssue('extraBlank', { index: 5, string: testStrings.singleSlashWithSpace })], + doubleSlashSurroundingSpace: [ + generateIssue('extraBlank', { index: 5, string: testStrings.doubleSlashSurroundingSpace }), + ], + doubleSlashThenSpace: [generateIssue('extraSlash', { index: 5, string: testStrings.doubleSlashThenSpace })], + sosPattern: [generateIssue('extraSlash', { index: 5, string: testStrings.sosPattern })], + alternatingSlashSpace: [generateIssue('extraBlank', { index: 5, string: testStrings.alternatingSlashSpace })], + leadingDoubleSlash: [generateIssue('extraSlash', { index: 0, string: testStrings.leadingDoubleSlash })], + trailingDoubleSlash: [generateIssue('extraSlash', { index: 15, string: testStrings.trailingDoubleSlash })], + leadingDoubleSlashWithSpace: [ + generateIssue('extraSlash', { index: 0, string: testStrings.leadingDoubleSlashWithSpace }), + ], + trailingDoubleSlashWithSpace: [ + generateIssue('extraBlank', { index: 15, string: testStrings.trailingDoubleSlashWithSpace }), + ], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + }) + + describe.skip('Short-to-long', () => { + const validator = function (testStrings, expectedResults, expectedIssues) { + return validatorBase(testStrings, expectedResults, expectedIssues, converter.convertHedStringToLong) + } + + it('should properly convert HED strings to long form', () => { + const testStrings = { + singleLevel: 'Event', + multiLevel: 'Sensory-event', + twoSingle: 'Event, Property', + oneExtension: 'Item/Extension', + threeMulti: 'Sensory-event, Train, RGB-red/0.5', + simpleGroup: '(Train, RGB-red/0.5)', + groupAndTag: '(Train, RGB-red/0.5), Car', + } + const expectedResults = { + singleLevel: 'Event', + multiLevel: 'Event/Sensory-event', + twoSingle: 'Event, Property', + oneExtension: 'Item/Extension', + threeMulti: + 'Event/Sensory-event, Item/Object/Man-made-object/Vehicle/Train, Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red/0.5', + simpleGroup: + '(Item/Object/Man-made-object/Vehicle/Train, Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red/0.5)', + groupAndTag: + '(Item/Object/Man-made-object/Vehicle/Train, Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red/0.5), Item/Object/Man-made-object/Vehicle/Car', + } + const expectedIssues = { + singleLevel: [], + multiLevel: [], + twoSingle: [], + oneExtension: [], + threeMulti: [], + simpleGroup: [], + groupAndTag: [], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + it('should raise an issue if an invalid node is found', () => { + const single = 'InvalidEvent' + const double = 'InvalidEvent/InvalidExtension' + const testStrings = { + single: single, + double: double, + both: single + ', ' + double, + singleWithTwoValid: 'Property, ' + single + ', Event', + doubleWithValid: double + ', Car/Minivan', + } + const expectedResults = { + single: single, + double: double, + both: single + ', ' + double, + singleWithTwoValid: 'Property, ' + single + ', Event', + doubleWithValid: double + ', Car/Minivan', + } + const expectedIssues = { + single: [generateIssue('invalidTag', { tag: single })], + double: [generateIssue('invalidTag', { tag: double })], + both: [generateIssue('invalidTag', { tag: single }), generateIssue('invalidTag', { tag: double })], + singleWithTwoValid: [generateIssue('invalidTag', { tag: single })], + doubleWithValid: [generateIssue('invalidTag', { tag: double })], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + it('should ignore leading and trailing spaces', () => { + const testStrings = { + leadingSpace: ' Environmental-sound/Unique Value', + trailingSpace: 'Environmental-sound/Unique Value ', + bothSpace: ' Environmental-sound/Unique Value ', + leadingSpaceTwo: ' Environmental-sound/Unique Value, Event', + trailingSpaceTwo: 'Event, Environmental-sound/Unique Value ', + bothSpaceTwo: ' Event, Environmental-sound/Unique Value ', + } + const expectedResults = { + leadingSpace: 'Item/Sound/Environmental-sound/Unique Value', + trailingSpace: 'Item/Sound/Environmental-sound/Unique Value', + bothSpace: 'Item/Sound/Environmental-sound/Unique Value', + leadingSpaceTwo: 'Item/Sound/Environmental-sound/Unique Value, Event', + trailingSpaceTwo: 'Event, Item/Sound/Environmental-sound/Unique Value', + bothSpaceTwo: 'Event, Item/Sound/Environmental-sound/Unique Value', + } + const expectedIssues = { + leadingSpace: [], + trailingSpace: [], + bothSpace: [], + leadingSpaceTwo: [], + trailingSpaceTwo: [], + bothSpaceTwo: [], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + it('should raise an issue if there are extra slashes', () => { + const testStrings = { + leadingSingle: '/Event', + leadingMultiLevel: '/Vehicle/Train', + trailingSingle: 'Event/', + trailingMultiLevel: 'Vehicle/Train/', + bothSingle: '/Event/', + bothMultiLevel: '/Vehicle/Train/', + twoMixedOuter: '/Event,Vehicle/Train/', + //twoMixedInner: 'Event/,/Vehicle/Train', + twoMixedBoth: '/Event/,/Vehicle/Train/', + twoMixedBothGroup: '(/Event/,/Vehicle/Train/)', + } + const expectedResults = testStrings + const expectedIssues = { + leadingSingle: [generateIssue('extraSlash', { index: 0, string: testStrings.leadingSingle })], + leadingMultiLevel: [generateIssue('extraSlash', { index: 0, string: testStrings.leadingMultiLevel })], + trailingSingle: [generateIssue('extraSlash', { index: 5, string: testStrings.trailingSingle })], + trailingMultiLevel: [generateIssue('extraSlash', { index: 13, string: testStrings.trailingMultiLevel })], + bothSingle: [generateIssue('extraSlash', { index: 0, string: testStrings.bothSingle })], + bothMultiLevel: [generateIssue('extraSlash', { index: 0, string: testStrings.bothMultiLevel })], + twoMixedOuter: [generateIssue('extraSlash', { index: 0, string: testStrings.twoMixedOuter })], + twoMixedInner: [generateIssue('extraSlash', { index: 0, string: testStrings.twoMixedOuter })], + twoMixedBoth: [generateIssue('extraSlash', { index: 0, string: testStrings.twoMixedBoth })], + twoMixedBothGroup: [generateIssue('extraSlash', { index: 1, string: testStrings.twoMixedBothGroup })], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + + it.skip('should replace extra spaces and slashes with single slashes', () => { + const testStrings = { + twoLevelDoubleSlash: 'Event//Extension', + threeLevelDoubleSlash: 'Vehicle//Boat//Tanker', + tripleSlashes: 'Vehicle///Boat///Tanker', + mixedSingleAndDoubleSlashes: 'Vehicle///Boat/Tanker', + singleSlashWithSpace: 'Event/ Extension', + doubleSlashSurroundingSpace: 'Event/ /Extension', + doubleSlashThenSpace: 'Event// Extension', + sosPattern: 'Event/// ///Extension', + alternatingSlashSpace: 'Vehicle/ / Boat/ / Tanker', + leadingDoubleSlash: '//Event/Extension', + trailingDoubleSlash: 'Event/Extension//', + leadingDoubleSlashWithSpace: '/ /Event/Extension', + trailingDoubleSlashWithSpace: 'Event/Extension/ /', + } + const expectedEventExtension = 'Event/Extension' + const expectedTanker = 'Item/Object/Man-made-object/Vehicle/Boat/Tanker' + const expectedResults = { + twoLevelDoubleSlash: expectedEventExtension, + threeLevelDoubleSlash: expectedTanker, + tripleSlashes: expectedTanker, + mixedSingleAndDoubleSlashes: expectedTanker, + singleSlashWithSpace: expectedEventExtension, + doubleSlashSurroundingSpace: expectedEventExtension, + doubleSlashThenSpace: expectedEventExtension, + sosPattern: expectedEventExtension, + alternatingSlashSpace: expectedTanker, + leadingDoubleSlash: expectedEventExtension, + trailingDoubleSlash: expectedEventExtension, + leadingDoubleSlashWithSpace: expectedEventExtension, + trailingDoubleSlashWithSpace: expectedEventExtension, + } + const expectedIssues = { + twoLevelDoubleSlash: [], + threeLevelDoubleSlash: [], + tripleSlashes: [], + mixedSingleAndDoubleSlashes: [], + singleSlashWithSpace: [], + doubleSlashSurroundingSpace: [], + doubleSlashThenSpace: [], + sosPattern: [], + alternatingSlashSpace: [], + leadingDoubleSlash: [], + trailingDoubleSlash: [], + leadingDoubleSlashWithSpace: [], + trailingDoubleSlashWithSpace: [], + } + return validator(testStrings, expectedResults, expectedIssues) + }) + }) + }) +}) diff --git a/tests/data/HED7.0.4.xml b/tests/data/HED7.0.4.xml new file mode 100644 index 00000000..891a677b --- /dev/null +++ b/tests/data/HED7.0.4.xml @@ -0,0 +1,3719 @@ + + + + Event + + Category + This is meant to designate the reason this event was recorded + + Initial context + The purpose is to set the starting context for the experiment --- and if there is no initial context event---this information would be stored as a dataset tag + + + Participant response + The purpose of this event was to record the state or response of a participant. Note: participant actions may occur in other kinds of events, such as the experimenter records a terrain change as an event and the participant happens to be walking. In this case the event category would be Environmental. In contrast, if the participant started walking in response to an instruction or to some other stimulus, the event would be recorded as Participant response + + + Technical error + Experimenters forgot to turn on something or the cord snagged and something may be wrong with the data + + # + Description as string + + + + Participant failure + Situation in which participant acts outside of the constraints of the experiment -- such as driving outside the boundary of a simulation experiment or using equipment incorrectly + + # + Description as string + + + + Environmental + Change in experimental context such as walking on dirt versus sidewalk + + # + Description as a string + + + + Experimental stimulus + + Instruction + + + + Experimental procedure + For example doing a saliva swab on the person + + # + Description as string + + + + Incidental + Not a part of the task as perceived by/instructed to the participant --- for example an airplane flew by and made noise or a random person showed up on the street + + # + Description as string + + + + Miscellaneous + Events that are only have informational value and cannot be put in other event categories + + # + Description as a string + + + + Experiment control + Information about states and events of the software program that controls the experiment + + Sequence + + Permutation ID + + # + Permutation number/code used for permuted experiment parts + + + + Experiment + Use Attribute/Onset and Attribute/Offset to indicate start and end of the experiment + + + Block + Each block has the same general context and contains several trials -- use Attribute/Onset and Attribute/Offset to specify start and end + + # + Block number or identifier + + + + Trial + Use Attribute/Onset and Attribute/Offset to specify start and end + + # + Trial number or identifier + + + + Pause + Use Attribute/Onset and Attribute/Offset to specify start and end + + + + Task + + # + Label here + + + + Activity + Experiment-specific actions such as moving a piece in a chess game + + Participant action + + + + Synchronization + An event used for synchronizing data streams + + Display refresh + + + Trigger + + + Tag + + # + Actual tag: string or integer + + + + + Status + + Waiting for input + + + Loading + + + Error + + + + Setup + + Parameters + + # + Experiment parameters in some a string. Do not used quotes. + + + + + + + ID + A number or string label that uniquely identifies an event instance from all others in the recording (a UUID is strongly preferred). + + # + ID of the event + + + + Group ID + A number or string label that uniquely identifies a group of events associated with each other. + + # + ID of the group + + + + Duration + An offset that is implicit after duration time passed from the onset + + # + + + + Description + Same as HED 1.0 description for human-readable text + + # + + + + Label + A label for the event that is less than 20 characters. For example /Label/Accept button. Please note that the information under this tag is primarily not for use in the analysis and is provided for the convenience in referring to events in the context of a single study. Please use Custom tag to define custom event hierarchies. Please do not mention the words Onset or Offset in the label. These should only be placed in Attribute/Onset and Attribute/Offset. Software automatically generates a final label with (onset) or (offset) in parentheses added to the original label. This makes it easier to automatically find onsets and offsets for the same event. + + # + + + + Long name + A long name for the event that could be over 100 characters and could contain characters like vertical bars as separators. Long names are used for cases when one wants to encode a lot of information in a single string such as Scenario | VehiclePassing | TravelLaneBLocked | Onset + + # + + + + + Item + + ID + Optional + + # + + + Local + For IDs with local scope --- that is IDs only defined in the scope of a single event. The local ID 5 in events 1 and 2 may refer to two different objects. The global IDs directly under ID/ tag refer to the same object through the whole experiment + + # + + + + + Group ID + Optional + + # + + + + Object + Visually discernable objects. This item excludes sounds that are Items but not objects + + Vehicle + + Bicycle + + + Car + + + Truck + + + Cart + + + Boat + + + Tractor + + + Train + + + Aircraft + + Airplane + + + Helicopter + + + + + Person + + Pedestrian + + + Cyclist + + + Mother-child + + + Experimenter + + + + Animal + + + Plant + + Flower + + + Tree + + Branch + + + Root + + + + + Building + + + Food + + Water + + + + Clothing + + Personal + clothing that is on the body of the subject + + + + Road sign + + + Barrel + + + Cone + + + Speedometer + + + Construction zone + + + 3D shape + + + Sphere + + + Box + + Cube + + + + + 2D shape + Geometric shapes + + Ellipse + + Circle + + + + Rectangle + + Square + + + + Star + + + Triangle + + + Gabor patch + + + Cross + By default a vertical-horizontal cross. For a rotated cross add Attribute/Object orientation/Rotated/ tag + + + Single point + + + Clock face + Used to study things like hemispheric neglect. The tag is related to the clock-drawing-test + + # + Hour:min + + + + + Pattern + + Checkerboard + + + Abstract + + + Fractal + + + LED + + + Dots + + Random dot + + + + Complex + + + + Face + + Whole face with hair + + + Whole face without hair + + + Cut-out + + + Parts only + + Nose + + + Lips + + + Chin + + + Eyes + + Left only + + + Right only + + + + + + Symbolic + Something that has a meaning, could be linguistic or not such as a stop signs. + + Braille character + + + Sign + Like the icon on a stop sign. This should not to be confused with the actual object itself. + + Traffic + + Speed limit + + # + Always give units e.g. mph or kph + + + + + + Character + + Digit + + + Pseudo-character + Alphabet-like but not really + + + Letter + Authograph or valid letters and numbers such as A or 5 + + # + + + + + Composite + + + + Natural scene + + Aerial + + Satellite + + + + + Drawing + Cartoon or sketch + + Line drawing + + + + Film clip + + Commercial TV + + + Animation + + + + IAPS + International Affective Picture System + + + IADS + International Affective Digital Sounds + + + SAM + The Self-Assessment Manikin + + + + Sensory presentation + Object manifestation + + Auditory + Sound + + Nameable + + + Cash register + + + Ding + Often associated with positive valence + + + Buzz + Often associated with negative valence + + + Fire alarm + + + Click + + ABR + Auditory Brainstem Response + + + + Tone + + + Siren + + + Music + + Chord sequence + + + Vocal + + + Instrumental + + + + Noise + + White + + + Colored + Not white --- for example a 1/f spectrum + + + + Human voice + + + Animal voice + + Bird + + + Dog + + + Insect + + + Squirrel + + + + Real world + For example people walking or machines operating + + Pedestrian + + + Footsteps + + Walking + + + Running + + + + Noisemaker + + + Construction noise + + + Machine + + + Vehicle + + Horn + + + Aircraft + + Airplane + + + Helicopter + + + + Train + + + Cart + + + Car alarm + + + Car + + + Bicycle + + + + + Nonverbal vocal + + Emotional + + Crying + + + Sighing + + + + Gulp + + + Gurgle + + + Sneeze + + + Cough + + + Yawn + + + + Nonvocal + A car engine or gears grinding --- anything that is not made by a human or an animal + + Engine + + + + + Olfactory + Odor + + + Taste + + + Tactile + Pressure + + + Visual + + Rendering type + + Screen + + View port + Two or more views on the same object --- for example one from top one from street view + + ID + + # + A descriptive label for the viewport + + + + + 2D + + + 3D + + + Movie + + Video-tape + + + Motion-capture + Stick figure of motion capture of someone else + + Point light + + + Stick figure + + + Outline + + + + Flickering + + + Steady state + + + + + Real-world + + + LED + Stimulus is turning on/off one or a few LEDs + + + + + + Attribute + + Onset + Default + + + Offset + + + Imagined + This is used to identity that the (sub)event only happened in participant's imagination, e.g. imagined movements in motor imagery paradigms. + + + State ID + This is used to identify a group of events that are changing the state of a variable where the onset means the offset of any other and a change in the state + + # + ID which could be a number or any string + + + + Repetition + When the same type of event such as a fixation on the exact same object happens multiple times and it might be necessary to distinguish the first look vs. others + + # + Number starting from 1 where 1 indicates the first occurrence and 2 indicates the second occurrence + + + + Temporal rate + + # + In Hz + + + + Condition + Specifies the value of an independent variable (number of letters, N, in an N-back task) or function of independent variables that is varied or controlled for in the experiment. This attribute is often specified at the task level and can be associated with specification of an experimental stimulus or an experiment context (e.g. 1-back, 2-back conditions in an N-back task, or changing the target type from faces to houses in a Rapid Serial Visual Presentation, or RSVP, task.) + + # + the condition + + + + Action judgment + External judgment (assumed to be ground truth, e.g. from an experiment control software or an annotator) about participant actions such as answering a question, failing to answer in time, etc. + + Correct + + + Incorrect + Wrong choice but not time out + + + Indeterminate + It cannot be determined that the action was correct or incorrect. + + + Time out + + Missed + Participant failed or could not have perceived the instruction due to their eyes being off-screen or a similar reason. Not easy to deduce this but it is possible + + + + Inappropriate + A choice that is not allowed such as moving a chess piece to a location it should not go based on game rules + + + + Response start delay + The time interval between this (stimulus) event and the start of the response event specified, usually by grouping with the event ID of the response start event. + + # + + + + Response end delay + The time interval between this (stimulus) event and the end of the response event specified, usually by grouping with the event ID of the response end event. + + # + + + + Social + Involving interactions among multiple agents such as humans or dogs or robots + + + Peak + Peak velocity or acceleration or jerk + + + Object side + Could be the left, right, or both sides of a person or a vehicle + + Reference object ID + Place object ID after this + + # + + + + Right + + + Left + + + Front + + + Back + + + Top + + + Bottom + + + Starboard + + + Port + + + Passenger side + Side of a car + + + Driver side + Side of a car + + + Bow + Front of a ship + + + Stern + Back of the ship + + + + Direction + Coordinate system is inferred from Attribute/Location. To specify a vector combine subnodes with number --- for example Attribute/Top/10, Attribute/Direction/Left/5 to create a vector with coordinates 10 and 5 + + Top + Combine Attribute/Direction/Top and Attribute/Direction/Left to mean the upper left + + # + + + + Bottom + + # + + + + Left + + # + + + + Right + + # + + + + Angle + Clockwise angle in degrees from vertical + + # + Clockwise angle in degrees from vertical + + + + North + + # + + + + South + + # + + + + East + + # + + + + West + + # + + + + Forward + Like a car moving forward + + + Backward + Like a car moving backward + + + + Location + Spot or center of an area. Use Area were you are referring to something with significant extent and emphasizing its boundaries, like a city + + # + location label + + + Screen + Specify displacements from each subnode in pixels or degrees or meters. Specify units such as Attribute/Location/Screen/Top/12 px + + Center + + # + + + + Top + You can combine Attribute/Location/Top and Attribute/Location/Left to designate UpperLeft and so on + + # + + + + Bottom + + # + + + + Left + + # + + + + Right + + # + + + + Angle + + # + Clockwise angle in degrees from vertical + + + + Center displacement + + # + displacement from screen center, in any direction, in degrees, cm, or other lengths + + + Horizontal + + # + Displacement from screen center in any direction + + + + Vertical + + # + Displacement from screen center in any direction + + + + + + Lane + For example a car lane + + Rightmost + + + Leftmost + + + Right of expected + + + Left of expected + + + Cruising + + + Passing + The lane that cars use to take over other cars + + + Oncoming + + + + Real-world coordinates + + Room + + xyz + have a subnode, e.g. Attribute/Location/Real-world coordinates/Room/xyz/10 50 30 + + # + + + + + + Reference frame + + Specified absolute reference + + + Relative to participant + + Participant ID + + # + + + + Left + + + Front + + + Right + + + Back + + + Distance + + # + Distance is in meters by default + + + Near + + + Moderate + + + Far + + + + Azimuth + + # + Clockwise with units preferably in degrees + + + + Elevation + + # + Preferably in degrees + + + + + + + Object orientation + + Rotated + + Degrees + + # + Preferably in degrees + + + + + + Size + + Length + + # + In meters or other units of length + + + + Width + + # + in meters + + + + Height + + # + Default units are meters + + + + Area + + # + + + + Volume + + # + In cubic-meters or other units of volume + + + + Angle + + # + In degrees or other units of angle + + + + + Item count + Number of items for example when there are 3 cars and they are identified as a single item + + # + Numeric value of number of items # + + + <=# + Number of items less than or equal to # + + + >=# + Number of items more than or equal to # + + + + Auditory + + Frequency + + # + In HZ + + + + Loudness + + # + in dB + + + + Ramp up + Increasing in amplitude + + + Ramp down + Decreasing in amplitude + + + + Blink + + Time shut + The amount of time the eyelid remains closed (typically measured as 90% of the blink amplitude), in seconds. + + # + + + + Duration + Duration of blink, usually the half-height blink duration in seconds taken either from base or zero of EEG signal. For eye-trackers, usually denotes interval when pupil covered by eyelid. + + # + + + + PAVR + Amplitude-Velocity ratio, in centiseconds + + # + + + + NAVR + Negative Amplitude-Velocity ratio, in centiseconds + + # + + + + + Visual + + Bistable + + + Background + + + Foreground + + + Up-down separated + Stimuli presented both at the top and the bottom of fovea + + # + Angle of separation in degrees by default + + + + Bilateral + For bilateral visual field stimulus presentations + + # + Angle of separation in degrees by default + + + + Motion + + Down + + # + e.g. 3 degrees-per-second + + + + Up + + # + e.g. 3 degrees-per-second + + + + Horizontal + + Right + + # + e.g. 3 degrees-per-second + + + + Left + + # + e.g. 3 degrees-per-second + + + + + Oblique + + Clock face + + # + For example 4:30 + + + + + + Fixation point + + + Luminance + + # + In candelas by default + + + + Color + + Dark + + + Light + + + Aqua + These are CSS 3 basic color names + + + Black + + + Fuchsia + + + Gray + + + Lime + + + Maroon + + + Navy + + + Olive + + + Purple + + + Silver + + + Teal + + + White + + + Yellow + + + Red + + # + R value of RGB between 0 and 1 + + + + Blue + + # + B value of RGB between 0 and 1 + + + + Green + + # + G value of RGB between 0 and 1 + + + + Hue + + # + H value of HSV between 0 and 1 + + + + Saturation + + # + S value of HSV between 0 and 1 + + + + Value + + # + V value of HSV between 0 and 1 + + + + Achromatic + Indicates gray scale + + # + White intensity between 0 and 1 + + + + + + Nonlinguistic + Something that conveys meaning without using words such as the iconic pictures of a man or a woman on the doors of restrooms. Another example is a deer crossing sign with just a picture of jumping deer. + + + Semantic + Like in priming or in congruence + + + Language + + Unit + + Phoneme + + + Syllable + + + Word + + Noun + + Proper + A proper noun that refers to a unique entity such as London or Jupiter + + + Common + A noun that refers to a class of entities such as cities or planets or corporations such as a Dog or a Skyscraper + + + + Verb + + + Adjective + + + Pseudoword + + + # + Actual word + + + + Sentence + + Full + + + Partial + + + # + Actual sentence + + + + Paragraph + + # + Actual paragraph + + + + Story + Multiple paragraphs making a detailed account + + + + Family + + Asian + + Chinese + + + Japanese + + + + Latin + + English + + + German + + + French + + + + + + Induced + Such as inducing emotions or keeping someone awake or in a coma with an external intervention + + + Emotional + + Arousal + Only in the context of 2D emotion representation + + # + A value between -1 and 1 + + + + Positive valence + Valence by itself can be the name of an emotion such as sadness so this tag distinguishes the type of emotion + + # + Ranges from 0 to 1 + + + + Negative valence + + # + Ranges from 0 to 1 + + + + + Priming + + Motoric + + + Emotional + + + Perceptual + + + + Subliminal + + Unmasked + + + Masked + + Forward + + + Backward + + + + + Supraliminal + By default this is assumed about each stimulus + + + Liminal + At the 75%-25% perception threshold + + + Probability + Use to specify the level of certainty about the occurrence of the event. Use either numerical values as the child node or 'low', 'high', etc. + + + Temporal uncertainty + Use to specify the amount of uncertainty in the timing of the event. Please notice that this is different from Attribute/Probability tag which relates to the occurrence of event and can be interpretative as the integral of probability density across a distribution whose shape (temporal extent) is specified by Attribute/Temporal uncertainty + + # + + + Standard deviation + implies that the distribution of temporal uncertainty is Gaussian with the provided standard deviation (in seconds). + + # + + + + + Presentation + Attributes associated with visual, auditory, tactile, etc. presentation of an stimulus + + Fraction + the fraction of presentation of an Oddball or Expected stimuli to the total number of same-class presentations, e.g. 10% of images in an RSVP being targets + + # + + + + Cued + what is presented is cued to the presentation of something else + + + Background + presented in the background such as background music, background image, etc. The main factor here is that background presentations are to be ignored, e.g. ignore math question auditory stimuli. + + + + Intended effect + This tag is to be grouped with Participant/Effect/Cognitive to specify the intended cognitive effect (of the experimenter). This is to differentiate the resulting group with Participant/Effect which specifies the actual effect on the participant. For example, in an RSVP experiment if an image is intended to be perceived as a target, (Participant/Effect/Cognitive/Target, Attribute/Intended effect) group is added. If the image was perceived by the subject as a target (e.g. they pressed a button to indicate so), then the tag Participant/Effect/Cognitive/Target is also added: Participant/Effect/Cognitive/Target, (Participant/Effect/Cognitive/Target, Attribute/Intended effect). otherwise the Participant/Effect/Cognitive/Target tag is not included outside of the group. + + + Instruction + This tag is placed in events of type Event/Category/Experimental stimulus/Instruction, grouped with one or more Action/ tags to replace the detailed specification XXX in Event/Category/Experimental stimulus/Instruction/XXX) in previous versions. Usage example: Event/Category/Experimental stimulus/Instruction, (Action/Fixate, Attribute/Instruction) + + + Participant indication + This tag is placed in events of type Event/Category/Participant response and grouped with Participant/Effect/Cognitive/.. tags to specify the type of cognitive effect the participant has experienced. For example, in an RSVP paradigm, the subject can indicate the detection of a target with a button press. The HED string associated with this button press must include (Attribute/Participant indication, Participant/Effect/Cognitive/Target,...) + + + Path + + Velocity + Use Attribute/Onset or Attribute/Offset to specify onset or offset + + # + Numeric value with default units of m-per-s + + + + Acceleration + + # + Numeric value with default units of m-per-s2 + + + + Jerk + + # + Numeric value with default units of m-per-s3 + + + + Constrained + For example a path cannot cross some region + + + + File + File attributes + + Name + + + Size + + # + Numeric value with default units of mb + + + + # + Number of files + + + + Object control + Specifies control such as for a vehicle + + Perturb + + + Collide + + + Near miss + Almost having an accident resulting in negative consequences + + + Correct position + After a lane deviation or side of the walkway + + + Halt + Time at which speed becomes exactly zero + + + Brake + + + Shift lane + + + Cross + Crossing in front of another object such as a vehicle + + + Pass by + Passing by another object or the participant + + + Accelerate + + + Decelerate + + + + Association + + Another person + Item such as a cup belonging to another person + + + Same person + Item such as a cup belonging to the participant + + + + Extraneous + Button presses that are not meaningful for example due to intrinsic mechanical causes after a meaningful press + + + Role + The role of the agent (participant, character, AI..) + + Leader + + + Follower + + + # + + + + + Action + May or may not be associated with a prior stimulus and can be extended + + Involuntary + Like sneezing or tripping on something or hiccuping + + Hiccup + + + Cough + + + Sneeze + + + Stumble + Temporary and involuntary loss of balance + + + Fall + + + Tether Jerk + When a tether attached to the subject is stuck/snagged and forces the participant to involuntary accommodate/react to it + + + Clear Throat + + + Yawn + + + Sniffle + + + Burp + + + Drop + For example something drops from subject’s hand + + + + Make fist + + Open and close + Continue to open and close the fist, for example in motor imagery paradigms. + + + + Curl toes + + Open and close + Continue to curl and uncurl toes, for example in motor imagery paradigms. + + + + Button press + + Touch screen + + + Keyboard + + + Mouse + + + Joystick + + + + Button hold + Press a button and keep it pressed + + + Button release + + + Cross boundary + + Arrive + + + Depart + + + + Speech + + + Hum + + + Eye saccade + Use Attribute/Peak for the middle of saccade and Attribute/Onset for the start of a saccade + + + Eye fixation + + + Eye blink + + Left base + The time of the first detectable eyelid movement on closing. + + + Left zero + The last time at which the EEG/EOG signal crosses zero during eyelid closing. + + + Left half height + The time at which the EEG/EOG signal reaches half maximum height during eyelid closing. + + + Max + The time at which the eyelid is the most closed. + + + Right half height + The time at which the EEG/EOG signal reaches half maximum height during eyelid opening. + + + Right zero + The first time at which the EEG/EOG signal crosses zero during eyelid opening. + + + Right base + The time of the last detectable eyelid movement on opening. + + + + Eye close + Close eyes and keep closed for more than approximately 0.1 s + + Keep + Keep the eye closed. If a value is provided it indicates the duration for this. + + # + the duration (by default in seconds) that they keep their eye closed. + + + + + Eye open + Open eyes and keep open for more than approximately 0.1 s + + Keep + Keep the eye open. If a value is provided it indicates the duration for this. + + # + the duration (by default in seconds) that they keep their eye open. + + + With blinking + Default. Allow blinking during the eye-open period. + + + Without blinking + Without blinking during the eye-open period. + + + + + Turn + Change in direction of movement or orientation. This includes both turn during movement on a path and also rotations such as head turns + + + Point + + + Push + + + Grab + + + Tap + When there is nothing to be pressed for example like tapping a finger on a chair surface to follow a rhythm + + + Lift + + + Reach + Requires a goal such as reaching to touch a button or to grab something. Stretching your body does not count as reach. + + To Grab + + + To Touch + + + + Course correction + Change the direction of a reach in the middle to adjust for a moving target. + + + Interact + + With human + + + + Take survey + + + Stretch + Stretch your body such as when you wake up + + + Bend + + + Deep breath + + + Laugh + + + Sigh + + + Groan + + + Scratch + + + Switch attention + + Intramodal + In the same modality but with a change in details such as changing from paying attention to red dots and instead of blue dots + + Visual + + + Auditory + + + Tactile + + + Taste + + + Smell + + + + Intermodal + Between modalities such as changing from audio to visual + + From modality + + Visual + + + Auditory + + + Tactile + + + Taste + + + Smell + + + + To modality + + Visual + + + Auditory + + + Tactile + + + Taste + + + Smell + + + + + + Walk + + Stride + Use onset and offset attributes to indicate different walking stride stages + + + Faster + increasing the speed of walking + + + Slower + decreasing the speed of walking + + + + Control vehicle + Controlling an object that you are aboard + + Drive + Driving a vehicle such as a car + + Correct + Correct for a perturbation + + + Near miss + + + Collide + + + + Stop + Brake a car + + + Pilot + Pilot a vehicle such as an airplane + + + + Teleoperate + Control an object that you are not aboard + + + Allow + Allow access to something such as allowing a car to pass + + + Deny + Deny access to something such as preventing someone to pass + + + Step around + + + Step over + + + Step on + + + Swallow + + + Flex + + + Evade + + + Shrug + + + Dance + + + Open mouth + + + Whistle + + + Read + + + Attend + + + Recall + + + Generate + + + Repeat + + + Hold breath + + + Breathe + + + Rest + + + Count + + + Move + + Upper torso + + + Lower torso + + + Whole body + + + + Speak + + + Sing + + + Detect + + + Name + + + Smile + + + Discriminate + + + Track + + + Encode + + + Eye-blink inhibit + + + + Participant + + ID + If not given assume 1 + + # + Numeric value of an ID + + + + Effect + How the stimulus effects the participants + + Cognitive + + Meaningful + + + Not meaningful + + + Newly learned meaning + + + Reward + + Low + + + Medium + + + High + + + # + Monetary values in some currency such as $10, or the ratio of the reward to the maximum possible (3 of max 10 becomes 0.3), or number Points + + + + Penalty + + Low + + + Medium + + + High + + + # + Absolute monetary values in some currency, for example $1, or the ratio of the reward to the maximum possible (3 of max 10 becomes 0.3), or number of Points + + + + Error + + Self originated + + + Other originated + + Human + + + Non-human + + + + Expected + + + Unexpected + + + Planned + The error feedback was given regardless of the validity of subject response as in a yoked design + + + + Threat + + To self + + + To others + + Close + + + + + Warning + As in a warning message that you are getting too close to the shoulder in a driving task + + + Oddball + Unexpected or infrequent + + One stimulus + Only oddballs are present but no frequent stimuli exist. See http://dx.doi.org/10.1016/0167-8760(96)00030-X + + + Two stimuli + There are non-targets and targets. See http://dx.doi.org/10.1016/0167-8760(96)00030-X + + + Three stimuli + There are regular non-targets and targets and infrequent non-targets, see http://dx.doi.org/10.1016/0167-8760(96)00030-X + + + Silent counting + + + Button pressing for target + + + Button pressing for all + + + + Target + Something the subject is looking for + + + Non-target + Make sure to tag Expected if the Non-target is frequent + + + Novel + Genuinely novel such as an event occurring once or so per experiment + + + Expected + Of low information value, for example frequent Non-targets in an RSVP paradigm + + Standard + + + Distractor + + + + Valid + Something that is understood to be valid such as an ID matches the person being displayed and it has all the correct information + + + Invalid + Something that is understood to not be valid such as like an ID with an impossible date-of-birth, or a photo not matching the person presenting it + + + Congruence + + Congruent + Like in Stroop paradigm when blue colored text displays the word blue + + + Incongruent + Like in Stroop paradigm whena blue colored text reading displays the word red + + + Temporal synchrony + + Synchronous + When a mouse click sound happens right after clicking it + + + Asynchronous + When a mouse click sound happens with significant delay which give could the person a strange feeling. Or if in a movie the sound of the explosion is heard before it appears visually. + + + + + Feedback + + Correct + Confirm something went well and last action was correct + + + Incorrect + Confirm something went wrong and last action was incorrect + + + Non-informative + Feedback that provides no information in regards to correct, incorrect, etc. + + + Expected + Feedback was expected as in a positive feedback after a response that was expected to be correct. + + + Unexpected + Feedback was unexpected as when positive feedback was received when response was expected to be incorrect. + + + On accuracy + Feedback was provided by evaluating response accuracy + + + On reaction time + Feedback was provided by evaluating subject reaction time + + + To self + Default + + + To other + Observed feedback to another person such as in a social paradigm + + + Deterministic + Feedback has a fixed relationship to what happened before + + + Stochastic + Feedback is non-deterministic and does not have fixed relationship with what has happened before in the experiment + + + False feedback + Feedback that was not honest for example as in feedback of correct on an incorrect response or vice versa + + Negative + Negative feedback was provided when it was not deserved + + + Positive + Positive feedback was provided when it was not deserved + + + + + Cue + An indicator of a future event, e.g. a sound cue that in 2-5 seconds a perturbation in driving will occur. Use (... Participant/Effect/Cognitive/Cue ~ hed tags for the event to follow, or main aspect of the event to follow) syntax. + + Constant delay + The cue is for an event that will happen after a constant delay. + + # + The delay, e.g. in seconds. + + + + Variable delay + The cue is for an event that will happen after a variable delay. + + # + The interval, e.g. between 2-5 seconds, of the variable delay. + + + + + + Visual + + Foveal + + + Peripheral + + + Perturbation + Sudden movement or perturbation of the virtual environment in a car driving or other scenario + + + + Auditory + + Stereo + + + Mono + + Left + + + Right + + + + + TMS + + With SPGS + SPGS stands for spatial position guiding system + + + Without SPGS + SPGS stands for spatial position guiding system + + + + Tactile + + Vibration + + + Acupuncture + + + Eye puff + + + Swab + Mouth swab + + + + Vestibular + + Shaking + being shaken or jerked around + + + + Pain + + Heat + + + Cold + + + Pressure + + + Electric shock + + + Laser-evoked + + + + Taste + + + Smell + + + Body part + + Whole Body + + + Eye + + + Arm + + Hand + + Finger + + Index + + + Thumb + + + Ring + + + Middle + + + Small + Pinkie or little finger + + + + + + Leg + + Feet + + Toes + + + + + Head + + Face + + Eyebrow + + + Lip + + + Forehead + + + Mouth + + + Nose + + + Chin + + + Cheek + + + + + Torso + + + + + State + + Level of consciousness + + Awake + + + Drowsy + + + Sleep + + Stage + + # + a number between 1 to 4, or 'REM' + + + + + Drunk + + + Anesthesia + + + Locked-in + + + Coma + + + Vegetative + + + Brain-dead + + + + Emotion + + Awe + + + Frustration + + + Joy + + + Anger + + + Happiness + + + Sadness + + + Love + + + Fear + + + Compassion + + + Jealousy + + + Contentment + + + Grief + + + Relief + + + Excitement + + + Disgust + + + Neutral + None of the above + + + + Sense of community + Primed to have an emotion such as patriotism + + # + SCI stands for Sense of Community Index + + + + Sense of social justice + + Distributive + + + Poverty + + + Inequality + + + Procedural + + + Interpersonal + + + Informational + + + + Stress level + + # + A number between 0 and 1 + + + + Task load + + # + A number between 0 and 1 + + + + Under time pressure + + Response window + + # + Default time is seconds + + + + Competitive + Subject is competing against an opponent as for example when the faster respondent wins + + + + Social interaction + Social + + Pseudo + Instructed so but actually not as when the other person may not exist in real world such as the case of a computer program agent + + + + Passive + There is a stimulus presentation but no behavioral measurements are collected from the subject. Subject is instructed not to make any behavioral outputs for example when told to carefully watch/listen/sense. The resting state is not considered passive. + + + Resting + State when there is no stimulus presentation and no behavioral outputs + + + Attention + + Top-down + Instructed to pay attention to something explicitly + + + Bottom-up + something captures your attention, like a big bang or your name + + Orienting + The lower state of the bottom-up or the pre-bottom up state + + + + Covert + Implicit + + + Overt + Explicit + + + Selective + If you have two circles but asked to pay attention to only one of them + + Divided + Attending to more than one object or location + + + + Focused + Paying a lot of attention + + + Sustained + Paying attention for a continuous time + + + Auditory + + + Visual + + + Tactile + + + Taste + + + Smell + + + To a location + Spatial -- use the location attribute to specify to where the attention is directed + + + Arousal + + + Alerting + Keeping the arousal up in order to respond quickly + + + Drowsy + + + Excited + + + Neutral + + + + + + Experiment context + Describes the context of the whole experiment or large portions of it and also includes tags that are common across all events + + # + Add common tags across all stimuli/ and/or responses here if all experimental events share /State/Drowsy, you can place it here instead of tagging each event individually + + + With chin rest + + + Sitting + + + Standing + + + Prone + As in on a bed + + + Running + + Treadmill + + + + Walking + + Treadmill + + + + Indoors + Default + + Clinic + Recording in a clinical setting such as in a hospital or doctor’s office + + + Dim Room + + + + Outdoors + + Terrain + + Grass + + + Uneven + + + Boardwalk + + + Dirt + + + Leaves + + + Mud + + + Woodchip + + + Rocky + + + Gravel + + + Downhill + + + Uphill + + + + + Motion platform + Subject is on a motion platform such as one that produces simulated car movements + + + Fixed screen + + Distance + Assuming static subject + + # + Distance from subject eyes to the presentation screen for 30 cm from subject eyes to the monitor + + + + Width resolution + + # + Default units are pixels + + + + Height resolution + + # + Default units are pixels + + + + + Real world + + + Virtual world + + + + Custom + This node can be used to organize events in an alternative (parallel) hierarchy. You can define your custom tags and hierarchies without any restriction under this node. These tags will still be matched to each other as for example /Custom/Dance/Waltz is considered a subtype of /Custom/DanceExample. + + + HED + Hierarchical Event Descriptor + + # + HED specification version number: normally there is no need to specify the version number in the HED string since it will be matched by default to the most recent compliant version, but this tag can be used to specify the exact HED version the HED string was based on. + + + + Paradigm + See Tasks in http://www.cognitiveatlas.org/tasks and CogPo definitions of paradigms + + Action imitation task + + + Action observation task + + + Acupuncture task + + + Adult attachment interview + + + Alternating runs paradigm + + + Animal naming task + + + Antisaccade-prosaccade task + + + Attention networks test + + + Attentional blink task + + + Audio-visual target-detection task + + + Autism diagnostic observation schedule + + + Ax-cpt task + + + Backward digit span task + + + Backward masking + + + Balloon analogue risk task - BART + + + Behavioral investment allocation strategy - BIAS + + + Behavioral rating inventory of executive function + + + Benton facial recognition test + + + Birmingham object recognition battery + + + Block design test + + + Block tapping test + + + Boston naming test + + + Braille reading task + + + Breath-holding + + + Breathhold paradigm + + + Brixton spatial anticipation test + + + California verbal learning test + + + California verbal learning test-ii + + + Cambridge face memory test + + + Cambridge gambling task + + + Cambridge neuropsychological test automated battery + + + Catbat task + + + Category fluency test + + + Cattell culture fair intelligence test + + + Chewing-swallowing + + + Chimeric animal stroop task + + + Choice reaction time task + + + Choice task between risky and non-risky options + + + Classical conditioning + + + Clinical evaluation of language fundamentals-3 + + + Color trails test + + + Color-discrimination task + + + Color-word stroop task + + + Complex span test + + + Conditional stop signal task + + + Conditioning paradigm + + Behavioral conditioning paradigm + + + Classical conditioning paradigm + + + + Continuous performance task + + + Continuous recognition paradigm + + + Counting stroop task + + + Counting-calculation + + + Cued explicit recognition + + + Cups task + + + Deception task + + + Deductive reasoning paradigm + + + Deductive reasoning task + + + Delayed discounting task + + + Delayed match to sample task + + + Delayed nonmatch to sample task + + + Delayed recall test + + + Delayed response task + + Delayed matching to sample paradigm + + Sternberg paradigm + + + + + Devils task + + + Dichotic listening task + + + Digit cancellation task + + + Digit span task + + + Digit-symbol coding test + + + Directed forgetting task + + + Divided auditory attention + + + Divided auditory attention paradigm + + + Doors and people test + + + Dot pattern expectancy task + + + Drawing + + + Drawing paradigm + + + Dual-task paradigm + + + Early social communications scales + + + Eating paradigm + + + Eating-drinking + + + Embedded figures test + + + Emotional regulation task + + + Encoding paradigm + + + Encoding task + + + Episodic recall + + + Episodic recall paradigm + + + Eriksen flanker task + + + Extradimensional shift task + + + Eye Saccade paradigm + + Anti saccade paradigm + + + Simple saccade paradigm + + + + Face monitor-discrimination + + + Face n-back task + + + Fagerstrom test for nicotine dependence + + + Film viewing + + + Finger tapping task + + + Fixation task + + + Flashing checkerboard + + + Flexion-extension + + + Forward digit span task + + + Free word list recall + + + Glasgow coma scale + + + Go-no-go task + + + Grasping task + + + Gray oral reading test - 4 + + + Haptic illusion task + + + Hayling sentence completion test + + + Heat sensitization-adaptation + + + Heat stimulation + + + Hooper visual organization test + + + ID screening + Visual examination of multiple fields of an ID or document to detect invalid or suspicious fields. For example at a security checkpoint. + + + Imagined emotion + + + Imagined movement + + + Imagined objects-scenes + + + Instructed movement + + + Immediate recall test + + + Inductive reasoning aptitude + + + International affective picture system + + + Intradimensional shift task + + + Ishihara plates for color blindness + + + Isometric force + + + Item recognition paradigm + + Serial item recognition paradigm + + + + Item recognition task + + + Kanizsa figures + + + Keep-track task + + + Letter comparison + + + Letter fluency test + + + Letter naming task + + + Letter number sequencing + + + Lexical decision task + + + Listening span task + + + Macauthur communicative development inventory + + + Machine failure detection task + + + Matching familiar figures test + + + Matching pennies game + + + Maudsley obsessive compulsive inventory + + + Mechanical stimulation + + + Memory span test + + + Mental rotation task + + + Micturition task + + + Mini mental state examination + + + Mirror tracing test + + + Mismatch negativity paradigm + + + Mixed gambles task + + + Modified erikson scale of communication attitudes + + + Morris water maze + + + Motor sequencing task + + + Music comprehension-production + + + N-back task + + Letter n-back task + + + + Naming + + Covert + + + Overt + + + + Nine-hole peg test + + + Non-choice task to study expected value and uncertainty + + + Non-painful electrical stimulation + + + Non-painful thermal stimulation + + + Nonword repetition task + + + Object alternation task + + + Object-discrimination task + + + Oculomotor delayed response + + + Oddball discrimination paradigm + + Auditory oddball paradigm + + + Visual oddball paradigm + + Rapid serial visual presentation + + + + + Oddball task + + + Olfactory monitor-discrimination + + + Operation span task + + + Orthographic discrimination + + + Paced auditory serial addition test + + + Pain monitor-discrimination task + + + Paired associate learning + + + Paired associate recall + + + Pantomime task + + + Parrott scale + + + Passive listening + + + Passive viewing + + + Pattern comparison + + + Perturbed driving + + + Phonological discrimination + + + Picture naming task + + + Picture set test + + + Picture-word stroop task + + + Pitch monitor-discrimination + + + Pointing + + + Porteus maze test + + + Positive and negative affect scale + + + Posner cueing task + + + Probabilistic classification task + + + Probabilistic gambling task + + + Probabilistic reversal learning + + + Pseudoword naming task + + + Psychomotor vigilance task + + + Pursuit rotor task + + + Pyramids and palm trees task + + + Rapid automatized naming test + + + Rapid serial object transformation + + + Reading - Covert + + + Reading - Overt + + + Reading paradigm + + Covert braille reading paradigm + + + Covert visual reading paradigm + + + + Reading span task + + + Recitation-repetition - Covert + + + Recitation-repetition - Overt + + + Remember-know task + + + Response mapping task + + + Rest + + Rest eyes open + + + Rest eyes closed + + + + Retrieval-induced forgetting task + + + Reversal learning task + + + Reward task + + + Rey auditory verbal learning task + + + Rey-ostereith complex figure test + + + Reynell developmental language scales + + + Rhyme verification task + + + Risky gains task + + + Rivermead behavioural memory test + + + + + acceleration + m-per-s2,cm-per-s2 + + + currency + dollars,$,points,fraction + + + angle + degrees,degree,radian,radians + + + frequency + Hz,mHz,Hertz,kHz + + + intensity + dB + + + jerk + m-per-s3,cm-per-s3 + + + luminousIntensity + candela,cd + + + memorySize + mb,kb,gb,tb + + + physicalLength + m,cm,km,mm,feet,foot,meter,meters,mile,miles + + + pixels + pixels,px,pixel + + + speed + m-per-s,mph,kph,cm-per-s + + + time + s,second,seconds,centiseconds,centisecond,cs,hour:min,day,days,ms,milliseconds,millisecond,minute,minutes,hour,hours + + + area + m2,cm2,km2,pixels2,px2,pixel2,mm2 + + + volume + m3,cm3,mm3,km3 + + + diff --git a/tests/data/HED7.1.1.xml b/tests/data/HED7.1.1.xml new file mode 100644 index 00000000..f166b8b2 --- /dev/null +++ b/tests/data/HED7.1.1.xml @@ -0,0 +1,3950 @@ + + + + Event + + Category + This is meant to designate the reason this event was recorded + + Initial context + The purpose is to set the starting context for the experiment --- and if there is no initial context event---this information would be stored as a dataset tag + + + Participant response + The purpose of this event was to record the state or response of a participant. Note: participant actions may occur in other kinds of events, such as the experimenter records a terrain change as an event and the participant happens to be walking. In this case the event category would be Environmental. In contrast, if the participant started walking in response to an instruction or to some other stimulus, the event would be recorded as Participant response + + + Technical error + Experimenters forgot to turn on something or the cord snagged and something may be wrong with the data + + # + Description as string + + + + Participant failure + Situation in which participant acts outside of the constraints of the experiment -- such as driving outside the boundary of a simulation experiment or using equipment incorrectly + + # + Description as string + + + + Environmental + Change in experimental context such as walking on dirt versus sidewalk + + # + Description as a string + + + + Experimental stimulus + + Instruction + + + + Experimental procedure + For example doing a saliva swab on the person + + # + Description as string + + + + Incidental + Not a part of the task as perceived by/instructed to the participant --- for example an airplane flew by and made noise or a random person showed up on the street + + # + Description as string + + + + Miscellaneous + Events that are only have informational value and cannot be put in other event categories + + # + Description as a string + + + + Experiment control + Information about states and events of the software program that controls the experiment + + Sequence + + Permutation ID + + # + Permutation number/code used for permuted experiment parts + + + + Experiment + Use Attribute/Onset and Attribute/Offset to indicate start and end of the experiment + + + Block + Each block has the same general context and contains several trials -- use Attribute/Onset and Attribute/Offset to specify start and end + + # + Block number or identifier + + + + Trial + Use Attribute/Onset and Attribute/Offset to specify start and end + + # + Trial number or identifier + + + + Pause + Use Attribute/Onset and Attribute/Offset to specify start and end + + + + Task + + # + Label here + + + + Activity + Experiment-specific actions such as moving a piece in a chess game + + Participant action + + + + Synchronization + An event used for synchronizing data streams + + Display refresh + + + Trigger + + + Tag + + # + Actual tag: string or integer + + + + + Status + + Waiting for input + + + Loading + + + Error + + + + Setup + + Parameters + + # + Experiment parameters in some a string. Do not used quotes. + + + + + + + ID + A number or string label that uniquely identifies an event instance from all others in the recording (a UUID is strongly preferred). + + # + ID of the event + + + + Group ID + A number or string label that uniquely identifies a group of events associated with each other. + + # + ID of the group + + + + Duration + An offset that is implicit after duration time passed from the onset + + # + + + + Description + Same as HED 1.0 description for human-readable text + + # + + + + Label + A label for the event that is less than 20 characters. For example /Label/Accept button. Please note that the information under this tag is primarily not for use in the analysis and is provided for the convenience in referring to events in the context of a single study. Please use Custom tag to define custom event hierarchies. Please do not mention the words Onset or Offset in the label. These should only be placed in Attribute/Onset and Attribute/Offset. Software automatically generates a final label with (onset) or (offset) in parentheses added to the original label. This makes it easier to automatically find onsets and offsets for the same event. + + # + + + + Long name + A long name for the event that could be over 100 characters and could contain characters like vertical bars as separators. Long names are used for cases when one wants to encode a lot of information in a single string such as Scenario | VehiclePassing | TravelLaneBLocked | Onset + + # + + + + + Item + + ID + Optional + + # + + + Local + For IDs with local scope --- that is IDs only defined in the scope of a single event. The local ID 5 in events 1 and 2 may refer to two different objects. The global IDs directly under ID/ tag refer to the same object through the whole experiment + + # + + + + + Group ID + Optional + + # + + + + Object + Visually discernable objects. This item excludes sounds that are Items but not objects + + Vehicle + + Bicycle + + + Car + + + Truck + + + Cart + + + Boat + + + Tractor + + + Train + + + Aircraft + + Airplane + + + Helicopter + + + + + Person + + Pedestrian + + + Cyclist + + + Mother-child + + + Experimenter + + + + Animal + + + Plant + + Flower + + + Tree + + Branch + + + Root + + + + + Building + + + Food + + Water + + + + Clothing + + Personal + clothing that is on the body of the subject + + + + Road sign + + + Barrel + + + Cone + + + Speedometer + + + Construction zone + + + 3D shape + + + Sphere + + + Box + + Cube + + + + + 2D shape + Geometric shapes + + Ellipse + + Circle + + + + Rectangle + + Square + + + + Star + + + Triangle + + + Gabor patch + + + Cross + By default a vertical-horizontal cross. For a rotated cross add Attribute/Object orientation/Rotated/ tag + + + Single point + + + Clock face + Used to study things like hemispheric neglect. The tag is related to the clock-drawing-test + + # + + + + + Pattern + + Checkerboard + + + Abstract + + + Fractal + + + LED + + + Dots + + Random dot + + + + Complex + + + + Face + + Whole face with hair + + + Whole face without hair + + + Cut-out + + + Parts only + + Nose + + + Lips + + + Chin + + + Eyes + + Left only + + + Right only + + + + + + Symbolic + Something that has a meaning, could be linguistic or not such as a stop signs. + + Braille character + + + Sign + Like the icon on a stop sign. This should not to be confused with the actual object itself. + + Traffic + + Speed limit + + # + Always give units e.g. mph or kph + + + + + + Character + + Digit + + + Pseudo-character + Alphabet-like but not really + + + Letter + Authograph or valid letters and numbers such as A or 5 + + # + + + + + Composite + + + + Natural scene + + Aerial + + Satellite + + + + + Drawing + Cartoon or sketch + + Line drawing + + + + Film clip + + Commercial TV + + + Animation + + + + IAPS + International Affective Picture System + + + IADS + International Affective Digital Sounds + + + SAM + The Self-Assessment Manikin + + + + Sensory presentation + Object manifestation + + Auditory + Sound + + Nameable + + + Cash register + + + Ding + Often associated with positive valence + + + Buzz + Often associated with negative valence + + + Fire alarm + + + Click + + ABR + Auditory Brainstem Response + + + + Tone + + + Siren + + + Music + + Chord sequence + + + Vocal + + + Instrumental + + + Tone + + + Genre + + + + Noise + + White + + + Colored + Not white --- for example a 1/f spectrum + + + + Human voice + + + Animal voice + + Bird + + + Dog + + + Insect + + + Squirrel + + + + Real world + For example people walking or machines operating + + Pedestrian + + + Footsteps + + Walking + + + Running + + + + Noisemaker + + + Construction noise + + + Machine + + + Vehicle + + Horn + + + Aircraft + + Airplane + + + Helicopter + + + + Train + + + Cart + + + Car alarm + + + Car + + + Bicycle + + + + + Nonverbal vocal + + Emotional + + Crying + + + Sighing + + + + Gulp + + + Gurgle + + + Sneeze + + + Cough + + + Yawn + + + + Nonvocal + A car engine or gears grinding --- anything that is not made by a human or an animal + + Engine + + + + + Olfactory + Odor + + + Taste + + + Tactile + Pressure + + + Visual + + Rendering type + + Screen + + Head-mounted display + + + View port + Two or more views on the same object --- for example one from top one from street view + + ID + + # + A descriptive label for the viewport + + + + + 2D + + + 3D + + + Movie + + Video-tape + + + Motion-capture + Stick figure of motion capture of someone else + + Point light + + + Stick figure + + + Outline + + + + Flickering + + + Steady state + + + + + Real-world + + + LED + Stimulus is turning on/off one or a few LEDs + + + + + + Attribute + + Onset + Default + + + Offset + + + Imagined + This is used to identity that the (sub)event only happened in the imagination of the participant, e.g. imagined movements in motor imagery paradigms. + + + State ID + This is used to identify a group of events that are changing the state of a variable where the onset means the offset of any other and a change in the state + + # + ID which could be a number or any string + + + + Repetition + When the same type of event such as a fixation on the exact same object happens multiple times and it might be necessary to distinguish the first look vs. others + + # + Number starting from 1 where 1 indicates the first occurrence and 2 indicates the second occurrence + + + + Temporal rate + + # + In Hz + + + + Condition + Specifies the value of an independent variable (number of letters, N, in an N-back task) or function of independent variables that is varied or controlled for in the experiment. This attribute is often specified at the task level and can be associated with specification of an experimental stimulus or an experiment context (e.g. 1-back, 2-back conditions in an N-back task, or changing the target type from faces to houses in a Rapid Serial Visual Presentation, or RSVP, task.) + + # + the condition + + + + Action judgment + External judgment (assumed to be ground truth, e.g. from an experiment control software or an annotator) about participant actions such as answering a question, failing to answer in time, etc. + + Correct + + + Incorrect + Wrong choice but not time out + + + Indeterminate + It cannot be determined that the action was correct or incorrect. + + + Time out + + Missed + Participant failed or could not have perceived the instruction due to their eyes being off-screen or a similar reason. Not easy to deduce this but it is possible + + + + Inappropriate + A choice that is not allowed such as moving a chess piece to a location it should not go based on game rules + + + + Response start delay + The time interval between this (stimulus) event and the start of the response event specified, usually by grouping with the event ID of the response start event. + + # + + + + Response end delay + The time interval between this (stimulus) event and the end of the response event specified, usually by grouping with the event ID of the response end event. + + # + + + + Social + Involving interactions among multiple agents such as humans or dogs or robots + + + Peak + Peak velocity or acceleration or jerk + + + Object side + Could be the left, right, or both sides of a person or a vehicle + + Reference object ID + Place object ID after this + + # + + + + Right + + + Left + + + Front + + + Back + + + Top + + + Bottom + + + Starboard + + + Port + + + Passenger side + Side of a car + + + Driver side + Side of a car + + + Bow + Front of a ship + + + Stern + Back of the ship + + + + Direction + Coordinate system is inferred from Attribute/Location. To specify a vector combine subnodes with number --- for example Attribute/Top/10, Attribute/Direction/Left/5 to create a vector with coordinates 10 and 5 + + Top + Combine Attribute/Direction/Top and Attribute/Direction/Left to mean the upper left + + # + + + + Bottom + + # + + + + Left + + # + + + + Right + + # + + + + Angle + Clockwise angle in degrees from vertical + + # + Clockwise angle in degrees from vertical + + + + North + + # + + + + South + + # + + + + East + + # + + + + West + + # + + + + Forward + Like a car moving forward + + + Backward + Like a car moving backward + + + + Location + Spot or center of an area. Use Area were you are referring to something with significant extent and emphasizing its boundaries, like a city + + # + location label + + + Screen + Specify displacements from each subnode in pixels or degrees or meters. Specify units such as Attribute/Location/Screen/Top/12 px + + Center + + # + + + + Top + You can combine Attribute/Location/Top and Attribute/Location/Left to designate UpperLeft and so on + + # + + + + Bottom + + # + + + + Left + + # + + + + Right + + # + + + + Angle + + # + Clockwise angle in degrees from vertical + + + + Center displacement + + # + displacement from screen center, in any direction, in degrees, cm, or other lengths + + + Horizontal + + # + Displacement from screen center in any direction + + + + Vertical + + # + Displacement from screen center in any direction + + + + + + Lane + For example a car lane + + Rightmost + + + Leftmost + + + Right of expected + + + Left of expected + + + Cruising + + + Passing + The lane that cars use to take over other cars + + + Oncoming + + + + Real-world coordinates + + Room + + xyz + have a subnode, e.g. Attribute/Location/Real-world coordinates/Room/xyz/10 50 30 + + # + + + + + + Reference frame + + Specified absolute reference + + + Relative to participant + + Participant ID + + # + + + + Left + + + Front + + + Right + + + Back + + + Distance + + # + Distance is in meters by default + + + Near + + + Moderate + + + Far + + + + Azimuth + + # + Clockwise with units preferably in degrees + + + + Elevation + + # + Preferably in degrees + + + + + + + Object orientation + + Rotated + + Degrees + + # + Preferably in degrees + + + + + + Size + + Length + + # + In meters or other units of length + + + + Width + + # + in meters + + + + Height + + # + Default units are meters + + + + Area + + # + + + + Volume + + # + In cubic-meters or other units of volume + + + + Angle + + # + In degrees or other units of angle + + + + + Item count + Number of items for example when there are 3 cars and they are identified as a single item + + # + Numeric value of number of items # + + + <=# + Number of items less than or equal to # + + + >=# + Number of items more than or equal to # + + + + Auditory + + Frequency + + # + In HZ + + + + Loudness + + # + in dB + + + + Ramp up + Increasing in amplitude + + + Ramp down + Decreasing in amplitude + + + + Blink + + Time shut + The amount of time the eyelid remains closed (typically measured as 90% of the blink amplitude), in seconds. + + # + + + + Duration + Duration of blink, usually the half-height blink duration in seconds taken either from base or zero of EEG signal. For eye-trackers, usually denotes interval when pupil covered by eyelid. + + # + + + + PAVR + Amplitude-Velocity ratio, in centiseconds + + # + + + + NAVR + Negative Amplitude-Velocity ratio, in centiseconds + + # + + + + + Visual + + Bistable + + + Background + + + Foreground + + + Up-down separated + Stimuli presented both at the top and the bottom of fovea + + # + Angle of separation in degrees by default + + + + Bilateral + For bilateral visual field stimulus presentations + + # + Angle of separation in degrees by default + + + + Motion + + Down + + # + e.g. 3 degrees-per-second + + + + Up + + # + e.g. 3 degrees-per-second + + + + Horizontal + + Right + + # + e.g. 3 degrees-per-second + + + + Left + + # + e.g. 3 degrees-per-second + + + + + Oblique + + Clock face + + # + For example 4:30 + + + + + + Fixation point + + + Luminance + + # + In candelas by default + + + + Color + + Dark + + + Light + + + Aqua + These are CSS 3 basic color names + + + Black + + + Fuchsia + + + Gray + + + Lime + + + Maroon + + + Navy + + + Olive + + + Purple + + + Silver + + + Teal + + + White + + + Yellow + + + Red + + # + R value of RGB between 0 and 1 + + + + Blue + + # + B value of RGB between 0 and 1 + + + + Green + + # + G value of RGB between 0 and 1 + + + + Hue + + # + H value of HSV between 0 and 1 + + + + Saturation + + # + S value of HSV between 0 and 1 + + + + Value + + # + V value of HSV between 0 and 1 + + + + Achromatic + Indicates gray scale + + # + White intensity between 0 and 1 + + + + + + Nonlinguistic + Something that conveys meaning without using words such as the iconic pictures of a man or a woman on the doors of restrooms. Another example is a deer crossing sign with just a picture of jumping deer. + + + Semantic + Like in priming or in congruence + + + Language + + Unit + + Phoneme + + + Syllable + + + Word + + Noun + + Proper + A proper noun that refers to a unique entity such as London or Jupiter + + + Common + A noun that refers to a class of entities such as cities or planets or corporations such as a Dog or a Skyscraper + + + + Verb + + + Adjective + + + Pseudoword + + + # + Actual word + + + + Sentence + + Full + + + Partial + + + # + Actual sentence + + + + Paragraph + + # + Actual paragraph + + + + Story + Multiple paragraphs making a detailed account + + + + Family + + Asian + + Chinese + + + Japanese + + + + Latin + + English + + + German + + + French + + + + + + Induced + Such as inducing emotions or keeping someone awake or in a coma with an external intervention + + + Emotional + + Arousal + Only in the context of 2D emotion representation + + # + A value between -1 and 1 + + + + Positive valence + Valence by itself can be the name of an emotion such as sadness so this tag distinguishes the type of emotion + + # + Ranges from 0 to 1 + + + + Negative valence + + # + Ranges from 0 to 1 + + + + + Priming + + Motoric + + + Emotional + + + Perceptual + + + + Subliminal + + Unmasked + + + Masked + + Forward + + + Backward + + + + + Supraliminal + By default this is assumed about each stimulus + + + Liminal + At the 75%-25% perception threshold + + + Probability + Use to specify the level of certainty about the occurrence of the event. Use either numerical values as the child node or low, high, etc. + + + Temporal uncertainty + Use to specify the amount of uncertainty in the timing of the event. Please notice that this is different from Attribute/Probability tag which relates to the occurrence of event and can be interpretative as the integral of probability density across a distribution whose shape (temporal extent) is specified by Attribute/Temporal uncertainty + + # + + + Standard deviation + implies that the distribution of temporal uncertainty is Gaussian with the provided standard deviation (in seconds). + + # + + + + + Presentation + Attributes associated with visual, auditory, tactile, etc. presentation of an stimulus + + Fraction + the fraction of presentation of an Oddball or Expected stimuli to the total number of same-class presentations, e.g. 10% of images in an RSVP being targets + + # + + + + Cued + what is presented is cued to the presentation of something else + + + Background + presented in the background such as background music, background image, etc. The main factor here is that background presentations are to be ignored, e.g. ignore math question auditory stimuli. + + + + Intended effect + This tag is to be grouped with Participant/Effect/Cognitive to specify the intended cognitive effect (of the experimenter). This is to differentiate the resulting group with Participant/Effect which specifies the actual effect on the participant. For example, in an RSVP experiment if an image is intended to be perceived as a target, (Participant/Effect/Cognitive/Target, Attribute/Intended effect) group is added. If the image was perceived by the subject as a target (e.g. they pressed a button to indicate so), then the tag Participant/Effect/Cognitive/Target is also added: Participant/Effect/Cognitive/Target, (Participant/Effect/Cognitive/Target, Attribute/Intended effect). otherwise the Participant/Effect/Cognitive/Target tag is not included outside of the group. + + + Instruction + This tag is placed in events of type Event/Category/Experimental stimulus/Instruction, grouped with one or more Action/ tags to replace the detailed specification XXX in Event/Category/Experimental stimulus/Instruction/XXX) in previous versions. Usage example: Event/Category/Experimental stimulus/Instruction, (Action/Fixate, Attribute/Instruction) + + + Participant indication + This tag is placed in events of type Event/Category/Participant response and grouped with Participant/Effect/Cognitive/.. tags to specify the type of cognitive effect the participant has experienced. For example, in an RSVP paradigm, the subject can indicate the detection of a target with a button press. The HED string associated with this button press must include (Attribute/Participant indication, Participant/Effect/Cognitive/Target,...) + + + Path + + Velocity + Use Attribute/Onset or Attribute/Offset to specify onset or offset + + # + Numeric value with default units of m-per-s + + + + Acceleration + + # + Numeric value with default units of m-per-s2 + + + + Jerk + + # + Numeric value with default units of m-per-s3 + + + + Constrained + For example a path cannot cross some region + + + + File + File attributes + + Name + + + Size + + # + Numeric value with default units of mb + + + + # + Number of files + + + + Object control + Specifies control such as for a vehicle + + Perturb + + + Collide + + + Near miss + Almost having an accident resulting in negative consequences + + + Correct position + After a lane deviation or side of the walkway + + + Halt + Time at which speed becomes exactly zero + + + Brake + + + Shift lane + + + Cross + Crossing in front of another object such as a vehicle + + + Pass by + Passing by another object or the participant + + + Accelerate + + + Decelerate + + + + Association + + Another person + Item such as a cup belonging to another person + + + Same person + Item such as a cup belonging to the participant + + + + Extraneous + Button presses that are not meaningful for example due to intrinsic mechanical causes after a meaningful press + + + Role + The role of the agent (participant, character, AI..) + + Leader + + + Follower + + + # + + + + + Action + May or may not be associated with a prior stimulus and can be extended + + Involuntary + Like sneezing or tripping on something or hiccuping + + Hiccup + + + Cough + + + Sneeze + + + Stumble + Temporary and involuntary loss of balance + + + Fall + + + Tether Jerk + When a tether attached to the subject is stuck/snagged and forces the participant to involuntary accommodate/react to it + + + Clear Throat + + + Yawn + + + Sniffle + + + Burp + + + Drop + For example something drops from the hand of the subject + + + + Make fist + + Open and close + Continue to open and close the fist, for example in motor imagery paradigms. + + + + Curl toes + + Open and close + Continue to curl and uncurl toes, for example in motor imagery paradigms. + + + + Button press + + Touch screen + + + Keyboard + + + Mouse + + + Joystick + + + + Button hold + Press a button and keep it pressed + + + Button release + + + Cross boundary + + Arrive + + + Depart + + + + Speech + + + Hum + + + Eye saccade + Use Attribute/Peak for the middle of saccade and Attribute/Onset for the start of a saccade + + + Eye fixation + + + Eye blink + + Left base + The time of the first detectable eyelid movement on closing. + + + Left zero + The last time at which the EEG/EOG signal crosses zero during eyelid closing. + + + Left half height + The time at which the EEG/EOG signal reaches half maximum height during eyelid closing. + + + Max + The time at which the eyelid is the most closed. + + + Right half height + The time at which the EEG/EOG signal reaches half maximum height during eyelid opening. + + + Right zero + The first time at which the EEG/EOG signal crosses zero during eyelid opening. + + + Right base + The time of the last detectable eyelid movement on opening. + + + + Eye close + Close eyes and keep closed for more than approximately 0.1 s + + Keep + Keep the eye closed. If a value is provided it indicates the duration for this. + + # + the duration (by default in seconds) that they keep their eye closed. + + + + + Eye open + Open eyes and keep open for more than approximately 0.1 s + + Keep + Keep the eye open. If a value is provided it indicates the duration for this. + + # + the duration (by default in seconds) that they keep their eye open. + + + With blinking + Default. Allow blinking during the eye-open period. + + + Without blinking + Without blinking during the eye-open period. + + + + + Turn + Change in direction of movement or orientation. This includes both turn during movement on a path and also rotations such as head turns + + + Point + + + Push + + + Grab + + + Tap + When there is nothing to be pressed for example like tapping a finger on a chair surface to follow a rhythm + + + Lift + + + Reach + Requires a goal such as reaching to touch a button or to grab something. Stretching your body does not count as reach. + + To Grab + + + To Touch + + + + Course correction + Change the direction of a reach in the middle to adjust for a moving target. + + + Interact + + With human + + + + Take survey + + + Stretch + Stretch your body such as when you wake up + + + Bend + + + Deep breath + + + Laugh + + + Sigh + + + Groan + + + Scratch + + + Switch attention + + Intramodal + In the same modality but with a change in details such as changing from paying attention to red dots and instead of blue dots + + Visual + + + Auditory + + + Tactile + + + Taste + + + Smell + + + + Intermodal + Between modalities such as changing from audio to visual + + From modality + + Visual + + + Auditory + + + Tactile + + + Taste + + + Smell + + + + To modality + + Visual + + + Auditory + + + Tactile + + + Taste + + + Smell + + + + + + Walk + + Stride + Use onset and offset attributes to indicate different walking stride stages + + + Faster + increasing the speed of walking + + + Slower + decreasing the speed of walking + + + + Control vehicle + Controlling an object that you are aboard + + Drive + Driving a vehicle such as a car + + Correct + Correct for a perturbation + + + Near miss + + + Collide + + + + Stop + Brake a car + + + Pilot + Pilot a vehicle such as an airplane + + + + Teleoperate + Control an object that you are not aboard + + + Allow + Allow access to something such as allowing a car to pass + + + Deny + Deny access to something such as preventing someone to pass + + + Step around + + + Step over + + + Step on + + + Swallow + + + Flex + + + Evade + + + Shrug + + + Dance + + + Open mouth + + + Whistle + + + Read + + + Attend + + + Recall + + + Generate + + + Repeat + + + Hold breath + + + Breathe + + + Rest + + + Count + + + Move + + Upper torso + + + Lower torso + + + Whole body + + + + Speak + + + Sing + + + Detect + + + Name + + + Smile + + + Discriminate + + + Track + + + Encode + + + Eye-blink inhibit + + + + Participant + + ID + If not given assume 1 + + # + Numeric value of an ID + + + + Effect + How the stimulus effects the participants + + Cognitive + + Meaningful + + + Not meaningful + + + Newly learned meaning + + + Reward + + Low + + + Medium + + + High + + + # + Monetary values in some currency such as $10, or the ratio of the reward to the maximum possible (3 of max 10 becomes 0.3), or number Points + + + + Penalty + + Low + + + Medium + + + High + + + # + Absolute monetary values in some currency, for example $1, or the ratio of the reward to the maximum possible (3 of max 10 becomes 0.3), or number of Points + + + + Error + + Self originated + + + Other originated + + Human + + + Non-human + + + + Expected + + + Unexpected + + + Planned + The error feedback was given regardless of the validity of subject response as in a yoked design + + + + Threat + + To self + + + To others + + Close + + + + + Warning + As in a warning message that you are getting too close to the shoulder in a driving task + + + Oddball + Unexpected or infrequent + + One stimulus + Only oddballs are present but no frequent stimuli exist. See http://dx.doi.org/10.1016/0167-8760(96)00030-X + + + Two stimuli + There are non-targets and targets. See http://dx.doi.org/10.1016/0167-8760(96)00030-X + + + Three stimuli + There are regular non-targets and targets and infrequent non-targets, see http://dx.doi.org/10.1016/0167-8760(96)00030-X + + + Silent counting + + + Button pressing for target + + + Button pressing for all + + + + Target + Something the subject is looking for + + + Non-target + Make sure to tag Expected if the Non-target is frequent + + + Novel + Genuinely novel such as an event occurring once or so per experiment + + + Expected + Of low information value, for example frequent Non-targets in an RSVP paradigm + + Standard + + + Distractor + + + + Valid + Something that is understood to be valid such as an ID matches the person being displayed and it has all the correct information + + + Invalid + Something that is understood to not be valid such as like an ID with an impossible date-of-birth, or a photo not matching the person presenting it + + + Congruence + + Congruent + Like in Stroop paradigm when blue colored text displays the word blue + + + Incongruent + Like in Stroop paradigm whena blue colored text reading displays the word red + + + Temporal synchrony + + Synchronous + When a mouse click sound happens right after clicking it + + + Asynchronous + When a mouse click sound happens with significant delay which give could the person a strange feeling. Or if in a movie the sound of the explosion is heard before it appears visually. + + + + + Feedback + + Correct + Confirm something went well and last action was correct + + + Incorrect + Confirm something went wrong and last action was incorrect + + + Non-informative + Feedback that provides no information in regards to correct, incorrect, etc. + + + Expected + Feedback was expected as in a positive feedback after a response that was expected to be correct. + + + Unexpected + Feedback was unexpected as when positive feedback was received when response was expected to be incorrect. + + + On accuracy + Feedback was provided by evaluating response accuracy + + + On reaction time + Feedback was provided by evaluating subject reaction time + + + To self + Default + + + To other + Observed feedback to another person such as in a social paradigm + + + Deterministic + Feedback has a fixed relationship to what happened before + + + Stochastic + Feedback is non-deterministic and does not have fixed relationship with what has happened before in the experiment + + + False feedback + Feedback that was not honest for example as in feedback of correct on an incorrect response or vice versa + + Negative + Negative feedback was provided when it was not deserved + + + Positive + Positive feedback was provided when it was not deserved + + + + + Cue + An indicator of a future event, e.g. a sound cue that in 2-5 seconds a perturbation in driving will occur. Use (... Participant/Effect/Cognitive/Cue ~ hed tags for the event to follow, or main aspect of the event to follow) syntax. + + Constant delay + The cue is for an event that will happen after a constant delay. + + # + The delay, e.g. in seconds. + + + + Variable delay + The cue is for an event that will happen after a variable delay. + + # + The interval, e.g. between 2-5 seconds, of the variable delay. + + + + + + Visual + + Foveal + + + Peripheral + + + Perturbation + Sudden movement or perturbation of the virtual environment in a car driving or other scenario + + + + Auditory + + Stereo + + + Mono + + Left + + + Right + + + + + TMS + + With SPGS + SPGS stands for spatial position guiding system + + + Without SPGS + SPGS stands for spatial position guiding system + + + + Tactile + + Vibration + + + Acupuncture + + + Eye puff + + + Swab + Mouth swab + + + + Vestibular + + Shaking + being shaken or jerked around + + + + Pain + + Heat + + + Cold + + + Pressure + + + Electric shock + + + Laser-evoked + + + + Taste + + + Smell + + + Body part + + Whole Body + + + Eye + + + Arm + + Hand + + Finger + + Index + + + Thumb + + + Ring + + + Middle + + + Small + Pinkie or little finger + + + + + + Leg + + Feet + + Toes + + + + + Head + + Face + + Eyebrow + + + Lip + + + Forehead + + + Mouth + + + Nose + + + Chin + + + Cheek + + + + + Torso + + + + + State + + Level of consciousness + + Awake + + + Drowsy + + + Sleep + + Stage + + # + a number between 1 to 4, or REM + + + + + Drunk + + + Anesthesia + + + Locked-in + + + Coma + + + Vegetative + + + Brain-dead + + + + Emotion + + Awe + + + Frustration + + + Joy + + + Anger + + + Happiness + + + Sadness + + + Love + + + Fear + + + Compassion + + + Jealousy + + + Contentment + + + Grief + + + Relief + + + Excitement + + + Disgust + + + Neutral + None of the above + + + + Sense of community + Primed to have an emotion such as patriotism + + # + SCI stands for Sense of Community Index + + + + Sense of social justice + + Distributive + + + Poverty + + + Inequality + + + Procedural + + + Interpersonal + + + Informational + + + + Stress level + + # + A number between 0 and 1 + + + + Task load + + # + A number between 0 and 1 + + + + Under time pressure + + Response window + + # + Default time is seconds + + + + Competitive + Subject is competing against an opponent as for example when the faster respondent wins + + + + Social interaction + Social + + Pseudo + Instructed so but actually not as when the other person may not exist in real world such as the case of a computer program agent + + + + Passive + There is a stimulus presentation but no behavioral measurements are collected from the subject. Subject is instructed not to make any behavioral outputs for example when told to carefully watch/listen/sense. The resting state is not considered passive. + + + Resting + State when there is no stimulus presentation and no behavioral outputs + + + Attention + + Top-down + Instructed to pay attention to something explicitly + + + Bottom-up + something captures your attention, like a big bang or your name + + Orienting + The lower state of the bottom-up or the pre-bottom up state + + + + Covert + Implicit + + + Overt + Explicit + + + Selective + If you have two circles but asked to pay attention to only one of them + + Divided + Attending to more than one object or location + + + + Focused + Paying a lot of attention + + + Sustained + Paying attention for a continuous time + + + Auditory + + + Visual + + + Tactile + + + Taste + + + Smell + + + To a location + Spatial -- use the location attribute to specify to where the attention is directed + + + Arousal + + + Alerting + Keeping the arousal up in order to respond quickly + + + Drowsy + + + Excited + + + Neutral + + + + + + Experiment context + Describes the context of the whole experiment or large portions of it and also includes tags that are common across all events + + # + Add common tags across all stimuli/ and/or responses here if all experimental events share /State/Drowsy, you can place it here instead of tagging each event individually + + + With chin rest + + + Sitting + + + Standing + + + Prone + As in on a bed + + + Running + + Treadmill + + + + Walking + + Treadmill + + + + Indoors + Default + + Clinic + Recording in a clinical setting such as in a hospital or medical office + + + Dim Room + + + + Outdoors + + Terrain + + Grass + + + Uneven + + + Boardwalk + + + Dirt + + + Leaves + + + Mud + + + Woodchip + + + Rocky + + + Gravel + + + Downhill + + + Uphill + + + + + Motion platform + Subject is on a motion platform such as one that produces simulated car movements + + + Fixed screen + + Distance + Assuming static subject + + # + Distance from subject eyes to the presentation screen for 30 cm from subject eyes to the monitor + + + + Width resolution + + # + Default units are pixels + + + + Height resolution + + # + Default units are pixels + + + + + Real world + + + Virtual world + + + + Custom + This node can be used to organize events in an alternative (parallel) hierarchy. You can define your custom tags and hierarchies without any restriction under this node. These tags will still be matched to each other as for example /Custom/Dance/Waltz is considered a subtype of /Custom/DanceExample. + + + HED + Hierarchical Event Descriptor + + # + HED specification version number: normally there is no need to specify the version number in the HED string since it will be matched by default to the most recent compliant version, but this tag can be used to specify the exact HED version the HED string was based on. + + + + Paradigm + See Tasks in http://www.cognitiveatlas.org/tasks and CogPo definitions of paradigms + + Action imitation task + + + Action observation task + + + Acupuncture task + + + Adult attachment interview + + + Alternating runs paradigm + + + Animal naming task + + + Antisaccade-prosaccade task + + + Attention networks test + + + Attentional blink task + + + Audio-visual target-detection task + + + Autism diagnostic observation schedule + + + Ax-cpt task + + + Backward digit span task + + + Backward masking + + + Balloon analogue risk task - BART + + + Behavioral investment allocation strategy - BIAS + + + Behavioral rating inventory of executive function + + + Benton facial recognition test + + + Birmingham object recognition battery + + + Block design test + + + Block tapping test + + + Boston naming test + + + Braille reading task + + + Breath-holding + + + Breathhold paradigm + + + Brixton spatial anticipation test + + + California verbal learning test + + + California verbal learning test-ii + + + Cambridge face memory test + + + Cambridge gambling task + + + Cambridge neuropsychological test automated battery + + + Catbat task + + + Category fluency test + + + Cattell culture fair intelligence test + + + Chewing-swallowing + + + Chimeric animal stroop task + + + Choice reaction time task + + + Choice task between risky and non-risky options + + + Classical conditioning + + + Clinical evaluation of language fundamentals-3 + + + Color trails test + + + Color-discrimination task + + + Color-word stroop task + + + Complex span test + + + Conditional stop signal task + + + Conditioning paradigm + + Behavioral conditioning paradigm + + + Classical conditioning paradigm + + + + Continuous performance task + + + Continuous recognition paradigm + + + Counting stroop task + + + Counting-calculation + + + Cued explicit recognition + + + Cups task + + + Deception task + + + Deductive reasoning paradigm + + + Deductive reasoning task + + + Delayed discounting task + + + Delayed match to sample task + + + Delayed nonmatch to sample task + + + Delayed recall test + + + Delayed response task + + Delayed matching to sample paradigm + + Sternberg paradigm + + + + + Devils task + + + Dichotic listening task + + + Digit cancellation task + + + Digit span task + + + Digit-symbol coding test + + + Directed forgetting task + + + Divided auditory attention + + + Divided auditory attention paradigm + + + Doors and people test + + + Dot pattern expectancy task + + + Drawing + + + Drawing paradigm + + + Dual-task paradigm + + + Early social communications scales + + + Eating paradigm + + + Eating-drinking + + + Embedded figures test + + + Emotional regulation task + + + Encoding paradigm + + + Encoding task + + + Episodic recall + + + Episodic recall paradigm + + + Eriksen flanker task + + + Extradimensional shift task + + + Eye Saccade paradigm + + Anti saccade paradigm + + + Simple saccade paradigm + + + + Face monitor-discrimination + + + Face n-back task + + + Fagerstrom test for nicotine dependence + + + Film viewing + + + Finger tapping task + + + Fixation task + + + Flashing checkerboard + + + Flexion-extension + + + Forward digit span task + + + Free word list recall + + + Glasgow coma scale + + + Go-no-go task + + + Grasping task + + + Gray oral reading test - 4 + + + Haptic illusion task + + + Hayling sentence completion test + + + Heat sensitization-adaptation + + + Heat stimulation + + + Hooper visual organization test + + + ID screening + Visual examination of multiple fields of an ID or document to detect invalid or suspicious fields. For example at a security checkpoint. + + + Imagined emotion + + + Imagined movement + + + Imagined objects-scenes + + + Instructed movement + + + Immediate recall test + + + Inductive reasoning aptitude + + + International affective picture system + + + Intradimensional shift task + + + Ishihara plates for color blindness + + + Isometric force + + + Item recognition paradigm + + Serial item recognition paradigm + + + + Item recognition task + + + Kanizsa figures + + + Keep-track task + + + Letter comparison + + + Letter fluency test + + + Letter naming task + + + Letter number sequencing + + + Lexical decision task + + + Listening span task + + + Macauthur communicative development inventory + + + Machine failure detection task + + + Matching familiar figures test + + + Matching pennies game + + + Maudsley obsessive compulsive inventory + + + Mechanical stimulation + + + Memory span test + + + Mental rotation task + + + Micturition task + + + Mini mental state examination + + + Mirror tracing test + + + Mismatch negativity paradigm + + + Mixed gambles task + + + Modified erikson scale of communication attitudes + + + Morris water maze + + + Motor sequencing task + + + Music comprehension-production + + + N-back task + + Letter n-back task + + + + Naming + + Covert + + + Overt + + + + Nine-hole peg test + + + Non-choice task to study expected value and uncertainty + + + Non-painful electrical stimulation + + + Non-painful thermal stimulation + + + Nonword repetition task + + + Object alternation task + + + Object-discrimination task + + + Oculomotor delayed response + + + Oddball discrimination paradigm + + Auditory oddball paradigm + + + Visual oddball paradigm + + Rapid serial visual presentation + + + + + Oddball task + + + Olfactory monitor-discrimination + + + Operation span task + + + Orthographic discrimination + + + Paced auditory serial addition test + + + Pain monitor-discrimination task + + + Paired associate learning + + + Paired associate recall + + + Pantomime task + + + Parrott scale + + + Passive listening + + + Passive viewing + + + Pattern comparison + + + Perturbed driving + + + Phonological discrimination + + + Picture naming task + + + Picture set test + + + Picture-word stroop task + + + Pitch monitor-discrimination + + + Pointing + + + Porteus maze test + + + Positive and negative affect scale + + + Posner cueing task + + + Probabilistic classification task + + + Probabilistic gambling task + + + Probabilistic reversal learning + + + Pseudoword naming task + + + Psychomotor vigilance task + + + Pursuit rotor task + + + Pyramids and palm trees task + + + Rapid automatized naming test + + + Rapid serial object transformation + + + Reading - Covert + + + Reading - Overt + + + Reading paradigm + + Covert braille reading paradigm + + + Covert visual reading paradigm + + + + Reading span task + + + Recitation-repetition - Covert + + + Recitation-repetition - Overt + + + Remember-know task + + + Response mapping task + + + Rest + + Rest eyes open + + + Rest eyes closed + + + + Retrieval-induced forgetting task + + + Reversal learning task + + + Reward task + + + Rey auditory verbal learning task + + + Rey-ostereith complex figure test + + + Reynell developmental language scales + + + Rhyme verification task + + + Risky gains task + + + Rivermead behavioural memory test + + + + + time + + second + s + day + minute + hour + + + + dateTime + + YYYY-MM-DDThh:mm:ss + + + + clockTime + + hour:min + hour:min:sec + + + + frequency + + hertz + Hz + + + + angle + + radian + rad + degree + + + + physicalLength + + metre + m + foot + mile + + + + pixels + + pixel + px + + + + area + + m^2 + px^2 + pixel^2 + + + + volume + + m^3 + + + + speed + + m-per-s + mph + kph + + + + acceleration + + m-per-s^2 + + + + jerk + + m-per-s^3 + + + + intensity + + dB + + + + luminousIntensity + + candela + cd + + + + memorySize + + byte + B + + + + currency + + dollar + $ + point + fraction + + + + + + deca + SI unit multiple representing 10^1 + + + da + SI unit multiple representing 10^1 + + + hecto + SI unit multiple representing 10^2 + + + h + SI unit multiple representing 10^2 + + + kilo + SI unit multiple representing 10^3 + + + k + SI unit multiple representing 10^3 + + + mega + SI unit multiple representing 10^6 + + + M + SI unit multiple representing 10^6 + + + giga + SI unit multiple representing 10^9 + + + G + SI unit multiple representing 10^9 + + + tera + SI unit multiple representing 10^12 + + + T + SI unit multiple representing 10^12 + + + peta + SI unit multiple representing 10^15 + + + P + SI unit multiple representing 10^15 + + + exa + SI unit multiple representing 10^18 + + + E + SI unit multiple representing 10^18 + + + zetta + SI unit multiple representing 10^21 + + + Z + SI unit multiple representing 10^21 + + + yotta + SI unit multiple representing 10^24 + + + Y + SI unit multiple representing 10^24 + + + deci + SI unit submultiple representing 10^-1 + + + d + SI unit submultiple representing 10^-1 + + + centi + SI unit submultiple representing 10^-2 + + + c + SI unit submultiple representing 10^-2 + + + milli + SI unit submultiple representing 10^-3 + + + m + SI unit submultiple representing 10^-3 + + + micro + SI unit submultiple representing 10^-6 + + + u + SI unit submultiple representing 10^-6 + + + nano + SI unit submultiple representing 10^-9 + + + n + SI unit submultiple representing 10^-9 + + + pico + SI unit submultiple representing 10^-12 + + + p + SI unit submultiple representing 10^-12 + + + femto + SI unit submultiple representing 10^-15 + + + f + SI unit submultiple representing 10^-15 + + + atto + SI unit submultiple representing 10^-18 + + + a + SI unit submultiple representing 10^-18 + + + zepto + SI unit submultiple representing 10^-21 + + + z + SI unit submultiple representing 10^-21 + + + yocto + SI unit submultiple representing 10^-24 + + + y + SI unit submultiple representing 10^-24 + + + diff --git a/tests/dataset.spec.js b/tests/dataset.spec.js new file mode 100644 index 00000000..008a5414 --- /dev/null +++ b/tests/dataset.spec.js @@ -0,0 +1,320 @@ +import chai from 'chai' +const assert = chai.assert +import { beforeAll, describe, it } from '@jest/globals' + +import * as hed from '../validator/dataset' +import { buildSchemas } from '../schema/init' +import { generateIssue as generateValidationIssue } from '../common/issues/issues' +import { SchemaSpec, SchemasSpec } from '../schema/specs' + +describe('HED dataset validation', () => { + const hedSchemaFile = 'tests/data/HED8.2.0.xml' + const hedLibrarySchemaFile = 'tests/data/HED_testlib_2.0.0.xml' + let hedSchemas + + beforeAll(async () => { + const spec1 = new SchemaSpec('', '8.2.0', '', hedSchemaFile) + const spec2 = new SchemaSpec('testlib', '2.0.0', 'testlib', hedLibrarySchemaFile) + const specs = new SchemasSpec().addSchemaSpec(spec1).addSchemaSpec(spec2) + hedSchemas = await buildSchemas(specs) + }) + + describe('Basic HED string lists', () => { + /** + * Test-validate a dataset. + * + * @param {Object} testDatasets The datasets to test. + * @param {Object} expectedIssues The expected issues. + */ + const validator = async function (testDatasets, expectedIssues) { + for (const [testDatasetKey, testDataset] of Object.entries(testDatasets)) { + assert.property(expectedIssues, testDatasetKey, testDatasetKey + ' is not in expectedIssues') + const [, testIssues] = hed.validateHedEvents(testDataset, hedSchemas, null, true) + assert.sameDeepMembers(testIssues, expectedIssues[testDatasetKey], testDataset.join(',')) + } + } + + it('should properly validate simple HED datasets', () => { + const testDatasets = { + empty: [], + singleValidLong: ['Event/Sensory-event'], + singleValidShort: ['Sensory-event'], + multipleValidLong: [ + 'Event/Sensory-event', + 'Item/Object/Man-made-object/Vehicle/Train', + 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red/0.5', + ], + multipleValidShort: ['Sensory-event', 'Train', 'RGB-red/0.5'], + multipleValidMixed: ['Event/Sensory-event', 'Train', 'RGB-red/0.5'], + multipleInvalid: ['Time-value/0.5 cm', 'InvalidEvent'], + } + + const expectedIssues = { + empty: [], + singleValidLong: [], + singleValidShort: [], + multipleValidLong: [], + multipleValidShort: [], + multipleValidMixed: [], + multipleInvalid: [ + generateValidationIssue('unitClassInvalidUnit', { + tag: testDatasets.multipleInvalid[0], + }), + generateValidationIssue('invalidTag', { tag: testDatasets.multipleInvalid[1] }), + ], + } + return validator(testDatasets, expectedIssues) + }) + }) + + describe('Full HED datasets', () => { + /** + * Test-validate a dataset. + * + * @param {Object} testDatasets The datasets to test. + * @param {Object} expectedIssues The expected issues. + */ + const validator = async function (testDatasets, expectedIssues) { + for (const [testDatasetKey, testDataset] of Object.entries(testDatasets)) { + assert.property(expectedIssues, testDatasetKey, testDatasetKey + ' is not in expectedIssues') + const [, testIssues] = hed.validateHedDataset(testDataset, hedSchemas, true) + assert.sameDeepMembers(testIssues, expectedIssues[testDatasetKey], testDataset.join(',')) + } + } + + it('should properly validate HED datasets without definitions', () => { + const testDatasets = { + empty: [], + singleValidLong: ['Event/Sensory-event'], + singleValidShort: ['Sensory-event'], + multipleValidLong: [ + 'Event/Sensory-event', + 'Item/Object/Man-made-object/Vehicle/Train', + 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red/0.5', + ], + multipleValidShort: ['Sensory-event', 'Train', 'RGB-red/0.5'], + multipleValidMixed: ['Event/Sensory-event', 'Train', 'RGB-red/0.5'], + multipleInvalid: ['Time-value/0.5 cm', 'InvalidEvent'], + } + const legalTimeUnits = ['s', 'second', 'day', 'minute', 'hour'] + const expectedIssues = { + empty: [], + singleValidLong: [], + singleValidShort: [], + multipleValidLong: [], + multipleValidShort: [], + multipleValidMixed: [], + multipleInvalid: [ + generateValidationIssue('unitClassInvalidUnit', { + tag: testDatasets.multipleInvalid[0], + }), + generateValidationIssue('invalidTag', { tag: testDatasets.multipleInvalid[1] }), + ], + } + return validator(testDatasets, expectedIssues) + }) + + it('should properly validate HED datasets with definitions', () => { + const testDatasets = { + valid: ['(Definition/BlueSquare,(Blue,Square))', '(Definition/RedCircle,(Red,Circle))'], + equalDuplicateDefinition: [ + '(Definition/BlueSquare,(Blue,Square))', + '(Definition/BlueSquare,(Blue,Square))', + '(Definition/RedCircle,(Red,Circle))', + ], + equalPartneredDuplicateDefinition: [ + '(Definition/BlueSquare,(Blue,Square))', + '(testlib:Definition/BlueSquare,(Blue,Square))', + '(Definition/RedCircle,(Red,Circle))', + ], + nonEqualDuplicateDefinition: [ + '(Definition/BlueSquare,(Blue,Square))', + '(Definition/BlueSquare,(RGB-blue/1.0,Square))', + '(Definition/RedCircle,(Red,Circle))', + ], + nonEqualPartneredDuplicateDefinition: [ + '(Definition/BlueSquare,(Blue,Square))', + '(testlib:Definition/BlueSquare,(RGB-blue/1.0,Square))', + '(Definition/RedCircle,(Red,Circle))', + ], + valueDuplicateDefinition: [ + '(Definition/BlueSquare,(Blue,Square))', + '(Definition/BlueSquare/#,(RGB-blue/#,Square))', + '(Definition/RedCircle,(Red,Circle))', + ], + valuePartneredDuplicateDefinition: [ + '(Definition/BlueSquare,(Blue,Square))', + '(testlib:Definition/BlueSquare/#,(RGB-blue/#,Square))', + '(Definition/RedCircle,(Red,Circle))', + ], + } + const expectedIssues = { + valid: [], + equalDuplicateDefinition: [], + equalPartneredDuplicateDefinition: [], + nonEqualDuplicateDefinition: [ + generateValidationIssue('duplicateDefinition', { + definition: 'BlueSquare', + tagGroup: '(Definition/BlueSquare,(Blue,Square))', + }), + generateValidationIssue('duplicateDefinition', { + definition: 'BlueSquare', + tagGroup: '(Definition/BlueSquare,(RGB-blue/1.0,Square))', + }), + ], + nonEqualPartneredDuplicateDefinition: [ + generateValidationIssue('duplicateDefinition', { + definition: 'BlueSquare', + tagGroup: '(Definition/BlueSquare,(Blue,Square))', + }), + generateValidationIssue('duplicateDefinition', { + definition: 'BlueSquare', + tagGroup: '(testlib:Definition/BlueSquare,(RGB-blue/1.0,Square))', + }), + ], + valueDuplicateDefinition: [ + generateValidationIssue('duplicateDefinition', { + definition: 'BlueSquare', + tagGroup: '(Definition/BlueSquare,(Blue,Square))', + }), + generateValidationIssue('duplicateDefinition', { + definition: 'BlueSquare', + tagGroup: '(Definition/BlueSquare/#,(RGB-blue/#,Square))', + }), + ], + valuePartneredDuplicateDefinition: [ + generateValidationIssue('duplicateDefinition', { + definition: 'BlueSquare', + tagGroup: '(Definition/BlueSquare,(Blue,Square))', + }), + generateValidationIssue('duplicateDefinition', { + definition: 'BlueSquare', + tagGroup: '(testlib:Definition/BlueSquare/#,(RGB-blue/#,Square))', + }), + ], + } + return validator(testDatasets, expectedIssues) + }) + }) + + describe('Full HED datasets with context', () => { + /** + * Test-validate a dataset. + * + * @param {Object} testDatasets The datasets to test. + * @param {string[]} testContext The context for the test datasets. + * @param {Object} expectedIssues The expected issues. + */ + const validator = async function (testDatasets, testContext, expectedIssues) { + for (const [testDatasetKey, testDataset] of Object.entries(testDatasets)) { + assert.property(expectedIssues, testDatasetKey, testDatasetKey + ' is not in expectedIssues') + const [, testIssues] = hed.validateHedDatasetWithContext(testDataset, testContext, hedSchemas, true) + assert.sameDeepMembers(testIssues, expectedIssues[testDatasetKey], testDataset.join(',')) + } + } + + it('should properly validate onset and offset ordering', () => { + const testContext = ['(Definition/Acc/#, (Acceleration/#, Red))', '(Definition/MyColor, (Label/Pie))'] + const testDatasets = { + singleOnsetOffset: ['(Def/MyColor, Onset)', '(Def/MyColor, Offset)', 'Red'], + singleOnsetInset: ['(Def/MyColor, Onset)', '(Def/MyColor, Inset)', 'Red'], + singleOnsetInsetOffset: ['(Def/MyColor, Onset)', '(Def/MyColor, Inset)', '(Def/MyColor, Offset)', 'Red'], + offsetForSameValue: ['(Def/Acc/4.2 m-per-s^2, Onset)', '(Def/Acc/4.2 m-per-s^2, Offset)', 'Red'], + insetForSameValue: ['(Def/Acc/4.2 m-per-s^2, Onset)', '(Def/Acc/4.2 m-per-s^2, Inset)', 'Red'], + insetOffsetForSameValue: [ + '(Def/Acc/4.2 m-per-s^2, Onset)', + '(Def/Acc/4.2 m-per-s^2, Inset)', + '(Def/Acc/4.2 m-per-s^2, Offset)', + 'Red', + ], + repeatedOnsetForNoValue: ['(Def/MyColor, Onset)', '(Def/MyColor, Onset)', 'Red', '(Def/MyColor, Offset)'], + repeatedOnsetForSameValue: [ + '(Def/Acc/4.2 m-per-s^2, Onset)', + 'Red', + '(Def/Acc/4.2 m-per-s^2, Onset)', + '(Def/Acc/4.2 m-per-s^2, Offset)', + ], + onsetOffsetForDifferentValues: [ + '(Def/Acc/4.2 m-per-s^2, Onset)', + '(Def/Acc/5.3 m-per-s^2, Onset)', + '(Def/Acc/4.2 m-per-s^2, Offset)', + '(Def/Acc/5.3 m-per-s^2, Offset)', + ], + onsetOffsetMixedInsetForDifferentValues: [ + '(Def/Acc/4.2 m-per-s^2, Onset)', + '(Def/Acc/5.3 m-per-s^2, Onset)', + '(Def/Acc/4.2 m-per-s^2, Offset)', + '(Def/Acc/5.3 m-per-s^2, Inset)', + '(Def/Acc/5.3 m-per-s^2, Offset)', + ], + repeatedInset: [ + '(Def/MyColor, Onset)', + '(Def/MyColor, Inset)', + '(Def/MyColor, Inset)', + '(Def/MyColor, Offset)', + 'Red', + ], + repeatedOffset: ['(Def/MyColor, Onset)', '(Def/MyColor, Offset)', 'Red', '(Def/MyColor, Offset)'], + offsetFirst: ['(Def/MyColor, Offset)', '(Def/MyColor, Onset)', 'Red', '(Def/MyColor, Offset)'], + insetFirst: ['(Def/MyColor, Inset)', '(Def/MyColor, Onset)', 'Red', '(Def/MyColor, Inset)'], + offsetForDifferentValue: ['(Def/Acc/4.2 m-per-s^2, Onset)', '(Def/Acc/5.3 m-per-s^2, Offset)', 'Red'], + insetForDifferentValue: ['(Def/Acc/4.2 m-per-s^2, Onset)', '(Def/Acc/5.3 m-per-s^2, Inset)', 'Red'], + duplicateTemporal: ['(Def/MyColor, Onset), (Def/MyColor, Offset)', '(Def/MyColor, Offset)', 'Red'], + } + const expectedIssues = { + singleOnsetOffset: [], + singleOnsetInset: [], + singleOnsetInsetOffset: [], + offsetForSameValue: [], + insetForSameValue: [], + insetOffsetForSameValue: [], + repeatedOnsetForNoValue: [], + repeatedOnsetForSameValue: [], + onsetOffsetForDifferentValues: [], + onsetOffsetMixedInsetForDifferentValues: [], + repeatedInset: [], + repeatedOffset: [ + generateValidationIssue('inactiveOnset', { + definition: 'MyColor', + tag: 'Offset', + }), + ], + offsetFirst: [ + generateValidationIssue('inactiveOnset', { + definition: 'MyColor', + tag: 'Offset', + }), + ], + insetFirst: [ + generateValidationIssue('inactiveOnset', { + definition: 'MyColor', + tag: 'Inset', + }), + ], + offsetForDifferentValue: [ + generateValidationIssue('inactiveOnset', { + definition: 'Acc/5.3 m-per-s^2', + tag: 'Offset', + }), + ], + insetForDifferentValue: [ + generateValidationIssue('inactiveOnset', { + definition: 'Acc/5.3 m-per-s^2', + tag: 'Inset', + }), + ], + duplicateTemporal: [ + generateValidationIssue('duplicateTemporal', { + string: testDatasets.duplicateTemporal[0], + definition: 'MyColor', + }), + generateValidationIssue('inactiveOnset', { + definition: 'MyColor', + tag: 'Offset', + }), + ], + } + return validator(testDatasets, testContext, expectedIssues) + }) + }) +}) diff --git a/tests/definitionManagerTests.spec.js b/tests/definitionManagerTests.spec.js deleted file mode 100644 index 7f59b1a5..00000000 --- a/tests/definitionManagerTests.spec.js +++ /dev/null @@ -1,89 +0,0 @@ -import chai from 'chai' -const assert = chai.assert -import { beforeAll, describe, afterAll } from '@jest/globals' -import path from 'path' - -import { buildSchemas } from '../schema/init' -import { SchemaSpec, SchemasSpec } from '../schema/specs' -import { parseHedString } from '../parser/parser' -import { definitionTestData } from './testData/definitionManagerTests.data' -import { shouldRun } from './testUtilities' -import { DefinitionManager } from '../parser/definitionManager' - -const skipMap = new Map() -const runAll = true -const runMap = new Map([['def-or-def-expand', ['valid-def-with-placeholder']]]) - -describe('DefinitionManager tests', () => { - const schemaMap = new Map([['8.3.0', undefined]]) - - beforeAll(async () => { - const spec3 = new SchemaSpec('', '8.3.0', '', path.join(__dirname, '../tests/data/HED8.3.0.xml')) - const specs3 = new SchemasSpec().addSchemaSpec(spec3) - const schemas3 = await buildSchemas(specs3) - schemaMap.set('8.3.0', schemas3) - }) - - afterAll(() => {}) - - describe.each(definitionTestData)('$name : $description', ({ name, schemaVersion, definitions, tests }) => { - let thisSchema - let defManager - - beforeAll(async () => { - thisSchema = schemaMap.get(schemaVersion) - const [defList, issues] = DefinitionManager.createDefinitions(definitions, thisSchema) - if (issues.length > 0) { - throw new Error(`Invalid test definitions: ${definitions}`) - } - defManager = new DefinitionManager() - const addIssues = defManager.addDefinitions(defList) - if (addIssues.length > 0) { - throw new Error(`Invalid test definitions: ${definitions}`) - } - }) - - afterAll(() => {}) - - const testDefinitions = function (test) { - const status = test.errors.length === 0 ? 'Expect pass' : 'Expect fail' - const header = `[${test.testname} (${status})]` - assert.isDefined(thisSchema, `header: ${test.schemaVersion} is not available in test ${test.testname}`) - - let thisDefManager - if (!test.definition) { - thisDefManager = defManager - } else { - thisDefManager = new DefinitionManager() - const [defsToAdd, defIssues] = DefinitionManager.createDefinitions([test.definition], thisSchema) - if (defIssues.length > 0) { - throw new Error(`Invalid test definitions: ${definitions}`) - } - thisDefManager.addDefinitions(defsToAdd) - } - const [parsedHed, issues] = parseHedString(test.stringIn, thisSchema, true, false, test.placeholderAllowed) - if (parsedHed === null && issues.length > 0) { - assert.deepStrictEqual( - issues, - test.errors, - `${header}: expected ${issues} errors but received ${test.errors}\n`, - ) - } - if (parsedHed === null) { - return - } - issues.push(...thisDefManager.validateDefs(parsedHed, thisSchema, test.placeholderAllowed)) - issues.push(...thisDefManager.validateDefExpands(parsedHed, thisSchema, test.placeholderAllowed)) - assert.deepStrictEqual(issues, test.errors, `${header}: expected ${issues} errors but received ${test.errors}\n`) - } - - test.each(tests)('$testname: $explanation ', (test) => { - if (shouldRun(name, test.testname, runAll, runMap, skipMap)) { - testDefinitions(test) - } else { - // eslint-disable-next-line no-console - console.log(`----Skipping definitionManagerTest ${name}: ${test.testname}`) - } - }) - }) -}) diff --git a/tests/event.spec.js b/tests/event.spec.js new file mode 100644 index 00000000..a2515f82 --- /dev/null +++ b/tests/event.spec.js @@ -0,0 +1,1594 @@ +import chai from 'chai' +const assert = chai.assert +import { beforeAll, describe, it } from '@jest/globals' + +import * as hed from '../validator/event' +import { buildSchemas } from '../schema/init' +import { parseHedString } from '../parser/parser' +import ParsedHedTag from '../parser/parsedHedTag' +import { HedValidator } from '../validator/event' +import { generateIssue } from '../common/issues/issues' +import { SchemaSpec, SchemasSpec } from '../schema/specs' +import { Schemas } from '../schema/containers' +import { TagSpec } from '../parser/tokenizer' + +describe('HED string and event validation', () => { + /** + * Validation base function. + * + * @param {Schemas} hedSchemas The HED schema collection used for testing. + * @param {typeof HedValidator} ValidatorClass A subclass of {@link HedValidator} to use for validation. + * @param {Object} testStrings A mapping of test strings. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorBase = function ( + hedSchemas, + ValidatorClass, + testStrings, + expectedIssues, + testFunction, + testOptions = {}, + ) { + for (const [testStringKey, testString] of Object.entries(testStrings)) { + assert.property(expectedIssues, testStringKey, testStringKey + ' is not in expectedIssues') + const [parsedTestString, parsingIssues] = parseHedString(testString, hedSchemas) + const validator = new ValidatorClass(parsedTestString, hedSchemas, testOptions) + const flattenedParsingIssues = Object.values(parsingIssues).flat() + if (flattenedParsingIssues.length === 0) { + testFunction(validator) + } + const issues = [].concat(flattenedParsingIssues, validator.issues) + assert.sameDeepMembers(issues, expectedIssues[testStringKey], testString) + } + } + + describe('HED generic syntax validation', () => { + /** + * Syntactic validation base function. + * + * This base function uses the generic {@link HedValidator} validator class. + * + * @param {Object} testStrings A mapping of test strings. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorSyntacticBase = function (testStrings, expectedIssues, testFunction, testOptions = {}) { + const dummySchema = new Schemas(null) + validatorBase(dummySchema, HedValidator, testStrings, expectedIssues, testFunction, testOptions) + } + + describe('Full HED Strings', () => { + const validatorSyntactic = validatorSyntacticBase + + //TODO: Remove this test -- the separate test for mismatched parentheses is no longer performed. + it.skip('(REMOVE) should not have mismatched parentheses', () => { + const testStrings = { + extraOpening: + 'Action/Reach/To touch,((Attribute/Object side/Left,Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px', + // The extra comma is needed to avoid a comma error. + extraClosing: + 'Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm),),Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px', + wrongOrder: + 'Action/Reach/To touch,((Attribute/Object side/Left),Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px),(Attribute/Location/Screen/Left/23 px', + valid: + 'Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px', + } + const expectedIssues = { + extraOpening: [generateIssue('parentheses', { opening: 2, closing: 1 })], + extraClosing: [generateIssue('parentheses', { opening: 1, closing: 2 })], + wrongOrder: [generateIssue('unopenedParenthesis', { index: 121, string: testStrings.wrongOrder })], + valid: [], + } + // No-op function as this check is done during the parsing stage. + // eslint-disable-next-line no-unused-vars + validatorSyntactic(testStrings, expectedIssues, (validator) => {}) + }) + + // TODO: This test is replaced by tokenizer tests and should be removed + it.skip('(REMOVE) should not have malformed delimiters', () => { + const testStrings = { + missingOpeningComma: + 'Action/Reach/To touch(Attribute/Object side/Left,Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px', + missingClosingComma: + 'Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm)Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px', + extraOpeningComma: + ',Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px', + extraClosingComma: + 'Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm),/Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px,', + multipleExtraOpeningDelimiter: + ',,Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px', + multipleExtraClosingDelimiter: + 'Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm),/Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px,,', + multipleExtraMiddleDelimiter: + 'Action/Reach/To touch,,(Attribute/Object side/Left,Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px,,Attribute/Location/Screen/Left/23 px', + valid: + 'Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px', + validDoubleOpeningParentheses: + 'Action/Reach/To touch,((Attribute/Object side/Left,Participant/Effect/Body part/Arm),Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px),Event/Duration/3 ms', + validDoubleClosingParentheses: + 'Action/Reach/To touch,(Attribute/Object side/Left,Participant/Effect/Body part/Arm,(Attribute/Location/Screen/Top/70 px,Attribute/Location/Screen/Left/23 px)),Event/Duration/3 ms', + } + const expectedIssues = { + missingOpeningComma: [generateIssue('commaMissing', { tag: 'Action/Reach/To touch(' })], + missingClosingComma: [ + generateIssue('commaMissing', { + tag: 'Participant/Effect/Body part/Arm)', + }), + ], + extraOpeningComma: [ + generateIssue('extraDelimiter', { + character: ',', + index: 0, + string: testStrings.extraOpeningComma, + }), + ], + extraClosingComma: [ + generateIssue('extraDelimiter', { + character: ',', + index: testStrings.extraClosingComma.length - 1, + string: testStrings.extraClosingComma, + }), + ], + multipleExtraOpeningDelimiter: [ + generateIssue('extraDelimiter', { + character: ',', + index: 0, + string: testStrings.multipleExtraOpeningDelimiter, + }), + generateIssue('extraDelimiter', { + character: ',', + index: 1, + string: testStrings.multipleExtraOpeningDelimiter, + }), + ], + multipleExtraClosingDelimiter: [ + generateIssue('extraDelimiter', { + character: ',', + index: testStrings.multipleExtraClosingDelimiter.length - 1, + string: testStrings.multipleExtraClosingDelimiter, + }), + generateIssue('extraDelimiter', { + character: ',', + index: testStrings.multipleExtraClosingDelimiter.length - 2, + string: testStrings.multipleExtraClosingDelimiter, + }), + ], + multipleExtraMiddleDelimiter: [ + generateIssue('extraDelimiter', { + character: ',', + index: 22, + string: testStrings.multipleExtraMiddleDelimiter, + }), + generateIssue('extraDelimiter', { + character: ',', + index: 121, + string: testStrings.multipleExtraMiddleDelimiter, + }), + ], + valid: [], + validDoubleOpeningParentheses: [], + validDoubleClosingParentheses: [], + } + // No-op function as this check is done during the parsing stage. + // eslint-disable-next-line no-unused-vars + validatorSyntactic(testStrings, expectedIssues, (validator) => {}) + }) + + // TODO: This test will be replaced by invalid character tests based on value classes + it.skip('should not have invalid characters', () => { + const testStrings = { + openingBrace: 'Attribute/Object side/Left,Participant/Effect{Body part/Arm', + closingBrace: 'Attribute/Object side/Left,Participant/Effect}/Body part/Arm', + openingBracket: 'Attribute/Object side/Left,Participant/Effect[Body part/Arm', + closingBracket: 'Attribute/Object side/Left,Participant/Effect]Body part/Arm', + tilde: 'Attribute/Object side/Left,Participant/Effect~/Body part/Arm', + doubleQuote: 'Attribute/Object side/Left,Participant/Effect"/Body part/Arm', + null: 'Attribute/Object side/Left,Participant/Effect/Body part/Arm\0', + tab: 'Attribute/Object side/Left,Participant/Effect/Body part/Arm\t', + } + const expectedIssues = { + openingBrace: [ + generateIssue('invalidCharacter', { + character: 'LEFT CURLY BRACKET', + index: 45, + string: testStrings.openingBrace, + }), + ], + closingBrace: [ + generateIssue('unopenedCurlyBrace', { + index: 45, + string: testStrings.closingBrace, + }), + ], + openingBracket: [ + generateIssue('invalidCharacter', { + character: 'LEFT SQUARE BRACKET', + index: 45, + string: testStrings.openingBracket, + }), + ], + closingBracket: [ + generateIssue('invalidCharacter', { + character: 'RIGHT SQUARE BRACKET', + index: 45, + string: testStrings.closingBracket, + }), + ], + tilde: [ + generateIssue('invalidCharacter', { + character: 'TILDE', + index: 45, + string: testStrings.tilde, + }), + ], + doubleQuote: [ + generateIssue('invalidCharacter', { + character: 'QUOTATION MARK', + index: 45, + string: testStrings.doubleQuote, + }), + ], + null: [ + generateIssue('invalidCharacter', { + character: 'NULL', + index: 59, + string: testStrings.null, + }), + ], + tab: [ + generateIssue('invalidCharacter', { + character: 'CHARACTER TABULATION', + index: 59, + string: testStrings.tab, + }), + ], + } + // No-op function as this check is done during the parsing stage. + // eslint-disable-next-line no-unused-vars + validatorSyntactic(testStrings, expectedIssues, (validator) => {}) + }) + }) + + describe.skip('HED Tag Levels', () => { + /** + * Tag level syntactic validation base function. + * + * @param {Object} testStrings A mapping of test strings. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator, ParsedHedSubstring[]): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorSyntactic = function (testStrings, expectedIssues, testFunction, testOptions = {}) { + validatorSyntacticBase( + testStrings, + expectedIssues, + (validator) => { + for (const tagGroup of validator.parsedString.tagGroups) { + for (const subGroup of tagGroup.subGroupArrayIterator()) { + testFunction(validator, subGroup) + } + } + testFunction(validator, validator.parsedString.parseTree) + }, + testOptions, + ) + } + + // TODO: Remove test as repeated in bidsTests + it.skip('(REMOVE) should not contain duplicates - now in bidsTests', () => { + const testStrings = { + noDuplicate: 'Event/Category/Experimental stimulus,Item/Object/Vehicle/Train,Attribute/Visual/Color/Purple', + legalDuplicate: 'Item/Object/Vehicle/Train,(Item/Object/Vehicle/Train,Event/Category/Experimental stimulus)', + nestedLegalDuplicate: + '(Item/Object/Vehicle/Train,(Item/Object/Vehicle/Train,Event/Category/Experimental stimulus))', + legalDuplicateDifferentValue: '(Attribute/Language/Unit/Word/Brain,Attribute/Language/Unit/Word/Study)', + twoNonDuplicateGroups: '(Red, Blue, (Green)), (Red, Blue, ((Green)))', + //topLevelDuplicate: 'Event/Category/Experimental stimulus,Event/Category/Experimental stimulus', + groupDuplicate: + 'Item/Object/Vehicle/Train,(Event/Category/Experimental stimulus,Attribute/Visual/Color/Purple,Event/Category/Experimental stimulus)', + nestedGroupDuplicate: + 'Item/Object/Vehicle/Train,(Attribute/Visual/Color/Purple,(Event/Category/Experimental stimulus,Event/Category/Experimental stimulus))', + twoDuplicateGroups: '(Red, Blue, (Green)), (Red, Blue, (Green))', + repeatedGroup: '(Red, (Blue, Green, (Yellow)), Red, (Blue, Green, (Yellow)))', + } + const expectedIssues = { + noDuplicate: [], + legalDuplicate: [], + nestedLegalDuplicate: [], + legalDuplicateDifferentValue: [], + twoNonDuplicateGroups: [], + topLevelDuplicate: [ + generateIssue('duplicateTag', { + tag: 'Event/Category/Experimental stimulus', + }), + generateIssue('duplicateTag', { + tag: 'Event/Category/Experimental stimulus', + }), + ], + groupDuplicate: [ + generateIssue('duplicateTag', { + tag: 'Event/Category/Experimental stimulus', + }), + generateIssue('duplicateTag', { + tag: 'Event/Category/Experimental stimulus', + }), + ], + nestedGroupDuplicate: [ + generateIssue('duplicateTag', { + tag: 'Event/Category/Experimental stimulus', + }), + generateIssue('duplicateTag', { + tag: 'Event/Category/Experimental stimulus', + }), + ], + twoDuplicateGroups: [ + generateIssue('duplicateTag', { + tag: '(Red, Blue, (Green))', + }), + generateIssue('duplicateTag', { + tag: '(Red, Blue, (Green))', + }), + ], + repeatedGroup: [ + generateIssue('duplicateTag', { + tag: 'Red', + }), + generateIssue('duplicateTag', { + tag: 'Red', + }), + generateIssue('duplicateTag', { + tag: '(Blue, Green, (Yellow))', + }), + generateIssue('duplicateTag', { + tag: '(Blue, Green, (Yellow))', + }), + ], + } + validatorSyntactic(testStrings, expectedIssues, (validator, tagLevel) => { + validator.checkForDuplicateTags(tagLevel) + }) + }) + }) + }) + + describe('HED-3G validation', () => { + const hedSchemaFile = 'tests/data/HED8.3.0.xml' + let hedSchemas + + beforeAll(async () => { + const spec3 = new SchemaSpec('', '8.3.0', '', hedSchemaFile) + const specs = new SchemasSpec().addSchemaSpec(spec3) + hedSchemas = await buildSchemas(specs) + }) + + /** + * Validation base function. + * + * This override is required due to incompatible constructor signatures between Hed3Validator and the other two classes. + * + * @param {Schemas} hedSchemas The HED schema collection used for testing. + * @param {Object} testStrings A mapping of test strings. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorBase = function (hedSchemas, testStrings, expectedIssues, testFunction, testOptions = {}) { + for (const [testStringKey, testString] of Object.entries(testStrings)) { + assert.property(expectedIssues, testStringKey, testStringKey + ' is not in expectedIssues') + const [parsedTestString, parsingIssues] = parseHedString(testString, hedSchemas) + const validator = new HedValidator(parsedTestString, hedSchemas, null, testOptions) + const flattenedParsingIssues = Object.values(parsingIssues).flat() + if (flattenedParsingIssues.length === 0) { + testFunction(validator) + } + const issues = [].concat(flattenedParsingIssues, validator.issues) + assert.sameDeepMembers(issues, expectedIssues[testStringKey], testString) + } + } + + /** + * HED 3 semantic validation base function. + * + * This base function uses the HED 3-specific {@link HedValidator} validator class. + * + * @param {Object} testStrings A mapping of test strings. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorSemanticBase = function (testStrings, expectedIssues, testFunction, testOptions = {}) { + validatorBase(hedSchemas, testStrings, expectedIssues, testFunction, testOptions) + } + + describe('Full HED Strings', () => { + const validatorSemantic = validatorSemanticBase + + // TODO: Remove - Units moved to bidsTests rest to stringParser tests + it.skip('REMOVE properly validate short tags', () => { + const testStrings = { + simple: 'Car', + groupAndValues: '(Train/Maglev,Age/15,RGB-red/0.5),Operate', + invalidUnit: 'Time-value/20 cm', + duplicateSame: 'Train,Train', + duplicateSimilar: 'Train,Vehicle/Train', + missingChild: 'Label', + } + const legalTimeUnits = ['s', 'second', 'day', 'minute', 'hour'] + const expectedIssues = { + simple: [], + groupAndValues: [], + invalidUnit: [ + generateIssue('unitClassInvalidUnit', { + tag: 'Time-value/20 cm', + unitClassUnits: legalTimeUnits.sort().join(','), + }), //Now in bidsTests + ], + duplicateSame: [ + generateIssue('duplicateTag', { + tag: 'Train', + }), + generateIssue('duplicateTag', { + tag: 'Train', + }), + ], + duplicateSimilar: [ + generateIssue('duplicateTag', { + tag: 'Train', + }), + generateIssue('duplicateTag', { + tag: 'Vehicle/Train', + }), + ], + missingChild: [ + generateIssue('childRequired', { + tag: 'Label', + }), + ], + } + return validatorSemantic(testStrings, expectedIssues, (validator) => { + validator.validateEventLevel() + }) + }) + + // TODO: REMOVE: The testing of units should be in the stringParser + it.skip('Test in string-parser should not validate strings with short-to-long conversion errors', () => { + const testStrings = { + // Duration/20 cm is an obviously invalid tag that should not be caught due to the first error. + red: 'Property/RGB-red, Duration/20 cm', + redAndBlue: 'Property/RGB-red, Property/RGB-blue/Blah, Duration/20 cm', + } + const expectedIssues = { + red: [ + generateIssue('invalidParentNode', { + tag: 'RGB-red', + parentTag: 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red', + }), + ], + redAndBlue: [ + generateIssue('invalidParentNode', { + tag: 'RGB-red', + parentTag: 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red', + }), + generateIssue('invalidParentNode', { + tag: 'RGB-blue', + parentTag: 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-blue', + }), + ], + } + // This is a no-op since short-to-long conversion errors are handled in the string parsing phase. + // eslint-disable-next-line no-unused-vars + return validatorSemantic(testStrings, expectedIssues, (validator) => {}) + }) + + it.skip('should not have overlapping onsets and offsets in the same string', () => { + const testStrings = { + onsetAndOffsetWithDifferentValues: '(Def/Acc/5.4, Offset), (Def/Acc/4.3, Onset)', + sameOffsetAndOnset: '(Def/MyColor, Offset), (Def/MyColor, Onset)', + sameOnsetAndOffset: '(Def/MyColor, Onset), (Def/MyColor, Offset)', + duplicateOnset: '(Def/MyColor, (Red), Onset), (Def/MyColor, Onset)', + } + const expectedIssues = { + onsetAndOffsetWithDifferentValues: [], + sameOffsetAndOnset: [ + generateIssue('duplicateTemporal', { string: testStrings.sameOffsetAndOnset, definition: 'MyColor' }), + ], + sameOnsetAndOffset: [ + generateIssue('duplicateTemporal', { string: testStrings.sameOnsetAndOffset, definition: 'MyColor' }), + ], + duplicateOnset: [ + generateIssue('duplicateTemporal', { string: testStrings.duplicateOnset, definition: 'MyColor' }), + ], + } + return validatorSemantic(testStrings, expectedIssues, (validator) => { + validator.validateEventLevel() + }) + }) + }) + + describe('Individual HED Tags', () => { + /** + * HED 3 individual tag semantic validation base function. + * + * @param {Object} testStrings A mapping of test strings. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator, ParsedHedTag, ParsedHedTag): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorSemantic = function (testStrings, expectedIssues, testFunction, testOptions) { + return validatorSemanticBase( + testStrings, + expectedIssues, + (validator) => { + let previousTag = new ParsedHedTag(new TagSpec('Sad', 0, 3, ''), validator.hedSchemas, 'Sad') + for (const tag of validator.parsedString.tags) { + testFunction(validator, tag, previousTag) + previousTag = tag + } + }, + testOptions, + ) + } + + /** + * HED 3 individual tag semantic validation base function. + * + * @param {Object} testStrings A mapping of test strings. + * @param {Object} testDefinitions A mapping of test definitions. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator, ParsedHedTag): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorSemanticWithDefinitions = function ( + testStrings, + testDefinitions, + expectedIssues, + testFunction, + testOptions, + ) { + const definitionMap = new Map() + return validatorSemanticBase( + testStrings, + expectedIssues, + (validator) => { + if (definitionMap.size === 0) { + for (const value of Object.values(testDefinitions)) { + const parsedString = parseHedString(value, validator.hedSchemas)[0] + for (const def of parsedString.definitionGroups) { + definitionMap.set(def.definitionName, def.definitionGroup) + } + } + } + validator.definitions = definitionMap + for (const tag of validator.parsedString.tags) { + testFunction(validator, tag) + } + }, + testOptions, + ) + } + + // TODO: Already covered in stringParserTests -- units still to move down. + /*it.skip('(REMOVE) should exist in the schema or be an allowed extension', () => { + const testStrings = { + takesValue: 'Time-value/3 ms', + full: 'Left-side-of', + extensionAllowed: 'Human/Driver', + leafExtension: 'Sensory-event/Something', + nonExtensionAllowed: 'Event/Nonsense', + illegalComma: 'Label/This_is_a_label,This/Is/A/Tag', + placeholder: 'Train/#', + } + const expectedIssues = { + takesValue: [], + full: [], + extensionAllowed: [generateIssue('extension', { tag: testStrings.extensionAllowed })], + leafExtension: [generateIssue('invalidExtension', { tag: 'Something', parentTag: 'Event/Sensory-event' })], + nonExtensionAllowed: [ + generateIssue('invalidExtension', { + tag: 'Nonsense', + parentTag: 'Event', + }), + ], + illegalComma: [ + generateIssue('invalidTag', { tag: 'This/Is/A/Tag' }), + /!* Intentionally not thrown (validation ends at parsing stage) + generateIssue('extraCommaOrInvalid', { + previousTag: 'Label/This_is_a_label', + tag: 'This/Is/A/Tag', + }), + *!/ + ], + placeholder: [ + generateIssue('invalidTag', { + tag: testStrings.placeholder, + }), + ], + } + return validatorSemantic( + testStrings, + expectedIssues, + (validator, tag, previousTag) => { + validator.checkIfTagIsValid(tag) + }, + { checkForWarnings: true }, + ) + })*/ + + /*// TODO: REMOVE as these tests have been moved to tagParserTests + it.skip('(REMOVE) now in tagParserTests - should have a proper unit when required', () => { + const testStrings = { + correctUnit: 'Time-value/3 ms', + correctUnitScientific: 'Time-value/3.5e1 ms', + correctSingularUnit: 'Time-value/1 millisecond', + correctPluralUnit: 'Time-value/3 milliseconds', + correctNoPluralUnit: 'Frequency/3 hertz', + incorrectNonSymbolCapitalizedUnit: 'Time-value/3 MilliSeconds', + correctSymbolCapitalizedUnit: 'Frequency/3 kHz', + incorrectUnit: 'Time-value/3 cm', + incorrectNonNumericValue: 'Time-value/A ms', + incorrectPluralUnit: 'Frequency/3 hertzs', + incorrectSymbolCapitalizedUnit: 'Frequency/3 hz', + incorrectSymbolCapitalizedUnitModifier: 'Frequency/3 KHz', + incorrectNonSIUnitModifier: 'Time-value/1 millihour', + incorrectNonSIUnitSymbolModifier: 'Speed/100 Mkph', + notRequiredNumber: 'RGB-red/0.5', + notRequiredScientific: 'RGB-red/5e-1', + /!*properTime: 'Clockface/08:30', + invalidTime: 'Clockface/54:54',*!/ + } + const expectedIssues = { + correctUnit: [], + correctUnitScientific: [], + correctSingularUnit: [], + correctPluralUnit: [], + correctNoPluralUnit: [], + incorrectNonSymbolCapitalizedUnit: [ + generateIssue('unitClassInvalidUnit', { + tag: testStrings.incorrectNonSymbolCapitalizedUnit, + }), + ], + correctSymbolCapitalizedUnit: [], + incorrectUnit: [ + generateIssue('unitClassInvalidUnit', { + tag: testStrings.incorrectUnit, + }), + ], + incorrectNonNumericValue: [ + generateIssue('invalidValue', { + tag: testStrings.incorrectNonNumericValue, + }), + ], + incorrectPluralUnit: [ + generateIssue('unitClassInvalidUnit', { + tag: testStrings.incorrectPluralUnit, + }), + ], + incorrectSymbolCapitalizedUnit: [ + generateIssue('unitClassInvalidUnit', { + tag: testStrings.incorrectSymbolCapitalizedUnit, + }), + ], + incorrectSymbolCapitalizedUnitModifier: [ + generateIssue('unitClassInvalidUnit', { + tag: testStrings.incorrectSymbolCapitalizedUnitModifier, + }), + ], + incorrectNonSIUnitModifier: [ + generateIssue('unitClassInvalidUnit', { + tag: testStrings.incorrectNonSIUnitModifier, + }), + ], + incorrectNonSIUnitSymbolModifier: [ + generateIssue('unitClassInvalidUnit', { + tag: testStrings.incorrectNonSIUnitSymbolModifier, + }), + ], + notRequiredNumber: [], + notRequiredScientific: [], + } + return validatorSemantic( + testStrings, + expectedIssues, + // eslint-disable-next-line no-unused-vars + (validator, tag, previousTag) => { + validator.checkIfTagUnitClassUnitsAreValid(tag) + }, + { checkForWarnings: true }, + ) + })*/ + + // TODO: BIDS sidecar validation does not detect missing definitions (under definition-tests in bidsTests) + it('should not contain undefined definitions', () => { + const testDefinitions = { + greenTriangle: '(Definition/GreenTriangleDefinition/#, (RGB-green/#, Triangle))', + train: '(Definition/TrainDefinition, (Train))', + yellowCube: '(Definition/CubeDefinition, (Yellow, Cube))', + } + const testStrings = { + greenTriangleDef: '(Def/GreenTriangleDefinition/0.5, Width/15 cm)', + trainDefExpand: '(Def-expand/TrainDefinition, (Age/20))', + yellowCubeDef: '(Def/CubeDefinition, Volume/50 m^3)', + invalidDef: '(Def/InvalidDefinition, Square)', + invalidDefExpand: '(Def-expand/InvalidDefExpand, (Circle))', + } + const expectedIssues = { + greenTriangleDef: [], + trainDefExpand: [], + yellowCubeDef: [], + invalidDef: [generateIssue('missingDefinitionForDef', { definition: 'InvalidDefinition' })], + invalidDefExpand: [generateIssue('missingDefinitionForDefExpand', { definition: 'InvalidDefExpand' })], + } + return validatorSemanticWithDefinitions(testStrings, testDefinitions, expectedIssues, (validator, tag) => { + validator.checkForMissingDefinitions(tag, 'Def') + validator.checkForMissingDefinitions(tag, 'Def-expand') + }) + }) + + // TODO: This has been fixed + it('should have a child when required', () => { + const testStrings = { + noRequiredChild: 'Red', + hasRequiredChild: '(Duration/5, (Red))', + missingChild: '(Duration)', + longMissingChild: '(Property/Data-property/Data-value/Spatiotemporal-value/Temporal-value/Duration)', + } + const expectedIssues = { + noRequiredChild: [], + hasRequiredChild: [], + missingChild: [ + generateIssue('childRequired', { + tag: 'Duration', + }), + ], + longMissingChild: [ + generateIssue('childRequired', { + tag: 'Property/Data-property/Data-value/Spatiotemporal-value/Temporal-value/Duration', + }), + ], + } + return validatorSemantic(testStrings, expectedIssues, (validator) => { + validator.validateEventLevel() + }) + }) + }) + + //TODO: Error codes have changed because errors are detected earlier + describe.skip('(TRANSFER):HED Tag Groups', () => { + /** + * HED 3 tag group semantic validation base function. + * + * @param {Object} testStrings A mapping of test strings. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator, ParsedHedGroup): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorSemantic = function (testStrings, expectedIssues, testFunction, testOptions = {}) { + return validatorSemanticBase( + testStrings, + expectedIssues, + (validator) => { + for (const parsedTagGroup of validator.parsedString.tagGroups) { + testFunction(validator, parsedTagGroup) + } + }, + testOptions, + ) + } + + // TODO: (REMOVE) Equivalent tests now in stringParser + it.skip('should have syntactically valid definitions', () => { + const testStrings = { + nonDefinition: 'Car', + nonDefinitionGroup: '(Train/Maglev, Age/15, RGB-red/0.5)', + definitionOnly: '(Definition/SimpleDefinition)', + tagGroupDefinition: '(Definition/TagGroupDefinition, (Square, RGB-blue))', + illegalSiblingDefinition: '(Definition/IllegalSiblingDefinition, Train, (Rectangle))', + nestedDefinition: '(Definition/NestedDefinition, (Touchscreen, (Definition/InnerDefinition, (Square))))', + multipleTagGroupDefinition: '(Definition/MultipleTagGroupDefinition, (Touchscreen), (Square))', + defNestedInDefinition: '(Definition/DefNestedInDefinition, (Def/Nested, Triangle))', + } + const expectedIssues = { + nonDefinition: [], + nonDefinitionGroup: [], + definitionOnly: [], + tagGroupDefinition: [], + illegalSiblingDefinition: [ + generateIssue('illegalDefinitionGroupTag', { + tag: 'Train', + definition: 'IllegalSiblingDefinition', + }), + ], + nestedDefinition: [ + generateIssue('invalidTopLevelTagGroupTag', { + tag: 'Definition/InnerDefinition', + string: '(Definition/NestedDefinition, (Touchscreen, (Definition/InnerDefinition, (Square))))', + }), + ], + multipleTagGroupDefinition: [ + generateIssue('multipleTagGroupsInDefinition', { + definition: 'MultipleTagGroupDefinition', + }), + ], + defNestedInDefinition: [ + generateIssue('nestedDefinition', { + definition: 'DefNestedInDefinition', + }), + ], + } + return validatorSemantic(testStrings, expectedIssues, (validator, tagGroup) => { + validator.checkDefinitionGroupSyntax(tagGroup) + }) + }) + + it.each(['Onset', 'Inset'])('should have syntactically valid %s tags', (temporalTagName) => { + const testStrings = { + simple: `(${temporalTagName}, Def/Acc/5.4)`, + defAndOneGroup: `(${temporalTagName}, Def/MyColor, (Red))`, + defExpandAndOneGroup: `(${temporalTagName}, (Def-expand/MyColor, (Label/Pie)), (Red))`, + noTag: `(${temporalTagName})`, + definition: `(${temporalTagName}, Definition/MyColor, (Label/Pie))`, + defAndTwoGroups: `(Def/DefAndTwoGroups, (Blue), (Green), ${temporalTagName})`, + defExpandAndTwoGroups: `((Def-expand/DefExpandAndTwoGroups, (Label/Pie)), (Green), (Red), ${temporalTagName})`, + tagAndNoDef: `(${temporalTagName}, Red)`, + tagGroupAndNoDef: `(${temporalTagName}, (Red))`, + defAndTag: `(${temporalTagName}, Def/DefAndTag, Red)`, + defTagAndTagGroup: `(${temporalTagName}, Def/DefTagAndTagGroup, (Red), Blue)`, + multipleDefs: `(${temporalTagName}, Def/MyColor, Def/Acc/5.4)`, + defAndDefExpand: `((Def-expand/MyColor, (Label/Pie)), Def/Acc/5.4, ${temporalTagName})`, + multipleDefinitionsAndExtraTagGroups: `((Def-expand/MyColor, (Label/Pie)), Def/Acc/5.4, ${temporalTagName}, (Blue), (Green))`, + } + const expectedIssues = { + simple: [], + defAndOneGroup: [], + defExpandAndOneGroup: [], + noTag: [generateIssue('temporalWithoutDefinition', { tagGroup: testStrings.noTag, tag: temporalTagName })], + definition: [ + generateIssue('temporalWithoutDefinition', { tagGroup: testStrings.definition, tag: temporalTagName }), + generateIssue('extraTagsInTemporal', { definition: null, tag: temporalTagName }), + ], + defAndTwoGroups: [ + generateIssue('extraTagsInTemporal', { definition: 'DefAndTwoGroups', tag: temporalTagName }), + ], + defExpandAndTwoGroups: [ + generateIssue('extraTagsInTemporal', { definition: 'DefExpandAndTwoGroups', tag: temporalTagName }), + ], + tagAndNoDef: [ + generateIssue('temporalWithoutDefinition', { tagGroup: testStrings.tagAndNoDef, tag: temporalTagName }), + generateIssue('extraTagsInTemporal', { definition: null, tag: temporalTagName }), + ], + tagGroupAndNoDef: [ + generateIssue('temporalWithoutDefinition', { + tagGroup: testStrings.tagGroupAndNoDef, + tag: temporalTagName, + }), + ], + defAndTag: [generateIssue('extraTagsInTemporal', { definition: 'DefAndTag', tag: temporalTagName })], + defTagAndTagGroup: [ + generateIssue('extraTagsInTemporal', { definition: 'DefTagAndTagGroup', tag: temporalTagName }), + ], + multipleDefs: [ + generateIssue('temporalWithMultipleDefinitions', { + tagGroup: testStrings.multipleDefs, + tag: temporalTagName, + }), + ], + defAndDefExpand: [ + generateIssue('temporalWithMultipleDefinitions', { + tagGroup: testStrings.defAndDefExpand, + tag: temporalTagName, + }), + ], + multipleDefinitionsAndExtraTagGroups: [ + generateIssue('temporalWithMultipleDefinitions', { + tagGroup: testStrings.multipleDefinitionsAndExtraTagGroups, + tag: temporalTagName, + }), + generateIssue('extraTagsInTemporal', { + definition: 'Multiple definition tags found', + tag: temporalTagName, + }), + ], + } + return validatorSemantic(testStrings, expectedIssues, (validator, tagGroup) => { + validator.checkTemporalSyntax(tagGroup) + }) + }) + + it('should have syntactically valid offsets', () => { + const testStrings = { + simple: '(Offset, Def/Acc/5.4)', + noTag: '(Offset)', + tagAndNoDef: '(Offset, Red)', + tagGroupAndNoDef: '(Offset, (Red))', + defAndTag: '(Offset, Def/DefAndTag, Red)', + defExpandAndTag: '((Def-expand/DefExpandAndTag, (Label/Pie)), Offset, Red)', + defAndTagGroup: '(Offset, Def/DefAndTagGroup, (Red))', + defTagAndTagGroup: '(Offset, Def/DefTagAndTagGroup, (Red), Blue)', + defExpandAndTagGroup: '((Def-expand/DefExpandAndTagGroup, (Label/Pie)), Offset, (Red))', + multipleDefs: '(Offset, Def/MyColor, Def/Acc/5.4)', + defAndDefExpand: '((Def-expand/MyColor, (Label/Pie)), Def/Acc/5.4, Offset)', + multipleDefinitionsAndExtraTagGroups: + '((Def-expand/MyColor, (Label/Pie)), Def/Acc/5.4, Offset, (Blue), (Green))', + } + const expectedIssues = { + simple: [], + noTag: [generateIssue('temporalWithoutDefinition', { tagGroup: testStrings.noTag, tag: 'Offset' })], + tagAndNoDef: [ + generateIssue('temporalWithoutDefinition', { tagGroup: testStrings.tagAndNoDef, tag: 'Offset' }), + generateIssue('extraTagsInTemporal', { definition: null, tag: 'Offset' }), + ], + tagGroupAndNoDef: [ + generateIssue('temporalWithoutDefinition', { tagGroup: testStrings.tagGroupAndNoDef, tag: 'Offset' }), + generateIssue('extraTagsInTemporal', { definition: null, tag: 'Offset' }), + ], + defAndTag: [generateIssue('invalidGroupTopTags', { tags: 'Offset, Def/DefAndTag, Red' })], + defExpandAndTag: [generateIssue('extraTagsInTemporal', { definition: 'DefExpandAndTag', tag: 'Offset' })], + defAndTagGroup: [generateIssue('extraTagsInTemporal', { definition: 'DefAndTagGroup', tag: 'Offset' })], + defTagAndTagGroup: [generateIssue('extraTagsInTemporal', { definition: 'DefTagAndTagGroup', tag: 'Offset' })], + defExpandAndTagGroup: [ + generateIssue('extraTagsInTemporal', { definition: 'DefExpandAndTagGroup', tag: 'Offset' }), + ], + multipleDefs: [ + generateIssue('temporalWithMultipleDefinitions', { tagGroup: testStrings.multipleDefs, tag: 'Offset' }), + ], + defAndDefExpand: [ + generateIssue('temporalWithMultipleDefinitions', { tagGroup: testStrings.defAndDefExpand, tag: 'Offset' }), + ], + multipleDefinitionsAndExtraTagGroups: [ + generateIssue('temporalWithMultipleDefinitions', { + tagGroup: testStrings.multipleDefinitionsAndExtraTagGroups, + tag: 'Offset', + }), + generateIssue('extraTagsInTemporal', { definition: 'Multiple definition tags found', tag: 'Offset' }), + ], + } + return validatorSemantic(testStrings, expectedIssues, (validator, tagGroup) => { + validator.checkTemporalSyntax(tagGroup) + }) + }) + }) + + describe('Top-level Tags', () => { + const validatorSemantic = validatorSemanticBase + + it('should not have invalid top-level tags', () => { + const testStrings = { + validDef: 'Def/TopLevelDefReference', + validDefExpand: '(Def-expand/ValidDefExpand)', + invalidDefExpand: 'Def-expand/InvalidDefExpand', + } + const expectedIssues = { + validDef: [], + validDefExpand: [], + invalidDefExpand: [ + generateIssue('missingTagGroup', { + tag: testStrings.invalidDefExpand, + string: testStrings.invalidDefExpand, + }), + ], + } + return validatorSemantic(testStrings, expectedIssues, (validator) => { + validator.validateTopLevelTags() + }) + }) + }) + + describe('Top-level Group Tags', () => { + const validatorSemantic = validatorSemanticBase + + it('should only have definitions, onsets, or offsets in top-level tag groups', () => { + const testStrings = { + validDefinition: '(Definition/SimpleDefinition)', + multipleDefinitions: '(Definition/FirstDefinition), (Definition/SecondDefinition)', + validOnset: '(Onset, Def/Acc/5.4 m-per-s^2)', + validOffset: '(Offset, Def/Acc/5.4 m-per-s^2)', + topLevelDefinition: 'Definition/TopLevelDefinition', + //topLevelPlaceholderDefinition: 'Definition/TopLevelPlaceholderDefinition/#', + topLevelOnset: 'Onset, Red', + topLevelOffset: 'Offset, Def/Acc/5.4 m-per-s^2', + nestedDefinition: '((Definition/SimpleDefinition), Red)', + nestedOnset: '((Onset, Def/MyColor), Red)', + nestedOffset: '((Offset, Def/MyColor), Red)', + } + const expectedIssues = { + validDefinition: [], + multipleDefinitions: [], + validOnset: [], + validOffset: [], + topLevelDefinition: [ + generateIssue('illegalInExclusiveContext', { + string: 'Definition/TopLevelDefinition', + tag: testStrings.topLevelDefinition, + }), + ], + topLevelPlaceholderDefinition: [ + generateIssue('invalidTopLevelTagGroupTag', { + tag: testStrings.topLevelPlaceholderDefinition, + }), + ], + topLevelOnset: [ + generateIssue('invalidTopLevelTagGroupTag', { + tag: 'Onset', + string: 'Onset, Red', + }), + ], + topLevelOffset: [ + generateIssue('invalidTopLevelTagGroupTag', { + tag: 'Offset', + string: 'Offset, Def/Acc/5.4 m-per-s^2', + }), + ], + nestedDefinition: [ + generateIssue('invalidTopLevelTagGroupTag', { + tag: 'Definition/SimpleDefinition', + string: '((Definition/SimpleDefinition), Red)', + }), + ], + nestedOnset: [ + generateIssue('invalidTopLevelTagGroupTag', { + tag: 'Onset', + string: '((Onset, Def/MyColor), Red)', + }), + ], + nestedOffset: [ + generateIssue('invalidTopLevelTagGroupTag', { + tag: 'Offset', + string: '((Offset, Def/MyColor), Red)', + }), + ], + } + return validatorSemantic(testStrings, expectedIssues, (validator) => { + validator.validateTopLevelTagGroups() + }) + }) + }) + + describe('HED Strings', () => { + const validatorSemantic = function (testStrings, expectedIssues, expectValuePlaceholderString = false) { + return validatorSemanticBase( + testStrings, + expectedIssues, + (validator) => { + validator.validateStringLevel() + }, + { expectValuePlaceholderString: expectValuePlaceholderString }, + ) + } + + // TODO: Remove -- now in tokenizer tests + it.skip('(REMOVE) should properly handle strings with placeholders', () => { + const testStrings = { + takesValue: 'RGB-red/#', + withUnit: 'Time-value/# ms', + child: 'Left-side-of/#', + extensionAllowed: 'Human/Driver/#', + extensionParent: 'Item/TestDef1/#', + missingRequiredUnit: 'Time-value/#', + wrongLocation: 'Item/#/OtherItem', + duplicatePlaceholder: 'Item/#/#', + } + const expectedIssues = { + takesValue: [], + withUnit: [], + child: [generateIssue('invalidPlaceholder', { tag: testStrings.child })], + extensionAllowed: [ + generateIssue('invalidPlaceholder', { + tag: testStrings.extensionAllowed, + }), + ], + extensionParent: [ + generateIssue('invalidPlaceholder', { + tag: testStrings.extensionParent, + }), + ], + missingRequiredUnit: [], + wrongLocation: [ + generateIssue('invalidPlaceholder', { + index: '16', + string: testStrings.wrongLocation, + tag: testStrings.wrongLocation, + }), + ], + duplicatePlaceholder: [ + generateIssue('invalidPlaceholder', { + index: '8', + string: testStrings.duplicatePlaceholder, + tag: testStrings.duplicatePlaceholder, + }), + ], + } + return validatorSemantic(testStrings, expectedIssues, true) + }) + + // TODO: Remove -- now in bidsTests as definition-tests -- but recheck + it.skip('should have valid placeholders in definitions', () => { + const expectedPlaceholdersTestStrings = { + noPlaceholders: 'Car', + noPlaceholderGroup: '(Train, Age/15, RGB-red/0.5)', + noPlaceholderTagGroupDefinition: '(Definition/TagGroupDefinition, (Square, RGB-blue))', + noPlaceholderDefinitionWithFixedValue: '(Definition/FixedTagGroupDefinition/Test, (Square, RGB-blue))', + singlePlaceholder: 'RGB-green/#', + definitionPlaceholder: '(Definition/PlaceholderDefinition/#, (RGB-green/#))', + definitionPlaceholderWithFixedValue: '(Definition/FixedPlaceholderDefinition/Test, (RGB-green/#))', + definitionPlaceholderWithTag: '(Definition/PlaceholderWithTagDefinition/#, (RGB-green/#))', + // singlePlaceholderWithValidDefinitionPlaceholder: + // '(Definition/SinglePlaceholderWithValidPlaceholderDefinition/#, (RGB-green/#))', + nestedDefinitionPlaceholder: + '(Definition/NestedPlaceholderDefinition/#, (Touchscreen, (Square, RGB-blue/#)))', + threePlaceholderDefinition: '(Definition/ThreePlaceholderDefinition/#, (RGB-green/#, RGB-blue/#))', + fourPlaceholderDefinition: + '(Definition/FourPlaceholderDefinition/#, (RGB-green/#, (Cube, Volume/#, RGB-blue/#)))', + multiPlaceholder: 'RGB-red/#, Circle, RGB-blue/#', + // multiPlaceholderWithValidDefinition: + // 'RGB-red/#, Circle, (Definition/MultiPlaceholderWithValidDefinition/#, (RGB-green/#)), RGB-blue/#', + // multiPlaceholderWithThreePlaceholderDefinition: + // 'RGB-red/#, Circle, (Definition/MultiPlaceholderWithThreePlaceholderDefinition/#, (RGB-green/#, RGB-blue/#)), Time-value/#', + } + const noExpectedPlaceholdersTestStrings = { + noPlaceholders: 'Car', + noPlaceholderGroup: '(Train, Age/15, RGB-red/0.5)', + noPlaceholderTagGroupDefinition: '(Definition/TagGroupDefinition, (Square, RGB-blue))', + noPlaceholderDefinitionWithFixedValue: '(Definition/FixedTagGroupDefinition/Test, (Square, RGB-blue))', + singlePlaceholder: 'RGB-green/#', + definitionPlaceholder: '(Definition/PlaceholderDefinition/#, (RGB-green/#))', + definitionPlaceholderWithFixedValue: '(Definition/FixedPlaceholderDefinition/Test, (RGB-green/#))', + // definitionPlaceholderWithTag: 'Car, (Definition/PlaceholderWithTagDefinition/#, (RGB-green/#))', + // singlePlaceholderWithValidDefinitionPlaceholder: + // 'Time-value/#, (Definition/SinglePlaceholderWithValidPlaceholderDefinition/#, (RGB-green/#))', + nestedDefinitionPlaceholder: + '(Definition/NestedPlaceholderDefinition/#, (Touchscreen, (Square, RGB-blue/#)))', + threePlaceholderDefinition: '(Definition/ThreePlaceholderDefinition/#, (RGB-green/#, RGB-blue/#))', + fourPlaceholderDefinition: + '(Definition/FourPlaceholderDefinition/#, (RGB-green/#, (Cube, Volume/#, RGB-blue/#)))', + multiPlaceholder: 'RGB-red/#, Circle, RGB-blue/#', + multiPlaceholderWithValidDefinition: + 'RGB-red/#, Circle, (Definition/MultiPlaceholderWithValidDefinition/#, (RGB-green/#)), RGB-blue/#', + multiPlaceholderWithThreePlaceholderDefinition: + 'RGB-red/#, Circle, (Definition/MultiPlaceholderWithThreePlaceholderDefinition/#, (RGB-green/#, RGB-blue/#)), Time-value/#', + } + const expectedPlaceholdersIssues = { + noPlaceholders: [ + generateIssue('missingPlaceholder', { + string: expectedPlaceholdersTestStrings.noPlaceholders, + }), + ], + noPlaceholderGroup: [ + generateIssue('missingPlaceholder', { + string: expectedPlaceholdersTestStrings.noPlaceholderGroup, + }), + ], + noPlaceholderDefinitionGroup: [ + generateIssue('missingPlaceholder', { + string: expectedPlaceholdersTestStrings.noPlaceholderDefinitionGroup, + }), + ], + noPlaceholderTagGroupDefinition: [ + generateIssue('missingPlaceholder', { + string: expectedPlaceholdersTestStrings.noPlaceholderTagGroupDefinition, + }), + ], + noPlaceholderDefinitionWithFixedValue: [ + generateIssue('missingPlaceholder', { + string: expectedPlaceholdersTestStrings.noPlaceholderDefinitionWithFixedValue, + }), + generateIssue('invalidPlaceholderInDefinition', { + definition: 'FixedTagGroupDefinition', + }), + ], + singlePlaceholder: [], + definitionPlaceholder: [ + generateIssue('missingPlaceholder', { + string: expectedPlaceholdersTestStrings.definitionPlaceholder, + }), + ], + definitionPlaceholderWithFixedValue: [ + generateIssue('missingPlaceholder', { + string: expectedPlaceholdersTestStrings.definitionPlaceholderWithFixedValue, + }), + generateIssue('invalidPlaceholderInDefinition', { + definition: 'FixedPlaceholderDefinition', + }), + ], + definitionPlaceholderWithTag: [ + generateIssue('missingPlaceholder', { + string: expectedPlaceholdersTestStrings.definitionPlaceholderWithTag, + }), + ], + singlePlaceholderWithValidDefinitionPlaceholder: [], + nestedDefinitionPlaceholder: [ + generateIssue('missingPlaceholder', { + string: expectedPlaceholdersTestStrings.nestedDefinitionPlaceholder, + }), + ], + threePlaceholderDefinition: [ + generateIssue('missingPlaceholder', { + string: expectedPlaceholdersTestStrings.threePlaceholderDefinition, + }), + generateIssue('invalidPlaceholderInDefinition', { + definition: 'ThreePlaceholderDefinition', + }), + ], + fourPlaceholderDefinition: [ + generateIssue('missingPlaceholder', { + string: expectedPlaceholdersTestStrings.fourPlaceholderDefinition, + }), + generateIssue('invalidPlaceholderInDefinition', { + definition: 'FourPlaceholderDefinition', + }), + ], + multiPlaceholder: [ + generateIssue('invalidPlaceholder', { tag: 'RGB-red/#' }), + generateIssue('invalidPlaceholder', { tag: 'RGB-blue/#' }), + ], + multiPlaceholderWithValidDefinition: [ + generateIssue('invalidPlaceholder', { tag: 'RGB-red/#' }), + generateIssue('invalidPlaceholder', { tag: 'RGB-blue/#' }), + ], + multiPlaceholderWithThreePlaceholderDefinition: [ + generateIssue('invalidPlaceholder', { tag: 'RGB-red/#' }), + generateIssue('invalidPlaceholderInDefinition', { + definition: 'MultiPlaceholderWithThreePlaceholderDefinition', + }), + generateIssue('invalidPlaceholder', { tag: 'Time-value/#' }), + ], + } + const noExpectedPlaceholdersIssues = { + noPlaceholders: [], + noPlaceholderGroup: [], + noPlaceholderDefinitionGroup: [], + noPlaceholderTagGroupDefinition: [], + noPlaceholderDefinitionWithFixedValue: [ + generateIssue('invalidPlaceholderInDefinition', { + definition: 'FixedTagGroupDefinition', + }), + ], + singlePlaceholder: [generateIssue('invalidPlaceholder', { tag: 'RGB-green/#' })], + definitionPlaceholder: [], + definitionPlaceholderWithFixedValue: [ + generateIssue('invalidPlaceholderInDefinition', { + definition: 'FixedPlaceholderDefinition', + }), + ], + //definitionPlaceholderWithTag: [], + // singlePlaceholderWithValidDefinitionPlaceholder: [ + // generateIssue('invalidPlaceholder', { tag: 'Time-value/#' }), + // ], + nestedDefinitionPlaceholder: [], + threePlaceholderDefinition: [ + generateIssue('invalidPlaceholderInDefinition', { + definition: 'ThreePlaceholderDefinition', + }), + ], + fourPlaceholderDefinition: [ + generateIssue('invalidPlaceholderInDefinition', { + definition: 'FourPlaceholderDefinition', + }), + ], + multiPlaceholder: [ + generateIssue('invalidPlaceholder', { tag: 'RGB-red/#' }), + generateIssue('invalidPlaceholder', { tag: 'RGB-blue/#' }), + ], + multiPlaceholderWithValidDefinition: [ + generateIssue('invalidPlaceholder', { tag: 'RGB-red/#' }), + generateIssue('invalidPlaceholder', { tag: 'RGB-blue/#' }), + ], + multiPlaceholderWithThreePlaceholderDefinition: [ + generateIssue('invalidPlaceholder', { tag: 'RGB-red/#' }), + generateIssue('invalidPlaceholderInDefinition', { + definition: 'MultiPlaceholderWithThreePlaceholderDefinition', + }), + generateIssue('invalidPlaceholder', { tag: 'Time-value/#' }), + ], + } + validatorSemantic(expectedPlaceholdersTestStrings, expectedPlaceholdersIssues, true) + validatorSemantic(noExpectedPlaceholdersTestStrings, noExpectedPlaceholdersIssues, false) + }) + }) + }) + + describe('HED-3G library and partnered schema validation', () => { + const hedLibrary2SchemaFile = 'tests/data/HED_testlib_2.0.0.xml' + const hedLibrary3SchemaFile = 'tests/data/HED_testlib_3.0.0.xml' + let hedSchemas, hedSchemas2 + + beforeAll(async () => { + const spec4 = new SchemaSpec('testlib', '2.0.0', 'testlib', hedLibrary2SchemaFile) + const spec5 = new SchemaSpec('testlib', '3.0.0', 'testlib', hedLibrary3SchemaFile) + const spec6 = new SchemaSpec('', '2.0.0', 'testlib', hedLibrary2SchemaFile) + const spec7 = new SchemaSpec('', '3.0.0', 'testlib', hedLibrary3SchemaFile) + const specs = new SchemasSpec().addSchemaSpec(spec4).addSchemaSpec(spec5) + const specs2 = new SchemasSpec().addSchemaSpec(spec6).addSchemaSpec(spec7) + hedSchemas = await buildSchemas(specs) + hedSchemas2 = await buildSchemas(specs2) + }) + + /** + * Validation base function. + * + * This override is required due to incompatible constructor signatures between Hed3Validator and the other two classes. + * + * @param {Schemas} hedSchemas The HED schema collection used for testing. + * @param {Object} testStrings A mapping of test strings. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorBase = function (hedSchemas, testStrings, expectedIssues, testFunction, testOptions = {}) { + for (const [testStringKey, testString] of Object.entries(testStrings)) { + assert.property(expectedIssues, testStringKey, testStringKey + ' is not in expectedIssues') + const [parsedTestString, parsingIssues] = parseHedString(testString, hedSchemas) + const validator = new HedValidator(parsedTestString, hedSchemas, null, testOptions) + const flattenedParsingIssues = Object.values(parsingIssues).flat() + if (flattenedParsingIssues.length === 0) { + testFunction(validator) + } + const issues = [].concat(flattenedParsingIssues, validator.issues) + assert.sameDeepMembers(issues, expectedIssues[testStringKey], testString) + } + } + + /** + * HED 3 semantic validation base function. + * + * This base function uses the HED 3-specific {@link HedValidator} validator class. + * + * @param {Object} testStrings A mapping of test strings. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorSemanticBase = function (testStrings, expectedIssues, testFunction, testOptions = {}) { + validatorBase(hedSchemas, testStrings, expectedIssues, testFunction, testOptions) + } + + describe('Full HED Strings', () => { + const validatorSemantic = validatorSemanticBase + + /** + * HED 3 semantic validation function using the alternative schema collection. + * + * This base function uses the HED 3-specific {@link HedValidator} validator class. + * + * @param {Object} testStrings A mapping of test strings. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorSemantic2 = function (testStrings, expectedIssues, testFunction, testOptions = {}) { + validatorBase(hedSchemas2, testStrings, expectedIssues, testFunction, testOptions) + } + + it('should allow combining tags from multiple partnered schemas', () => { + const testStrings = { + music: 'testlib:Piano-sound, testlib:Violin-sound', + test2: 'testlib:SubnodeA3, testlib:Frown', + test3: 'testlib:SubnodeF1, testlib:Car', + } + const expectedIssues = { + music: [], + test2: [], + test3: [], + } + return validatorSemantic(testStrings, expectedIssues, (validator) => { + validator.validateEventLevel() + }) + }) + + it('should allow combining tags from multiple partnered schemas as the unprefixed schema', () => { + const testStrings = { + music: 'Piano-sound, Violin-sound', + test2: 'SubnodeA3, Frown', + test3: 'SubnodeF1, Car', + } + const expectedIssues = { + music: [], + test2: [], + test3: [], + } + return validatorSemantic2(testStrings, expectedIssues, (validator) => { + validator.validateEventLevel() + }) + }) + }) + + describe.skip('HED Tag Groups', () => { + /** + * HED 3 tag group semantic validation base function. + * + * @param {Object} testStrings A mapping of test strings. + * @param {Object} expectedIssues The expected issues for each test string. + * @param {function(HedValidator, ParsedHedGroup): void} testFunction A test-specific function that executes the required validation check. + * @param {Object?} testOptions Any needed custom options for the validator. + */ + const validatorSemantic = function (testStrings, expectedIssues, testFunction, testOptions = {}) { + return validatorSemanticBase( + testStrings, + expectedIssues, + (validator) => { + for (const parsedTagGroup of validator.parsedString.tagGroups) { + testFunction(validator, parsedTagGroup) + } + }, + testOptions, + ) + } + + it('should have syntactically valid definitions', () => { + const testStrings = { + nonDefinition: 'testlib:Car', + nonDefinitionGroup: '(testlib:Hold-breath, testlib:Sit-down)', + definitionOnly: '(testlib:Definition/SimpleDefinition)', + tagGroupDefinition: '(testlib:Definition/TagGroupDefinition, (testlib:Hold-breath, testlib:Sit-down))', + illegalSiblingDefinition: '(testlib:Definition/IllegalSiblingDefinition, testlib:Train, (testlib:Rectangle))', + nestedDefinition: + '(testlib:Definition/NestedDefinition, (testlib:Touchscreen, (testlib:Definition/InnerDefinition, (testlib:Square))))', + multipleTagGroupDefinition: + '(testlib:Definition/MultipleTagGroupDefinition, (testlib:Touchscreen), (testlib:Square))', + defNestedInDefinition: '(testlib:Definition/DefNestedInDefinition, (testlib:Def/Nested, testlib:Triangle))', + } + const expectedIssues = { + nonDefinition: [], + nonDefinitionGroup: [], + definitionOnly: [], + tagGroupDefinition: [], + illegalSiblingDefinition: [ + generateIssue('illegalDefinitionGroupTag', { + tag: 'testlib:Train', + definition: 'IllegalSiblingDefinition', + }), + ], + nestedDefinition: [ + generateIssue('invalidTopLevelTagGroupTag', { + tag: 'Definition/InnerDefinition', + }), + ], + multipleTagGroupDefinition: [ + generateIssue('multipleTagGroupsInDefinition', { + definition: 'MultipleTagGroupDefinition', + }), + ], + defNestedInDefinition: [ + generateIssue('nestedDefinition', { + definition: 'DefNestedInDefinition', + }), + ], + } + return validatorSemantic(testStrings, expectedIssues, (validator, tagGroup) => { + validator.checkDefinitionGroupSyntax(tagGroup) + }) + }) + + it.each(['Onset', 'Inset'])('should have syntactically valid %s tags', (temporalTagName) => { + const testStrings = { + simple: `(testlib:${temporalTagName}, testlib:Def/Acc/5.4)`, + defAndOneGroup: `(testlib:${temporalTagName}, testlib:Def/ShowFace, (testlib:Hold-breath, testlib:Sit-down))`, + defExpandAndOneGroup: `(testlib:${temporalTagName}, (testlib:Def-expand/ShowFace, (testlib:Label/Pie)), (testlib:Hold-breath, testlib:Sit-down))`, + noTag: `(testlib:${temporalTagName})`, + definition: `(testlib:${temporalTagName}, testlib:Definition/ShowFace, (testlib:Label/Pie))`, + defAndTwoGroups: `(testlib:Def/DefAndTwoGroups, (testlib:Blue), (testlib:Green), testlib:${temporalTagName})`, + defExpandAndTwoGroups: `((testlib:Def-expand/DefExpandAndTwoGroups, (testlib:Label/Pie)), (testlib:Green), (testlib:Red), testlib:${temporalTagName})`, + tagAndNoDef: `(testlib:${temporalTagName}, testlib:Red)`, + tagGroupAndNoDef: `(testlib:${temporalTagName}, (testlib:Red))`, + defAndTag: `(testlib:${temporalTagName}, testlib:Def/DefAndTag, testlib:Red)`, + defTagAndTagGroup: `(testlib:${temporalTagName}, testlib:Def/DefTagAndTagGroup, (testlib:Red), testlib:Blue)`, + multipleDefs: `(testlib:${temporalTagName}, testlib:Def/ShowFace, testlib:Def/Acc/5.4)`, + defAndDefExpand: `((testlib:Def-expand/ShowFace, (testlib:Label/Pie)), testlib:Def/Acc/5.4, testlib:${temporalTagName})`, + multipleDefinitionsAndExtraTagGroups: `((testlib:Def-expand/ShowFace, (testlib:Label/Pie)), testlib:Def/Acc/5.4, testlib:${temporalTagName}, (testlib:Blue), (testlib:Green))`, + } + const expectedIssues = { + simple: [], + defAndOneGroup: [], + defExpandAndOneGroup: [], + noTag: [generateIssue('temporalWithoutDefinition', { tagGroup: testStrings.noTag, tag: temporalTagName })], + definition: [ + generateIssue('temporalWithoutDefinition', { tagGroup: testStrings.definition, tag: temporalTagName }), + generateIssue('extraTagsInTemporal', { definition: null, tag: temporalTagName }), + ], + defAndTwoGroups: [ + generateIssue('extraTagsInTemporal', { definition: 'DefAndTwoGroups', tag: temporalTagName }), + ], + defExpandAndTwoGroups: [ + generateIssue('extraTagsInTemporal', { definition: 'DefExpandAndTwoGroups', tag: temporalTagName }), + ], + tagAndNoDef: [ + generateIssue('temporalWithoutDefinition', { tagGroup: testStrings.tagAndNoDef, tag: temporalTagName }), + generateIssue('extraTagsInTemporal', { definition: null, tag: temporalTagName }), + ], + tagGroupAndNoDef: [ + generateIssue('temporalWithoutDefinition', { + tagGroup: testStrings.tagGroupAndNoDef, + tag: temporalTagName, + }), + ], + defAndTag: [generateIssue('extraTagsInTemporal', { definition: 'DefAndTag', tag: temporalTagName })], + defTagAndTagGroup: [ + generateIssue('extraTagsInTemporal', { definition: 'DefTagAndTagGroup', tag: temporalTagName }), + ], + multipleDefs: [ + generateIssue('temporalWithMultipleDefinitions', { + tagGroup: testStrings.multipleDefs, + tag: temporalTagName, + }), + ], + defAndDefExpand: [ + generateIssue('temporalWithMultipleDefinitions', { + tagGroup: testStrings.defAndDefExpand, + tag: temporalTagName, + }), + ], + multipleDefinitionsAndExtraTagGroups: [ + generateIssue('temporalWithMultipleDefinitions', { + tagGroup: testStrings.multipleDefinitionsAndExtraTagGroups, + tag: temporalTagName, + }), + generateIssue('extraTagsInTemporal', { + definition: 'Multiple definition tags found', + tag: temporalTagName, + }), + ], + } + return validatorSemantic(testStrings, expectedIssues, (validator, tagGroup) => { + validator.checkTemporalSyntax(tagGroup) + }) + }) + + // TODO: Transfer to new format the internal error codes have changed because error is detected earlier + it.skip('(REVISIT) should have syntactically valid offsets', () => { + const testStrings = { + simple: '(testlib:Offset, testlib:Def/Acc/5.4)', + noTag: '(testlib:Offset)', + tagAndNoDef: '(testlib:Offset, testlib:Red)', + tagGroupAndNoDef: '(testlib:Offset, (testlib:Red))', + defAndTag: '(testlib:Offset, testlib:Def/DefAndTag, testlib:Red)', + defExpandAndTag: '((testlib:Def-expand/DefExpandAndTag, (testlib:Label/Pie)), testlib:Offset, testlib:Red)', + defAndTagGroup: '(testlib:Offset, testlib:Def/DefAndTagGroup, (testlib:Red))', + defTagAndTagGroup: '(testlib:Offset, testlib:Def/DefTagAndTagGroup, (testlib:Red), testlib:Blue)', + defExpandAndTagGroup: + '((testlib:Def-expand/DefExpandAndTagGroup, (testlib:Label/Pie)), testlib:Offset, (testlib:Red))', + multipleDefs: '(testlib:Offset, testlib:Def/MyColor, testlib:Def/Acc/5.4)', + defAndDefExpand: '((testlib:Def-expand/MyColor, (testlib:Label/Pie)), testlib:Def/Acc/5.4, testlib:Offset)', + multipleDefinitionsAndExtraTagGroups: + '((testlib:Def-expand/MyColor, (testlib:Label/Pie)), testlib:Def/Acc/5.4, testlib:Offset, (testlib:Blue), (testlib:Green))', + } + const expectedIssues = { + simple: [], + noTag: [generateIssue('temporalWithoutDefinition', { tagGroup: testStrings.noTag, tag: 'Offset' })], + tagAndNoDef: [ + generateIssue('temporalWithoutDefinition', { tagGroup: testStrings.tagAndNoDef, tag: 'Offset' }), + generateIssue('extraTagsInTemporal', { definition: null, tag: 'Offset' }), + ], + tagGroupAndNoDef: [ + generateIssue('temporalWithoutDefinition', { tagGroup: testStrings.tagGroupAndNoDef, tag: 'Offset' }), + generateIssue('extraTagsInTemporal', { definition: null, tag: 'Offset' }), + ], + defAndTag: [generateIssue('extraTagsInTemporal', { definition: 'DefAndTag', tag: 'Offset' })], + defExpandAndTag: [generateIssue('extraTagsInTemporal', { definition: 'DefExpandAndTag', tag: 'Offset' })], + defAndTagGroup: [generateIssue('extraTagsInTemporal', { definition: 'DefAndTagGroup', tag: 'Offset' })], + defTagAndTagGroup: [generateIssue('extraTagsInTemporal', { definition: 'DefTagAndTagGroup', tag: 'Offset' })], + defExpandAndTagGroup: [ + generateIssue('extraTagsInTemporal', { definition: 'DefExpandAndTagGroup', tag: 'Offset' }), + ], + multipleDefs: [ + generateIssue('temporalWithMultipleDefinitions', { tagGroup: testStrings.multipleDefs, tag: 'Offset' }), + ], + defAndDefExpand: [ + generateIssue('temporalWithMultipleDefinitions', { tagGroup: testStrings.defAndDefExpand, tag: 'Offset' }), + ], + multipleDefinitionsAndExtraTagGroups: [ + generateIssue('temporalWithMultipleDefinitions', { + tagGroup: testStrings.multipleDefinitionsAndExtraTagGroups, + tag: 'Offset', + }), + generateIssue('extraTagsInTemporal', { definition: 'Multiple definition tags found', tag: 'Offset' }), + ], + } + return validatorSemantic(testStrings, expectedIssues, (validator, tagGroup) => { + validator.checkTemporalSyntax(tagGroup) + }) + }) + }) + }) +}) diff --git a/tests/normalizerTests.spec.js b/tests/normalizerTests.spec.js deleted file mode 100644 index b2a29cec..00000000 --- a/tests/normalizerTests.spec.js +++ /dev/null @@ -1,66 +0,0 @@ -import chai from 'chai' -const assert = chai.assert -import { beforeAll, describe, afterAll } from '@jest/globals' -import path from 'path' - -import { buildSchemas } from '../schema/init' -import { SchemaSpec, SchemasSpec } from '../schema/specs' -import { normalizerTestData } from './testData/normalizerTests.data' -import { shouldRun, getHedString } from './testUtilities' - -const skipMap = new Map() -const runAll = true -const runMap = new Map([['simple-tags', 'duplicate-tags']]) - -describe('Normalize HED string tests', () => { - const schemaMap = new Map([['8.3.0', undefined]]) - - beforeAll(async () => { - const spec3 = new SchemaSpec('', '8.3.0', '', path.join(__dirname, '../tests/data/HED8.3.0.xml')) - const specs3 = new SchemasSpec().addSchemaSpec(spec3) - const schemas3 = await buildSchemas(specs3) - schemaMap.set('8.3.0', schemas3) - }) - - afterAll(() => {}) - - describe.each(normalizerTestData)('$name : $description', ({ name, tests }) => { - const testConvert = function (test) { - const header = `${test.testname}` - const thisSchema = schemaMap.get(test.schemaVersion) - assert.isDefined(thisSchema, `header: ${test.schemaVersion} is not available in test ${test.testname}`) - - let issues - // Parse the string before converting - try { - const [parsedString, errorIssues, warningIssues] = getHedString(test.string, thisSchema, true, false, true) - issues = errorIssues - assert.strictEqual( - parsedString?.normalized, - test.stringNormalized, - `${header}: received normalized string: ${parsedString?.normalized} but expected ${test.stringNormalized}`, - ) - assert.equal(warningIssues.length, 0, `${header}: expects errors not warnings`) - } catch (error) { - issues = [error.issue] - } - // Check for errors - assert.deepStrictEqual(issues, test.errors, `${header}: expected ${issues} errors but received ${test.errors}\n`) - } - - beforeAll(async () => {}) - - afterAll(() => {}) - - if (tests && tests.length > 0) { - test.each(tests)('$testname: $explanation ', (test) => { - if (shouldRun(name, test.testname, runAll, runMap, skipMap)) { - testConvert(test) - } else { - // eslint-disable-next-line no-console - console.log(`----Skipping stringParserTest ${name}: ${test.testname}`) - } - }) - } - }) -}) diff --git a/tests/schema.spec.js b/tests/schema.spec.js index a3091636..4e34f211 100644 --- a/tests/schema.spec.js +++ b/tests/schema.spec.js @@ -115,6 +115,222 @@ describe('HED schemas', () => { }) }) + describe.skip('HED-2G schemas', () => { + const localHedSchemaFile = 'tests/data/HED7.1.1.xml' + let hedSchemas + + beforeAll(async () => { + const spec1 = new SchemaSpec('', '7.1.1', '', localHedSchemaFile) + const specs = new SchemasSpec().addSchemaSpec(spec1) + hedSchemas = await buildSchemas(specs) + }) + + it('should have tag dictionaries for all required tag attributes', () => { + const tagDictionaryKeys = [ + 'default', + 'extensionAllowed', + 'isNumeric', + 'position', + 'predicateType', + 'recommended', + 'required', + 'requireChild', + 'takesValue', + 'unique', + ] + const dictionaries = hedSchemas.baseSchema.attributes.tagAttributes + assert.hasAllKeys(dictionaries, tagDictionaryKeys) + }) + + it('should have unit dictionaries for all required unit attributes', () => { + const unitDictionaryKeys = ['SIUnit', 'unitSymbol'] + const dictionaries = hedSchemas.baseSchema.attributes.unitAttributes + assert.hasAllKeys(dictionaries, unitDictionaryKeys) + }) + + it('should contain all of the required tags', () => { + const requiredTags = ['event/category', 'event/description', 'event/label'] + const dictionariesRequiredTags = hedSchemas.baseSchema.attributes.tagAttributes['required'] + assert.hasAllKeys(dictionariesRequiredTags, requiredTags) + }) + + it('should contain all of the positioned tags', () => { + const positionedTags = ['event/category', 'event/description', 'event/label', 'event/long name'] + const dictionariesPositionedTags = hedSchemas.baseSchema.attributes.tagAttributes['position'] + assert.hasAllKeys(dictionariesPositionedTags, positionedTags) + }) + + it('should contain all of the unique tags', () => { + const uniqueTags = ['event/description', 'event/label', 'event/long name'] + const dictionariesUniqueTags = hedSchemas.baseSchema.attributes.tagAttributes['unique'] + assert.hasAllKeys(dictionariesUniqueTags, uniqueTags) + }) + + it('should contain all of the tags with default units', () => { + const defaultUnitTags = { + 'attribute/blink/time shut/#': 's', + 'attribute/blink/duration/#': 's', + 'attribute/blink/pavr/#': 'centiseconds', + 'attribute/blink/navr/#': 'centiseconds', + } + const dictionariesDefaultUnitTags = hedSchemas.baseSchema.attributes.tagAttributes['default'] + assert.deepStrictEqual(dictionariesDefaultUnitTags, defaultUnitTags) + }) + + it('should contain all of the unit classes with their units and default units', () => { + const defaultUnits = { + acceleration: 'm-per-s^2', + currency: '$', + angle: 'radian', + frequency: 'Hz', + intensity: 'dB', + jerk: 'm-per-s^3', + luminousIntensity: 'cd', + memorySize: 'B', + physicalLength: 'm', + pixels: 'px', + speed: 'm-per-s', + time: 's', + clockTime: 'hour:min', + dateTime: 'YYYY-MM-DDThh:mm:ss', + area: 'm^2', + volume: 'm^3', + } + const allUnits = { + acceleration: ['m-per-s^2'], + currency: ['dollar', '$', 'point', 'fraction'], + angle: ['radian', 'rad', 'degree'], + frequency: ['hertz', 'Hz'], + intensity: ['dB'], + jerk: ['m-per-s^3'], + luminousIntensity: ['candela', 'cd'], + memorySize: ['byte', 'B'], + physicalLength: ['metre', 'm', 'foot', 'mile'], + pixels: ['pixel', 'px'], + speed: ['m-per-s', 'mph', 'kph'], + time: ['second', 's', 'day', 'minute', 'hour'], + clockTime: ['hour:min', 'hour:min:sec'], + dateTime: ['YYYY-MM-DDThh:mm:ss'], + area: ['m^2', 'px^2', 'pixel^2'], + volume: ['m^3'], + } + + const dictionariesUnitAttributes = hedSchemas.baseSchema.attributes.unitClassAttributes + const dictionariesAllUnits = hedSchemas.baseSchema.attributes.unitClasses + for (const [unitClass, unitClassAttributes] of Object.entries(dictionariesUnitAttributes)) { + const defaultUnit = unitClassAttributes.defaultUnits + assert.deepStrictEqual(defaultUnit[0], defaultUnits[unitClass], `Default unit for unit class ${unitClass}`) + } + assert.deepStrictEqual(dictionariesAllUnits, allUnits, 'All units') + }) + + it('should contain the correct (large) numbers of tags with certain attributes', () => { + const expectedAttributeTagCount = { + isNumeric: 80, + predicateType: 20, + recommended: 0, + requireChild: 64, + takesValue: 119, + } + + const dictionaries = hedSchemas.baseSchema.attributes.tagAttributes + for (const [attribute, count] of Object.entries(expectedAttributeTagCount)) { + assert.lengthOf(Object.keys(dictionaries[attribute]), count, 'Mismatch on attribute ' + attribute) + } + + const expectedTagCount = 1116 - 119 + 2 + const expectedUnitClassCount = 63 + assert.lengthOf( + Object.keys(hedSchemas.baseSchema.attributes.tags), + expectedTagCount, + 'Mismatch on overall tag count', + ) + assert.lengthOf( + Object.keys(hedSchemas.baseSchema.attributes.tagUnitClasses), + expectedUnitClassCount, + 'Mismatch on unit class tag count', + ) + }) + + it('should identify if a tag has a certain attribute', () => { + const testStrings = { + value: 'Attribute/Location/Reference frame/Relative to participant/Azimuth/#', + valueParent: 'Attribute/Location/Reference frame/Relative to participant/Azimuth', + extensionAllowed: 'Item/Object/Road sign', + } + const expectedResults = { + value: { + default: false, + extensionAllowed: false, + isNumeric: true, + position: false, + predicateType: false, + recommended: false, + required: false, + requireChild: false, + tags: false, + takesValue: true, + unique: false, + unitClass: true, + }, + valueParent: { + default: false, + extensionAllowed: true, + isNumeric: false, + position: false, + predicateType: false, + recommended: false, + required: false, + requireChild: true, + tags: true, + takesValue: false, + unique: false, + unitClass: false, + }, + extensionAllowed: { + default: false, + extensionAllowed: true, + isNumeric: false, + position: false, + predicateType: false, + recommended: false, + required: false, + requireChild: false, + tags: true, + takesValue: false, + unique: false, + unitClass: false, + }, + } + + for (const [testStringKey, testString] of Object.entries(testStrings)) { + const testStringLowercase = testString.toLowerCase() + const expected = expectedResults[testStringKey] + for (const [expectedKey, expectedResult] of Object.entries(expected)) { + if (expectedKey === 'tags') { + assert.strictEqual( + hedSchemas.baseSchema.attributes.tags.includes(testStringLowercase), + expectedResult, + `Test string: ${testString}. Attribute: ${expectedKey}`, + ) + } else if (expectedKey === 'unitClass') { + assert.strictEqual( + testStringLowercase in hedSchemas.baseSchema.attributes.tagUnitClasses, + expectedResult, + `Test string: ${testString}. Attribute: ${expectedKey}`, + ) + } else { + assert.strictEqual( + hedSchemas.baseSchema.attributes.tagHasAttribute(testStringLowercase, expectedKey), + expectedResult, + `Test string: ${testString}. Attribute: ${expectedKey}.`, + ) + } + } + } + }) + }) + describe('HED-3G schemas', () => { const localHedSchemaFile = 'tests/data/HED8.0.0.xml' let hedSchemas diff --git a/tests/schemaBuildTests.spec.js b/tests/schemaBuildTests.spec.js index 49d9469a..eb383533 100644 --- a/tests/schemaBuildTests.spec.js +++ b/tests/schemaBuildTests.spec.js @@ -48,7 +48,7 @@ describe('Schema build validation', () => { let schema = undefined let caughtError = null try { - schema = await buildBidsSchemas(desc, null) + schema = await buildBidsSchemas(desc) } catch (error) { caughtError = error } @@ -60,7 +60,6 @@ describe('Schema build validation', () => { if (shouldRun(name, test.testname, runAll, runMap, skipMap)) { await testSchema(test) } else { - // eslint-disable-next-line no-console console.log(`----Skipping schemaBuildTest ${name}: ${test.testname}`) } }) diff --git a/tests/schemaSpecTests.spec.js b/tests/schemaSpecTests.spec.js index 84d3c1de..4933e536 100644 --- a/tests/schemaSpecTests.spec.js +++ b/tests/schemaSpecTests.spec.js @@ -52,7 +52,6 @@ describe('Schema validation', () => { if (shouldRun(name, test.testname, runAll, runMap, skipMap)) { validateSpec(test) } else { - // eslint-disable-next-line no-console console.log(`----Skipping schemaSpecTest ${name}: ${test.testname}`) } }) diff --git a/tests/splitterTests.spec.js b/tests/splitterTests.spec.js index 75200180..2004e222 100644 --- a/tests/splitterTests.spec.js +++ b/tests/splitterTests.spec.js @@ -8,24 +8,43 @@ import { SchemaSpec, SchemasSpec } from '../schema/specs' import { splitterTestData } from './testData/splitterTests.data' import { shouldRun } from './testUtilities' import ParsedHedGroup from '../parser/parsedHedGroup' -import { filterByClass } from '../parser/parseUtils' import HedStringSplitter from '../parser/splitter' const skipMap = new Map() const runAll = true -const runMap = new Map([['valid-strings', ['empty-string']]]) +const runMap = new Map([['valid-strings', ['multiple-nested-groups']]]) describe('Parse HED string tests', () => { - const schemaMap = new Map([['8.3.0', undefined]]) + const schemaMap = new Map([ + ['8.2.0', undefined], + ['8.3.0', undefined], + ]) beforeAll(async () => { + const spec2 = new SchemaSpec('', '8.2.0', '', path.join(__dirname, '../tests/data/HED8.2.0.xml')) + const specs2 = new SchemasSpec().addSchemaSpec(spec2) + const schemas2 = await buildSchemas(specs2) const spec3 = new SchemaSpec('', '8.3.0', '', path.join(__dirname, '../tests/data/HED8.3.0.xml')) const specs3 = new SchemasSpec().addSchemaSpec(spec3) const schemas3 = await buildSchemas(specs3) + schemaMap.set('8.2.0', schemas2) schemaMap.set('8.3.0', schemas3) }) afterAll(() => {}) + describe('testExperiment', () => { + /*it('Should give experiment', () => { + const thisSchema = schemaMap.get('8.3.0') + // const w = new TagSpec('Speed/5 mph', 0, 10, '') + // const g = new ParsedHedTag(w, thisSchema, 'Speed/5 mph') + const stringIn = 'Item, Sensory-event, (Red, Blue, {help}, (Definition/Blech, (Green, Black))), (Orange, ((Definition/Blech1, (White))))' + //const stringIn = 'Item, ((Def-expand/Apple, (Purple)), (((Def-expand/Banana, (Orange)), Item)), Sensory-event), Red' + //const stringIn = 'Item/Object, (Length/5 m, (Green)), (Green, Object), (Sensory-event, Green), Red' + const [parsedString, issues] = parseHedString(stringIn, thisSchema) + console.log(issues) +*/ + }) + describe.each(splitterTestData)('$name : $description', ({ name, tests }) => { const testSplit = function (test) { const header = `[${test.testname}]: ${test.explanation}` @@ -33,9 +52,9 @@ describe('Parse HED string tests', () => { assert.isDefined(thisSchema, `header: ${test.schemaVersion} is not available in test ${test.testname}`) const [parsedTags, parsingIssues] = new HedStringSplitter(test.stringIn, thisSchema).splitHedString() - assert.isEmpty([...parsingIssues], `${header} expects no splitting issues`) + assert.isEmpty([...parsingIssues.syntax, ...parsingIssues.conversion], `${header} expects no splitting issues`) - const parsedGroups = filterByClass(parsedTags, ParsedHedGroup) + const parsedGroups = parsedTags.filter((obj) => obj instanceof ParsedHedGroup) assert.equal(parsedGroups.length, test.allSubgroupCount, `[${header}] should have correct number of subgroups)`) for (let i = 0; i < parsedGroups.length; i++) { assert.equal( @@ -55,7 +74,6 @@ describe('Parse HED string tests', () => { if (shouldRun(name, test.testname, runAll, runMap, skipMap)) { testSplit(test) } else { - // eslint-disable-next-line no-console console.log(`----Skipping stringParserTest ${name}: ${test.testname}`) } }) diff --git a/tests/stringParser.spec.js b/tests/stringParser.spec.js new file mode 100644 index 00000000..968b2212 --- /dev/null +++ b/tests/stringParser.spec.js @@ -0,0 +1,431 @@ +import chai from 'chai' +const assert = chai.assert +import { beforeAll, describe, it } from '@jest/globals' + +import { generateIssue } from '../common/issues/issues' +import { SchemaSpec, SchemasSpec } from '../schema/specs' +import { recursiveMap } from '../utils/array' +import { parseHedString } from '../parser/parser' +import ParsedHedTag from '../parser/parsedHedTag' +import HedStringSplitter from '../parser/splitter' +import { buildSchemas } from '../schema/init' +import ColumnSplicer from '../parser/columnSplicer' +import ParsedHedGroup from '../parser/parsedHedGroup' +import { Schemas } from '../schema/containers' + +describe.skip('HED string parsing', () => { + const nullSchema = new Schemas(null) + /** + * Retrieve the original tag from a parsed HED tag object. + * @param {ParsedHedTag} parsedTag The parsed tag. + * @returns {string} The original tag. + */ + const originalMap = (parsedTag) => parsedTag.originalTag + + const splitHedString = (hedString, hedSchemas) => new HedStringSplitter(hedString, hedSchemas).splitHedString() + + const hedSchemaFile = 'tests/data/HED8.0.0.xml' + let hedSchemas + + beforeAll(async () => { + const spec2 = new SchemaSpec('', '8.0.0', '', hedSchemaFile) + const specs = new SchemasSpec().addSchemaSpec(spec2) + hedSchemas = await buildSchemas(specs) + }) + + /** + * Test-validate a list of strings without issues. + * + * @template T + * @param {Object} testStrings The strings to test. + * @param {Object} expectedResults The expected results. + * @param {function (string): T} testFunction The testing function. + */ + const validatorWithoutIssues = function (testStrings, expectedResults, testFunction) { + for (const [testStringKey, testString] of Object.entries(testStrings)) { + assert.property(expectedResults, testStringKey, testStringKey + ' is not in expectedResults') + const testResult = testFunction(testString) + assert.deepStrictEqual(testResult, expectedResults[testStringKey], testString) + } + } + + /** + * Test-validate a list of strings with issues. + * + * @template T + * @param {Object} testStrings The strings to test. + * @param {Object} expectedResults The expected results. + * @param {Object>} expectedIssues The expected issues. + * @param {function (string): [T, Object>]} testFunction The testing function. + */ + const validatorWithIssues = function (testStrings, expectedResults, expectedIssues, testFunction) { + for (const [testStringKey, testString] of Object.entries(testStrings)) { + assert.property(expectedResults, testStringKey, testStringKey + ' is not in expectedResults') + assert.property(expectedIssues, testStringKey, testStringKey + ' is not in expectedIssues') + const [testResult, testIssues] = testFunction(testString) + assert.sameDeepMembers(testResult, expectedResults[testStringKey], testString) + assert.deepOwnInclude(testIssues, expectedIssues[testStringKey], testString) + } + } + + describe('HED strings', () => { + it.skip('cannot have invalid characters', () => { + const testStrings = { + openingSquare: 'Relation/Spatial-relation/Left-side-of,/Action/Move/Bend[/Upper-extremity/Elbow', + closingSquare: 'Relation/Spatial-relation/Left-side-of,/Action/Move/Bend]/Upper-extremity/Elbow', + tilde: 'Relation/Spatial-relation/Left-side-of,/Action/Move/Bend~/Upper-extremity/Elbow', + } + const expectedResults = { + openingSquare: null, + closingSquare: null, + tilde: null, + } + const expectedIssues = { + openingSquare: { + syntax: [ + generateIssue('invalidCharacter', { + character: 'LEFT SQUARE BRACKET', + index: 56, + string: testStrings.openingSquare, + }), + ], + }, + closingSquare: { + syntax: [ + generateIssue('invalidCharacter', { + character: 'RIGHT SQUARE BRACKET', + index: 56, + string: testStrings.closingSquare, + }), + ], + }, + tilde: { + syntax: [ + generateIssue('invalidCharacter', { + character: 'TILDE', + index: 56, + string: testStrings.tilde, + }), + ], + }, + } + for (const [testStringKey, testString] of Object.entries(testStrings)) { + assert.property(expectedResults, testStringKey, testStringKey + ' is not in expectedResults') + assert.property(expectedIssues, testStringKey, testStringKey + ' is not in expectedIssues') + const [testResult, testIssues] = splitHedString(testString, nullSchema) + assert.strictEqual(testResult, expectedResults[testStringKey], testString) + assert.deepStrictEqual(testIssues, expectedIssues[testStringKey], testString) + } + }) + }) + + describe('Lists of HED tags', () => { + it('should be an array', () => { + const hedString = + 'Event/Category/Sensory-event,Item/Object/Man-made-object/Vehicle/Train,Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Purple-color/Purple' + const [result] = splitHedString(hedString, nullSchema) + assert.isTrue(Array.isArray(result)) + }) + + it('should include each top-level tag as its own single element', () => { + const hedString = + 'Event/Category/Sensory-event,Item/Object/Man-made-object/Vehicle/Train,Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Purple-color/Purple' + const [result, issues] = splitHedString(hedString, nullSchema) + assert.isEmpty(Object.values(issues).flat(), 'Parsing issues occurred') + assert.deepStrictEqual(result, [ + new ParsedHedTag('Event/Category/Sensory-event', [0, 28]), + new ParsedHedTag('Item/Object/Man-made-object/Vehicle/Train', [29, 70]), + new ParsedHedTag( + 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Purple-color/Purple', + [71, 167], + ), + ]) + }) + + it('should include each group as its own single element', () => { + const hedString = + 'Action/Move/Flex,(Relation/Spatial-relation/Left-side-of,Action/Move/Bend,Upper-extremity/Elbow),Position/X-position/70 px,Position/Y-position/23 px' + const [result, issues] = splitHedString(hedString, nullSchema) + assert.isEmpty(Object.values(issues).flat(), 'Parsing issues occurred') + assert.deepStrictEqual(result, [ + new ParsedHedTag('Action/Move/Flex', [0, 16]), + new ParsedHedGroup( + [ + new ParsedHedTag('Relation/Spatial-relation/Left-side-of', [18, 56]), + new ParsedHedTag('Action/Move/Bend', [57, 73]), + new ParsedHedTag('Upper-extremity/Elbow', [74, 95]), + ], + nullSchema, + hedString, + [17, 96], + ), + new ParsedHedTag('Position/X-position/70 px', [97, 122]), + new ParsedHedTag('Position/Y-position/23 px', [123, 148]), + ]) + }) + + it('should not include blanks', () => { + const testStrings = { + okay: 'Item/Object/Man-made-object/Vehicle/Car, Action/Perform/Operate', + internalBlank: 'Item Object', + } + const expectedList = [ + new ParsedHedTag('Item/Object/Man-made-object/Vehicle/Car', [0, 39]), + new ParsedHedTag('Action/Perform/Operate', [41, 63]), + ] + const expectedResults = { + okay: expectedList, + internalBlank: [new ParsedHedTag('Item Object', [0, 11])], + } + const expectedIssues = { + okay: {}, + internalBlank: {}, + } + validatorWithIssues(testStrings, expectedResults, expectedIssues, (string) => { + return splitHedString(string, nullSchema) + }) + }) + }) + + describe('Formatted HED tags', () => { + it('should be lowercase and not have leading or trailing double quotes', () => { + // Correct formatting + const formattedHedTag = 'event/category/sensory-event' + const testStrings = { + formatted: formattedHedTag, + openingDoubleQuote: '"Event/Category/Sensory-event', + closingDoubleQuote: 'Event/Category/Sensory-event"', + openingAndClosingDoubleQuote: '"Event/Category/Sensory-event"', + // openingSlash: '/Event/Category/Sensory-event', + // closingSlash: 'Event/Category/Sensory-event/', + // openingAndClosingSlash: '/Event/Category/Sensory-event/', + // openingDoubleQuotedSlash: '"Event/Category/Sensory-event', + // closingDoubleQuotedSlash: 'Event/Category/Sensory-event"', + // openingSlashClosingDoubleQuote: '/Event/Category/Sensory-event"', + // closingSlashOpeningDoubleQuote: '"Event/Category/Sensory-event/', + // openingAndClosingDoubleQuotedSlash: '"/Event/Category/Sensory-event/"', + } + const expectedResults = { + formatted: formattedHedTag, + openingDoubleQuote: formattedHedTag, + closingDoubleQuote: formattedHedTag, + openingAndClosingDoubleQuote: formattedHedTag, + // openingSlash: formattedHedTag, + // closingSlash: formattedHedTag, + // openingAndClosingSlash: formattedHedTag, + // openingDoubleQuotedSlash: formattedHedTag, + // closingDoubleQuotedSlash: formattedHedTag, + // openingSlashClosingDoubleQuote: formattedHedTag, + // closingSlashOpeningDoubleQuote: formattedHedTag, + // openingAndClosingDoubleQuotedSlash: formattedHedTag, + } + validatorWithoutIssues(testStrings, expectedResults, (string) => { + const parsedTag = new ParsedHedTag(string, []) + return parsedTag.formattedTag + }) + }) + }) + + describe('Parsed HED strings', () => { + it('must have the correct number of tags, top-level tags, and groups', () => { + const hedString = + 'Action/Move/Flex,(Relation/Spatial-relation/Left-side-of,Action/Move/Bend,Upper-extremity/Elbow),Position/X-position/70 px,Position/Y-position/23 px' + const [parsedString, issues] = parseHedString(hedString, nullSchema) + assert.isEmpty(Object.values(issues).flat(), 'Parsing issues occurred') + assert.sameDeepMembers(parsedString.tags.map(originalMap), [ + 'Action/Move/Flex', + 'Relation/Spatial-relation/Left-side-of', + 'Action/Move/Bend', + 'Upper-extremity/Elbow', + 'Position/X-position/70 px', + 'Position/Y-position/23 px', + ]) + assert.sameDeepMembers(parsedString.topLevelTags.map(originalMap), [ + 'Action/Move/Flex', + 'Position/X-position/70 px', + 'Position/Y-position/23 px', + ]) + assert.sameDeepMembers( + parsedString.tagGroups.map((group) => group.tags.map(originalMap)), + [['Relation/Spatial-relation/Left-side-of', 'Action/Move/Bend', 'Upper-extremity/Elbow']], + ) + }) + + it('must include properly formatted tags', () => { + const hedString = + 'Action/Move/Flex,(Relation/Spatial-relation/Left-side-of,Action/Move/Bend,Upper-extremity/Elbow),Position/X-position/70 px,Position/Y-position/23 px' + const formattedHedString = + 'action/move/flex,(relation/spatial-relation/left-side-of,action/move/bend,upper-extremity/elbow),position/x-position/70 px,position/y-position/23 px' + const [parsedString, issues] = parseHedString(hedString, nullSchema) + const [parsedFormattedString, formattedIssues] = parseHedString(formattedHedString, nullSchema) + const formattedMap = (parsedTag) => { + return parsedTag.formattedTag + } + assert.isEmpty(Object.values(issues).flat(), 'Parsing issues occurred') + assert.isEmpty(Object.values(formattedIssues).flat(), 'Parsing issues occurred in the formatted string') + assert.deepStrictEqual(parsedString.tags.map(formattedMap), parsedFormattedString.tags.map(originalMap)) + assert.deepStrictEqual( + parsedString.topLevelTags.map(formattedMap), + parsedFormattedString.topLevelTags.map(originalMap), + ) + }) + + it('must correctly handle multiple levels of parentheses', () => { + const testStrings = { + shapes: 'Square,(Definition/RedCircle,(Circle,Red)),Rectangle', + vehicles: + 'Car,(Definition/TrainVelocity/#,(Train,(Measurement-device/Odometer,Data-maximum/160,Speed/# kph),Blue,Age/12,(Navigational-object/Railway,Data-maximum/150)))', + typing: '((Human-agent,Joyful),Press,Keyboard-key/F),(Braille,Character/A,Screen-window)', + } + const expectedTags = { + shapes: ['Square', 'Definition/RedCircle', 'Circle', 'Red', 'Rectangle'], + vehicles: [ + 'Car', + 'Definition/TrainVelocity/#', + 'Train', + 'Measurement-device/Odometer', + 'Data-maximum/160', + 'Speed/# kph', + 'Blue', + 'Age/12', + 'Navigational-object/Railway', + 'Data-maximum/150', + ], + typing: ['Human-agent', 'Joyful', 'Press', 'Keyboard-key/F', 'Braille', 'Character/A', 'Screen-window'], + } + const expectedGroups = { + shapes: [['Definition/RedCircle', ['Circle', 'Red']]], + vehicles: [ + [ + 'Definition/TrainVelocity/#', + [ + 'Train', + ['Measurement-device/Odometer', 'Data-maximum/160', 'Speed/# kph'], + 'Blue', + 'Age/12', + ['Navigational-object/Railway', 'Data-maximum/150'], + ], + ], + ], + typing: [ + [['Human-agent', 'Joyful'], 'Press', 'Keyboard-key/F'], + ['Braille', 'Character/A', 'Screen-window'], + ], + } + + for (const [testStringKey, testString] of Object.entries(testStrings)) { + const [parsedString, issues] = parseHedString(testString, hedSchemas) + assert.isEmpty(Object.values(issues).flat(), 'Parsing issues occurred') + assert.sameDeepMembers(parsedString.tags.map(originalMap), expectedTags[testStringKey], testString) + assert.deepStrictEqual( + recursiveMap( + originalMap, + parsedString.tagGroups.map((tagGroup) => tagGroup.nestedGroups()), + ), + expectedGroups[testStringKey], + testString, + ) + } + }) + }) + + describe('Canonical HED tags', () => { + it('should convert HED 3 tags into canonical form', () => { + const testStrings = { + simple: 'Car', + groupAndTag: '(Train, RGB-red/0.5), Car', + invalidTag: 'InvalidTag', + invalidParentNode: 'Car/Train/Maglev', + } + const expectedResults = { + simple: ['Item/Object/Man-made-object/Vehicle/Car'], + groupAndTag: [ + 'Item/Object/Man-made-object/Vehicle/Train', + 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red/0.5', + 'Item/Object/Man-made-object/Vehicle/Car', + ], + invalidTag: [], + invalidParentNode: [], + } + const expectedIssues = { + simple: {}, + groupAndTag: {}, + invalidTag: { + conversion: [generateIssue('invalidTag', { tag: testStrings.invalidTag })], + }, + invalidParentNode: { + conversion: [ + generateIssue('invalidParentNode', { + parentTag: 'Item/Object/Man-made-object/Vehicle/Train', + tag: 'Train', + }), + ], + }, + } + + return validatorWithIssues(testStrings, expectedResults, expectedIssues, (string) => { + const [parsedString, issues] = parseHedString(string, hedSchemas) + const canonicalTags = parsedString.tags.map((parsedTag) => { + return parsedTag.canonicalTag + }) + return [canonicalTags, issues] + }) + }) + }) + + describe('HED column splicing', () => { + it('must properly splice columns', () => { + const hedStrings = [ + 'Sensory-event, Visual-presentation, {stim_file}, (Face, {stim_file})', + '(Image, Face, Pathname/#)', + 'Sensory-event, Visual-presentation, (Image, Face, Pathname/abc.bmp), (Face, (Image, Face, Pathname/abc.bmp))', + ] + const issues = [] + const parsedStrings = [] + for (const hedString of hedStrings) { + const [parsedString, parsingIssues] = parseHedString(hedString, hedSchemas) + parsedStrings.push(parsedString) + issues.push(...Object.values(parsingIssues).flat()) + } + assert.isEmpty(issues, 'Parsing issues') + const [baseString, refString, correctString] = parsedStrings + const replacementMap = new Map([['stim_file', refString]]) + const columnSplicer = new ColumnSplicer( + baseString, + replacementMap, + new Map([['stim_file', 'abc.bmp']]), + hedSchemas, + ) + const splicedString = columnSplicer.splice() + const splicingIssues = columnSplicer.issues + assert.strictEqual(splicedString.format(), correctString.format(), 'Full string') + assert.isEmpty(splicingIssues, 'Splicing issues') + }) + + it('must properly detect recursive curly braces', () => { + const hedStrings = ['Sensory-event, Visual-presentation, {stim_file}', '(Image, {body_part}, Pathname/#)', 'Face'] + const issues = [] + const parsedStrings = [] + for (const hedString of hedStrings) { + const [parsedString, parsingIssues] = parseHedString(hedString, hedSchemas) + parsedStrings.push(parsedString) + issues.push(...Object.values(parsingIssues).flat()) + } + const [baseString, refString, doubleRefString] = parsedStrings + const replacementMap = new Map([ + ['stim_file', refString], + ['body_part', doubleRefString], + ]) + const columnSplicer = new ColumnSplicer( + baseString, + replacementMap, + new Map([['stim_file', 'abc.bmp']]), + hedSchemas, + ) + columnSplicer.splice() + const splicingIssues = columnSplicer.issues + issues.push(...splicingIssues) + assert.deepStrictEqual(issues, [generateIssue('recursiveCurlyBraces', { column: 'stim_file' })]) + }) + }) +}) diff --git a/tests/stringParserTests.spec.js b/tests/stringParserTests.spec.js index fbadf4cb..7d6b9774 100644 --- a/tests/stringParserTests.spec.js +++ b/tests/stringParserTests.spec.js @@ -1,6 +1,6 @@ import chai from 'chai' const assert = chai.assert -import { beforeAll, describe, afterAll, it } from '@jest/globals' +import { beforeAll, describe, afterAll } from '@jest/globals' import path from 'path' import { buildSchemas } from '../schema/init' @@ -12,49 +12,54 @@ import { shouldRun, getHedString } from './testUtilities' const skipMap = new Map() const runAll = true -const runMap = new Map([['tag-strings', ['multiple-complex-duplicates']]]) +const runMap = new Map([['special-tag-group-tests', ['definition-with-deep-defs-inside']]]) describe('Null schema objects should cause parsing to bail', () => { it('Should not proceed if no schema and valid string', () => { const stringIn = 'Item, Red' - const [parsedString, errorIssues, warningIssues] = getHedString(stringIn, null, true, true, true) + const [parsedString, errorIssues, warningIssues] = getHedString(stringIn, null, true) assert.isNull(parsedString, `Parsed HED string of ${stringIn} is null although string is valid`) const expectedIssues = [generateIssue('missingSchemaSpecification', {})] assert.deepStrictEqual(errorIssues, expectedIssues, `A SCHEMA_LOAD_FAILED issue should be generated`) - const [directParsed, directIssues] = parseHedString(stringIn, null, false, true, true) + const [directParsed, directIssues] = parseHedString(stringIn, null) assert.isNull(directParsed, `Parsed HED string of ${stringIn} is null for invalid string`) - assert.deepStrictEqual(directIssues, expectedIssues) - assert.equal(warningIssues.length, 0, `Null schema produces errors, not warnings`) + assert.deepStrictEqual(directIssues.syntaxIssues, expectedIssues) }) it('Should not proceed if no schema and invalid string', () => { - const stringIn = 'Item/Blue, Red' - const [parsedString, errorIssues, warningIssues] = getHedString(stringIn, null, true, false, false) + const stringIn = 'Item/#, Red' + const [parsedString, errorIssues, warningIssues] = getHedString(stringIn, null, true) assert.isNull(parsedString, `Parsed HED string of ${stringIn} is null for invalid string`) const expectedIssues = [generateIssue('missingSchemaSpecification', {})] assert.deepStrictEqual(errorIssues, expectedIssues, `A SCHEMA_LOAD_FAILED issue should be generated`) - const [directParsed, directIssues] = parseHedString(stringIn, null, true, true, true) + const [directParsed, directIssues] = parseHedString(stringIn, null) assert.isNull(directParsed, `Parsed HED string of ${stringIn} is null for invalid string`) - assert.deepStrictEqual(directIssues, expectedIssues) - assert.equal(warningIssues.length, 0, `Null schema produces errors, not warnings`) + assert.deepStrictEqual(directIssues.syntaxIssues, expectedIssues) }) it('Should not proceed if no schema and valid array of strings', () => { - const arrayIn = ['Item, Red', 'Blue'] + const arrayIn = new Array('Item, Red', 'Blue') const expectedIssues = [generateIssue('missingSchemaSpecification', {})] - const [directParsed, directIssues] = parseHedStrings(arrayIn, null, true, false, false) + const [directParsed, directIssues] = parseHedStrings(arrayIn, null) assert.isNull(directParsed, `Parsed HED string of ${arrayIn} is null for invalid string`) - assert.deepStrictEqual(directIssues, expectedIssues) + assert.deepStrictEqual(directIssues.syntaxIssues, expectedIssues) }) }) describe('Parse HED string tests', () => { - const schemaMap = new Map([['8.3.0', undefined]]) + const schemaMap = new Map([ + ['8.2.0', undefined], + ['8.3.0', undefined], + ]) beforeAll(async () => { + const spec2 = new SchemaSpec('', '8.2.0', '', path.join(__dirname, '../tests/data/HED8.2.0.xml')) + const specs2 = new SchemasSpec().addSchemaSpec(spec2) + const schemas2 = await buildSchemas(specs2) const spec3 = new SchemaSpec('', '8.3.0', '', path.join(__dirname, '../tests/data/HED8.3.0.xml')) const specs3 = new SchemasSpec().addSchemaSpec(spec3) const schemas3 = await buildSchemas(specs3) + schemaMap.set('8.2.0', schemas2) schemaMap.set('8.3.0', schemas3) }) @@ -68,29 +73,26 @@ describe('Parse HED string tests', () => { assert.isDefined(thisSchema, `header: ${test.schemaVersion} is not available in test ${test.testname}`) // Parse the string before converting - let issues - let warnings = [] - let parsedString = null - try { - ;[parsedString, issues, warnings] = getHedString( - test.stringIn, - thisSchema, - test.fullCheck, - test.definitionsAllowed, - test.placeholdersAllowed, - ) - } catch (error) { - issues = [error.issue] - } + const [parsedString, errorIssues, warningIssues] = getHedString(test.stringIn, thisSchema, test.fullCheck) + + // Check for errors + assert.deepStrictEqual( + errorIssues, + test.errors, + `${header}: expected ${errorIssues} errors but received ${test.errors}\n`, + ) // Check if warning match assert.deepStrictEqual( - warnings, + warningIssues, test.warnings, - `${header}: expected ${warnings} warnings but received ${test.warnings}\n`, + `${header}: expected ${warningIssues} warnings but received ${test.warnings}\n`, ) + if (parsedString === null) { + return + } // Check the conversion to long - const longString = parsedString ? parsedString.format(true) : null + const longString = parsedString.format(true) assert.strictEqual( longString, test.stringLong, @@ -98,14 +100,12 @@ describe('Parse HED string tests', () => { ) // Check the conversion to short - const shortString = parsedString ? parsedString.format(false) : null + const shortString = parsedString.format(false) assert.strictEqual( shortString, test.stringShort, `${header}: expected ${test.stringShort} but received ${shortString}`, ) - // Check for errors - assert.deepStrictEqual(issues, test.errors, `${header}: expected ${issues} errors but received ${test.errors}\n`) } beforeAll(async () => {}) @@ -117,7 +117,6 @@ describe('Parse HED string tests', () => { if (shouldRun(name, test.testname, runAll, runMap, skipMap)) { testConvert(test) } else { - // eslint-disable-next-line no-console console.log(`----Skipping stringParserTest ${name}: ${test.testname}`) } }) diff --git a/tests/tagParserTests.spec.js b/tests/tagParserTests.spec.js index 59f73eb3..00b4b96d 100644 --- a/tests/tagParserTests.spec.js +++ b/tests/tagParserTests.spec.js @@ -1,6 +1,6 @@ import chai from 'chai' const assert = chai.assert -import { beforeAll, describe, afterAll } from '@jest/globals' +import { beforeAll, describe, afterAll, it } from '@jest/globals' import ParsedHedTag from '../parser/parsedHedTag' import { shouldRun } from './testUtilities' @@ -8,20 +8,30 @@ import { parsedHedTagTests } from './testData/tagParserTests.data' import { SchemaSpec, SchemasSpec } from '../schema/specs' import path from 'path' import { buildSchemas } from '../schema/init' -import { SchemaValueTag } from '../schema/entries' +import { SchemaTag, SchemaValueTag } from '../schema/entries' +import { TagSpec } from '../parser/tokenizer' +import TagConverter from '../parser/tagConverter' +import { BidsSidecar, BidsTsvFile } from '../bids' // Ability to select individual tests to run const skipMap = new Map() const runAll = true -const runMap = new Map([['valid-tags', ['valid-tag-with-extension-and-blanks']]]) +const runMap = new Map([['valid-tags', ['valid-tag-with-extension']]]) describe('TagSpec converter tests using JSON tests', () => { - const schemaMap = new Map([['8.3.0', undefined]]) + const schemaMap = new Map([ + ['8.2.0', undefined], + ['8.3.0', undefined], + ]) beforeAll(async () => { + const spec2 = new SchemaSpec('', '8.2.0', '', path.join(__dirname, '../tests/data/HED8.2.0.xml')) + const specs2 = new SchemasSpec().addSchemaSpec(spec2) + const schemas2 = await buildSchemas(specs2) const spec3 = new SchemaSpec('', '8.3.0', '', path.join(__dirname, '../tests/data/HED8.3.0.xml')) const specs3 = new SchemasSpec().addSchemaSpec(spec3) const schemas3 = await buildSchemas(specs3) + schemaMap.set('8.2.0', schemas2) schemaMap.set('8.3.0', schemas3) }) @@ -42,12 +52,12 @@ describe('TagSpec converter tests using JSON tests', () => { } catch (error) { issue = error.issue } - assert.deepEqual(issue, test.error, `${header}: expected ${issue} but received ${test.error}`) + assert.deepEqual(issue, test.error, `${header}: wrong issue`) assert.strictEqual(tag?.format(false), test.tagShort, `${header}: wrong short version`) assert.strictEqual(tag?.format(true), test.tagLong, `${header}: wrong long version`) assert.strictEqual(tag?.formattedTag, test.formattedTag, `${header}: wrong formatted version`) assert.strictEqual(tag?.canonicalTag, test.canonicalTag, `${header}: wrong canonical version`) - if (test.error || !tag) { + if (test.error) { return } if (test.takesValue) { @@ -66,7 +76,6 @@ describe('TagSpec converter tests using JSON tests', () => { if (shouldRun(name, test.testname, runAll, runMap, skipMap)) { hedTagTest(test) } else { - // eslint-disable-next-line no-console console.log(`----Skipping tagParserTest ${name}: ${test.testname}`) } }) diff --git a/tests/testData/bids.spec.data.js b/tests/testData/bids.spec.data.js new file mode 100644 index 00000000..5d8b2476 --- /dev/null +++ b/tests/testData/bids.spec.data.js @@ -0,0 +1,782 @@ +import { BidsEventFile, BidsJsonFile, BidsSidecar } from '../../bids' +import { recursiveMap } from '../../utils/array' + +const sidecars = [ + // sub01 - Valid sidecars + [ + { + color: { + HED: { + red: 'RGB-red', + green: 'RGB-green', + blue: 'RGB-blue', + }, + }, + }, + { + vehicle: { + HED: { + car: 'Car', + train: 'Train', + boat: 'Boat', + }, + }, + speed: { + HED: 'Speed/# mph', + }, + }, + { + duration: { + HED: 'Duration/# s', + }, + age: { + HED: 'Age/#', + }, + }, + ], + // sub02 - Invalid sidecars + [ + { + transport: { + HED: { + car: 'Car', + train: 'Train', + boat: 'Boat', + maglev: 'Train/Maglev', // Extension. + }, + }, + }, + { + emotion: { + HED: { + happy: 'Happy', + sad: 'Sad', + angry: 'Angry', + confused: 'Confused', // Not in schema. + }, + }, + }, + ], + // sub03 - Placeholders + [ + { + valid_definition: { + HED: { definition: '(Definition/ValidDefinition, (Square))' }, + }, + }, + { + valid_placeholder_definition: { + HED: { + definition: '(Definition/ValidPlaceholderDefinition/#, (RGB-red/#))', + }, + }, + }, + { + invalid_definition_group: { + HED: { definition: '(Definition/InvalidDefinitionGroup, (Age/#))' }, + }, + }, + { + invalid_definition_tag: { + HED: { definition: '(Definition/InvalidDefinitionTag/#, (Age))' }, + }, + }, + { + multiple_placeholders_in_group: { + HED: { + definition: '(Definition/MultiplePlaceholdersInGroupDefinition/#, (Age/#, Duration/# s))', + }, + }, + }, + { + multiple_value_tags: { + HED: 'Label/#, Description/#', + }, + }, + { + no_value_tags: { + HED: 'Sad', + }, + }, + { + value_in_categorical: { + HED: { + purple: 'Purple', + yellow: 'Yellow', + orange: 'Orange', + green: 'RGB-green/#', + }, + }, + }, + ], + // sub04 - HED 2 sidecars + [ + { + test: { + HED: { + first: 'Event/Label/Test,Event/Category/Miscellaneous/Test,Event/Description/Test', + }, + }, + }, + ], + // sub05 - HED 3 sidecars with libraries + [ + { + // Library and base and defs + event_type: { + HED: { + show_face: 'Sensory-event, ts:Visual-presentation', + left_press: 'Press, Def/My-def1, ts:Def/My-def2/3', + }, + }, + dummy_defs: { + HED: { + def1: '(Definition/My-def1, (Red, Blue))', + def2: '(ts:Definition/My-def2/#, (Green, Label/#))', + }, + }, + }, + { + // Just library no defs + event_type: { + HED: { + show_face: 'ts:Sensory-event, ts:Visual-presentation', + left_press: 'ts:Push-button', + }, + }, + }, + { + // Just base + event_type: { + HED: { + show_face: 'Sensory-event, Visual-presentation', + left_press: 'Push-button', + }, + }, + }, + { + // Just score as base + event_type: { + HED: { + show_face: 'Manual-eye-closure, Drowsiness', + left_press: 'Wicket-spikes, Finding-frequency', + }, + }, + }, + { + // Just score as a library + event_type: { + HED: { + show_face: 'sc:Manual-eye-closure, sc:Drowsiness', + left_press: 'sc:Wicket-spikes, sc:Finding-frequency', + }, + }, + }, + { + // Testlib with Defs as base + event_type: { + HED: { + show_face: 'Sensory-event, Visual-presentation', + left_press: 'Press, Def/My-def1, Def/My-def2/3', + }, + }, + dummy_defs: { + HED: { + def1: '(Definition/My-def1, (Red, Blue))', + def2: '(Definition/My-def2/#, (Green, Label/#))', + }, + }, + }, + { + // Testlib with defs with as library + event_type: { + HED: { + show_face: 'ts:Sensory-event, ts:Visual-presentation', + left_press: 'ts:Press, ts:Def/My-def1, ts:Def/My-def2/3', + }, + }, + dummy_defs: { + HED: { + def1: '(ts:Definition/My-def1, (ts:Red, ts:Blue))', + def2: '(ts:Definition/My-def2/#, (ts:Green, ts:Label/#))', + }, + }, + }, + ], + // sub06 - HED definitions + [ + { + // Valid + defs: { + HED: { + face: '(Definition/myDef, (Label/Red, Blue)), (Definition/myDef2, (Label/Red, Blue))', + ball: '(Definition/myDef1, (Label/Red, Blue))', + }, + }, + }, + { + // Valid + defs: { + HED: { + def1: '(Definition/Apple/#, (Label/#))', + def2: '(Definition/Blech/#, (Red, Label/#))', + }, + }, + }, + { + // Invalid "value" column with definition + event_code: { + HED: '(Definition/myDef, (Label/Red, Blue))', + }, + }, + { + // Invalid mix of definitions and non-definitions in categorical column + event_code: { + HED: { + face: 'Red, Blue, (Definition/myDef, (Label/Red, Blue))', + ball: 'Def/Acc/5.4', + }, + }, + }, + { + // Invalid mix of definitions and non-definitions in categorical column + event_code: { + HED: { + face: '(Definition/myDef, (Label/Red, Blue))', + ball: 'Def/Acc/4.5', + }, + }, + }, + ], + // sub07 - Standalone HED curly brace tests + [ + { + // Valid definitions + defs: { + HED: { + def1: '(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', + def2: '(Definition/MyColor, (Label/Pie))', + }, + }, + }, + { + // Invalid definitions + defs: { + HED: { + def1: '(Definition/Acc/#, {event_code}, (Acceleration/#, Red))', + def2: '(Definition/MyColor, (Label/Pie, {response_time}))', + }, + }, + }, + { + // Valid reference to named column with HED + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{response_time}, (Def/Acc/3.5)', + }, + }, + response_time: { + Description: 'Has description with HED', + HED: 'Label/#', + }, + }, + { + // Valid reference to HED column + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{HED}, (Def/Acc/3.5)', + }, + }, + response_action: { + Description: 'Does not correspond to curly braces', + }, + }, + { + // Valid references to named column and HED column + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{response_time}, (Def/Acc/3.5), ({HED})', + }, + }, + response_time: { + Description: 'Has description with HED', + HED: 'Label/#', + }, + }, + { + // Valid use of shared curly brace column references + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{response_time}, (Def/Acc/3.5)', + }, + }, + response_time: { + Description: 'Has description with HED', + HED: 'Label/#', + }, + response_count: { + Description: 'A count used to test curly braces in value columns.', + HED: '(Item-count/#, {response_time})', + }, + }, + { + // Invalid use of mutually recursive curly brace references + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow)), {HED}', + ball: '{response_time}, (Def/Acc/3.5)', + }, + }, + response_time: { + HED: 'Label/#, {event_code}', + }, + }, + { + // Invalid use of recursive curly brace references + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow)), {HED}', + ball: '(Def/Acc/3.5 m-per-s^2)', + dog: 'Orange, {event_type}', + }, + }, + response_time: { + HED: 'Label/#, {response_time2}', + }, + response_time2: { + HED: 'Label/#', + }, + event_type: { + HED: { + banana: 'Blue, {event_code}', + apple: 'Green', + }, + }, + }, + { + // Invalid use of self-recursive curly braces + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow)), {HED}', + ball: '{HED}, (Def/Acc/3.5)', + }, + }, + response_time: { + HED: 'Label/#, {response_time}', + }, + }, + { + // Invalid syntax errors in curly braces + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{response_time}{, (Def/Acc/3.5)', + }, + }, + event_code2: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{{response_time}, (Def/Acc/3.5 m-per-s^2)', + }, + }, + event_code3: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{response_time}}, (Def/Acc/3.5)', + }, + }, + event_code4: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{}, (Def/Acc/3.5 m-per-s^2)', + }, + }, + response_time: { + Description: 'Has description but no HED', + HED: 'Label/#', + }, + }, + ], + // sub08 - Combined HED curly brace tests + [ + { + // Valid definitions + defs: { + HED: { + def1: '(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', + def2: '(Definition/MyColor, (Label/Pie))', + }, + }, + }, + { + // Invalid definitions + defs: { + HED: { + def1: '(Definition/Acc/#, {event_code}, (Acceleration/# m-per-s^2, Red))', + def2: '(Definition/MyColor, (Label/Pie, {response_time}))', + }, + }, + }, + { + // Valid reference to named column with HED + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{response_time}, (Def/Acc/3.5)', + }, + }, + response_time: { + Description: 'Has description with HED', + HED: 'Label/#', + }, + }, + { + // Valid reference to HED column + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{HED}, (Def/Acc/3.5)', + }, + }, + response_action: { + Description: 'Does not correspond to curly braces', + }, + }, + { + // Valid references to named column and HED column + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{response_time}, (Def/Acc/3.5), ({HED})', + }, + }, + response_time: { + Description: 'Has description with HED', + HED: 'Label/#', + }, + }, + { + // Invalid reference to column with no HED + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{response_time}, (Def/Acc/3.5)', + }, + }, + response_time: { + Description: 'Has description but no HED', + }, + }, + { + // Invalid reference to non-existent column + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{response_time}, (Def/Acc/3.5)', + }, + }, + response_action: { + Description: 'Does not correspond to curly braces', + }, + }, + { + // Invalid duplicate reference to existing column + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '{response_time}, {response_time}, (Def/Acc/3.5)', + }, + }, + response_time: { + Description: 'Has description with HED', + HED: 'Label/#', + }, + }, + ], + // sub09 - Syntax errors + [ + { + // Invalid location of curly braces + event_code: { + HED: { + face: '(Red, Blue), (Green, (Yellow))', + ball: '(Def/Acc/{response_time})', + }, + }, + response_time: { + Description: 'Has description with HED', + HED: 'Label/#', + }, + }, + ], + // sub10 - Lazy partnered schemas + [ + { + // Valid partnered schemas + instruments: { + HED: { + piano_and_violin: '(Piano-sound, Violin-sound)', + flute_and_oboe: '(Flute-sound, Oboe-sound)', + choral_piano: '(Piano-sound, Vocalized-sound)', + }, + }, + }, + ], +] + +const hedColumnOnlyHeader = 'onset\tduration\tHED\n' +const tsvFiles = [ + // sub01 - Valid TSV-only data + [ + [{}, hedColumnOnlyHeader + '7\tsomething\tCellphone'], + [{}, hedColumnOnlyHeader + '7\tsomething\tCellphone\n' + '11\telse\tDesktop-computer'], + [{}, hedColumnOnlyHeader + '7\tsomething\tCeramic, Pink'], + ], + // sub02 - Invalid TSV-only data + [ + [{}, hedColumnOnlyHeader + '11\telse\tSpeed/300 miles'], + [{}, hedColumnOnlyHeader + '7\tsomething\tTrain/Maglev'], + [{}, hedColumnOnlyHeader + '7\tsomething\tTrain\n' + '11\telse\tSpeed/300 miles'], + [{}, hedColumnOnlyHeader + '7\tsomething\tMaglev\n' + '11\telse\tSpeed/300 miles'], + [{}, hedColumnOnlyHeader + '7\tsomething\tTrain/Maglev\n' + '11\telse\tSpeed/300 miles'], + ], + // sub03 - Valid combined sidecar/TSV data + [ + [sidecars[2][0], 'onset\tduration\n' + '7\t4'], + [sidecars[0][0], 'onset\tduration\tcolor\n' + '7\t4\tred'], + [sidecars[0][1], 'onset\tduration\tspeed\n' + '7\t4\t60'], + [sidecars[2][0], hedColumnOnlyHeader + '7\t4\tLaptop-computer'], + [sidecars[0][0], 'onset\tduration\tcolor\tHED\n' + '7\t4\tgreen\tLaptop-computer'], + [ + Object.assign({}, sidecars[0][0], sidecars[0][1]), + 'onset\tduration\tcolor\tvehicle\tspeed\n' + '7\tsomething\tblue\ttrain\t150', + ], + [ + Object.assign({}, sidecars[0][0], sidecars[0][1]), + 'onset\tduration\tcolor\tvehicle\tspeed\n' + + '7\tsomething\tred\ttrain\t150\n' + + '11\telse\tblue\tboat\t15\n' + + '15\tanother\tgreen\tcar\t70', + ], + ], + // sub04 - Invalid combined sidecar/TSV data + [ + [ + sidecars[1][1], + 'onset\tduration\temotion\tHED\n' + + '7\thigh\thappy\tYellow\n' + + '11\tlow\tsad\tBlue\n' + + '15\tmad\tangry\tRed\n' + + '19\thuh\tconfused\tGray', + ], + [ + sidecars[1][0], + 'onset\tduration\ttransport\n' + + '7\twet\tboat\n' + + '11\tsteam\ttrain\n' + + '15\ttires\tcar\n' + + '19\tspeedy\tmaglev', + ], + [ + Object.assign({}, sidecars[0][1], sidecars[1][0]), + 'onset\tduration\tvehicle\ttransport\tspeed\n' + + '7\tferry\ttrain\tboat\t20\n' + + '11\tautotrain\tcar\ttrain\t79\n' + + '15\ttowing\tboat\tcar\t30\n' + + '19\ttugboat\tboat\tboat\t5\n', + ], + [sidecars[0][2], 'onset\tduration\tage\tHED\n' + '7\tferry\t30\tAge/30\n'], + [sidecars[0][0], 'onset\tduration\tcolor\n' + '7\troyal\tpurple\n'], + ], + // sub05 - Valid combined sidecar/TSV data from HED 2 - Deprecated + [], + // sub06 - Valid combined sidecar/TSV data with library + [ + [sidecars[4][0], 'onset\tduration\tevent_type\tsize\n' + '7\tn/a\tshow_face\t6\n' + '7\tn/a\tleft_press\t7\n'], + [sidecars[4][1], 'onset\tduration\tevent_type\tsize\n' + '7\tn/a\tshow_face\t6\n' + '7\tn/a\tleft_press\t7\n'], + [sidecars[4][2], 'onset\tduration\tevent_type\tsize\n' + '7\tn/a\tshow_face\t6\n' + '7\tn/a\tleft_press\t7\n'], + [sidecars[4][3], 'onset\tduration\tevent_type\tsize\n' + '7\tn/a\tshow_face\t6\n' + '7\tn/a\tleft_press\t7\n'], + [sidecars[4][4], 'onset\tduration\tevent_type\tsize\n' + '7\tn/a\tshow_face\t6\n' + '7\tn/a\tleft_press\t7\n'], + [sidecars[4][5], 'onset\tduration\tevent_type\tsize\n' + '7\tn/a\tshow_face\t6\n' + '7\tn/a\tleft_press\t7\n'], + [sidecars[4][6], 'onset\tduration\tevent_type\tsize\n' + '7\tn/a\tshow_face\t6\n' + '7\tn/a\tleft_press\t7\n'], + ], + // sub07 - Definitions + [[sidecars[5][0], hedColumnOnlyHeader + '7\tsomething\t(Definition/myDef, (Label/Red, Green))']], + // sub08 - Standalone curly brace tests + [ + [ + Object.assign({}, sidecars[6][0], sidecars[6][4]), + 'onset\tduration\tevent_code\tHED\tresponse_time\n' + + '4.5\t0\tface\tBlue\t0\n' + + '5.0\t0\tball\tGreen,Def/MyColor\t1\n' + + '5.5\t0\tface\t\t2\n' + + '5.7\t0\tface\tn/a\t3', + ], + [ + Object.assign({}, sidecars[6][0], sidecars[6][4]), + 'onset\tduration\tevent_code\tHED\n' + + '4.5\t0\tface\tBlue, {response_time}\n' + + '5.0\t0\tball\tGreen, Def/MyColor\n' + + '5.2\t0\tface\t\n' + + '5.5\t0\tface\tn/a', + ], + ], + // sub09 - Combined curly brace tests + [ + [ + Object.assign({}, sidecars[7][0], sidecars[7][5]), + 'onset\tduration\tevent_code\tHED\tresponse_time\n' + + '4.5\t0\tface\tBlue\t0\n' + + '5.0\t0\tball\tGreen,Def/MyColor\t1\n' + + '5.5\t0\tface\t\t2\n' + + '5.7\t0\tface\tn/a\t3', + ], + [ + Object.assign({}, sidecars[7][0], sidecars[7][6]), + 'onset\tduration\tevent_code\tHED\tresponse_action\n' + + '4.5\t0\tface\tBlue\t0\n' + + '5.0\t0\tball\tGreen,Def/MyColor\t1\n' + + '5.5\t0\tface\t\t2\n' + + '5.7\t0\tface\tn/a\t3', + ], + [ + Object.assign({}, sidecars[7][0], sidecars[7][7]), + 'onset\tduration\tevent_code\tHED\tresponse_time\n' + + '4.5\t0\tface\tBlue\t0\n' + + '5.0\t0\tball\tGreen,Def/MyColor\t1\n' + + '5.5\t0\tface\t\t2\n' + + '5.7\t0\tface\tn/a\t3', + ], + ], + // sub10 - HED column curly brace tests + [ + [ + Object.assign({}, sidecars[6][0], sidecars[6][4]), + 'onset\tduration\tevent_code\tHED\n' + + '4.5\t0\tface\tBlue, {response_time}\n' + + '5.0\t0\tball\tGreen, Def/MyColor\n' + + '5.2\t0\tface\t\n' + + '5.5\t0\tface\tn/a', + ], + ], + // sub11 - 'n/a' curly brace tests + [ + [ + // Control + Object.assign({}, sidecars[6][0], sidecars[6][5]), + 'onset\tduration\tevent_code\tresponse_time\tresponse_count\n' + '5.0\t0\tball\t1\t2\n', + ], + [ + Object.assign({}, sidecars[6][0], sidecars[6][2]), + 'onset\tduration\tevent_code\tresponse_time\n' + '5.0\t0\tball\tn/a\n', + ], + [ + Object.assign({}, sidecars[6][0], sidecars[6][4]), + 'onset\tduration\tevent_code\tHED\tresponse_time\n' + '5.0\t0\tball\tGreen,Def/MyColor\tn/a\n', + ], + [ + Object.assign({}, sidecars[6][0], sidecars[6][4]), + 'onset\tduration\tevent_code\tHED\tresponse_time\n' + '5.0\t0\tball\tn/a\t1\n', + ], + [ + Object.assign({}, sidecars[6][0], sidecars[6][4]), + 'onset\tduration\tevent_code\tHED\tresponse_time\n' + '5.0\t0\tball\tn/a\tn/a\n', + ], + [ + Object.assign({}, sidecars[6][0], sidecars[6][5]), + 'onset\tduration\tevent_code\tresponse_time\tresponse_count\n' + '5.0\t0\tface\t1\tn/a\n', + ], + ], + // sub12 - Lazy partnered schemas + [ + [ + sidecars[9][0], + 'onset\tduration\tinstruments\n' + + '4.5\t0\tpiano_and_violin\n' + + '5.0\t0\tflute_and_oboe\n' + + '5.2\t0\tchoral_piano\n', + ], + ], +] + +const datasetDescriptions = [ + // Good datasetDescription.json files + [ + { Name: 'OnlyBase', BIDSVersion: '1.10.0', HEDVersion: '8.3.0' }, + { Name: 'BaseAndTest', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 'ts:testlib_1.0.2'] }, + { Name: 'OnlyTestAsLib', BIDSVersion: '1.10.0', HEDVersion: ['ts:testlib_1.0.2'] }, + { Name: 'BaseAndTwoTests', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 'ts:testlib_1.0.2', 'bg:testlib_1.0.2'] }, + { Name: 'TwoTests', BIDSVersion: '1.10.0', HEDVersion: ['ts:testlib_1.0.2', 'bg:testlib_1.0.2'] }, + { Name: 'OnlyScoreAsBase', BIDSVersion: '1.10.0', HEDVersion: 'score_1.0.0' }, + { Name: 'OnlyScoreAsLib', BIDSVersion: '1.10.0', HEDVersion: 'sc:score_1.0.0' }, + { Name: 'OnlyTestAsBase', BIDSVersion: '1.10.0', HEDVersion: 'testlib_1.0.2' }, + { Name: 'GoodLazyPartneredSchemas', BIDSVersion: '1.10.0', HEDVersion: ['testlib_2.0.0', 'testlib_3.0.0'] }, + { + Name: 'GoodLazyPartneredSchemasWithStandard', + BIDSVersion: '1.10.0', + HEDVersion: ['testlib_2.0.0', 'testlib_3.0.0', '8.2.0'], + }, + ], + // Bad datasetDescription.json files + [ + { Name: 'NonExistentLibrary', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 'ts:badlib_1.0.2'] }, + { Name: 'LeadingColon', BIDSVersion: '1.10.0', HEDVersion: [':testlib_1.0.2', '8.3.0'] }, + { Name: 'BadNickName', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 't-s:testlib_1.0.2'] }, + { Name: 'MultipleColons1', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 'ts::testlib_1.0.2'] }, + { Name: 'MultipleColons2', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', ':ts:testlib_1.0.2'] }, + { Name: 'NoLibraryName', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 'ts:_1.0.2'] }, + { Name: 'BadVersion1', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 'ts:testlib1.0.2'] }, + { Name: 'BadVersion2', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 'ts:testlib_1.a.2'] }, + { Name: 'BadRemote1', BIDSVersion: '1.10.0', HEDVersion: ['8.3.0', 'ts:testlib_1.800.2'] }, + { Name: 'BadRemote2', BIDSVersion: '1.10.0', HEDVersion: '8.828.0' }, + { Name: 'NoHedVersion', BIDSVersion: '1.10.0' }, + { Name: 'BadLazyPartneredSchema1', BIDSVersion: '1.10.0', HEDVersion: ['testlib_2.0.0', 'testlib_2.1.0'] }, + { Name: 'BadLazyPartneredSchema2', BIDSVersion: '1.10.0', HEDVersion: ['testlib_2.1.0', 'testlib_3.0.0'] }, + { + Name: 'LazyPartneredSchemasWithWrongStandard', + BIDSVersion: '1.10.0', + HEDVersion: ['testlib_2.0.0', 'testlib_3.0.0', '8.1.0'], + }, + ], +] + +/** + * @type {BidsSidecar[][]} + */ +export const bidsSidecars = sidecars.map((subData, sub) => { + return subData.map((runData, run) => { + const name = `/sub0${sub + 1}/sub0${sub + 1}_task-test_run-${run + 1}_events.json` + return new BidsSidecar(name, runData, { + relativePath: name, + path: name, + }) + }) +}) + +/** + * @type {BidsEventFile[][]} + */ +export const bidsTsvFiles = tsvFiles.map((subData, sub) => { + return subData.map((runData, run) => { + const [mergedDictionary, tsvData] = runData + const name = `/sub0${sub + 1}/sub0${sub + 1}_task-test_run-${run + 1}_events.tsv` + return new BidsEventFile(name, [], mergedDictionary, tsvData, { + relativePath: name, + path: name, + }) + }) +}) + +/** + * @type {BidsJsonFile[][]} + */ +export const bidsDatasetDescriptions = recursiveMap((datasetDescriptionData) => { + return new BidsJsonFile('/dataset_description.json', datasetDescriptionData, { + relativePath: '/dataset_description.json', + path: '/dataset_description.json', + }) +}, datasetDescriptions) diff --git a/tests/testData/bidsTests.data.js b/tests/testData/bidsTests.data.js index 56bb7f4e..f76eb3a8 100644 --- a/tests/testData/bidsTests.data.js +++ b/tests/testData/bidsTests.data.js @@ -5,12 +5,12 @@ export const bidsTestData = [ { name: 'valid-bids-datasets-with-limited-hed', description: 'HED or data is missing in various places', + definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], tests: [ { testname: 'no-hed-at-all-but-both-tsv-json-non-empty', explanation: 'Neither the sidecar or tsv has HED but neither non-empty', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { duration: { description: 'Duration of the event in seconds.', @@ -25,7 +25,6 @@ export const bidsTestData = [ testname: 'only-header-in-tsv-with-return', explanation: 'TSV only has header and trailing return and white space', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { duration: { description: 'Duration of the event in seconds.', @@ -40,7 +39,6 @@ export const bidsTestData = [ testname: 'empty-json-empty-tsv', explanation: 'Both sidecar and tsv are empty except for white space', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: {}, eventsString: '\n \n', sidecarErrors: [], @@ -52,12 +50,12 @@ export const bidsTestData = [ { name: 'invalid-syntax', description: 'Syntax errors in various places', + definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], tests: [ { testname: 'mismatched-parentheses-in-tsv', explanation: 'HED column has mismatched parentheses', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -69,7 +67,7 @@ export const bidsTestData = [ sidecarErrors: [], tsvErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('unclosedParenthesis', { index: '0', string: '(Red, Def/MyColor', tsvLine: '2' }), + generateIssue('unclosedParenthesis', { index: '0', string: '(Red, Def/MyColor', tsvLine: 2 }), { path: 'mismatched-parentheses-in-tsv.tsv', relativePath: 'mismatched-parentheses-in-tsv.tsv', @@ -78,7 +76,7 @@ export const bidsTestData = [ ], comboErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('unclosedParenthesis', { index: '0', string: '(Red, Def/MyColor', tsvLine: '2' }), + generateIssue('unclosedParenthesis', { index: '0', string: '(Red, Def/MyColor', tsvLine: 2 }), { path: 'mismatched-parentheses-in-tsv.tsv', relativePath: 'mismatched-parentheses-in-tsv.tsv' }, { tsvLine: 2 }, ), @@ -89,12 +87,12 @@ export const bidsTestData = [ { name: 'invalid-tag-tests', description: 'JSON is valid but tsv is invalid', + definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], tests: [ { testname: 'invalid-bad-tag-in-tsv', explanation: 'Unrelated sidecar is valid but HED column tag is invalid', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -105,7 +103,7 @@ export const bidsTestData = [ eventsString: 'onset\tduration\tHED\n' + '7\t4\tBaloney', sidecarErrors: [], tsvErrors: [ - BidsHedIssue.fromHedIssue(generateIssue('invalidTag', { tag: 'Baloney', tsvLine: '2' }), { + BidsHedIssue.fromHedIssue(generateIssue('invalidTag', { tag: 'Baloney', tsvLine: 2 }), { path: 'invalid-bad-tag-in-tsv.tsv', relativePath: 'invalid-bad-tag-in-tsv.tsv', }), @@ -122,7 +120,6 @@ export const bidsTestData = [ testname: 'invalid-bad-tag-in-JSON', explanation: 'Sidecar has a bad tag but tsv HED column tag is valid', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -149,7 +146,6 @@ export const bidsTestData = [ testname: 'invalid-bad-tag-in-JSON', explanation: 'Bad tag in JSON', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -176,7 +172,6 @@ export const bidsTestData = [ testname: 'valid-sidecar-tsv-curly-brace', explanation: 'The sidecar is valid, but tsv HED column has braces}', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -187,16 +182,17 @@ export const bidsTestData = [ eventsString: 'onset\tduration\tevent_code\tHED\n' + '7\t4\tface\tRed,{blue}', sidecarErrors: [], tsvErrors: [ - BidsHedIssue.fromHedIssue(generateIssue('curlyBracesInHedColumn', { string: 'Red,{blue}', tsvLine: '2' }), { + BidsHedIssue.fromHedIssue(generateIssue('curlyBracesNotAllowed', { string: 'Red,{blue}', tsvLine: 2 }), { path: 'valid-sidecar-tsv-curly-brace.tsv', relativePath: 'valid-sidecar-tsv-curly-brace.tsv', }), ], comboErrors: [ - BidsHedIssue.fromHedIssue(generateIssue('curlyBracesInHedColumn', { string: 'Red,{blue}', tsvLine: '2' }), { - path: 'valid-sidecar-tsv-curly-brace.tsv', - relativePath: 'valid-sidecar-tsv-curly-brace.tsv', - }), + BidsHedIssue.fromHedIssue( + generateIssue('curlyBracesNotAllowed', { string: 'Red,{blue}' }), + { path: 'valid-sidecar-tsv-curly-brace.tsv', relativePath: 'valid-sidecar-tsv-curly-brace.tsv' }, + { tsvLine: 2 }, + ), ], }, ], @@ -204,12 +200,12 @@ export const bidsTestData = [ { name: 'duplicate-tag-tests', description: 'Duplicate tags can appear in isolation or in combination', + definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], tests: [ { testname: 'valid-no-duplicate-tsv', explanation: 'No duplicates in tsv, no groups, no JSON', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: {}, eventsString: 'onset\tduration\tHED\n' + '19\t6\tEvent/Sensory-event,Item/Object, Purple\n', sidecarErrors: [], @@ -220,7 +216,6 @@ export const bidsTestData = [ testname: 'valid-duplicate-tsv', explanation: 'Duplicate at different level in tsv', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -239,7 +234,6 @@ export const bidsTestData = [ testname: 'valid-repeats-different-nesting-tsv', explanation: 'Duplicate groups not at same level in tsv', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: {}, eventsString: 'onset\tduration\tHED\n' + '19\t6\t(Red, Blue, (Green)), (Red, Blue, ((Green)))\n', sidecarErrors: [], @@ -250,7 +244,6 @@ export const bidsTestData = [ testname: 'invalid-duplicate-groups-first-level-tsv', explanation: 'The HED string has first level duplicate groups', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { vehicle: { HED: { @@ -263,21 +256,31 @@ export const bidsTestData = [ eventsString: 'onset\tduration\tvehicle\tHED\n' + '19\t6\tboat\t(Green, Blue),(Green, Blue)\n', sidecarErrors: [], tsvErrors: [ + BidsHedIssue.fromHedIssue(generateIssue('duplicateTag', { tag: '(Green, Blue)', tsvLine: 2 }), { + path: 'invalid-duplicate-groups-first-level-tsv.tsv', + relativePath: 'invalid-duplicate-groups-first-level-tsv.tsv', + }), + BidsHedIssue.fromHedIssue(generateIssue('duplicateTag', { tag: '(Green, Blue)', tsvLine: 2 }), { + path: 'invalid-duplicate-groups-first-level-tsv.tsv', + relativePath: 'invalid-duplicate-groups-first-level-tsv.tsv', + }), + ], + comboErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('duplicateTag', { tags: '[(Blue,Green)]', string: '(Green, Blue),(Green, Blue)' }), + generateIssue('duplicateTag', { tag: '(Green, Blue)' }), { path: 'invalid-duplicate-groups-first-level-tsv.tsv', relativePath: 'invalid-duplicate-groups-first-level-tsv.tsv', }, + { tsvLine: 2 }, ), - ], - comboErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('duplicateTag', { tags: '[(Blue,Green)]', string: '(Green, Blue),(Green, Blue)' }), + generateIssue('duplicateTag', { tag: '(Green, Blue)' }), { path: 'invalid-duplicate-groups-first-level-tsv.tsv', relativePath: 'invalid-duplicate-groups-first-level-tsv.tsv', }, + { tsvLine: 2 }, ), ], }, @@ -285,28 +288,42 @@ export const bidsTestData = [ testname: 'invalid-different-forms-same-tag-tsv', explanation: 'Duplicate tags in different forms', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: {}, eventsString: 'onset\tduration\tHED\n' + '19\t6\tTrain,Vehicle/Train\n', sidecarErrors: [], tsvErrors: [ - BidsHedIssue.fromHedIssue(generateIssue('duplicateTag', { tags: '[Train]', string: 'Train,Vehicle/Train' }), { + BidsHedIssue.fromHedIssue(generateIssue('duplicateTag', { tag: 'Train', tsvLine: 2 }), { path: 'invalid-different-forms-same-tag-tsv.tsv', relativePath: 'invalid-different-forms-same-tag-tsv.tsv', }), - ], - comboErrors: [ - BidsHedIssue.fromHedIssue(generateIssue('duplicateTag', { tags: '[Train]', string: 'Train,Vehicle/Train' }), { + BidsHedIssue.fromHedIssue(generateIssue('duplicateTag', { tag: 'Vehicle/Train', tsvLine: 2 }), { path: 'invalid-different-forms-same-tag-tsv.tsv', relativePath: 'invalid-different-forms-same-tag-tsv.tsv', }), ], + comboErrors: [ + BidsHedIssue.fromHedIssue( + generateIssue('duplicateTag', { tag: 'Train' }), + { + path: 'invalid-different-forms-same-tag-tsv.tsv', + relativePath: 'invalid-different-forms-same-tag-tsv.tsv', + }, + { tsvLine: 2 }, + ), + BidsHedIssue.fromHedIssue( + generateIssue('duplicateTag', { tag: 'Vehicle/Train' }), + { + path: 'invalid-different-forms-same-tag-tsv.tsv', + relativePath: 'invalid-different-forms-same-tag-tsv.tsv', + }, + { tsvLine: 2 }, + ), + ], }, { testname: 'invalid-repeated-nested-groups-tsv', explanation: 'The HED string has first level repeated nested groups', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { vehicle: { HED: { @@ -321,27 +338,55 @@ export const bidsTestData = [ '19\t6\tboat\t(Red, (Blue, Green, (Yellow)), Red, (Blue, Green, (Yellow)))\n', sidecarErrors: [], tsvErrors: [ + BidsHedIssue.fromHedIssue(generateIssue('duplicateTag', { tag: 'Red', tsvLine: 2 }), { + path: 'invalid-repeated-nested-groups-tsv.tsv', + relativePath: 'invalid-repeated-nested-groups-tsv.tsv', + }), + BidsHedIssue.fromHedIssue(generateIssue('duplicateTag', { tag: 'Red', tsvLine: 2 }), { + path: 'invalid-repeated-nested-groups-tsv.tsv', + relativePath: 'invalid-repeated-nested-groups-tsv.tsv', + }), + BidsHedIssue.fromHedIssue(generateIssue('duplicateTag', { tag: '(Blue, Green, (Yellow))', tsvLine: 2 }), { + path: 'invalid-repeated-nested-groups-tsv.tsv', + relativePath: 'invalid-repeated-nested-groups-tsv.tsv', + }), + BidsHedIssue.fromHedIssue(generateIssue('duplicateTag', { tag: '(Blue, Green, (Yellow))', tsvLine: 2 }), { + path: 'invalid-repeated-nested-groups-tsv.tsv', + relativePath: 'invalid-repeated-nested-groups-tsv.tsv', + }), + ], + comboErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('duplicateTag', { - tags: '[((Yellow),Blue,Green)],[Red]', - string: '(Red, (Blue, Green, (Yellow)), Red, (Blue, Green, (Yellow)))', - }), + generateIssue('duplicateTag', { tag: 'Red' }), { path: 'invalid-repeated-nested-groups-tsv.tsv', relativePath: 'invalid-repeated-nested-groups-tsv.tsv', }, + { tsvLine: 2 }, ), - ], - comboErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('duplicateTag', { - tags: '[((Yellow),Blue,Green)],[Red]', - string: '(Red, (Blue, Green, (Yellow)), Red, (Blue, Green, (Yellow)))', - }), + generateIssue('duplicateTag', { tag: 'Red' }), { path: 'invalid-repeated-nested-groups-tsv.tsv', relativePath: 'invalid-repeated-nested-groups-tsv.tsv', }, + { tsvLine: 2 }, + ), + BidsHedIssue.fromHedIssue( + generateIssue('duplicateTag', { tag: '(Blue, Green, (Yellow))' }), + { + path: 'invalid-repeated-nested-groups-tsv.tsv', + relativePath: 'invalid-repeated-nested-groups-tsv.tsv', + }, + { tsvLine: 2 }, + ), + BidsHedIssue.fromHedIssue( + generateIssue('duplicateTag', { tag: '(Blue, Green, (Yellow))' }), + { + path: 'invalid-repeated-nested-groups-tsv.tsv', + relativePath: 'invalid-repeated-nested-groups-tsv.tsv', + }, + { tsvLine: 2 }, ), ], }, @@ -349,7 +394,6 @@ export const bidsTestData = [ testname: 'invalid-first-level-duplicate-combo', explanation: 'Each is okay but when combined, duplicate tag', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { vehicle: { HED: { @@ -375,11 +419,20 @@ export const bidsTestData = [ tsvErrors: [], comboErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('duplicateTag', { tags: '[Boat]', string: 'Boat,Boat,Speed/5 mph' }), + generateIssue('duplicateTag', { tag: 'Boat' }), + { + path: 'invalid-first-level-duplicate-combo.tsv', + relativePath: 'invalid-first-level-duplicate-combo.tsv', + }, + { tsvLine: 2 }, + ), + BidsHedIssue.fromHedIssue( + generateIssue('duplicateTag', { tag: 'Boat' }), { path: 'invalid-first-level-duplicate-combo.tsv', relativePath: 'invalid-first-level-duplicate-combo.tsv', }, + { tsvLine: 2 }, ), ], }, @@ -387,7 +440,6 @@ export const bidsTestData = [ testname: 'invalid-first-level-duplicate-combo-reordered', explanation: 'Each is okay but when combined, duplicate group in different order', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -400,14 +452,20 @@ export const bidsTestData = [ tsvErrors: [], comboErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('duplicateTag', { - tags: '[(Green,Purple)]', - string: '(Green, Purple), Blue, Orange,White,(Purple, Green), (Orange)', - }), + generateIssue('duplicateTag', { tag: '(Green, Purple)' }), { path: 'invalid-first-level-duplicate-combo-reordered.tsv', relativePath: 'invalid-first-level-duplicate-combo-reordered.tsv', }, + { tsvLine: 2 }, + ), + BidsHedIssue.fromHedIssue( + generateIssue('duplicateTag', { tag: '(Purple, Green)' }), + { + path: 'invalid-first-level-duplicate-combo-reordered.tsv', + relativePath: 'invalid-first-level-duplicate-combo-reordered.tsv', + }, + { tsvLine: 2 }, ), ], }, @@ -415,7 +473,6 @@ export const bidsTestData = [ testname: 'invalid-nested-duplicate-json-reordered', explanation: 'Deeply nested duplicates in JSON entry reordered', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -427,9 +484,18 @@ export const bidsTestData = [ sidecarErrors: [ BidsHedIssue.fromHedIssue( generateIssue('duplicateTag', { - tags: '[((((Black,Purple),Blue,Orange)),Green,White)]', - string: - '(Green, ((Blue, Orange, (Black, Purple))), White), Blue, Orange, (White, (((Purple, Black), Blue, Orange)), Green)', + sidecarKey: 'event_code', + tag: '(Green, ((Blue, Orange, (Black, Purple))), White)', + }), + { + path: 'invalid-nested-duplicate-json-reordered.json', + relativePath: 'invalid-nested-duplicate-json-reordered.json', + }, + ), + BidsHedIssue.fromHedIssue( + generateIssue('duplicateTag', { + sidecarKey: 'event_code', + tag: '(White, (((Purple, Black), Blue, Orange)), Green)', }), { path: 'invalid-nested-duplicate-json-reordered.json', @@ -440,15 +506,20 @@ export const bidsTestData = [ tsvErrors: [], comboErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('duplicateTag', { - tags: '[((((Black,Purple),Blue,Orange)),Green,White)]', - string: - '(Green, ((Blue, Orange, (Black, Purple))), White), Blue, Orange, (White, (((Purple, Black), Blue, Orange)), Green)', - }), + generateIssue('duplicateTag', { tag: '(Green, ((Blue, Orange, (Black, Purple))), White)' }), + { + path: 'invalid-nested-duplicate-json-reordered.tsv', + relativePath: 'invalid-nested-duplicate-json-reordered.tsv', + }, + { tsvLine: 2 }, + ), + BidsHedIssue.fromHedIssue( + generateIssue('duplicateTag', { tag: '(White, (((Purple, Black), Blue, Orange)), Green)' }), { path: 'invalid-nested-duplicate-json-reordered.tsv', relativePath: 'invalid-nested-duplicate-json-reordered.tsv', }, + { tsvLine: 2 }, ), ], }, @@ -456,7 +527,6 @@ export const bidsTestData = [ testname: 'invalid-nested-duplicate-combo-reordered', explanation: 'Deeply nested duplicates reordered', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -470,15 +540,20 @@ export const bidsTestData = [ tsvErrors: [], comboErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('duplicateTag', { - tags: '[((((Black,Purple),Blue,Orange)),Green,White)]', - string: - '(Green, ((Blue, Orange, (Black, Purple))), White), Blue, Orange,(White, (((Purple, Black), Blue, Orange)), Green)', - }), + generateIssue('duplicateTag', { tag: '(Green, ((Blue, Orange, (Black, Purple))), White)' }), + { + path: 'invalid-nested-duplicate-combo-reordered.tsv', + relativePath: 'invalid-nested-duplicate-combo-reordered.tsv', + }, + { tsvLine: 2 }, + ), + BidsHedIssue.fromHedIssue( + generateIssue('duplicateTag', { tag: '(White, (((Purple, Black), Blue, Orange)), Green)' }), { path: 'invalid-nested-duplicate-combo-reordered.tsv', relativePath: 'invalid-nested-duplicate-combo-reordered.tsv', }, + { tsvLine: 2 }, ), ], }, @@ -487,12 +562,12 @@ export const bidsTestData = [ { name: 'curly-brace-tests', description: 'Curly braces tested in various places', + definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], tests: [ { testname: 'valid-curly-brace-in-sidecar-with-value-splice', explanation: 'Valid curly brace in sidecar and valid value is spliced in', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -510,34 +585,10 @@ export const bidsTestData = [ tsvErrors: [], comboErrors: [], }, - { - testname: 'valid-curly-brace-in-sidecar-with-tsv-n/a', - explanation: 'Valid curly brace in sidecar and valid tsv with n/a', - schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], - sidecar: { - event_code: { - HED: { - face: '(Red, Blue), (Green, (Yellow)), ({HED})', - ball: '{response_time}, (Def/Acc/3.5)', - }, - }, - response_time: { - Description: 'Has description with HED', - HED: 'Parameter-value/#', - }, - }, - eventsString: - 'onset\tduration\tresponse_time\tevent_code\tHED\n4.5\t 0\t3.4\tface\tBlue\n5.0\t0\t6.8\tball\tGreen, Def/MyColor\n5.2\t0\tn/a\tface\t\n5.5\t0\t7.3\tface\tn/a\n', - sidecarErrors: [], - tsvErrors: [], - comboErrors: [], - }, { testname: 'valid-curly-brace-in-sidecar-with-category-splice', explanation: 'Valid curly brace in sidecar and valid value is spliced in', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -561,7 +612,6 @@ export const bidsTestData = [ testname: 'valid-curly-brace-in-sidecar-with-n/a-splice', explanation: 'Valid curly brace in sidecar and but tsv splice entry is n/a', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -583,7 +633,6 @@ export const bidsTestData = [ testname: 'valid-HED-column-splice', explanation: 'Valid curly brace in sidecar with valid HED column splice', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -605,7 +654,6 @@ export const bidsTestData = [ testname: 'valid-HED-column-splice-with-n/a', explanation: 'Valid curly brace in sidecar with HED column entry n/a', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -622,7 +670,6 @@ export const bidsTestData = [ testname: 'valid-HED-curly-brace-but-tsv-has-no-HED-column', explanation: 'A {HED} column splice is used in a sidecar but the tsv has no HED column', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -640,7 +687,6 @@ export const bidsTestData = [ testname: 'invalid-curly-brace-column-slice-has-no hed', explanation: 'A column name is used in a splice but does not have a HED key', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -668,7 +714,6 @@ export const bidsTestData = [ testname: 'invalid-curly-brace-in-HED-tsv-column', explanation: 'Curly braces are used in the HED column of a tsv.', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -680,14 +725,14 @@ export const bidsTestData = [ eventsString: 'onset\tduration\tHED\n' + '19\t6\t{event_code}\n', sidecarErrors: [], tsvErrors: [ - BidsHedIssue.fromHedIssue(generateIssue('curlyBracesInHedColumn', { string: '{event_code}', tsvLine: '2' }), { + BidsHedIssue.fromHedIssue(generateIssue('curlyBracesNotAllowed', { string: '{event_code}', tsvLine: 2 }), { path: 'invalid-curly-brace-in-HED-tsv-column.tsv', relativePath: 'invalid-curly-brace-in-HED-tsv-column.tsv', }), ], comboErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('curlyBracesInHedColumn', { string: '{event_code}' }), + generateIssue('curlyBracesNotAllowed', { string: '{event_code}' }), { path: 'invalid-curly-brace-in-HED-tsv-column.tsv', relativePath: 'invalid-curly-brace-in-HED-tsv-column.tsv', @@ -700,7 +745,6 @@ export const bidsTestData = [ testname: 'invalid-recursive-curly-braces', explanation: 'Mutually recursive curly braces in sidecar.', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -753,7 +797,6 @@ export const bidsTestData = [ testname: 'invalid-self-recursive-curly-braces', explanation: 'Mutually recursive curly braces in sidecar.', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -787,7 +830,6 @@ export const bidsTestData = [ testname: 'invalid-recursive-curly-brace-chain', explanation: 'Curly braces column A -> column B -> Column C.', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -829,12 +871,12 @@ export const bidsTestData = [ { name: 'placeholder-tests', description: 'Various placeholder tests', + definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], tests: [ { testname: 'valid-placeholder-used-in-tsv', explanation: 'The sidecar has a placeholder that is used in the tsv', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { vehicle: { HED: { @@ -855,7 +897,6 @@ export const bidsTestData = [ testname: 'valid-placeholder-not-used', explanation: 'The sidecar has a placeholder that is not used in the tsv', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { vehicle: { HED: { @@ -876,7 +917,6 @@ export const bidsTestData = [ testname: 'invalid-no-placeholder-value-column', explanation: 'The sidecar has a value column with no placeholder tag', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { vehicle: { HED: { @@ -890,24 +930,21 @@ export const bidsTestData = [ }, eventsString: 'onset\tduration\tvehicle\tspeed\n' + '19\t6\ttrain\t5\n', sidecarErrors: [ - BidsHedIssue.fromHedIssue(generateIssue('missingPlaceholder', { string: 'Blue,Speed', column: 'speed' }), { - path: 'invalid-no-placeholder-value-column.json', - relativePath: 'invalid-no-placeholder-value-column.json', - }), + BidsHedIssue.fromHedIssue( + generateIssue('missingPlaceholder', { string: 'Blue,Speed', sidecarKey: 'speed' }), + { + path: 'invalid-no-placeholder-value-column.json', + relativePath: 'invalid-no-placeholder-value-column.json', + }, + ), ], tsvErrors: [], - comboErrors: [ - BidsHedIssue.fromHedIssue(generateIssue('missingPlaceholder', { string: 'Blue,Speed', column: 'speed' }), { - path: 'invalid-no-placeholder-value-column.tsv', - relativePath: 'invalid-no-placeholder-value-column.tsv', - }), - ], + comboErrors: [], }, { testname: 'invalid-multiple-placeholders-in-value-column', explanation: 'The sidecar has a value column with no placeholder tag', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { speed: { HED: 'Label/#, Speed/# mph', @@ -915,36 +952,29 @@ export const bidsTestData = [ }, eventsString: 'onset\tduration\tvehicle\tspeed\n' + '19\t6\ttrain\t5\n', sidecarErrors: [ - BidsHedIssue.fromHedIssue( - generateIssue('invalidSidecarPlaceholder', { column: 'speed', string: 'Label/#, Speed/# mph' }), - { - path: 'invalid-multiple-placeholders-in-value-column.json', - relativePath: 'invalid-multiple-placeholders-in-value-column.json', - }, - ), + BidsHedIssue.fromHedIssue(generateIssue('invalidPlaceholder', { sidecarKey: 'speed', tag: 'Label/#' }), { + path: 'invalid-multiple-placeholders-in-value-column.json', + relativePath: 'invalid-multiple-placeholders-in-value-column.json', + }), + BidsHedIssue.fromHedIssue(generateIssue('invalidPlaceholder', { sidecarKey: 'speed', tag: 'Speed/# mph' }), { + path: 'invalid-multiple-placeholders-in-value-column.json', + relativePath: 'invalid-multiple-placeholders-in-value-column.json', + }), ], tsvErrors: [], - comboErrors: [ - BidsHedIssue.fromHedIssue( - generateIssue('invalidSidecarPlaceholder', { column: 'speed', string: 'Label/#, Speed/# mph' }), - { - path: 'invalid-multiple-placeholders-in-value-column.tsv', - relativePath: 'invalid-multiple-placeholders-in-value-column.tsv', - }, - ), - ], + comboErrors: [], }, ], }, { name: 'unit-tests', description: 'Various unit tests (limited for now)', + definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], tests: [ { testname: 'valid-units-on-a-placeholder', explanation: 'The sidecar has invalid units on a placeholder', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { speed: { HED: 'Speed/# mph', @@ -959,7 +989,6 @@ export const bidsTestData = [ testname: 'wrong-units-on-a-placeholder', explanation: 'The sidecar has wrong units on a placeholder', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { speed: { HED: 'Speed/# Hz', @@ -990,12 +1019,12 @@ export const bidsTestData = [ { name: 'definition-tests', description: 'Various definition tests (limited for now)', + definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], tests: [ { testname: 'valid-definition-no-placeholder', explanation: 'Simple definition in sidecar', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { speed: { HED: 'Speed/# mph, Def/TrainDef', @@ -1015,7 +1044,6 @@ export const bidsTestData = [ testname: 'valid-definition-with-placeholder', explanation: 'Definition in sidecar has a placeholder', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { speed: { HED: 'Speed/# mph, Def/GreenDef/0.5', @@ -1031,26 +1059,10 @@ export const bidsTestData = [ tsvErrors: [], comboErrors: [], }, - { - testname: 'valid-def-with-placeholder', - explanation: 'Def in sidecar has a placeholder', - schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], - sidecar: { - speed: { - HED: 'Def/Acc/#', - }, - }, - eventsString: 'onset\tduration\tspeed\n' + '19\t6\t5\n', - sidecarErrors: [], - tsvErrors: [], - comboErrors: [], - }, { testname: 'valid-definition-with-nested-placeholder', explanation: 'Definition in sidecar has nested placeholder', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { speed: { HED: 'Speed/# mph, Def/GreenDef/0.5', @@ -1070,7 +1082,6 @@ export const bidsTestData = [ testname: 'valid-definition-no-group', explanation: 'The sidecar with definition that has no internal group.', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { speed: { HED: 'Speed/# mph, Def/BlueDef', @@ -1086,104 +1097,26 @@ export const bidsTestData = [ tsvErrors: [], comboErrors: [], }, - { - testname: 'invalid-def-expand-no-group', - explanation: 'The sidecar with definition that has no internal group.', - schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], - sidecar: { - speed: { - HED: 'Speed/# mph, (Def-expand/Acc/4.5)', - }, - }, - eventsString: 'onset\tduration\tspeed\n' + '19\t6\t5\n', - sidecarErrors: [ - BidsHedIssue.fromHedIssue( - generateIssue('defExpandContentsInvalid', { - contents: '', - defContents: '(Acceleration/4.5 m-per-s^2,Red)', - sidecarKeyName: 'speed', - }), - { - path: 'invalid-def-expand-no-group.json', - relativePath: 'invalid-def-expand-no-group.json', - }, - ), - ], - tsvErrors: [], - comboErrors: [ - BidsHedIssue.fromHedIssue( - generateIssue('defExpandContentsInvalid', { - contents: '', - defContents: '(Acceleration/4.5 m-per-s^2,Red)', - sidecarKeyName: 'speed', - }), - { - path: 'invalid-def-expand-no-group.tsv', - relativePath: 'invalid-def-expand-no-group.tsv', - }, - ), - ], - }, { testname: 'invalid-missing-definition-for-def', explanation: 'The sidecar uses a def with no definition', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { speed: { HED: 'Speed/# mph, Def/MissingDef', }, }, eventsString: 'onset\tduration\tspeed\n' + '19\t6\t5\n', - sidecarErrors: [ - BidsHedIssue.fromHedIssue( - generateIssue('missingDefinitionForDef', { definition: 'missingdef', sidecarKeyName: 'speed' }), - { - path: 'invalid-missing-definition-for-def.json', - relativePath: 'invalid-missing-definition-for-def.json', - }, - ), - ], + sidecarErrors: [], tsvErrors: [], comboErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('missingDefinitionForDef', { definition: 'missingdef', sidecarKeyName: 'speed' }), + generateIssue('missingDefinitionForDef', { definition: 'MissingDef' }), { path: 'invalid-missing-definition-for-def.tsv', relativePath: 'invalid-missing-definition-for-def.tsv', }, - ), - ], - }, - { - testname: 'invalid-missing-definition-for-def-expand', - explanation: 'The sidecar uses a def-expand with no definition', - schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], - sidecar: { - speed: { - HED: 'Speed/# mph, (Def-expand/MissingDefExpand, (Red))', - }, - }, - eventsString: 'onset\tduration\tspeed\n' + '19\t6\t5\n', - sidecarErrors: [ - BidsHedIssue.fromHedIssue( - generateIssue('missingDefinitionForDefExpand', { definition: 'missingdefexpand', sidecarKeyName: 'speed' }), - { - path: 'invalid-missing-definition-for-def-expand.json', - relativePath: 'invalid-missing-definition-for-def-expand.json', - }, - ), - ], - tsvErrors: [], - comboErrors: [ - BidsHedIssue.fromHedIssue( - generateIssue('missingDefinitionForDefExpand', { definition: 'missingdefexpand', sidecarKeyName: 'speed' }), - { - path: 'invalid-missing-definition-for-def-expand.tsv', - relativePath: 'invalid-missing-definition-for-def-expand.tsv', - }, + { tsvLine: 2 }, ), ], }, @@ -1191,7 +1124,6 @@ export const bidsTestData = [ testname: 'invalid-nested-definition', explanation: 'The sidecar has a definition inside a definition', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { speed: { HED: 'Speed/# mph, Def/NestedDef', @@ -1233,7 +1165,6 @@ export const bidsTestData = [ testname: 'invalid-multiple-definition-tags', explanation: 'The sidecar has multiple definition tags in same definition', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { speed: { HED: 'Speed/# mph, Def/Apple', @@ -1247,9 +1178,8 @@ export const bidsTestData = [ eventsString: 'onset\tduration\tspeed\n' + '19\t6\t5\n', sidecarErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('invalidGroupTopTags', { - string: '(Definition/Apple, Definition/Banana, (Blue))', - tags: 'Definition/Apple, Definition/Banana', + generateIssue('invalidTagGroup', { + tagGroup: '(Definition/Apple, Definition/Banana, (Blue))', }), { path: 'invalid-multiple-definition-tags.json', @@ -1260,9 +1190,8 @@ export const bidsTestData = [ tsvErrors: [], comboErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('invalidGroupTopTags', { - string: '(Definition/Apple, Definition/Banana, (Blue))', - tags: 'Definition/Apple, Definition/Banana', + generateIssue('invalidTagGroup', { + tagGroup: '(Definition/Apple, Definition/Banana, (Blue))', }), { path: 'invalid-multiple-definition-tags.tsv', @@ -1275,7 +1204,6 @@ export const bidsTestData = [ testname: 'invalid-definition-with-extra-groups', explanation: 'The sidecar has a definition with extra internal group', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { speed: { HED: 'Speed/# mph, Def/ExtraGroupDef', @@ -1289,9 +1217,8 @@ export const bidsTestData = [ eventsString: 'onset\tduration\tspeed\n' + '19\t6\t5\n', sidecarErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('invalidNumberOfSubgroups', { - tag: 'Definition/ExtraGroupDef', - string: '(Definition/ExtraGroupDef, (Red), (Blue))', + generateIssue('invalidTagGroup', { + tagGroup: '(Definition/ExtraGroupDef, (Red), (Blue))', }), { path: 'invalid-definition-with-extra-groups.json', @@ -1302,9 +1229,8 @@ export const bidsTestData = [ tsvErrors: [], comboErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('invalidNumberOfSubgroups', { - tag: 'Definition/ExtraGroupDef', - string: '(Definition/ExtraGroupDef, (Red), (Blue))', + generateIssue('invalidTagGroup', { + tagGroup: '(Definition/ExtraGroupDef, (Red), (Blue))', }), { path: 'invalid-definition-with-extra-groups.tsv', @@ -1317,7 +1243,6 @@ export const bidsTestData = [ testname: 'invalid-definition-with-extra-sibling', explanation: 'The sidecar has a definition with an extra internal sibling', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { speed: { HED: 'Speed/# mph, Def/ExtraSiblingDef', @@ -1331,9 +1256,8 @@ export const bidsTestData = [ eventsString: 'onset\tduration\tspeed\n' + '19\t6\t5\n', sidecarErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('invalidGroupTopTags', { - string: '(Definition/ExtraSiblingDef, Red, (Blue))', - tags: 'Definition/ExtraSiblingDef, Red', + generateIssue('invalidTagGroup', { + tagGroup: '(Definition/ExtraSiblingDef, Red, (Blue))', }), { path: 'invalid-definition-with-extra-sibling.json', @@ -1344,9 +1268,8 @@ export const bidsTestData = [ tsvErrors: [], comboErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('invalidGroupTopTags', { - string: '(Definition/ExtraSiblingDef, Red, (Blue))', - tags: 'Definition/ExtraSiblingDef, Red', + generateIssue('invalidTagGroup', { + tagGroup: '(Definition/ExtraSiblingDef, Red, (Blue))', }), { path: 'invalid-definition-with-extra-sibling.tsv', @@ -1359,7 +1282,6 @@ export const bidsTestData = [ testname: 'invalid-definition-in-HED-column', explanation: 'The tsv has a definition in HED column', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { speed: { HED: 'Speed/# mph', @@ -1369,21 +1291,13 @@ export const bidsTestData = [ sidecarErrors: [], tsvErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('illegalDefinitionContext', { - definition: 'Definition/TsvDef', - string: '(Definition/TsvDef)', - tsvLine: '2', - }), + generateIssue('illegalDefinitionContext', { string: '(Definition/TsvDef)', tsvLine: '2' }), { path: 'invalid-definition-in-HED-column.tsv', relativePath: 'invalid-definition-in-HED-column.tsv' }, ), ], comboErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('illegalDefinitionContext', { - definition: 'Definition/TsvDef', - string: '(Definition/TsvDef)', - tsvLine: '2', - }), + generateIssue('illegalDefinitionContext', { string: '(Definition/TsvDef)', tsvLine: '2' }), { path: 'invalid-definition-in-HED-column.tsv', relativePath: 'invalid-definition-in-HED-column.tsv' }, ), ], @@ -1392,7 +1306,6 @@ export const bidsTestData = [ testname: 'invalid-definition-with-missing-placeholder', explanation: 'Definition in sidecar has missing placeholder', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { speed: { HED: 'Def/MySpeed/#', @@ -1406,7 +1319,7 @@ export const bidsTestData = [ eventsString: 'onset\tduration\tspeed\n' + '19\t6\t5\n', sidecarErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('invalidPlaceholderInDefinition', { definition: '(Definition/MySpeed/#, (Speed, Red))' }), + generateIssue('invalidPlaceholderInDefinition', { definition: 'MySpeed', sidecarKey: 'mydefs' }), { path: 'invalid-definition-with-missing-placeholder.json', relativePath: 'invalid-definition-with-missing-placeholder.json', @@ -1414,21 +1327,12 @@ export const bidsTestData = [ ), ], tsvErrors: [], - comboErrors: [ - BidsHedIssue.fromHedIssue( - generateIssue('invalidPlaceholderInDefinition', { definition: '(Definition/MySpeed/#, (Speed, Red))' }), - { - path: 'invalid-definition-with-missing-placeholder.tsv', - relativePath: 'invalid-definition-with-missing-placeholder.tsv', - }, - ), - ], + comboErrors: [], }, { testname: 'invalid-definition-with-fixed-placeholder', explanation: 'Definition in sidecar has a fixed value instead of placeholder', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { speed: { HED: 'Def/GreenDef/Test', @@ -1442,33 +1346,27 @@ export const bidsTestData = [ eventsString: 'onset\tduration\tspeed\n' + '19\t6\t5\n', sidecarErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('invalidPlaceholderInDefinition', { - definition: '(Definition/GreenDef/Test, (Red, Triangle))', - }), + generateIssue('missingPlaceholder', { sidecarKey: 'speed', string: 'Def/GreenDef/Test' }), { path: 'invalid-definition-with-fixed-placeholder.json', relativePath: 'invalid-definition-with-fixed-placeholder.json', }, ), - ], - tsvErrors: [], - comboErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('invalidPlaceholderInDefinition', { - definition: '(Definition/GreenDef/Test, (Red, Triangle))', - }), + generateIssue('invalidPlaceholderInDefinition', { definition: 'GreenDef', sidecarKey: 'mydefs' }), { - path: 'invalid-definition-with-fixed-placeholder.tsv', - relativePath: 'invalid-definition-with-fixed-placeholder.tsv', + path: 'invalid-definition-with-fixed-placeholder.json', + relativePath: 'invalid-definition-with-fixed-placeholder.json', }, ), ], + tsvErrors: [], + comboErrors: [], }, { testname: 'invalid-definition-has-multiple-placeholders', explanation: 'Definition in sidecar has multiple placeholders', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { speed: { HED: 'Def/SpeedDef/#', @@ -1482,9 +1380,7 @@ export const bidsTestData = [ eventsString: 'onset\tduration\tspeed\n' + '19\t6\t5\n', sidecarErrors: [ BidsHedIssue.fromHedIssue( - generateIssue('invalidPlaceholderInDefinition', { - definition: '(Definition/SpeedDef/#, (Speed/# mph, (Label/#, Red, Triangle)))', - }), + generateIssue('invalidPlaceholderInDefinition', { definition: 'SpeedDef', sidecarKey: 'mydefs' }), { path: 'invalid-definition-has-multiple-placeholders.json', relativePath: 'invalid-definition-has-multiple-placeholders.json', @@ -1492,23 +1388,12 @@ export const bidsTestData = [ ), ], tsvErrors: [], - comboErrors: [ - BidsHedIssue.fromHedIssue( - generateIssue('invalidPlaceholderInDefinition', { - definition: '(Definition/SpeedDef/#, (Speed/# mph, (Label/#, Red, Triangle)))', - }), - { - path: 'invalid-definition-has-multiple-placeholders.tsv', - relativePath: 'invalid-definition-has-multiple-placeholders.tsv', - }, - ), - ], + comboErrors: [], }, { testname: 'invalid-definition-not-isolated', explanation: 'Definition in sidecar appears with other tags', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { speed: { HED: 'Def/SpeedDef/#', @@ -1545,12 +1430,12 @@ export const bidsTestData = [ { name: 'delay-tests', description: 'Tests with delay', + definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], tests: [ { testname: 'nested-delay', explanation: 'A delay tag with nesting', schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], sidecar: { event_code: { HED: { @@ -1630,187 +1515,4 @@ export const bidsTestData = [ }, ], }, - { - name: 'temporal-tests', - description: 'Dataset level tests with temporal groups.', - tests: [ - { - testname: 'valid-offset-after-onset', - explanation: 'An offset after an inset at an earlier time', - schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], - sidecar: {}, - eventsString: 'onset\tduration\tHED\n4.5\t0\t(Onset, Def/MyColor)\n6.0\t0\t(Offset, Def/MyColor)\n', - sidecarErrors: [], - tsvErrors: [], - comboErrors: [], - }, - { - testname: 'valid-offset-after-onset-with-def-expand', - explanation: 'An offset after an inset at an earlier time but one is a Def and the other is a Def-expand', - schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], - sidecar: {}, - eventsString: - 'onset\tduration\tHED\n4.5\t0\t(Onset, Def/MyColor)\n6.0\t0\t(Offset, (Def-expand/MyColor, (Label/Pie)))\n', - sidecarErrors: [], - tsvErrors: [], - comboErrors: [], - }, - { - testname: 'valid-offset-after-onset-with-def-expand-with-value', - explanation: 'An offset after an inset, the defs have a value and one is a Def and the other is a Def-expand', - schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], - sidecar: {}, - eventsString: - 'onset\tduration\tHED\n4.5\t0\t(Onset, Def/Acc/5.4)\n6.0\t0\t(Offset, (Def-expand/Acc/5.4, (Acceleration/5.4 m-per-s^2, Red)))\n', - sidecarErrors: [], - tsvErrors: [], - comboErrors: [], - }, - { - testname: 'simultaneous-temporal-onset', - explanation: 'temporal onsets at same time', - schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], - sidecar: {}, - eventsString: 'onset\tduration\tHED\n4.5\t0\t(Onset, Def/MyColor, (Red)),(Onset, Def/MyColor, (Blue))\n', - sidecarErrors: [], - tsvErrors: [ - BidsHedIssue.fromHedIssue( - generateIssue('simultaneousDuplicateEvents', { - onset1: '4.5', - onset2: '4.5', - tagGroup1: '(Onset, Def/MyColor, (Blue))', - tagGroup2: '(Onset, Def/MyColor, (Red))', - tsvLine1: '2', - tsvLine2: '2', - }), - { - path: 'simultaneous-temporal-onset.tsv', - relativePath: 'simultaneous-temporal-onset.tsv', - }, - ), - ], - comboErrors: [ - BidsHedIssue.fromHedIssue( - generateIssue('simultaneousDuplicateEvents', { - onset1: '4.5', - onset2: '4.5', - tagGroup1: '(Onset, Def/MyColor, (Blue))', - tagGroup2: '(Onset, Def/MyColor, (Red))', - tsvLine1: '2', - tsvLine2: '2', - }), - { - path: 'simultaneous-temporal-onset.tsv', - relativePath: 'simultaneous-temporal-onset.tsv', - }, - ), - ], - }, - { - testname: 'missing-temporal-onset', - explanation: 'offset appears before an onset', - schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], - sidecar: {}, - eventsString: 'onset\tduration\tHED\n4.5\t0\t(Offset, Def/MyColor)\n', - sidecarErrors: [], - tsvErrors: [ - BidsHedIssue.fromHedIssue( - generateIssue('inactiveOnset', { tag: 'Offset', definition: 'mycolor' }), - { - path: 'missing-temporal-onset.tsv', - relativePath: 'missing-temporal-onset.tsv', - }, - { tsvLine: '2' }, - ), - ], - comboErrors: [ - BidsHedIssue.fromHedIssue( - generateIssue('inactiveOnset', { tag: 'Offset', definition: 'mycolor' }), - { - path: 'missing-temporal-onset.tsv', - relativePath: 'missing-temporal-onset.tsv', - }, - { tsvLine: '2' }, - ), - ], - }, - { - testname: 'delayed-onset-with-offset-okay', - explanation: 'onset has delay, but offset appears after anyway', - schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], - sidecar: {}, - eventsString: - 'onset\tduration\tHED\n4.5\t0\t(Onset, Delay/1.0 s, Def/MyColor)\n7.0\t\0\t(Offset, Def/MyColor)\n', - sidecarErrors: [], - tsvErrors: [], - comboErrors: [], - }, - { - testname: 'delayed-onset-with-offset-before', - explanation: 'offset appears before an onset in delay scenario', - schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], - sidecar: {}, - eventsString: - 'onset\tduration\tHED\n4.5\t0\t(Onset, Delay/5.0 s, Def/MyColor)\n6.0\t\0\t(Offset, Def/MyColor)\n', - sidecarErrors: [], - tsvErrors: [ - BidsHedIssue.fromHedIssue( - generateIssue('inactiveOnset', { tag: 'Offset', definition: 'mycolor' }), - { - path: 'delayed-onset-with-offset-before.tsv', - relativePath: 'delayed-onset-with-offset-before.tsv', - }, - { tsvLine: '3' }, - ), - ], - comboErrors: [ - BidsHedIssue.fromHedIssue( - generateIssue('inactiveOnset', { tag: 'Offset', definition: 'mycolor' }), - { - path: 'delayed-onset-with-offset-before.tsv', - relativePath: 'delayed-onset-with-offset-before.tsv', - }, - { tsvLine: '3' }, - ), - ], - }, - { - testname: 'delayed-onset-with-offset-before-with-sidecar', - explanation: 'offset appears before an onset with a sidecar in complex delayed scenario', - schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], - sidecar: { - event_code: { - HED: { - face: '(Delay/5.0 s, Onset, Def/MyColor)', - ball: '(Delay/5.0 s, (Def-expand/MyColor, (Label/Pie)), Onset)', - square: '(Delay/5.0 s, Offset, Def/MyColor)', - circle: '(Delay/5.0 s, (Def-expand/MyColor, (Label/Pie)), Offset)', - }, - }, - }, - eventsString: - 'onset\tduration\tevent_code\tHED\n4.5\t0\tface\tn/a\n4.8\t\0\tsquare\tn/a\n4.9\t\0\tball\tGreen\n10.0\t\0\tn/a\t(Delay/5.0 s, (Def-expand/MyColor, (Label/Pie)), Offset)\n', - sidecarErrors: [], - tsvErrors: [ - BidsHedIssue.fromHedIssue( - generateIssue('inactiveOnset', { tag: 'Offset', definition: 'mycolor' }), - { - path: 'delayed-onset-with-offset-before-with-sidecar.tsv', - relativePath: 'delayed-onset-with-offset-before-with-sidecar.tsv', - }, - { tsvLine: '5' }, - ), - ], - comboErrors: [], - }, - ], - }, ] diff --git a/tests/testData/definitionManagerTests.data.js b/tests/testData/definitionManagerTests.data.js deleted file mode 100644 index e1512e06..00000000 --- a/tests/testData/definitionManagerTests.data.js +++ /dev/null @@ -1,156 +0,0 @@ -import { generateIssue } from '../../common/issues/issues' - -export const definitionTestData = [ - { - name: 'def-or-def-expand', - description: '"Event" is a single level tag"', - schemaVersion: '8.3.0', - definitions: ['(Definition/Acc/#, (Acceleration/# m-per-s^2, Red))', '(Definition/MyColor, (Label/Pie))'], - tests: [ - { - testname: 'valid-def', - explanation: '"Def/MyColor" is a valid def tag', - definition: null, - stringIn: 'Def/MyColor', - fullCheck: true, - placeholderAllowed: false, - errors: [], - }, - { - testname: 'valid-def-different-case', - explanation: '"def/myColor" is a valid def tag although different cases', - definition: null, - stringIn: 'def/myColor', - fullCheck: true, - placeholderAllowed: false, - errors: [], - }, - { - testname: 'valid-def-no-group', - explanation: '"Def/Blech" has a definition without a group', - definition: '(Definition/Blech)', - stringIn: 'Def/Blech, (Red, Green)', - fullCheck: true, - placeholderAllowed: false, - errors: [], - }, - { - testname: 'valid-def-with-placeholder', - explanation: '"def/Acc/#" is def tag with an allowed placeholder', - definition: null, - stringIn: 'def/Acc/#', - fullCheck: true, - placeholderAllowed: true, - errors: [], - }, - { - testname: 'invalid-def-with-placeholder', - explanation: '"def/Acc/#" is def tag with disallowed placeholder', - definition: null, - stringIn: 'def/Acc/#', - fullCheck: true, - placeholderAllowed: false, - errors: [generateIssue('invalidPlaceholderContext', { string: 'def/Acc/#' })], - }, - { - testname: 'valid-def-expand-with-placeholder', - explanation: - '"(Def-expand/Acc/#, (Acceleration/# m-per-s^2, Red))" is def-expand tag with an allowed placeholder', - definition: null, - stringIn: '(Def-expand/Acc/#, (Acceleration/# m-per-s^2, Red))', - fullCheck: true, - placeholderAllowed: true, - errors: [], - }, - { - testname: 'invalid-def-with-placeholder', - explanation: - '"(Def-expand/Acc/#, (Acceleration/# m-per-s^2, Red))" is def-expand tag with disallowed placeholder', - definition: null, - stringIn: '(Def-expand/Acc/#, (Acceleration/# m-per-s^2, Red))', - fullCheck: true, - placeholderAllowed: false, - errors: [ - generateIssue('invalidPlaceholderContext', { string: '(Def-expand/Acc/#, (Acceleration/# m-per-s^2, Red))' }), - ], - }, - { - testname: 'invalid-def-expand-should-have-a-group', - explanation: '"(Def-expand/Acc/4.5)" has a definition without a group', - definition: null, - stringIn: '(Def-expand/Acc/4.5)', - fullCheck: true, - errors: [ - generateIssue('defExpandContentsInvalid', { contents: '', defContents: '(Acceleration/4.5 m-per-s^2,Red)' }), - ], - }, - { - testname: 'missing-definition', - explanation: '"def/Blech" does not have a corresponding definition', - definition: null, - stringIn: 'def/Blech', - fullCheck: true, - placeholderAllowed: false, - errors: [generateIssue('missingDefinitionForDef', { definition: 'blech' })], - }, - { - testname: 'invalid-def-extra-level', - explanation: '"def/Blech/5" is def tag with unexpected second level', - definition: '(Definition/Blech, (Label/Cake))', - stringIn: 'def/Blech/5', - fullCheck: true, - placeholderAllowed: false, - errors: [generateIssue('missingDefinitionForDef', { definition: 'blech' })], - }, - { - testname: 'invalid-def-invalid-value', - explanation: '"def/Acc/4.5/3" is def tag with invalid value', - definition: null, - stringIn: 'def/Acc/4.5/3', - fullCheck: true, - placeholderAllowed: false, - errors: [generateIssue('invalidValue', { tag: 'Acceleration/4.5/3 m-per-s^2' })], - }, - { - testname: 'invalid-def-expand-invalid-value', - explanation: - '"(Def-expand/Acc/4.5/3, (Acceleration/4.5/3 m-per-s^2, Red))" is def-expand tag with invalid substituted value', - definition: null, - stringIn: '(Def-expand/Acc/4.5/3, (Acceleration/4.5/3 m-per-s^2, Red))', - fullCheck: true, - placeholderAllowed: false, - errors: [generateIssue('invalidValue', { tag: 'Acceleration/4.5/3 m-per-s^2' })], - }, - { - testname: 'invalid-def-expand-invalid-substitution', - explanation: - '"(Def-expand/Acc/4.5, (Acceleration/6 m-per-s^2, Red))" has def-expand tag with invalid substitution', - definition: null, - stringIn: '(Def-expand/Acc/4.5, (Acceleration/6 m-per-s^2, Red))', - fullCheck: true, - placeholderAllowed: false, - errors: [ - generateIssue('defExpandContentsInvalid', { - contents: '(Acceleration/6 m-per-s^2,Red)', - defContents: '(Acceleration/4.5 m-per-s^2,Red)', - }), - ], - }, - { - testname: 'invalid-def-expand-invalid-value', - explanation: - '"(Def-expand/Acc/Blech, (Acceleration/Blech m-per-s^2, Red))" is def-expand tag with invalid substitution', - definition: null, - stringIn: '(Def-expand/Acc/4.5, (Acceleration/6 m-per-s^2, Red))', - fullCheck: true, - placeholderAllowed: false, - errors: [ - generateIssue('defExpandContentsInvalid', { - contents: '(Acceleration/6 m-per-s^2,Red)', - defContents: '(Acceleration/4.5 m-per-s^2,Red)', - }), - ], - }, - ], - }, -] diff --git a/tests/testData/normalizerTests.data.js b/tests/testData/normalizerTests.data.js deleted file mode 100644 index 20f1647d..00000000 --- a/tests/testData/normalizerTests.data.js +++ /dev/null @@ -1,42 +0,0 @@ -import { generateIssue } from '../../common/issues/issues' - -export const normalizerTestData = [ - { - name: 'simple-tags', - description: 'Simple tags requirements', - tests: [ - { - testname: 'single-tag', - explanation: '"" is a single level tag"', - schemaVersion: '8.3.0', - string: 'Item', - stringNormalized: 'Item', - errors: [], - }, - { - testname: 'empty-string', - explanation: '"" is an empty string"', - schemaVersion: '8.3.0', - string: '', - stringNormalized: '', - errors: [], - }, - { - testname: 'non-duplicate-tags', - explanation: '"Red, Blue, Green" is a simple list of non-duplicate tags', - schemaVersion: '8.3.0', - string: 'Red, Blue, Green', - stringNormalized: 'Blue,Green,Red', - errors: [], - }, - { - testname: 'duplicate-tags', - explanation: '"Red, Blue, Red" has duplicate tags', - schemaVersion: '8.3.0', - string: 'Red, Blue, Red', - stringNormalized: null, - errors: [generateIssue('duplicateTag', { tags: '[Red]', string: 'Red, Blue, Red' })], - }, - ], - }, -] diff --git a/tests/testData/splitterTests.data.js b/tests/testData/splitterTests.data.js index f2db1533..198d1096 100644 --- a/tests/testData/splitterTests.data.js +++ b/tests/testData/splitterTests.data.js @@ -23,9 +23,6 @@ export const splitterTestData = [ stringIn: 'Event, (Item, Red, (Blue, (Green))), (Item, Blue)', allGroupTagCount: [4, 2], allSubgroupCount: 2, - fullCheck: true, - errors: [], - warnings: [], }, { testname: 'single-multiple-nested-groups', @@ -34,20 +31,6 @@ export const splitterTestData = [ stringIn: '(((Event, (Item, Red, (Blue, (Green))), (Item, Blue))))', allGroupTagCount: [7], allSubgroupCount: 1, - fullCheck: true, - errors: [], - warnings: [], - }, - { - testname: 'empty-string', - explanation: '"" is a an empty string', - schemaVersion: '8.3.0', - stringIn: '', - allGroupTagCount: [0], - allSubgroupCount: 0, - fullCheck: true, - errors: [], - warnings: [], }, ], }, diff --git a/tests/testData/stringParserTests.data.js b/tests/testData/stringParserTests.data.js index f5264461..a71d6b86 100644 --- a/tests/testData/stringParserTests.data.js +++ b/tests/testData/stringParserTests.data.js @@ -13,8 +13,6 @@ export const parseTestData = [ stringLong: 'Event', stringShort: 'Event', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -26,8 +24,6 @@ export const parseTestData = [ stringLong: 'Event/Sensory-event', stringShort: 'Sensory-event', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -39,8 +35,6 @@ export const parseTestData = [ stringLong: 'Item/Object/Geometric-object', stringShort: 'Geometric-object', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -52,8 +46,6 @@ export const parseTestData = [ stringLong: 'Item/Object/Geometric-object', stringShort: 'Geometric-object', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -65,8 +57,6 @@ export const parseTestData = [ stringLong: 'Item/Object/Geometric-object', stringShort: 'Geometric-object', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -78,8 +68,6 @@ export const parseTestData = [ stringLong: 'Item/Sound/Environmental-sound/Unique-value', stringShort: 'Environmental-sound/Unique-value', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -92,8 +80,6 @@ export const parseTestData = [ stringShort: 'Environmental-sound/Unique-value/Junk', operation: 'toShort', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -105,8 +91,6 @@ export const parseTestData = [ stringLong: 'Item/Sound/Environmental-sound/Unique-value/Junk', stringShort: 'Environmental-sound/Unique-value/Junk', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -124,8 +108,6 @@ export const parseTestData = [ stringLong: 'Property/Agent-property/Agent-trait/Age/15', stringShort: 'Age/15', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -136,8 +118,6 @@ export const parseTestData = [ stringIn: 'Agent-trait/Age', stringLong: 'Property/Agent-property/Agent-trait/Age', stringShort: 'Age', - placeholdersAllowed: false, - definitionsAllowed: false, fullCheck: true, errors: [], warnings: [], @@ -149,8 +129,6 @@ export const parseTestData = [ stringIn: 'Label', stringLong: 'Property/Informational-property/Label', stringShort: 'Label', - placeholdersAllowed: false, - definitionsAllowed: false, fullCheck: true, errors: [], warnings: [], @@ -170,8 +148,6 @@ export const parseTestData = [ '(Item/Object/Man-made-object/Vehicle/Train/Maglev, Property/Agent-property/Agent-trait/Age/15, Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red/0.5), Action/Perform/Operate', stringShort: '(Train/Maglev, Age/15, RGB-red/0.5), Operate', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -184,8 +160,6 @@ export const parseTestData = [ '(Property/Data-property/Data-value/Spatiotemporal-value/Temporal-value/Time-value/20 ms), Action/Perform/Operate', stringShort: '(Time-value/20 ms), Operate', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -203,8 +177,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [generateIssue('invalidParentNode', { tag: 'Event', parentTag: 'Item/Sound' })], warnings: [], }, @@ -216,8 +188,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [generateIssue('invalidParentNode', { parentTag: 'Item/Sound/Environmental-sound', tag: 'Event' })], warnings: [], }, @@ -229,8 +199,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [generateIssue('invalidParentNode', { parentTag: 'Item/Sound', tag: 'Event' })], warnings: [], }, @@ -242,8 +210,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [generateIssue('invalidTag', { tag: 'Junk/Item/Sound/Event/Sensory-event/Environmental-sound' })], warnings: [], }, @@ -255,8 +221,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [generateIssue('invalidTag', { tag: 'Junk' })], warnings: [], }, @@ -268,8 +232,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [generateIssue('invalidTag', { tag: 'Junk/Blech' })], warnings: [], }, @@ -282,8 +244,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [generateIssue('invalidParentNode', { parentTag: 'Item/Object/Junk', tag: 'Geometric-object' })], warnings: [], }, @@ -295,8 +255,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [generateIssue('invalidExtension', { parentTag: 'Event/Agent-action', tag: 'Baloney' })], warnings: [], }, @@ -315,8 +273,6 @@ export const parseTestData = [ 'Property/Informational-property/Label/#, Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Red-color/Red', stringShort: 'Label/#, Red', fullCheck: true, - placeholdersAllowed: true, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -329,8 +285,6 @@ export const parseTestData = [ 'Property/Data-property/Data-value/Spatiotemporal-value/Temporal-value/Time-value/# ms, Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Red-color/Red', stringShort: 'Time-value/# ms, Red', fullCheck: true, - placeholdersAllowed: true, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -342,8 +296,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: true, - definitionsAllowed: false, errors: [generateIssue('invalidExtension', { tag: '#', parentTag: 'Object' })], warnings: [], }, @@ -355,8 +307,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: true, - definitionsAllowed: false, errors: [generateIssue('invalidExtension', { parentTag: 'Object/Thingie', tag: '#' })], warnings: [], }, @@ -368,8 +318,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: true, - definitionsAllowed: false, errors: [ generateIssue('invalidPlaceholder', { index: '13', string: 'Label/#/Blech, Red', tag: 'Label/#/Blech' }), ], @@ -383,8 +331,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: true, - definitionsAllowed: false, errors: [generateIssue('invalidPlaceholder', { index: '8', string: 'Label/##, Red', tag: 'Label/##' })], warnings: [], }, @@ -402,8 +348,6 @@ export const parseTestData = [ stringLong: 'Property/Agent-property/Agent-trait/Age/5', stringShort: 'Age/5', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -415,8 +359,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [generateIssue('childRequired', { tag: 'Duration' })], warnings: [], }, @@ -429,8 +371,6 @@ export const parseTestData = [ '(Property/Data-property/Data-value/Spatiotemporal-value/Temporal-value/Time-value/20 ms), Action/Perform/Operate', stringShort: '(Time-value/20 ms), Operate', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -448,8 +388,6 @@ export const parseTestData = [ stringLong: 'Property/Agent-property/Agent-trait/Age, Action/Perform/Operate', stringShort: 'Age, Operate', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -461,8 +399,6 @@ export const parseTestData = [ stringLong: '', stringShort: '', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -475,8 +411,6 @@ export const parseTestData = [ 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Red-color/Red, (Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Blue-color/Blue, Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Green-color/Green)', stringShort: 'Red, (Blue, Green)', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -489,8 +423,6 @@ export const parseTestData = [ 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Red-color/Red, (Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Blue-color/Blue, (Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Green-color/Green))', stringShort: 'Red, (Blue, (Green))', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -502,8 +434,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [generateIssue('invalidTagString', {})], warnings: [], }, @@ -515,32 +445,9 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [generateIssue('invalidTagString', {})], warnings: [], }, - { - testname: 'multiple-complex-duplicates', - explanation: - '"(Green, ((Blue, Orange, (Black, Purple))), White), Blue, Orange, (White, (((Purple, Black), Blue, Orange)), Green)" is a complex group with multiple duplicates', - schemaVersion: '8.3.0', - stringIn: - '(Green, ((Blue, Orange, (Black, Purple))), White), Blue, Orange, (White, (((Purple, Black), Blue, Orange)), Green)', - stringLong: null, - stringShort: null, - fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, - errors: [ - generateIssue('duplicateTag', { - tags: '[((((Black,Purple),Blue,Orange)),Green,White)]', - string: - '(Green, ((Blue, Orange, (Black, Purple))), White), Blue, Orange, (White, (((Purple, Black), Blue, Orange)), Green)', - }), - ], - warnings: [], - }, ], }, { @@ -555,8 +462,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: false, - placeholdersAllowed: true, - definitionsAllowed: true, errors: [ generateIssue('invalidTopLevelTagGroupTag', { tag: 'Definition/Green1', @@ -573,8 +478,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: false, - placeholdersAllowed: true, - definitionsAllowed: true, errors: [ generateIssue('illegalInExclusiveContext', { tag: 'Definition/Green1', @@ -591,16 +494,14 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: false, - placeholdersAllowed: true, - definitionsAllowed: true, errors: [ - generateIssue('invalidGroupTopTags', { - string: '(Definition/IllegalSibling, Train, (Circle))', - tags: 'Definition/IllegalSibling, Train', + generateIssue('invalidTagGroup', { + tagGroup: '(Definition/IllegalSibling, Train, (Circle))', }), ], warnings: [], }, + { testname: 'definition-with-deep-defs-inside', explanation: @@ -610,8 +511,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: false, - placeholdersAllowed: true, - definitionsAllowed: true, errors: [ generateIssue('invalidGroupTags', { string: '(Definition/DefNested, (Def/Nested, (Red, Blue, (Def/Blech)), Triangle))', @@ -620,6 +519,7 @@ export const parseTestData = [ ], warnings: [], }, + { testname: 'definition-with-nested-definition', explanation: @@ -629,8 +529,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: false, - placeholdersAllowed: true, - definitionsAllowed: true, errors: [ generateIssue('invalidTopLevelTagGroupTag', { string: '(Definition/NestedDefinition, (Touchscreen, (Definition/InnerDefinition, (Square))))', @@ -648,12 +546,9 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: false, - placeholdersAllowed: true, - definitionsAllowed: true, errors: [ - generateIssue('invalidNumberOfSubgroups', { - tag: 'Definition/MultipleTagGroupDefinition', - string: '(Definition/MultipleTagGroupDefinition, (Touchscreen), (Square))', + generateIssue('invalidTagGroup', { + tagGroup: '(Definition/MultipleTagGroupDefinition, (Touchscreen), (Square))', }), ], warnings: [], @@ -666,12 +561,9 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: false, - placeholdersAllowed: true, - definitionsAllowed: true, errors: [ - generateIssue('invalidGroupTopTags', { - string: '(Definition/Apple, Definition/Banana, (Blue))', - tags: 'Definition/Apple, Definition/Banana', + generateIssue('invalidTagGroup', { + tagGroup: '(Definition/Apple, Definition/Banana, (Blue))', }), ], warnings: [], @@ -684,31 +576,11 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: true, - definitionsAllowed: true, errors: [ generateIssue('missingTagGroup', { tag: 'Def-expand/Green1', string: 'Def-expand/Green1, (Red, Blue)' }), ], warnings: [], }, - { - testname: 'def-expand-tag-with-extra-group-tag', - explanation: '"(Def-expand/Acc/5.4, (Acceleration/5.4 m-per-s^2, Red), Blue)" has an extra tag in the group', - schemaVersion: '8.3.0', - stringIn: '(Def-expand/Acc/5.4, (Acceleration/5.4 m-per-s^2, Red), Blue)', - stringLong: null, - stringShort: null, - fullCheck: false, - placeholdersAllowed: true, - definitionsAllowed: true, - errors: [ - generateIssue('invalidGroupTopTags', { - string: '(Def-expand/Acc/5.4, (Acceleration/5.4 m-per-s^2, Red), Blue)', - tags: 'Def-expand/Acc/5.4, Blue', - }), - ], - warnings: [], - }, { testname: 'valid-def-expand-in-second-level-group', explanation: '"(Agent-action, (Def-expand/Blech, (Item, Sensory-event)))" is a valid group', @@ -718,8 +590,6 @@ export const parseTestData = [ '(Event/Agent-action, (Property/Organizational-property/Def-expand/Blech, (Item, Event/Sensory-event)))', stringShort: '(Agent-action, (Def-expand/Blech, (Item, Sensory-event)))', fullCheck: true, - placeholdersAllowed: true, - definitionsAllowed: true, errors: [], warnings: [], }, @@ -733,8 +603,6 @@ export const parseTestData = [ 'Item, (Event, Item/Object, (Item, (Property/Organizational-property/Def-expand/Blech, (Event/Agent-action, Item))))', stringShort: 'Item, (Event, Object, (Item, (Def-expand/Blech, (Agent-action, Item))))', fullCheck: true, - placeholdersAllowed: true, - definitionsAllowed: true, errors: [], warnings: [], }, @@ -747,8 +615,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: false, - placeholdersAllowed: true, - definitionsAllowed: true, errors: [ generateIssue('invalidGroupTags', { tags: 'Def-expand/Temp', @@ -757,19 +623,6 @@ export const parseTestData = [ ], warnings: [], }, - { - testname: 'event-context-in-group', - explanation: '"(Event-context, (Item))" is a valid event context', - schemaVersion: '8.3.0', - stringIn: '(Event-context, (Item))', - stringLong: '(Property/Organizational-property/Event-context, (Item))', - stringShort: '(Event-context, (Item))', - fullCheck: false, - placeholdersAllowed: true, - definitionsAllowed: true, - errors: [], - warnings: [], - }, { testname: 'event-context-in-subgroup', explanation: '"(Red, (Event-context, (Blue)))" has Event-context not in a top-level-tag group', @@ -778,8 +631,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: false, - placeholdersAllowed: true, - definitionsAllowed: true, errors: [ generateIssue('invalidTopLevelTagGroupTag', { tag: 'Event-context', @@ -796,8 +647,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: false, - placeholdersAllowed: true, - definitionsAllowed: true, errors: [ generateIssue('invalidTopLevelTagGroupTag', { tag: 'Event-context', @@ -814,8 +663,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [ generateIssue('multipleUniqueTags', { tag: 'Event-context', @@ -833,8 +680,6 @@ export const parseTestData = [ '(Property/Data-property/Data-value/Spatiotemporal-value/Temporal-value/Delay/5, Property/Data-property/Data-value/Spatiotemporal-value/Temporal-value/Duration/10, (Event))', stringShort: '(Delay/5, Duration/10, (Event))', fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -847,8 +692,6 @@ export const parseTestData = [ '(Property/Data-property/Data-value/Spatiotemporal-value/Temporal-value/Delay/5, Property/Data-property/Data-value/Spatiotemporal-value/Temporal-value/Duration/10, (Event))', stringShort: '(Delay/5, Duration/10, (Event))', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -860,9 +703,7 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, - errors: [generateIssue('temporalWithWrongNumberDefs', { tagGroup: '(Offset, Item)', tag: 'Offset' })], + errors: [generateIssue('invalidTagGroup', { tagGroup: '(Offset, Item)' })], warnings: [], }, { @@ -874,8 +715,6 @@ export const parseTestData = [ 'Property/Informational-property/Label/1, (Property/Organizational-property/Def/Acc/3.5), (Property/Data-property/Data-value/Quantitative-value/Item-count/2, Property/Informational-property/Label/1)', stringShort: 'Label/1, (Def/Acc/3.5), (Item-count/2, Label/1)', fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -888,8 +727,6 @@ export const parseTestData = [ '(Property/Organizational-property/Def/MyColor, Property/Data-property/Data-marker/Temporal-marker/Onset)', stringShort: '(Def/MyColor, Onset)', fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -902,8 +739,6 @@ export const parseTestData = [ '(Property/Organizational-property/Def/MyColor, Property/Data-property/Data-marker/Temporal-marker/Onset)', stringShort: '(Def/MyColor, Onset)', fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -914,23 +749,19 @@ export const parseTestData = [ stringIn: '(Onset, Delay/5 s)', stringLong: null, stringShort: null, - fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, - errors: [generateIssue('temporalWithWrongNumberDefs', { tagGroup: '(Onset, Delay/5 s)', tag: 'Onset' })], + fullCheck: false, + errors: [generateIssue('temporalWithoutDefinition', { tagGroup: '(Onset, Delay/5 s)', tag: 'Onset' })], warnings: [], }, { - testname: 'inset-delay-def', - explanation: '"(Inset, Delay/5 s, Def/myDef)" has an Inset, Delay and Def.', + testname: 'onset-delay-with-def', + explanation: '"(Onset, Delay/5 s, Def/myDef)" has an Onset, Delay and Def.', schemaVersion: '8.3.0', - stringIn: '(Inset, Delay/5 s, Def/myDef)', + stringIn: '(Onset, Delay/5 s, Def/myDef)', stringLong: - '(Property/Data-property/Data-marker/Temporal-marker/Inset, Property/Data-property/Data-value/Spatiotemporal-value/Temporal-value/Delay/5 s, Property/Organizational-property/Def/myDef)', - stringShort: '(Inset, Delay/5 s, Def/myDef)', + '(Property/Data-property/Data-marker/Temporal-marker/Onset, Property/Data-property/Data-value/Spatiotemporal-value/Temporal-value/Delay/5 s, Property/Organizational-property/Def/myDef)', + stringShort: '(Onset, Delay/5 s, Def/myDef)', fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -942,9 +773,7 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, - errors: [generateIssue('temporalWithWrongNumberDefs', { tagGroup: '(Inset, Delay/5 s)', tag: 'Inset' })], + errors: [generateIssue('temporalWithoutDefinition', { tagGroup: '(Inset, Delay/5 s)', tag: 'Inset' })], warnings: [], }, { @@ -956,8 +785,6 @@ export const parseTestData = [ '(Property/Data-property/Data-marker/Temporal-marker/Inset, Property/Data-property/Data-value/Spatiotemporal-value/Temporal-value/Delay/5 s, Property/Organizational-property/Def/myDef)', stringShort: '(Inset, Delay/5 s, Def/myDef)', fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -970,8 +797,6 @@ export const parseTestData = [ '(Property/Data-property/Data-marker/Temporal-marker/Inset, Property/Organizational-property/Def/myDef, (Property/Organizational-property/Def/Blech, Item))', stringShort: '(Inset, Def/myDef, (Def/Blech, Item))', fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -984,8 +809,6 @@ export const parseTestData = [ '(Property/Data-property/Data-marker/Temporal-marker/Inset, (Property/Organizational-property/Def-expand/myDef), (Property/Organizational-property/Def/Blech, Item))', stringShort: '(Inset, (Def-expand/myDef), (Def/Blech, Item))', fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -997,8 +820,6 @@ export const parseTestData = [ stringIn: '(Inset, (Def-expand/myDef, (Item,(Def/Temp))), (Def/Blech, Item))', stringLong: null, stringShort: null, - placeholdersAllowed: false, - definitionsAllowed: false, fullCheck: false, errors: [ generateIssue('invalidGroupTags', { string: '(Def-expand/myDef, (Item,(Def/Temp)))', tags: 'Def/Temp' }), @@ -1006,34 +827,14 @@ export const parseTestData = [ warnings: [], }, { - testname: 'offset-with-delay-no-def', + testname: 'Offset-with delay-no-def', explanation: '"(Offset, Delay/5 s)" does not have a Def for Inset.', schemaVersion: '8.3.0', stringIn: '(Offset, Delay/5 s)', stringLong: null, stringShort: null, fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, - errors: [generateIssue('temporalWithWrongNumberDefs', { tagGroup: '(Offset, Delay/5 s)', tag: 'Offset' })], - warnings: [], - }, - { - testname: 'offset-with-extra-group', - explanation: '"((Def-expand/MyColor, (Label/Pie)), Offset, (Red))" does not have a Def for Inset.', - schemaVersion: '8.3.0', - stringIn: '((Def-expand/MyColor, (Label/Pie)), Offset, (Red))', - stringLong: null, - stringShort: null, - fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, - errors: [ - generateIssue('invalidNumberOfSubgroups', { - string: '((Def-expand/MyColor, (Label/Pie)), Offset, (Red))', - tag: 'Offset', - }), - ], + errors: [generateIssue('temporalWithoutDefinition', { tagGroup: '(Offset, Delay/5 s)', tag: 'Offset' })], warnings: [], }, { @@ -1045,11 +846,10 @@ export const parseTestData = [ '(Property/Data-property/Data-marker/Temporal-marker/Offset, Property/Data-property/Data-value/Spatiotemporal-value/Temporal-value/Delay/5 s, Property/Organizational-property/Def/myDef)', stringShort: '(Offset, Delay/5 s, Def/myDef)', fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, + { testname: 'onset-with-def-expand', explanation: '"(Onset, (Def-expand/MyColor, (Label/Pie)), (Red))" is okay.', @@ -1059,8 +859,6 @@ export const parseTestData = [ '(Property/Data-property/Data-marker/Temporal-marker/Onset, (Property/Organizational-property/Def-expand/MyColor, (Property/Informational-property/Label/Pie)), (Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Red-color/Red))', stringShort: '(Onset, (Def-expand/MyColor, (Label/Pie)), (Red))', fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -1073,8 +871,6 @@ export const parseTestData = [ '(Property/Organizational-property/Def-expand/Acc/4.5, (Property/Data-property/Data-value/Spatiotemporal-value/Rate-of-change/Acceleration/4.5, Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Red-color/Red))', stringShort: '(Def-expand/Acc/4.5, (Acceleration/4.5, Red))', fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -1086,8 +882,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [ generateIssue('missingTagGroup', { string: 'Def-expand/Acc/4.5, (Acceleration/4.5 m-per-s^2, Red)', @@ -1096,24 +890,6 @@ export const parseTestData = [ ], warnings: [], }, - { - testname: 'onset-offset-in-same-group', - explanation: '"(Def/MyColor, Onset, Offset)" has multiple temporal tags in same group.', - schemaVersion: '8.3.0', - stringIn: '(Def/MyColor, Onset, Offset)', - stringLong: null, - stringShort: null, - fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, - errors: [ - generateIssue('multipleRequiresDefTags', { - string: '(Def/MyColor, Onset, Offset)', - tags: 'Offset, Onset', - }), - ], - warnings: [], - }, { testname: 'def-expand-not-in-group', explanation: '"Def-expand/Acc/4.5, (Acceleration/4.5, Red)" does not have group detected if not full check.', @@ -1122,8 +898,6 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [ generateIssue('missingTagGroup', { string: 'Def-expand/Acc/4.5, (Acceleration/4.5 m-per-s^2, Red)', @@ -1141,38 +915,6 @@ export const parseTestData = [ '(Property/Data-property/Data-value/Spatiotemporal-value/Temporal-value/Delay/5.0 s, Property/Data-property/Data-marker/Temporal-marker/Onset, Property/Organizational-property/Def/MyColor)', stringShort: '(Delay/5.0 s, Onset, Def/MyColor)', fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, - errors: [], - warnings: [], - }, - { - testname: 'onset-delay-def-expand-in-same-group', - explanation: - '"(Delay/5.0 s, Onset, (Def-expand/MyColor, (Label/Pie)))" does not have group detected if not full check.', - schemaVersion: '8.3.0', - stringIn: '(Delay/5.0 s, Onset, (Def-expand/MyColor, (Label/Pie)))', - stringLong: - '(Property/Data-property/Data-value/Spatiotemporal-value/Temporal-value/Delay/5.0 s, Property/Data-property/Data-marker/Temporal-marker/Onset, (Property/Organizational-property/Def-expand/MyColor, (Property/Informational-property/Label/Pie)))', - stringShort: '(Delay/5.0 s, Onset, (Def-expand/MyColor, (Label/Pie)))', - fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, - errors: [], - warnings: [], - }, - { - testname: 'onset-delay-def-expand-extra-tag-group', - explanation: - '"(Delay/5.0 s, Onset, (Def-expand/MyColor, (Label/Pie)), (Item))" does not have group detected if not full check.', - schemaVersion: '8.3.0', - stringIn: '(Delay/5.0 s, Onset, (Def-expand/MyColor, (Label/Pie)), (Item))', - stringLong: - '(Property/Data-property/Data-value/Spatiotemporal-value/Temporal-value/Delay/5.0 s, Property/Data-property/Data-marker/Temporal-marker/Onset, (Property/Organizational-property/Def-expand/MyColor, (Property/Informational-property/Label/Pie)), (Item))', - stringShort: '(Delay/5.0 s, Onset, (Def-expand/MyColor, (Label/Pie)), (Item))', - fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [], warnings: [], }, @@ -1184,12 +926,9 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: false, - placeholdersAllowed: false, - definitionsAllowed: false, errors: [ - generateIssue('invalidGroupTopTags', { - string: '((Def-expand/MyColor, (Label/Pie)), Inset, Blue)', - tags: 'Inset, Blue', + generateIssue('invalidTagGroup', { + tagGroup: '((Def-expand/MyColor, (Label/Pie)), Inset, Blue)', }), ], warnings: [], @@ -1209,8 +948,6 @@ export const parseTestData = [ 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Red-color/Red, (({event_code}, Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Blue-color/Blue), Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Green-color/Green)', stringShort: 'Red, (({event_code}, Blue), Green)', fullCheck: false, - placeholdersAllowed: true, - definitionsAllowed: true, errors: [], warnings: [], }, @@ -1222,58 +959,20 @@ export const parseTestData = [ stringLong: null, stringShort: null, fullCheck: true, - placeholdersAllowed: true, - definitionsAllowed: true, errors: [generateIssue('curlyBracesNotAllowed', { string: 'Red, (({event_code}, Blue), Green)' })], warnings: [], }, { testname: 'column-splice-in-definition', - explanation: '"(Definition/Blech, ({event_code}, Blue))" cannot have a column splice in definition', + explanation: '"(Definition/Blech, ({event_code}, Blue))" is not allowed if full check', schemaVersion: '8.3.0', stringIn: '(Definition/Blech, ({event_code}, Blue))', stringLong: null, stringShort: null, fullCheck: false, - placeholdersAllowed: true, - definitionsAllowed: true, errors: [generateIssue('curlyBracesNotAllowed', { string: '(Definition/Blech, ({event_code}, Blue))' })], warnings: [], }, - { - testname: 'column-splice-in-deep-def-expand', - explanation: - '"(Red, ((Def-expand/Blech, ({event_code}, Label/Pie)), (Blue))), Green))" cannot have a column splice in def-expand', - schemaVersion: '8.3.0', - stringIn: '(Red, ((Def-expand/Blech, ({event_code}, Label/Pie), (Blue))), Green)', - stringLong: null, - stringShort: null, - fullCheck: false, - placeholdersAllowed: true, - definitionsAllowed: true, - errors: [ - generateIssue('curlyBracesNotAllowed', { - string: '(Red, ((Def-expand/Blech, ({event_code}, Label/Pie), (Blue))), Green)', - }), - ], - warnings: [], - }, - { - testname: 'column-splice-with-def-expand', - explanation: - '"(Red, ((Def-expand/Blech, ({event_code}, Label/Pie))))" cannot have a column splice with def-expand', - schemaVersion: '8.3.0', - stringIn: '(Red, ((Def-expand/Blech, ({event_code}, Label/Pie))))', - stringLong: null, - stringShort: null, - fullCheck: false, - placeholdersAllowed: true, - definitionsAllowed: true, - errors: [ - generateIssue('curlyBracesNotAllowed', { string: '(Red, ((Def-expand/Blech, ({event_code}, Label/Pie))))' }), - ], - warnings: [], - }, ], }, ] diff --git a/tests/testUtilities.js b/tests/testUtilities.js index 85d0b9c8..41dd9b96 100644 --- a/tests/testUtilities.js +++ b/tests/testUtilities.js @@ -29,19 +29,14 @@ export function extractHedCodes(issues) { } // Parse the HED string -export function getHedString(hedString, hedSchemas, fullCheck, definitionsAllowed, placeholdersAllowed) { - const [parsedString, issues] = parseHedString( - hedString, - hedSchemas, - fullCheck, - definitionsAllowed, - placeholdersAllowed, - ) +export function getHedString(hedString, hedSchemas, fullCheck) { + const [parsedString, issues] = parseHedString(hedString, hedSchemas, fullCheck) + const flattenedIssues = Object.values(issues).flat() let errorIssues = [] let warningIssues = [] - if (issues.length !== 0) { - errorIssues = issues.filter((obj) => obj.level === 'error') - warningIssues = issues.filter((obj) => obj.level !== 'error') + if (flattenedIssues.length !== 0) { + errorIssues = flattenedIssues.filter((obj) => obj.level === 'error') + warningIssues = flattenedIssues.filter((obj) => obj.level !== 'error') } if (errorIssues.length > 0) { return [null, errorIssues, warningIssues] diff --git a/tests/tokenizerTests.spec.js b/tests/tokenizerTests.spec.js index cf7b5240..f1b26304 100644 --- a/tests/tokenizerTests.spec.js +++ b/tests/tokenizerTests.spec.js @@ -21,7 +21,8 @@ describe('Tokenizer validation using JSON tests', () => { const status = test.errors.length > 0 ? 'Expect fail' : 'Expect pass' const tokenizer = new HedStringTokenizer(test.string) const header = `\n[${test.testname}](${status}): ${test.explanation}` - const [tagSpecs, groupSpec, issues] = tokenizer.tokenize() + const [tagSpecs, groupSpec, tokenizingIssues] = tokenizer.tokenize() + const issues = Object.values(tokenizingIssues).flat() assert.sameDeepMembers(issues, test.errors, `${header} should have the same errors`) assert.sameDeepMembers(tagSpecs, test.tagSpecs, `${header} should generate the same tagSpecs`) assert.deepEqual(groupSpec, test.groupSpec, `${header} should generate the same groupSpec`) @@ -36,7 +37,6 @@ describe('Tokenizer validation using JSON tests', () => { if (shouldRun(name, test.testname, runAll, runMap, skipMap)) { stringTokenizer(test) } else { - // eslint-disable-next-line no-console console.log(`----Skipping tokenizerTest ${name}: ${test.testname}`) } }) diff --git a/utils/array.js b/utils/array.js index 546e7cba..cc5cfd5e 100644 --- a/utils/array.js +++ b/utils/array.js @@ -30,20 +30,3 @@ export function recursiveMap(fn, array) { return fn(array) } } - -/** - * Apply a function recursively to an array. - * - * @template T,U - * @param {function(T, U[]): U} fn The function to apply. - * @param {T[]} array The array to map. - * @param {U[]} [issues] An optional array to collect issues. - * @returns {U[]} The mapped array. - */ -export function recursiveMapNew(fn, array, issues = []) { - if (Array.isArray(array)) { - return array.map((element) => recursiveMap(fn, element, issues)) - } else { - return fn(array, issues) - } -} diff --git a/utils/map.js b/utils/map.js index fb45f915..a9864f9f 100644 --- a/utils/map.js +++ b/utils/map.js @@ -35,7 +35,7 @@ export const filterNonEqualDuplicates = function (list, equalityFunction = isEqu * @template T, U * @param {T[]} list The list to group. * @param {function (T): U} groupingFunction A function mapping a list value to the key it is to be grouped under. - * @returns {Map} The grouped map. + * @return {Map} The grouped map. */ export const groupBy = function (list, groupingFunction = identity) { const groupingMap = new Map() diff --git a/validator/dataset.js b/validator/dataset.js index 014f5adb..f54d30d7 100644 --- a/validator/dataset.js +++ b/validator/dataset.js @@ -13,9 +13,9 @@ import { filterNonEqualDuplicates } from '../utils/map' */ export const parseDefinitions = function (parsedHedStrings) { const issues = [] - const parsedHedStringDefinitions = parsedHedStrings.flatMap((parsedHedString) => - parsedHedString ? parsedHedString.definitions : [], - ) + const parsedHedStringDefinitions = parsedHedStrings.flatMap((parsedHedString) => { + return parsedHedString.definitions + }) const [definitionMap, definitionDuplicates] = filterNonEqualDuplicates( parsedHedStringDefinitions, (definition, other) => definition.definitionGroup.equivalent(other.definitionGroup), @@ -39,10 +39,10 @@ export const parseDefinitions = function (parsedHedStrings) { * @returns {Issue[]} Any issues found. */ const checkGroupForTemporalOrder = (parsedGroup, activeScopes) => { - if (parsedGroup.isSpecialGroup('Onset')) { + if (parsedGroup.isOnsetGroup) { activeScopes.add(parsedGroup.defNameAndValue) } - if (parsedGroup.isSpecialGroup('Inset') && !activeScopes.has(parsedGroup.defNameAndValue)) { + if (parsedGroup.isInsetGroup && !activeScopes.has(parsedGroup.defNameAndValue)) { return [ generateIssue('inactiveOnset', { definition: parsedGroup.defNameAndValue, @@ -50,7 +50,7 @@ const checkGroupForTemporalOrder = (parsedGroup, activeScopes) => { }), ] } - if (parsedGroup.isSpecialGroup('Offset') && !activeScopes.delete(parsedGroup.defNameAndValue)) { + if (parsedGroup.isOffsetGroup && !activeScopes.delete(parsedGroup.defNameAndValue)) { return [ generateIssue('inactiveOnset', { definition: parsedGroup.defNameAndValue, @@ -112,7 +112,7 @@ export const validateDataset = function (definitions, hedStrings, hedSchemas) { * * @param {(string[]|ParsedHedString[])} parsedHedStrings The dataset's parsed HED strings. * @param {Schemas} hedSchemas The HED schema container object. - * @param {DefinitionManager} definitions The dataset's parsed definitions. + * @param {Map} definitions The dataset's parsed definitions. * @param {Object} settings The configuration settings for validation. * @returns {[boolean, Issue[]]} Whether the HED strings are valid and any issues found. */ @@ -158,7 +158,7 @@ export const validateHedDataset = function (hedStrings, hedSchemas, ...args) { if (stringsValid && settings.validateDatasetLevel) { datasetIssues = validateDataset(definitions, parsedHedStrings, hedSchemas) } - const issues = [...parsingIssues, ...definitionIssues, ...stringIssues, ...datasetIssues] + const issues = stringIssues.concat(...Object.values(parsingIssues), definitionIssues, datasetIssues) return Issue.issueListWithValidStatus(issues) } @@ -167,12 +167,12 @@ export const validateHedDataset = function (hedStrings, hedSchemas, ...args) { * Validate a HED dataset with additional context. * * @param {string[]|ParsedHedString[]} hedStrings The dataset's HED strings. - * @param {BidsSidecar} contextHedStrings The dataset's context HED strings. + * @param {string[]|ParsedHedString[]} contextHedStrings The dataset's context HED strings. * @param {Schemas} hedSchemas The HED schema container object. * @param {boolean} checkForWarnings Whether to check for warnings or only errors. * @returns {[boolean, Issue[]]} Whether the HED dataset is valid and any issues found. */ -export const validateHedDatasetWithContext = function (hedStrings, context, hedSchemas, ...args) { +export const validateHedDatasetWithContext = function (hedStrings, contextHedStrings, hedSchemas, ...args) { let settings if (args[0] === Object(args[0])) { settings = { @@ -185,20 +185,24 @@ export const validateHedDatasetWithContext = function (hedStrings, context, hedS validateDatasetLevel: true, } } - if (hedStrings.length + context.hedStrings.length === 0) { + if (hedStrings.length + contextHedStrings.length === 0) { return [true, []] } - const [parsedHedStrings, issues] = parseHedStrings(hedStrings, hedSchemas, true, false, false) - //const [parsedContextHedStrings, contextParsingIssues] = parseHedStrings(contextHedStrings, hedSchemas, false) - //const combinedParsedHedStrings = parsedHedStrings.concat(parsedContextHedStrings) - //const [definitions, definitionIssues] = parseDefinitions(combinedParsedHedStrings) - const [stringsValid, stringIssues] = validateHedEvents(parsedHedStrings, hedSchemas, context.definitions, settings) - issues.push(...stringIssues) + const [parsedHedStrings, parsingIssues] = parseHedStrings(hedStrings, hedSchemas, false) + const [parsedContextHedStrings, contextParsingIssues] = parseHedStrings(contextHedStrings, hedSchemas, false) + const combinedParsedHedStrings = parsedHedStrings.concat(parsedContextHedStrings) + const [definitions, definitionIssues] = parseDefinitions(combinedParsedHedStrings) + const [stringsValid, stringIssues] = validateHedEvents(parsedHedStrings, hedSchemas, definitions, settings) let datasetIssues = [] if (stringsValid && settings.validateDatasetLevel) { - datasetIssues = validateDataset(context.definitions, parsedHedStrings, hedSchemas) + datasetIssues = validateDataset(definitions, parsedHedStrings, hedSchemas) } - issues.push(...datasetIssues) + const issues = stringIssues.concat( + ...Object.values(parsingIssues), + ...Object.values(contextParsingIssues), + definitionIssues, + datasetIssues, + ) return Issue.issueListWithValidStatus(issues) } diff --git a/validator/event/init.js b/validator/event/init.js index d744d87c..c5e33b09 100644 --- a/validator/event/init.js +++ b/validator/event/init.js @@ -39,16 +39,16 @@ export const validateHedString = function (hedString, hedSchemas, ...args) { settings = { checkForWarnings: settingsArg.checkForWarnings ?? false, expectValuePlaceholderString: settingsArg.expectValuePlaceholderString ?? false, - definitionsAllowed: settingsArg.definitionsAllowed ?? true, + definitionsAllowed: settingsArg.definitionsAllowed ?? 'yes', } } else { settings = { checkForWarnings: args[0] ?? false, expectValuePlaceholderString: args[1] ?? false, - definitionsAllowed: true, + definitionsAllowed: 'yes', } } - const [parsedString, parsingIssues] = parseHedString(hedString, hedSchemas, false, settings.definitionsAllowed) + const [parsedString, parsingIssues] = parseHedString(hedString, hedSchemas) if (parsedString === null) { return [false, [].concat(...Object.values(parsingIssues))] } @@ -85,7 +85,7 @@ export const validateHedEvent = function (hedString, hedSchemas, ...args) { } } //const [parsedString, parsedStringIssues, hedValidator] = initiallyValidateHedString(hedString, hedSchemas, settings) - const [parsedString, parsingIssues] = parseHedString(hedString, hedSchemas, true, false) + const [parsedString, parsingIssues] = parseHedString(hedString, hedSchemas) if (parsedString === null) { return [false, [].concat(...Object.values(parsingIssues))] } @@ -100,7 +100,7 @@ export const validateHedEvent = function (hedString, hedSchemas, ...args) { * * @param {string|ParsedHedString} hedString The HED event string to validate. * @param {Schemas} hedSchemas The HED schemas to validate against. - * @param {DefinitionManager} definitions The dataset's parsed definitions. + * @param {Map} definitions The dataset's parsed definitions. * @param {boolean} checkForWarnings Whether to check for warnings or only errors. * @returns {[boolean, Issue[]]} Whether the HED string is valid and any issues found. */ @@ -117,7 +117,7 @@ export const validateHedEventWithDefinitions = function (hedString, hedSchemas, } //const [parsedString, parsedStringIssues, hedValidator] = initiallyValidateHedString( // hedString, hedSchemas, settings, definitions,) - const [parsedString, parsingIssues] = parseHedString(hedString, hedSchemas, true, false) + const [parsedString, parsingIssues] = parseHedString(hedString, hedSchemas) if (parsedString === null) { return [false, [].concat(...Object.values(parsingIssues))] } diff --git a/validator/event/validator.js b/validator/event/validator.js index ca9ad6f5..cc83e17b 100644 --- a/validator/event/validator.js +++ b/validator/event/validator.js @@ -14,7 +14,7 @@ const topLevelTagGroupType = 'topLevelTagGroup' const uniqueType = 'unique' const requiredType = 'required' -const specialTags = require('../../data/json/reservedTags.json') +const specialTags = require('../../data/json/specialTags.json') /** * HED validator. @@ -43,7 +43,7 @@ export default class HedValidator { /** * The parsed definitions. * - * @type {DefinitionManager} + * @type {Map} */ definitions @@ -52,7 +52,7 @@ export default class HedValidator { * * @param {ParsedHedString} parsedString The parsed HED string to be validated. * @param {Schemas} hedSchemas The collection of HED schemas. - * @param {DefinitionManager} definitions The parsed definitions. + * @param {Map} definitions The parsed definitions. * @param {Object} options The validation options. */ constructor(parsedString, hedSchemas, definitions, options) { @@ -64,70 +64,70 @@ export default class HedValidator { } validateStringLevel() { - // this.options.isEventLevel = false - // this.validateIndividualHedTags() - // this.validateHedTagLevels() - // this.validateHedTagGroups() - // this.validateFullParsedHedString() + this.options.isEventLevel = false + this.validateIndividualHedTags() + this.validateHedTagLevels() + this.validateHedTagGroups() + this.validateFullParsedHedString() } validateEventLevel() { - // this.options.isEventLevel = true - // this.validateTopLevelTags() - // this.validateIndividualHedTags() - // this.validateHedTagLevels() - // this.validateHedTagGroups() - // this.validateTopLevelTagGroups() + this.options.isEventLevel = true + this.validateTopLevelTags() + this.validateIndividualHedTags() + this.validateHedTagLevels() + this.validateHedTagGroups() + this.validateTopLevelTagGroups() } // Categories - // /** - // * Validate the individual HED tags in a parsed HED string object. - // */ - // validateIndividualHedTags() { - // for (const tag of this.parsedString.tags) { - // this.validateIndividualHedTag(tag) - // } - // } - - // /** - // * Validate an individual HED tag. - // */ - // validateIndividualHedTag(tag) { - // //this.checkIfTagIsValid(tag, previousTag) - // //this.checkIfTagUnitClassUnitsAreValid(tag) - // if (!this.options.isEventLevel) { - // //this.checkValueTagSyntax(tag) - // } - // if (this.definitions !== null) { - // const [definition, missingIssues] = this.definitions.findDefinition(tag) - // this.issues.push(...missingIssues) - // } - // // if (this.options.expectValuePlaceholderString) { - // // this.checkPlaceholderTagSyntax(tag) - // // } - // } - // - // /** - // * Validate the HED tag levels in a parsed HED string object. - // */ - // validateHedTagLevels() { - // for (const tagGroup of this.parsedString.tagGroups) { - // for (const subGroup of tagGroup.subGroupArrayIterator()) { - // this.validateHedTagLevel(subGroup) - // } - // } - // this.validateHedTagLevel(this.parsedString.parseTree) - // } - // - // /** - // * Validate a HED tag level. - // */ - // validateHedTagLevel(tagList) { - // //this.checkForMultipleUniqueTags(tagList) - // //this.checkForDuplicateTags(tagList) - // } + /** + * Validate the individual HED tags in a parsed HED string object. + */ + validateIndividualHedTags() { + for (const tag of this.parsedString.tags) { + this.validateIndividualHedTag(tag) + } + } + + /** + * Validate an individual HED tag. + */ + validateIndividualHedTag(tag) { + //this.checkIfTagIsValid(tag, previousTag) + //this.checkIfTagUnitClassUnitsAreValid(tag) + if (!this.options.isEventLevel) { + //this.checkValueTagSyntax(tag) + } + if (this.definitions !== null) { + this.checkForMissingDefinitions(tag, 'Def') + this.checkForMissingDefinitions(tag, 'Def-expand') + } + if (this.options.expectValuePlaceholderString) { + this.checkPlaceholderTagSyntax(tag) + } + } + + /** + * Validate the HED tag levels in a parsed HED string object. + */ + validateHedTagLevels() { + for (const tagGroup of this.parsedString.tagGroups) { + for (const subGroup of tagGroup.subGroupArrayIterator()) { + this.validateHedTagLevel(subGroup) + } + } + this.validateHedTagLevel(this.parsedString.parseTree) + } + + /** + * Validate a HED tag level. + */ + validateHedTagLevel(tagList) { + //this.checkForMultipleUniqueTags(tagList) + this.checkForDuplicateTags(tagList) + } /** * Validate the HED tag groups in a parsed HED string. @@ -140,90 +140,90 @@ export default class HedValidator { } } - // /** - // * Validate a HED tag group. - // */ - // // eslint-disable-next-line no-unused-vars - // validateHedTagGroup(parsedTagGroup) { - // //this.checkDefinitionGroupSyntax(parsedTagGroup) - // //this.checkTemporalSyntax(parsedTagGroup) - // } - // - // /** - // * Validate the top-level HED tags in a parsed HED string. - // */ - // validateTopLevelTags() { - // for (const topLevelTag of this.parsedString.topLevelTags) { - // if ( - // !hedStringIsAGroup(topLevelTag.formattedTag) && - // (topLevelTag.hasAttribute(tagGroupType) || topLevelTag.parentHasAttribute(tagGroupType)) - // ) { - // this.pushIssue('invalidTopLevelTag', { - // tag: topLevelTag, - // }) - // } - // } - // } - // - // /** - // * Validate the top-level HED tag groups in a parsed HED string. - // */ - // validateTopLevelTagGroups() { - // for (const tag of this.parsedString.tags) { - // if (!tag.hasAttribute(topLevelTagGroupType) && !tag.parentHasAttribute(topLevelTagGroupType)) { - // continue - // } - // if (!this.parsedString.topLevelGroupTags.some((topLevelTagGroup) => topLevelTagGroup.includes(tag))) { - // this.pushIssue('invalidTopLevelTagGroupTag', { - // tag: tag, - // string: this.parsedString.hedString, - // }) - // } - // } - // } + /** + * Validate a HED tag group. + */ + // eslint-disable-next-line no-unused-vars + validateHedTagGroup(parsedTagGroup) { + //this.checkDefinitionGroupSyntax(parsedTagGroup) + this.checkTemporalSyntax(parsedTagGroup) + } + + /** + * Validate the top-level HED tags in a parsed HED string. + */ + validateTopLevelTags() { + for (const topLevelTag of this.parsedString.topLevelTags) { + if ( + !hedStringIsAGroup(topLevelTag.formattedTag) && + (topLevelTag.hasAttribute(tagGroupType) || topLevelTag.parentHasAttribute(tagGroupType)) + ) { + this.pushIssue('invalidTopLevelTag', { + tag: topLevelTag, + }) + } + } + } + + /** + * Validate the top-level HED tag groups in a parsed HED string. + */ + validateTopLevelTagGroups() { + for (const tag of this.parsedString.tags) { + if (!tag.hasAttribute(topLevelTagGroupType) && !tag.parentHasAttribute(topLevelTagGroupType)) { + continue + } + if (!this.parsedString.topLevelGroupTags.some((topLevelTagGroup) => topLevelTagGroup.includes(tag))) { + this.pushIssue('invalidTopLevelTagGroupTag', { + tag: tag, + string: this.parsedString.hedString, + }) + } + } + } /** * Validate the full parsed HED string. */ validateFullParsedHedString() { this.checkPlaceholderStringSyntax() - //this.checkDefinitionStringSyntax() + this.checkDefinitionStringSyntax() } // Individual checks - // /** - // * Check for duplicate tags at the top level or within a single group. - // */ - // checkForDuplicateTags(tagList) { - // const duplicateTags = new Set() - // - // const addIssue = (tag) => { - // if (duplicateTags.has(tag)) { - // return - // } - // this.pushIssue('duplicateTag', { - // tag: tag, - // }) - // duplicateTags.add(tag) - // } - // - // tagList.forEach((firstTag, firstIndex) => { - // tagList.forEach((secondTag, secondIndex) => { - // if (firstIndex !== secondIndex && firstTag.equivalent(secondTag)) { - // // firstIndex and secondIndex are not the same (i.e. comparing a tag with itself), - // // but they are equivalent tags or tag groups (i.e. have the same members up to order). - // addIssue(firstTag) - // addIssue(secondTag) - // } - // }) - // }) - //} - - /* /!** + /** + * Check for duplicate tags at the top level or within a single group. + */ + checkForDuplicateTags(tagList) { + const duplicateTags = new Set() + + const addIssue = (tag) => { + if (duplicateTags.has(tag)) { + return + } + this.pushIssue('duplicateTag', { + tag: tag, + }) + duplicateTags.add(tag) + } + + tagList.forEach((firstTag, firstIndex) => { + tagList.forEach((secondTag, secondIndex) => { + if (firstIndex !== secondIndex && firstTag.equivalent(secondTag)) { + // firstIndex and secondIndex are not the same (i.e. comparing a tag with itself), + // but they are equivalent tags or tag groups (i.e. have the same members up to order). + addIssue(firstTag) + addIssue(secondTag) + } + }) + }) + } + + /** * Validation check based on a tag attribute. * * @param {string} attribute The name of the attribute. * @param {function (string): void} fn The actual validation code. - *!/ + */ _checkForTagAttribute(attribute, fn) { const schemas = this.hedSchemas.schemas.values() for (const schema of schemas) { @@ -232,14 +232,14 @@ export default class HedValidator { fn(tag.longName) } } - }*/ + } /** * Check basic placeholder tag syntax. * * @param {ParsedHedTag} tag A HED tag. */ - /* checkPlaceholderTagSyntax(tag) { + checkPlaceholderTagSyntax(tag) { // TODO: Refactor or eliminate after column splicing completed const placeholderCount = getCharacterCount(tag.formattedTag, '#') if (placeholderCount === 1) { @@ -255,102 +255,188 @@ export default class HedValidator { tag: tag, }) } - }*/ - - // /** - // * Check full-string placeholder syntax. - // */ - // checkPlaceholderStringSyntax() { - // const standalonePlaceholders = { - // // Count of placeholders not in Definition groups. - // placeholders: 0, - // // Whether an Issue has already been generated for an excess placeholder outside a Definition group. - // issueGenerated: false, - // } - // this._checkStandalonePlaceholderStringSyntaxInGroup(this.parsedString.topLevelTags, standalonePlaceholders) - // // Loop over the top-level tag groups. - // for (const tagGroup of this.parsedString.tagGroups) { - // if (tagGroup.isDefinitionGroup) { - // this._checkDefinitionPlaceholderStringSyntaxInGroup(tagGroup) - // } else if (!standalonePlaceholders.issueGenerated) { - // this._checkStandalonePlaceholderStringSyntaxInGroup(tagGroup.tagIterator(), standalonePlaceholders) - // } - // } - // if (this.options.expectValuePlaceholderString && standalonePlaceholders.placeholders === 0) { - // this.pushIssue('missingPlaceholder', { - // string: this.parsedString.hedString, - // }) - // } - // } - - // /** - // * Check Definition-related placeholder syntax in a tag group. - // * - // * @param {ParsedHedGroup} tagGroup A HED tag group. - // * @private - // */ - // _checkDefinitionPlaceholderStringSyntaxInGroup(tagGroup) { - // // Count of placeholders within this Definition group. - // let definitionPlaceholders = 0 - // const definitionValue = tagGroup.definitionValue - // const definitionHasPlaceholder = definitionValue === '#' - // const definitionName = tagGroup.definitionName - // for (const tag of tagGroup.tagIterator()) { - // if (!definitionHasPlaceholder || tag !== tagGroup.definitionTag) { - // definitionPlaceholders += getCharacterCount(tag.formattedTag, '#') - // } - // } - // const isValid = - // (definitionValue === '' && definitionPlaceholders === 0) || - // (definitionHasPlaceholder && definitionPlaceholders === 1) - // if (!isValid) { - // this.pushIssue('invalidPlaceholderInDefinition', { - // definition: definitionName, - // }) - // } - // } - - // /** - // * Check non-Definition-related placeholder syntax in a tag group. - // * - // * @param {ParsedHedTag[]|Generator} tags A HED tag iterator. - // * @param {{placeholders: number, issueGenerated: boolean}} standalonePlaceholders The validator's standalone placeholder context. - // * @private - // */ - // _checkStandalonePlaceholderStringSyntaxInGroup(tags, standalonePlaceholders) { - // let firstStandaloneTag - // for (const tag of tags) { - // const tagString = tag.formattedTag - // const tagPlaceholders = getCharacterCount(tagString, '#') - // standalonePlaceholders.placeholders += tagPlaceholders - // if (!firstStandaloneTag && tagPlaceholders > 0) { - // firstStandaloneTag = tag - // } - // if ( - // tagPlaceholders === 0 || - // (standalonePlaceholders.placeholders <= 1 && - // (this.options.expectValuePlaceholderString || standalonePlaceholders.placeholders === 0)) - // ) { - // continue - // } - // if (this.options.expectValuePlaceholderString && !standalonePlaceholders.issueGenerated) { - // this.pushIssue('invalidPlaceholder', { - // tag: firstStandaloneTag, - // }) - // } - // this.pushIssue('invalidPlaceholder', { - // tag: tag, - // }) - // standalonePlaceholders.issueGenerated = true - // } - // } - - /* /!** + } + + /** + * Check full-string placeholder syntax. + */ + checkPlaceholderStringSyntax() { + const standalonePlaceholders = { + // Count of placeholders not in Definition groups. + placeholders: 0, + // Whether an Issue has already been generated for an excess placeholder outside a Definition group. + issueGenerated: false, + } + this._checkStandalonePlaceholderStringSyntaxInGroup(this.parsedString.topLevelTags, standalonePlaceholders) + // Loop over the top-level tag groups. + for (const tagGroup of this.parsedString.tagGroups) { + if (tagGroup.isDefinitionGroup) { + this._checkDefinitionPlaceholderStringSyntaxInGroup(tagGroup) + } else if (!standalonePlaceholders.issueGenerated) { + this._checkStandalonePlaceholderStringSyntaxInGroup(tagGroup.tagIterator(), standalonePlaceholders) + } + } + if (this.options.expectValuePlaceholderString && standalonePlaceholders.placeholders === 0) { + this.pushIssue('missingPlaceholder', { + string: this.parsedString.hedString, + }) + } + } + + /** + * Check Definition-related placeholder syntax in a tag group. + * + * @param {ParsedHedGroup} tagGroup A HED tag group. + * @private + */ + _checkDefinitionPlaceholderStringSyntaxInGroup(tagGroup) { + // Count of placeholders within this Definition group. + let definitionPlaceholders = 0 + const definitionValue = tagGroup.definitionValue + const definitionHasPlaceholder = definitionValue === '#' + const definitionName = tagGroup.definitionName + for (const tag of tagGroup.tagIterator()) { + if (!definitionHasPlaceholder || tag !== tagGroup.definitionTag) { + definitionPlaceholders += getCharacterCount(tag.formattedTag, '#') + } + } + const isValid = + (definitionValue === '' && definitionPlaceholders === 0) || + (definitionHasPlaceholder && definitionPlaceholders === 1) + if (!isValid) { + this.pushIssue('invalidPlaceholderInDefinition', { + definition: definitionName, + }) + } + } + + /** + * Check non-Definition-related placeholder syntax in a tag group. + * + * @param {ParsedHedTag[]|Generator} tags A HED tag iterator. + * @param {{placeholders: number, issueGenerated: boolean}} standalonePlaceholders The validator's standalone placeholder context. + * @private + */ + _checkStandalonePlaceholderStringSyntaxInGroup(tags, standalonePlaceholders) { + let firstStandaloneTag + for (const tag of tags) { + const tagString = tag.formattedTag + const tagPlaceholders = getCharacterCount(tagString, '#') + standalonePlaceholders.placeholders += tagPlaceholders + if (!firstStandaloneTag && tagPlaceholders > 0) { + firstStandaloneTag = tag + } + if ( + tagPlaceholders === 0 || + (standalonePlaceholders.placeholders <= 1 && + (this.options.expectValuePlaceholderString || standalonePlaceholders.placeholders === 0)) + ) { + continue + } + if (this.options.expectValuePlaceholderString && !standalonePlaceholders.issueGenerated) { + this.pushIssue('invalidPlaceholder', { + tag: firstStandaloneTag, + }) + } + this.pushIssue('invalidPlaceholder', { + tag: tag, + }) + standalonePlaceholders.issueGenerated = true + } + } + + /** + * Check full-string Definition syntax. + */ + checkDefinitionStringSyntax() { + if (this.parsedString.definitionGroups.length === 0) { + return + } + switch (this.options.definitionsAllowed) { + case 'no': + this.pushIssue('illegalDefinitionContext', { + string: this.parsedString.hedString, + }) + break + case 'exclusive': + if ( + !isEqual(this.parsedString.definitionGroups, this.parsedString.tagGroups) || + this.parsedString.topLevelTags.length > 0 + ) { + this.pushIssue('illegalInExclusiveContext', { + string: this.parsedString.hedString, + }) + } + break + } + } + + /** + * Check the syntax of HED 3 definitions. + * + * @param {ParsedHedGroup} tagGroup The tag group. + */ + checkDefinitionGroupSyntax(tagGroup) { + if (!tagGroup.isDefinitionGroup) { + return + } + + const definitionShortTag = 'Definition' + const defExpandShortTag = 'Def-expand' + const defShortTag = 'Def' + + const definitionName = tagGroup.definitionNameAndValue + + let tagGroupValidated = false + let tagGroupIssueGenerated = false + for (const tag of tagGroup.tags) { + if (tag instanceof ParsedHedGroup) { + if (tagGroupValidated && !tagGroupIssueGenerated) { + this.pushIssue('multipleTagGroupsInDefinition', { + definition: definitionName, + }) + tagGroupIssueGenerated = true + continue + } + tagGroupValidated = true + for (const columnSplice of tag.columnSpliceIterator()) { + this.pushIssue('curlyBracesInDefinition', { + definition: definitionName, + column: columnSplice.originalTag, + }) + } + for (const innerTag of tag.tagIterator()) { + const nestedDefinitionParentTags = [definitionShortTag, defExpandShortTag, defShortTag] + if ( + nestedDefinitionParentTags.some((parentTag) => { + return innerTag.schemaTag?.name === parentTag + }) + ) { + this.pushIssue('nestedDefinition', { + definition: definitionName, + }) + } + } + } else if (tag instanceof ParsedHedColumnSplice) { + this.pushIssue('curlyBracesInDefinition', { + definition: definitionName, + column: tag.originalTag, + }) + } else if (tag.schemaTag?.name !== 'Definition') { + this.pushIssue('illegalDefinitionGroupTag', { + tag: tag, + definition: definitionName, + }) + } + } + } + + /** * Check for missing HED 3 definitions. * * @param {ParsedHedTag} tag The HED tag. * @param {string} defShortTag The short tag to check for. - *!/ + */ checkForMissingDefinitions(tag, defShortTag = 'Def') { if (tag.schemaTag?.name !== defShortTag) { return @@ -361,76 +447,75 @@ export default class HedValidator { this.pushIssue(code, { definition: defName }) } } -*/ - - // /** - // * Check the syntax of HED 3 onsets and offsets. - // * - // * @param {ParsedHedGroup} tagGroup The tag group. - // */ - // checkTemporalSyntax(tagGroup) { - // if (!tagGroup.isTemporalGroup) { - // return - // } - // const definitionName = this._getTemporalDefinitionName(tagGroup) - // - // const defExpandChildren = tagGroup.defExpandChildren - // const defTags = tagGroup.getSpecial('Def') - // if (tagGroup.defCount === 0) { - // this.pushIssue('temporalWithoutDefinition', { - // tagGroup: tagGroup, - // tag: tagGroup.temporalGroupName, - // }) - // } - // /** - // * The Onset/Offset tag plus the definition tag/tag group. - // * @type {(ParsedHedTag|ParsedHedGroup)[]} - // */ - // const allowedTags = [ - // ...getParsedParentTags(this.hedSchemas, tagGroup.temporalGroupName).values(), - // ...defExpandChildren, - // ...defTags, - // ] - // const remainingTags = differenceWith(tagGroup.tags, allowedTags, (ours, theirs) => ours.equivalent(theirs)) - // const allowedTagGroups = tagGroup.isSpecialGroup('Onset') || tagGroup.isSpecialGroup('Inset') ? 1 : 0 - // if ( - // remainingTags.length > allowedTagGroups || - // remainingTags.filter((tag) => tag instanceof ParsedHedTag).length > 0 - // ) { - // this.pushIssue('extraTagsInTemporal', { - // definition: definitionName, - // tagGroup: tagGroup.originalTag, - // }) - // } - // } - - // /** - // * Determine the definition name for an Onset- or Offset-type tag group. - // * - // * Normally, this simply returns the tag group's {@link ParsedHedGroup.defNameAndValue} return value. However, - // * if this throws an {@link IssueError}, we add the embedded {@link Issue} to our issue list and return a string - // * stating that multiple definitions were found. - // * - // * @param {ParsedHedGroup} tagGroup The onset or offset group. - // * @returns {string} The group's definition name and (optional) value, if any, or a string noting that multiple definitions were found. - // * @throws {Error} If passed a {@link ParsedHedGroup} that is not an Onset- or Offset-type group. - // * @private - // */ - // _getTemporalDefinitionName(tagGroup) { - // if (!tagGroup.isTemporalGroup) { - // throw new Error( - // 'Internal validator function "Hed3Validator._getTemporalDefinitionName()" called outside of its intended context', - // ) - // } - // try { - // return tagGroup.defNameAndValue - // } catch (e) { - // if (e instanceof IssueError) { - // this.issues.push(e.issue) - // return 'Multiple definition tags found' - // } - // } - // } + + /** + * Check the syntax of HED 3 onsets and offsets. + * + * @param {ParsedHedGroup} tagGroup The tag group. + */ + checkTemporalSyntax(tagGroup) { + if (!tagGroup.isTemporalGroup) { + return + } + const definitionName = this._getTemporalDefinitionName(tagGroup) + + const defExpandChildren = tagGroup.defExpandChildren + const defTags = tagGroup.defTags ?? [] + if (tagGroup.defCount === 0) { + this.pushIssue('temporalWithoutDefinition', { + tagGroup: tagGroup, + tag: tagGroup.temporalGroupName, + }) + } + /** + * The Onset/Offset tag plus the definition tag/tag group. + * @type {(ParsedHedTag|ParsedHedGroup)[]} + */ + const allowedTags = [ + ...getParsedParentTags(this.hedSchemas, tagGroup.temporalGroupName).values(), + ...defExpandChildren, + ...defTags, + ] + const remainingTags = differenceWith(tagGroup.tags, allowedTags, (ours, theirs) => ours.equivalent(theirs)) + const allowedTagGroups = tagGroup.isOnsetGroup || tagGroup.isInsetGroup ? 1 : 0 + if ( + remainingTags.length > allowedTagGroups || + remainingTags.filter((tag) => tag instanceof ParsedHedTag).length > 0 + ) { + this.pushIssue('extraTagsInTemporal', { + definition: definitionName, + tag: tagGroup.temporalGroupName, + }) + } + } + + /** + * Determine the definition name for an Onset- or Offset-type tag group. + * + * Normally, this simply returns the tag group's {@link ParsedHedGroup.defNameAndValue} return value. However, + * if this throws an {@link IssueError}, we add the embedded {@link Issue} to our issue list and return a string + * stating that multiple definitions were found. + * + * @param {ParsedHedGroup} tagGroup The onset or offset group. + * @returns {string} The group's definition name and (optional) value, if any, or a string noting that multiple definitions were found. + * @throws {Error} If passed a {@link ParsedHedGroup} that is not an Onset- or Offset-type group. + * @private + */ + _getTemporalDefinitionName(tagGroup) { + if (!tagGroup.isTemporalGroup) { + throw new Error( + 'Internal validator function "Hed3Validator._getTemporalDefinitionName()" called outside of its intended context', + ) + } + try { + return tagGroup.defNameAndValue + } catch (e) { + if (e instanceof IssueError) { + this.issues.push(e.issue) + return 'Multiple definition tags found' + } + } + } /** * Generate a new issue object and push it to the end of the issues array.