diff --git a/bids/tsvParser.js b/bids/tsvParser.js index e7bc228b..9e9e5c6a 100644 --- a/bids/tsvParser.js +++ b/bids/tsvParser.js @@ -20,7 +20,7 @@ export function parseTSV(contents) { const rows = stripBOM(normalizeEOL(contents)) .split('\n') .filter(isContentfulRow) - .map((str) => str.split('\t')) + .map((str) => str.split('\t').map((cell) => cell.trim())) const headers = rows.length ? rows[0] : [] headers.forEach((x) => { diff --git a/bids/types/json.js b/bids/types/json.js index 80520d58..c82d1583 100644 --- a/bids/types/json.js +++ b/bids/types/json.js @@ -75,7 +75,7 @@ export class BidsSidecar extends BidsJsonFile { const sidecarHedTags = Object.entries(this.jsonData) .map(([sidecarKey, sidecarValue]) => { if (sidecarValueHasHed(sidecarValue)) { - return [sidecarKey, new BidsSidecarKey(sidecarKey, sidecarValue.HED)] + return [sidecarKey.trim(), new BidsSidecarKey(sidecarKey.trim(), sidecarValue.HED)] } else { return null } diff --git a/common/issues/data.js b/common/issues/data.js index b038cf7d..115cfd93 100644 --- a/common/issues/data.js +++ b/common/issues/data.js @@ -185,10 +185,15 @@ export default { level: 'error', message: stringTemplate`"${'tag'}" appears as "${'parentTag'}" and cannot be used as an extension. Indices (${0}, ${1}).`, }, + invalidExtension: { + hedCode: 'TAG_EXTENSION_INVALID', + level: 'error', + message: stringTemplate`"${'tag'}" appears as an extension of "${'parentTag'}", which does not allow tag extensions.`, + }, emptyTagFound: { hedCode: 'TAG_EMPTY', level: 'error', - message: stringTemplate`Empty tag cannot be converted.`, + message: stringTemplate`Empty tag at index ${'index'} cannot be converted.`, }, duplicateTagsInSchema: { hedCode: 'SCHEMA_DUPLICATE_NODE', @@ -309,4 +314,9 @@ export default { level: 'error', message: stringTemplate`Unknown HED error "${'internalCode'}" - parameters: "${'parameters'}".`, }, + internalConsistencyError: { + hedCode: 'GENERIC_ERROR', + level: 'error', + message: stringTemplate`Internal consistency error - message: "${'message'}".`, + }, } diff --git a/common/schema/types.js b/common/schema/types.js index cf443537..1477b48b 100644 --- a/common/schema/types.js +++ b/common/schema/types.js @@ -106,11 +106,6 @@ export class Hed3Schema extends Schema { * @type {SchemaEntries} */ entries - /** - * The mapping between short and long tags. - * @type {Mapping} - */ - mapping /** * The standard HED schema version this schema is linked to. * @type {string} @@ -122,9 +117,8 @@ export class Hed3Schema extends Schema { * * @param {object} xmlData The schema XML data. * @param {SchemaEntries} entries A collection of schema entries. - * @param {Mapping} mapping A mapping between short and long tags. */ - constructor(xmlData, entries, mapping) { + constructor(xmlData, entries) { super(xmlData) if (!this.library) { @@ -133,7 +127,6 @@ export class Hed3Schema extends Schema { this.withStandard = xmlData.HED?.$?.withStandard } this.entries = entries - this.mapping = mapping } /** @@ -164,7 +157,7 @@ export class PartneredSchema extends Hed3Schema { * @param {Hed3Schema} actualSchema The actual HED 3 schema underlying this partnered schema. */ constructor(actualSchema) { - super({}, actualSchema.entries, actualSchema.mapping) + super({}, actualSchema.entries) this.actualSchema = actualSchema this.withStandard = actualSchema.withStandard this.library = undefined diff --git a/converter/__tests__/converter.spec.js b/converter/__tests__/converter.spec.js index dfa4ad9c..e75c27b1 100644 --- a/converter/__tests__/converter.spec.js +++ b/converter/__tests__/converter.spec.js @@ -1,7 +1,9 @@ import chai from 'chai' const assert = chai.assert +import { beforeAll, describe, it } from '@jest/globals' + import * as converter from '../converter' -import generateIssue from '../issues' +import { generateIssue } from '../../common/issues/issues' import { SchemaSpec, SchemasSpec } from '../../common/schema/types' import { buildSchemas } from '../../validator/schema/init' @@ -22,13 +24,13 @@ describe('HED string conversion', () => { * @param {Object} testStrings The test strings. * @param {Object} expectedResults The expected results. * @param {Object} expectedIssues The expected issues. - * @param {function (Schema, string, string, number): [string, Issue[]]} testFunction The test function. + * @param {function (Schema, string): [string, Issue[]]} testFunction The test function. * @returns {Promise} */ const validatorBase = async function (testStrings, expectedResults, expectedIssues, testFunction) { const hedSchemas = await hedSchemaPromise for (const [testStringKey, testString] of Object.entries(testStrings)) { - const [testResult, issues] = testFunction(hedSchemas.baseSchema, testString, testString, 0) + const [testResult, issues] = testFunction(hedSchemas, testString) assert.strictEqual(testResult, expectedResults[testStringKey], testString) assert.sameDeepMembers(issues, expectedIssues[testStringKey], testString) } @@ -36,7 +38,7 @@ describe('HED string conversion', () => { describe('Long-to-short', () => { const validator = function (testStrings, expectedResults, expectedIssues) { - return validatorBase(testStrings, expectedResults, expectedIssues, converter.convertTagToShort) + return validatorBase(testStrings, expectedResults, expectedIssues, converter.convertHedStringToShort) } it('should convert basic HED tags to short form', () => { @@ -95,31 +97,22 @@ describe('HED string conversion', () => { mixed: 'Item/Sound/Event/Sensory-event/Environmental-sound', } const expectedIssues = { - singleLevel: [generateIssue('invalidParentNode', testStrings.singleLevel, { parentTag: 'Event' }, [31, 36])], - multiLevel: [ - generateIssue('invalidParentNode', testStrings.multiLevel, { parentTag: 'Event/Sensory-event' }, [37, 50]), - ], - mixed: [ - generateIssue( - 'invalidParentNode', - testStrings.mixed, - { parentTag: 'Item/Sound/Environmental-sound' }, - [31, 50], - ), - ], + singleLevel: [generateIssue('invalidParentNode', { tag: 'Event', parentTag: 'Event' })], + multiLevel: [generateIssue('invalidParentNode', { tag: 'Event', parentTag: 'Event' })], + mixed: [generateIssue('invalidParentNode', { tag: 'Event', parentTag: 'Event' })], } return validator(testStrings, expectedResults, expectedIssues) }) it('should convert HED tags with extensions to short form', () => { const testStrings = { - singleLevel: 'Event/Experiment-control/extended lvl1', - multiLevel: 'Event/Experiment-control/extended lvl1/Extension2', + singleLevel: 'Item/Object/extended lvl1', + multiLevel: 'Item/Object/extended lvl1/Extension2', partialPath: 'Object/Man-made-object/Vehicle/Boat/Yacht', } const expectedResults = { - singleLevel: 'Experiment-control/extended lvl1', - multiLevel: 'Experiment-control/extended lvl1/Extension2', + singleLevel: 'Object/extended lvl1', + multiLevel: 'Object/extended lvl1/Extension2', partialPath: 'Boat/Yacht', } const expectedIssues = { @@ -132,62 +125,54 @@ describe('HED string conversion', () => { it('should raise an issue if an "extension" is already a valid node', () => { const testStrings = { - validThenInvalid: 'Event/Experiment-control/valid extension followed by invalid/Event', - singleLevel: 'Event/Experiment-control/Geometric-object', - singleLevelAlreadyShort: 'Experiment-control/Geometric-object', - twoLevels: 'Event/Experiment-control/Geometric-object/Event', + validThenInvalid: 'Item/Object/valid extension followed by invalid/Event', + singleLevel: 'Item/Object/Visual-presentation', + singleLevelAlreadyShort: 'Object/Visual-presentation', + twoLevels: 'Item/Object/Visual-presentation/Event', duplicate: 'Item/Object/Geometric-object/Item/Object/Geometric-object', } const expectedResults = { - validThenInvalid: 'Event/Experiment-control/valid extension followed by invalid/Event', - singleLevel: 'Event/Experiment-control/Geometric-object', - singleLevelAlreadyShort: 'Experiment-control/Geometric-object', - twoLevels: 'Event/Experiment-control/Geometric-object/Event', + validThenInvalid: 'Item/Object/valid extension followed by invalid/Event', + singleLevel: 'Item/Object/Visual-presentation', + singleLevelAlreadyShort: 'Object/Visual-presentation', + twoLevels: 'Item/Object/Visual-presentation/Event', duplicate: 'Item/Object/Geometric-object/Item/Object/Geometric-object', } const expectedIssues = { - validThenInvalid: [ - generateIssue('invalidParentNode', testStrings.validThenInvalid, { parentTag: 'Event' }, [61, 66]), - ], + validThenInvalid: [generateIssue('invalidParentNode', { tag: 'Event', parentTag: 'Event' })], singleLevel: [ - generateIssue( - 'invalidParentNode', - testStrings.singleLevel, - { parentTag: 'Item/Object/Geometric-object' }, - [25, 41], - ), + generateIssue('invalidParentNode', { + tag: 'Visual-presentation', + parentTag: 'Property/Sensory-property/Sensory-presentation/Visual-presentation', + }), ], singleLevelAlreadyShort: [ - generateIssue( - 'invalidParentNode', - testStrings.singleLevelAlreadyShort, - { parentTag: 'Item/Object/Geometric-object' }, - [19, 35], - ), + generateIssue('invalidParentNode', { + tag: 'Visual-presentation', + parentTag: 'Property/Sensory-property/Sensory-presentation/Visual-presentation', + }), ], - twoLevels: [generateIssue('invalidParentNode', testStrings.twoLevels, { parentTag: 'Event' }, [42, 47])], - duplicate: [ - generateIssue( - 'invalidParentNode', - testStrings.duplicate, - { parentTag: 'Item/Object/Geometric-object' }, - [41, 57], - ), + twoLevels: [ + generateIssue('invalidParentNode', { + tag: 'Visual-presentation', + parentTag: 'Property/Sensory-property/Sensory-presentation/Visual-presentation', + }), ], + duplicate: [generateIssue('invalidParentNode', { tag: 'Item', parentTag: 'Item' })], } return validator(testStrings, expectedResults, expectedIssues) }) it('should raise an issue if an invalid node is found', () => { const testStrings = { - invalidParentWithExistingGrandchild: 'InvalidEvent/Experiment-control/Geometric-object', + invalidParentWithExistingGrandchild: 'InvalidItem/Object/Visual-presentation', invalidChildWithExistingGrandchild: 'Event/InvalidEvent/Geometric-object', invalidParentWithExistingChild: 'InvalidEvent/Geometric-object', invalidSingle: 'InvalidEvent', invalidWithExtension: 'InvalidEvent/InvalidExtension', } const expectedResults = { - invalidParentWithExistingGrandchild: 'InvalidEvent/Experiment-control/Geometric-object', + invalidParentWithExistingGrandchild: 'InvalidItem/Object/Visual-presentation', invalidChildWithExistingGrandchild: 'Event/InvalidEvent/Geometric-object', invalidParentWithExistingChild: 'InvalidEvent/Geometric-object', invalidSingle: 'InvalidEvent', @@ -195,36 +180,21 @@ describe('HED string conversion', () => { } const expectedIssues = { invalidParentWithExistingGrandchild: [ - generateIssue( - 'invalidParentNode', - testStrings.invalidParentWithExistingGrandchild, - { parentTag: 'Item/Object/Geometric-object' }, - [32, 48], - ), + generateIssue('invalidTag', { tag: testStrings.invalidParentWithExistingGrandchild }), ], invalidChildWithExistingGrandchild: [ - generateIssue( - 'invalidParentNode', - testStrings.invalidChildWithExistingGrandchild, - { parentTag: 'Item/Object/Geometric-object' }, - [19, 35], - ), + generateIssue('invalidExtension', { tag: 'InvalidEvent', parentTag: 'Event' }), ], invalidParentWithExistingChild: [ - generateIssue( - 'invalidParentNode', - testStrings.invalidParentWithExistingChild, - { parentTag: 'Item/Object/Geometric-object' }, - [13, 29], - ), + generateIssue('invalidTag', { tag: testStrings.invalidParentWithExistingChild }), ], - invalidSingle: [generateIssue('invalidTag', testStrings.invalidSingle, {}, [0, 12])], - invalidWithExtension: [generateIssue('invalidTag', testStrings.invalidWithExtension, {}, [0, 12])], + invalidSingle: [generateIssue('invalidTag', { tag: testStrings.invalidSingle })], + invalidWithExtension: [generateIssue('invalidTag', { tag: testStrings.invalidWithExtension })], } return validator(testStrings, expectedResults, expectedIssues) }) - it('should not validate whether a node actually allows extensions', () => { + it('should validate whether a node actually allows extensions', () => { const testStrings = { validTakesValue: 'Property/Agent-property/Agent-trait/Age/15', cascadeExtension: 'Property/Agent-property/Agent-state/Agent-emotional-state/Awed/Cascade Extension', @@ -233,12 +203,12 @@ describe('HED string conversion', () => { const expectedResults = { validTakesValue: 'Age/15', cascadeExtension: 'Awed/Cascade Extension', - invalidExtension: 'Agent-action/Good/Time', + invalidExtension: 'Event/Agent-action/Good/Time', } const expectedIssues = { validTakesValue: [], cascadeExtension: [], - invalidExtension: [], + invalidExtension: [generateIssue('invalidExtension', { tag: 'Good', parentTag: 'Event/Agent-action' })], } return validator(testStrings, expectedResults, expectedIssues) }) @@ -249,24 +219,17 @@ describe('HED string conversion', () => { trailingSpace: 'Item/Sound/Environmental-sound/Unique Value ', } const expectedResults = { - leadingSpace: ' Item/Sound/Environmental-sound/Unique Value', - trailingSpace: 'Environmental-sound/Unique Value ', + leadingSpace: 'Environmental-sound/Unique Value', + trailingSpace: 'Environmental-sound/Unique Value', } const expectedIssues = { - leadingSpace: [ - generateIssue( - 'invalidParentNode', - testStrings.leadingSpace, - { parentTag: 'Item/Sound/Environmental-sound' }, - [12, 31], - ), - ], + leadingSpace: [], trailingSpace: [], } return validator(testStrings, expectedResults, expectedIssues) }) - it('should strip leading and trailing slashes', () => { + it.skip('should strip leading and trailing slashes', () => { const testStrings = { leadingSingle: '/Event', leadingExtension: '/Event/Extension', @@ -315,7 +278,7 @@ describe('HED string conversion', () => { describe('Short-to-long', () => { const validator = function (testStrings, expectedResults, expectedIssues) { - return validatorBase(testStrings, expectedResults, expectedIssues, converter.convertTagToLong) + return validatorBase(testStrings, expectedResults, expectedIssues, converter.convertHedStringToLong) } it('should convert basic HED tags to long form', () => { @@ -364,13 +327,13 @@ describe('HED string conversion', () => { it('should convert HED tags with extensions to long form', () => { const testStrings = { - singleLevel: 'Experiment-control/extended lvl1', - multiLevel: 'Experiment-control/extended lvl1/Extension2', + singleLevel: 'Object/extended lvl1', + multiLevel: 'Object/extended lvl1/Extension2', partialPath: 'Vehicle/Boat/Yacht', } const expectedResults = { - singleLevel: 'Event/Experiment-control/extended lvl1', - multiLevel: 'Event/Experiment-control/extended lvl1/Extension2', + singleLevel: 'Item/Object/extended lvl1', + multiLevel: 'Item/Object/extended lvl1/Extension2', partialPath: 'Item/Object/Man-made-object/Vehicle/Boat/Yacht', } const expectedIssues = { @@ -383,50 +346,40 @@ describe('HED string conversion', () => { it('should raise an issue if an "extension" is already a valid node', () => { const testStrings = { - validThenInvalid: 'Experiment-control/valid extension followed by invalid/Event', - singleLevel: 'Experiment-control/Geometric-object', - singleLevelAlreadyLong: 'Event/Experiment-control/Geometric-object', - twoLevels: 'Experiment-control/Geometric-object/Event', + validThenInvalid: 'Object/valid extension followed by invalid/Event', + singleLevel: 'Object/Visual-presentation', + singleLevelAlreadyLong: 'Item/Object/Visual-presentation', + twoLevels: 'Object/Visual-presentation/Event', partialDuplicate: 'Geometric-object/Item/Object/Geometric-object', } const expectedResults = { - validThenInvalid: 'Experiment-control/valid extension followed by invalid/Event', - singleLevel: 'Experiment-control/Geometric-object', - singleLevelAlreadyLong: 'Event/Experiment-control/Geometric-object', - twoLevels: 'Experiment-control/Geometric-object/Event', + validThenInvalid: 'Object/valid extension followed by invalid/Event', + singleLevel: 'Object/Visual-presentation', + singleLevelAlreadyLong: 'Item/Object/Visual-presentation', + twoLevels: 'Object/Visual-presentation/Event', partialDuplicate: 'Geometric-object/Item/Object/Geometric-object', } const expectedIssues = { - validThenInvalid: [ - generateIssue('invalidParentNode', testStrings.validThenInvalid, { parentTag: 'Event' }, [55, 60]), - ], + validThenInvalid: [generateIssue('invalidParentNode', { tag: 'Event', parentTag: 'Event' })], singleLevel: [ - generateIssue( - 'invalidParentNode', - testStrings.singleLevel, - { parentTag: 'Item/Object/Geometric-object' }, - [19, 35], - ), + generateIssue('invalidParentNode', { + tag: 'Visual-presentation', + parentTag: 'Property/Sensory-property/Sensory-presentation/Visual-presentation', + }), ], singleLevelAlreadyLong: [ - generateIssue( - 'invalidParentNode', - testStrings.singleLevelAlreadyLong, - { parentTag: 'Item/Object/Geometric-object' }, - [25, 41], - ), + generateIssue('invalidParentNode', { + tag: 'Visual-presentation', + parentTag: 'Property/Sensory-property/Sensory-presentation/Visual-presentation', + }), ], twoLevels: [ - generateIssue( - 'invalidParentNode', - testStrings.twoLevels, - { parentTag: 'Item/Object/Geometric-object' }, - [19, 35], - ), - ], - partialDuplicate: [ - generateIssue('invalidParentNode', testStrings.partialDuplicate, { parentTag: 'Item' }, [17, 21]), + generateIssue('invalidParentNode', { + tag: 'Visual-presentation', + parentTag: 'Property/Sensory-property/Sensory-presentation/Visual-presentation', + }), ], + partialDuplicate: [generateIssue('invalidParentNode', { tag: 'Item', parentTag: 'Item' })], } return validator(testStrings, expectedResults, expectedIssues) }) @@ -443,14 +396,14 @@ describe('HED string conversion', () => { validChild: 'InvalidEvent/Event', } const expectedIssues = { - single: [generateIssue('invalidTag', testStrings.single, {}, [0, 12])], - invalidChild: [generateIssue('invalidTag', testStrings.invalidChild, {}, [0, 12])], - validChild: [generateIssue('invalidTag', testStrings.validChild, {}, [0, 12])], + single: [generateIssue('invalidTag', { tag: testStrings.single })], + invalidChild: [generateIssue('invalidTag', { tag: testStrings.invalidChild })], + validChild: [generateIssue('invalidTag', { tag: testStrings.validChild })], } return validator(testStrings, expectedResults, expectedIssues) }) - it('should not validate whether a node actually allows extensions', () => { + it('should validate whether a node actually allows extensions', () => { const testStrings = { validTakesValue: 'Age/15', cascadeExtension: 'Awed/Cascade Extension', @@ -459,12 +412,12 @@ describe('HED string conversion', () => { const expectedResults = { validTakesValue: 'Property/Agent-property/Agent-trait/Age/15', cascadeExtension: 'Property/Agent-property/Agent-state/Agent-emotional-state/Awed/Cascade Extension', - invalidExtension: 'Event/Agent-action/Good/Time', + invalidExtension: 'Agent-action/Good/Time', } const expectedIssues = { validTakesValue: [], cascadeExtension: [], - invalidExtension: [], + invalidExtension: [generateIssue('invalidExtension', { tag: 'Good', parentTag: 'Event/Agent-action' })], } return validator(testStrings, expectedResults, expectedIssues) }) @@ -475,17 +428,17 @@ describe('HED string conversion', () => { trailingSpace: 'Environmental-sound/Unique Value ', } const expectedResults = { - leadingSpace: ' Environmental-sound/Unique Value', - trailingSpace: 'Item/Sound/Environmental-sound/Unique Value ', + leadingSpace: 'Item/Sound/Environmental-sound/Unique Value', + trailingSpace: 'Item/Sound/Environmental-sound/Unique Value', } const expectedIssues = { - leadingSpace: [generateIssue('invalidTag', testStrings.leadingSpace, {}, [0, 20])], + leadingSpace: [], trailingSpace: [], } return validator(testStrings, expectedResults, expectedIssues) }) - it('should strip leading and trailing slashes', () => { + it.skip('should strip leading and trailing slashes', () => { const testStrings = { leadingSingle: '/Event', leadingExtension: '/Event/Extension', @@ -531,7 +484,7 @@ describe('HED string conversion', () => { return validator(testStrings, expectedResults, expectedIssues) }) - it.skip('should properly handle node names in value-taking strings', () => { + it('should properly handle node names in value-taking strings', () => { const testStrings = { valueTaking: 'Label/Red', nonValueTaking: 'Train/Car', @@ -546,32 +499,17 @@ describe('HED string conversion', () => { definitionName: 'Property/Organizational-property/Definition/Blue', definitionNameWithPlaceholder: 'Property/Organizational-property/Definition/BlueCircle/#', definitionNameWithNodeValue: 'Property/Organizational-property/Definition/BlueSquare/SteelBlue', - definitionNodeNameWithValue: 'Definition/Blue/Cobalt', + definitionNodeNameWithValue: 'Property/Organizational-property/Definition/Blue/Cobalt', } const expectedIssues = { valueTaking: [], nonValueTaking: [ - generateIssue( - 'invalidParentNode', - testStrings.nonValueTaking, - { parentTag: 'Item/Object/Man-made-object/Vehicle/Car' }, - [6, 9], - ), + generateIssue('invalidParentNode', { tag: 'Car', parentTag: 'Item/Object/Man-made-object/Vehicle/Car' }), ], definitionName: [], // To be caught in validation. definitionNameWithPlaceholder: [], definitionNameWithNodeValue: [], - definitionNodeNameWithValue: [ - generateIssue( - 'invalidParentNode', - testStrings.definitionNodeNameWithValue, - { - parentTag: - 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Blue-color/Blue', - }, - [11, 15], - ), - ], + definitionNodeNameWithValue: [], // To be caught in validation. } return validator(testStrings, expectedResults, expectedIssues) }) @@ -607,7 +545,7 @@ describe('HED string conversion', () => { singleLevel: 'Event', multiLevel: 'Event/Sensory-event', twoSingle: 'Event, Property', - oneExtension: 'Event/Extension', + oneExtension: 'Item/Extension', threeMulti: 'Event/Sensory-event, Item/Object/Man-made-object/Vehicle/Train, Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red/0.5', simpleGroup: @@ -619,7 +557,7 @@ describe('HED string conversion', () => { singleLevel: 'Event', multiLevel: 'Sensory-event', twoSingle: 'Event, Property', - oneExtension: 'Event/Extension', + oneExtension: 'Item/Extension', threeMulti: 'Sensory-event, Train, RGB-red/0.5', simpleGroup: '(Train, RGB-red/0.5)', groupAndTag: '(Train, RGB-red/0.5), Car', @@ -651,17 +589,14 @@ describe('HED string conversion', () => { double: double, both: single + ', ' + double, singleWithTwoValid: 'Property, ' + single + ', Event', - doubleWithValid: double + ', Car/Minivan', + doubleWithValid: double + ', Item/Object/Man-made-object/Vehicle/Car/Minivan', } const expectedIssues = { - single: [generateIssue('invalidTag', single, {}, [0, 12])], - double: [generateIssue('invalidTag', double, {}, [0, 12])], - both: [ - generateIssue('invalidTag', testStrings.both, {}, [0, 12]), - generateIssue('invalidTag', testStrings.both, {}, [14, 26]), - ], - singleWithTwoValid: [generateIssue('invalidTag', testStrings.singleWithTwoValid, {}, [10, 22])], - doubleWithValid: [generateIssue('invalidTag', testStrings.doubleWithValid, {}, [0, 12])], + single: [generateIssue('invalidTag', { tag: single })], + double: [generateIssue('invalidTag', { tag: double })], + both: [generateIssue('invalidTag', { tag: single }), generateIssue('invalidTag', { tag: double })], + singleWithTwoValid: [generateIssue('invalidTag', { tag: single })], + doubleWithValid: [generateIssue('invalidTag', { tag: double })], } return validator(testStrings, expectedResults, expectedIssues) }) @@ -676,12 +611,12 @@ describe('HED string conversion', () => { bothSpaceTwo: ' Event, Item/Sound/Environmental-sound/Unique Value ', } const expectedResults = { - leadingSpace: ' Environmental-sound/Unique Value', - trailingSpace: 'Environmental-sound/Unique Value ', - bothSpace: ' Environmental-sound/Unique Value ', - leadingSpaceTwo: ' Environmental-sound/Unique Value, Event', - trailingSpaceTwo: 'Event, Environmental-sound/Unique Value ', - bothSpaceTwo: ' Event, Environmental-sound/Unique Value ', + leadingSpace: 'Environmental-sound/Unique Value', + trailingSpace: 'Environmental-sound/Unique Value', + bothSpace: 'Environmental-sound/Unique Value', + leadingSpaceTwo: 'Environmental-sound/Unique Value, Event', + trailingSpaceTwo: 'Event, Environmental-sound/Unique Value', + bothSpaceTwo: 'Event, Environmental-sound/Unique Value', } const expectedIssues = { leadingSpace: [], @@ -697,106 +632,64 @@ describe('HED string conversion', () => { it('should strip leading and trailing slashes', () => { const testStrings = { leadingSingle: '/Event', - leadingMultiLevel: '/Object/Man-made-object/Vehicle/Train', + leadingMultiLevel: '/Item/Object/Man-made-object/Vehicle/Train', trailingSingle: 'Event/', - trailingMultiLevel: 'Object/Man-made-object/Vehicle/Train/', + trailingMultiLevel: 'Item/Object/Man-made-object/Vehicle/Train/', bothSingle: '/Event/', - bothMultiLevel: '/Object/Man-made-object/Vehicle/Train/', - twoMixedOuter: '/Event,Object/Man-made-object/Vehicle/Train/', - twoMixedInner: 'Event/,/Object/Man-made-object/Vehicle/Train', - twoMixedBoth: '/Event/,/Object/Man-made-object/Vehicle/Train/', - twoMixedBothGroup: '(/Event/,/Object/Man-made-object/Vehicle/Train/)', - } - const expectedEvent = 'Event' - const expectedTrain = 'Train' - const expectedMixed = expectedEvent + ',' + expectedTrain - const expectedResults = { - leadingSingle: expectedEvent, - leadingMultiLevel: expectedTrain, - trailingSingle: expectedEvent, - trailingMultiLevel: expectedTrain, - bothSingle: expectedEvent, - bothMultiLevel: expectedTrain, - twoMixedOuter: expectedMixed, - twoMixedInner: expectedMixed, - twoMixedBoth: expectedMixed, - twoMixedBothGroup: '(' + expectedMixed + ')', + bothMultiLevel: '/Item/Object/Man-made-object/Vehicle/Train/', + twoMixedOuter: '/Event,Item/Object/Man-made-object/Vehicle/Train/', + twoMixedInner: 'Event/,/Item/Object/Man-made-object/Vehicle/Train', + twoMixedBoth: '/Event/,/Item/Object/Man-made-object/Vehicle/Train/', + twoMixedBothGroup: '(/Event/,/Item/Object/Man-made-object/Vehicle/Train/)', } + const expectedResults = testStrings const expectedIssues = { - leadingSingle: [], - leadingMultiLevel: [], - trailingSingle: [], - trailingMultiLevel: [], - bothSingle: [], - bothMultiLevel: [], - twoMixedOuter: [], - twoMixedInner: [], - twoMixedBoth: [], - twoMixedBothGroup: [], + leadingSingle: [generateIssue('invalidTag', { tag: testStrings.leadingSingle })], + leadingMultiLevel: [generateIssue('invalidTag', { tag: testStrings.leadingMultiLevel })], + trailingSingle: [generateIssue('invalidTag', { tag: testStrings.trailingSingle })], + trailingMultiLevel: [generateIssue('invalidTag', { tag: testStrings.trailingMultiLevel })], + bothSingle: [generateIssue('invalidTag', { tag: testStrings.bothSingle })], + bothMultiLevel: [generateIssue('invalidTag', { tag: testStrings.bothMultiLevel })], + twoMixedOuter: [ + generateIssue('invalidTag', { tag: '/Event' }), + generateIssue('invalidTag', { tag: 'Item/Object/Man-made-object/Vehicle/Train/' }), + ], + twoMixedInner: [ + generateIssue('invalidTag', { tag: 'Event/' }), + generateIssue('invalidTag', { tag: '/Item/Object/Man-made-object/Vehicle/Train' }), + ], + twoMixedBoth: [ + generateIssue('invalidTag', { tag: '/Event/' }), + generateIssue('invalidTag', { tag: '/Item/Object/Man-made-object/Vehicle/Train/' }), + ], + twoMixedBothGroup: [ + generateIssue('invalidTag', { tag: '/Event/' }), + generateIssue('invalidTag', { tag: '/Item/Object/Man-made-object/Vehicle/Train/' }), + ], } return validator(testStrings, expectedResults, expectedIssues) }) it('should replace extra spaces and slashes with single slashes', () => { const testStrings = { - twoLevelDoubleSlash: 'Event//Extension', + twoLevelDoubleSlash: 'Item//Extension', threeLevelDoubleSlash: 'Item//Object//Geometric-object', tripleSlashes: 'Item///Object///Geometric-object', mixedSingleAndDoubleSlashes: 'Item///Object/Geometric-object', - singleSlashWithSpace: 'Event/ Extension', - doubleSlashSurroundingSpace: 'Event/ /Extension', - doubleSlashThenSpace: 'Event// Extension', - sosPattern: 'Event/// ///Extension', + singleSlashWithSpace: 'Item/ Extension', + doubleSlashSurroundingSpace: 'Item/ /Extension', + doubleSlashThenSpace: 'Item// Extension', + sosPattern: 'Item/// ///Extension', alternatingSlashSpace: 'Item/ / Object/ / Geometric-object', - leadingDoubleSlash: '//Event/Extension', - trailingDoubleSlash: 'Event/Extension//', - leadingDoubleSlashWithSpace: '/ /Event/Extension', - trailingDoubleSlashWithSpace: 'Event/Extension/ /', - } - const expectedEventExtension = 'Event/Extension' - const expectedGeometric = 'Geometric-object' - const expectedResults = { - twoLevelDoubleSlash: expectedEventExtension, - threeLevelDoubleSlash: expectedGeometric, - tripleSlashes: expectedGeometric, - mixedSingleAndDoubleSlashes: expectedGeometric, - singleSlashWithSpace: expectedEventExtension, - doubleSlashSurroundingSpace: expectedEventExtension, - doubleSlashThenSpace: expectedEventExtension, - sosPattern: expectedEventExtension, - alternatingSlashSpace: expectedGeometric, - leadingDoubleSlash: expectedEventExtension, - trailingDoubleSlash: expectedEventExtension, - leadingDoubleSlashWithSpace: expectedEventExtension, - trailingDoubleSlashWithSpace: expectedEventExtension, + leadingDoubleSlash: '//Item/Extension', + trailingDoubleSlash: 'Item/Extension//', + leadingDoubleSlashWithSpace: '/ /Item/Extension', + trailingDoubleSlashWithSpace: 'Item/Extension/ /', } - const expectedIssues = { - twoLevelDoubleSlash: [], - threeLevelDoubleSlash: [], - tripleSlashes: [], - mixedSingleAndDoubleSlashes: [], - singleSlashWithSpace: [], - doubleSlashSurroundingSpace: [], - doubleSlashThenSpace: [], - sosPattern: [], - alternatingSlashSpace: [], - leadingDoubleSlash: [], - trailingDoubleSlash: [], - leadingDoubleSlashWithSpace: [], - trailingDoubleSlashWithSpace: [], - } - return validator(testStrings, expectedResults, expectedIssues) - }) - - it('should raise an error if an empty string is passed', () => { - const testStrings = { - emptyString: '', - } - const expectedResults = { - emptyString: '', - } - const expectedIssues = { - emptyString: [generateIssue('emptyTagFound', testStrings.emptyString)], + const expectedResults = testStrings + const expectedIssues = {} + for (const [testStringKey, testString] of Object.entries(testStrings)) { + expectedIssues[testStringKey] = [generateIssue('invalidTag', { tag: testString })] } return validator(testStrings, expectedResults, expectedIssues) }) @@ -812,7 +705,7 @@ describe('HED string conversion', () => { singleLevel: 'Event', multiLevel: 'Sensory-event', twoSingle: 'Event, Property', - oneExtension: 'Event/Extension', + oneExtension: 'Item/Extension', threeMulti: 'Sensory-event, Train, RGB-red/0.5', simpleGroup: '(Train, RGB-red/0.5)', groupAndTag: '(Train, RGB-red/0.5), Car', @@ -821,7 +714,7 @@ describe('HED string conversion', () => { singleLevel: 'Event', multiLevel: 'Event/Sensory-event', twoSingle: 'Event, Property', - oneExtension: 'Event/Extension', + oneExtension: 'Item/Extension', threeMulti: 'Event/Sensory-event, Item/Object/Man-made-object/Vehicle/Train, Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red/0.5', simpleGroup: @@ -856,17 +749,14 @@ describe('HED string conversion', () => { double: double, both: single + ', ' + double, singleWithTwoValid: 'Property, ' + single + ', Event', - doubleWithValid: double + ', Item/Object/Man-made-object/Vehicle/Car/Minivan', + doubleWithValid: double + ', Car/Minivan', } const expectedIssues = { - single: [generateIssue('invalidTag', single, {}, [0, 12])], - double: [generateIssue('invalidTag', double, {}, [0, 12])], - both: [ - generateIssue('invalidTag', testStrings.both, {}, [0, 12]), - generateIssue('invalidTag', testStrings.both, {}, [14, 26]), - ], - singleWithTwoValid: [generateIssue('invalidTag', testStrings.singleWithTwoValid, {}, [10, 22])], - doubleWithValid: [generateIssue('invalidTag', testStrings.doubleWithValid, {}, [0, 12])], + single: [generateIssue('invalidTag', { tag: single })], + double: [generateIssue('invalidTag', { tag: double })], + both: [generateIssue('invalidTag', { tag: single }), generateIssue('invalidTag', { tag: double })], + singleWithTwoValid: [generateIssue('invalidTag', { tag: single })], + doubleWithValid: [generateIssue('invalidTag', { tag: double })], } return validator(testStrings, expectedResults, expectedIssues) }) @@ -881,12 +771,12 @@ describe('HED string conversion', () => { bothSpaceTwo: ' Event, Environmental-sound/Unique Value ', } const expectedResults = { - leadingSpace: ' Item/Sound/Environmental-sound/Unique Value', - trailingSpace: 'Item/Sound/Environmental-sound/Unique Value ', - bothSpace: ' Item/Sound/Environmental-sound/Unique Value ', - leadingSpaceTwo: ' Item/Sound/Environmental-sound/Unique Value, Event', - trailingSpaceTwo: 'Event, Item/Sound/Environmental-sound/Unique Value ', - bothSpaceTwo: ' Event, Item/Sound/Environmental-sound/Unique Value ', + leadingSpace: 'Item/Sound/Environmental-sound/Unique Value', + trailingSpace: 'Item/Sound/Environmental-sound/Unique Value', + bothSpace: 'Item/Sound/Environmental-sound/Unique Value', + leadingSpaceTwo: 'Item/Sound/Environmental-sound/Unique Value, Event', + trailingSpaceTwo: 'Event, Item/Sound/Environmental-sound/Unique Value', + bothSpaceTwo: 'Event, Item/Sound/Environmental-sound/Unique Value', } const expectedIssues = { leadingSpace: [], @@ -899,7 +789,7 @@ describe('HED string conversion', () => { return validator(testStrings, expectedResults, expectedIssues) }) - it('should strip leading and trailing slashes', () => { + it('should raise an issue if there are extra slashes', () => { const testStrings = { leadingSingle: '/Event', leadingMultiLevel: '/Vehicle/Train', @@ -912,37 +802,35 @@ describe('HED string conversion', () => { twoMixedBoth: '/Event/,/Vehicle/Train/', twoMixedBothGroup: '(/Event/,/Vehicle/Train/)', } - const expectedEvent = 'Event' - const expectedTrain = 'Item/Object/Man-made-object/Vehicle/Train' - const expectedMixed = expectedEvent + ',' + expectedTrain - const expectedResults = { - leadingSingle: expectedEvent, - leadingMultiLevel: expectedTrain, - trailingSingle: expectedEvent, - trailingMultiLevel: expectedTrain, - bothSingle: expectedEvent, - bothMultiLevel: expectedTrain, - twoMixedOuter: expectedMixed, - twoMixedInner: expectedMixed, - twoMixedBoth: expectedMixed, - twoMixedBothGroup: '(' + expectedMixed + ')', - } + const expectedResults = testStrings const expectedIssues = { - leadingSingle: [], - leadingMultiLevel: [], - trailingSingle: [], - trailingMultiLevel: [], - bothSingle: [], - bothMultiLevel: [], - twoMixedOuter: [], - twoMixedInner: [], - twoMixedBoth: [], - twoMixedBothGroup: [], + leadingSingle: [generateIssue('invalidTag', { tag: testStrings.leadingSingle })], + leadingMultiLevel: [generateIssue('invalidTag', { tag: testStrings.leadingMultiLevel })], + trailingSingle: [generateIssue('invalidTag', { tag: testStrings.trailingSingle })], + trailingMultiLevel: [generateIssue('invalidTag', { tag: testStrings.trailingMultiLevel })], + bothSingle: [generateIssue('invalidTag', { tag: testStrings.bothSingle })], + bothMultiLevel: [generateIssue('invalidTag', { tag: testStrings.bothMultiLevel })], + twoMixedOuter: [ + generateIssue('invalidTag', { tag: '/Event' }), + generateIssue('invalidTag', { tag: 'Vehicle/Train/' }), + ], + twoMixedInner: [ + generateIssue('invalidTag', { tag: 'Event/' }), + generateIssue('invalidTag', { tag: '/Vehicle/Train' }), + ], + twoMixedBoth: [ + generateIssue('invalidTag', { tag: '/Event/' }), + generateIssue('invalidTag', { tag: '/Vehicle/Train/' }), + ], + twoMixedBothGroup: [ + generateIssue('invalidTag', { tag: '/Event/' }), + generateIssue('invalidTag', { tag: '/Vehicle/Train/' }), + ], } return validator(testStrings, expectedResults, expectedIssues) }) - it('should replace extra spaces and slashes with single slashes', () => { + it.skip('should replace extra spaces and slashes with single slashes', () => { const testStrings = { twoLevelDoubleSlash: 'Event//Extension', threeLevelDoubleSlash: 'Vehicle//Boat//Tanker', @@ -992,19 +880,6 @@ describe('HED string conversion', () => { } return validator(testStrings, expectedResults, expectedIssues) }) - - it('should raise an error if an empty string is passed', () => { - const testStrings = { - emptyString: '', - } - const expectedResults = { - emptyString: '', - } - const expectedIssues = { - emptyString: [generateIssue('emptyTagFound', testStrings.emptyString)], - } - return validator(testStrings, expectedResults, expectedIssues) - }) }) }) }) diff --git a/converter/__tests__/splitHedString.spec.js b/converter/__tests__/splitHedString.spec.js deleted file mode 100644 index a0195721..00000000 --- a/converter/__tests__/splitHedString.spec.js +++ /dev/null @@ -1,45 +0,0 @@ -import chai from 'chai' -const assert = chai.assert -import splitHedString from '../splitHedString' - -describe('HED string delimiter splitting', () => { - /** - * Validation function. - * - * @param {Object} testStrings The test strings. - * @param {Object} expectedResults The expected results. - */ - const validator = function (testStrings, expectedResults) { - for (const [testStringKey, testString] of Object.entries(testStrings)) { - const testResult = splitHedString(testString) - const testResultParts = testResult.map(([, [startPosition, endPosition]]) => { - return testString.slice(startPosition, endPosition) - }) - assert.deepStrictEqual(testResultParts, expectedResults[testStringKey], testStrings[testStringKey]) - } - } - - it('should property split a HED string into tags and delimiters', () => { - const testStrings = { - single: 'Event', - double: 'Event, Event/Extension', - singleAndGroup: 'Event/Extension, (Event/Extension2, Event/Extension3)', - singleAndGroupWithBlank: 'Event/Extension, (Event, ,Event/Extension3)', - manyParens: 'Event/Extension,(((((Event/Extension2, )(Event)', - manyParensEndingSpace: 'Event/Extension,(((((Event/Extension2, )(Event) ', - manyParensOpeningSpace: ' Event/Extension,(((((Event/Extension2, )(Event)', - manyParensBothSpace: ' Event/Extension,(((((Event/Extension2, )(Event ', - } - const expectedResults = { - single: ['Event'], - double: ['Event', ', ', 'Event/Extension'], - singleAndGroup: ['Event/Extension', ', (', 'Event/Extension2', ', ', 'Event/Extension3', ')'], - singleAndGroupWithBlank: ['Event/Extension', ', (', 'Event', ', ,', 'Event/Extension3', ')'], - manyParens: ['Event/Extension', ',(((((', 'Event/Extension2', ', )(', 'Event', ')'], - manyParensEndingSpace: ['Event/Extension', ',(((((', 'Event/Extension2', ', )(', 'Event', ') '], - manyParensOpeningSpace: [' ', 'Event/Extension', ',(((((', 'Event/Extension2', ', )(', 'Event', ')'], - manyParensBothSpace: [' ', 'Event/Extension', ',(((((', 'Event/Extension2', ', )(', 'Event', ' '], - } - return validator(testStrings, expectedResults) - }) -}) diff --git a/converter/converter.js b/converter/converter.js index ae39c90f..6c7b1f59 100644 --- a/converter/converter.js +++ b/converter/converter.js @@ -1,267 +1,21 @@ -import castArray from 'lodash/castArray' - -import { TagEntry } from './types' -import generateIssue from './issues' -import splitHedString from './splitHedString' - -const doubleSlashPattern = /[\s/]*\/+[\s/]*/g - -/** - * Remove extra slashes and spaces from a HED string. - * - * @param {string} hedString The HED string to clean. - * @returns {string} The cleaned HED string. - */ -export const removeSlashesAndSpaces = function (hedString) { - return hedString.replace(doubleSlashPattern, '/') -} - -/** - * Convert a HED tag to long form. - * - * The seemingly redundant code for duplicate tag entries (which are errored out - * on for HED 3 schemas) allow for similar HED 2 validation with minimal code - * duplication. - * - * @param {Schema} schema The schema object containing a short-to-long mapping. - * @param {string} hedTag The HED tag to convert. - * @param {string} hedString The full HED string (for error messages). - * @param {number} offset The offset of this tag within the HED string. - * @returns {[string, Issue[]]} The long-form tag and any issues. - */ -export const convertTagToLong = function (schema, hedTag, hedString, offset) { - const mapping = schema.mapping - - if (hedTag.startsWith('/')) { - hedTag = hedTag.slice(1) - } - if (hedTag.endsWith('/')) { - hedTag = hedTag.slice(0, -1) - } - - const cleanedTag = hedTag.toLowerCase() - const splitTag = cleanedTag.split('/') - - /** - * @type {TagEntry} - */ - let foundTagEntry = null - let takesValueTag = false - let endingIndex = 0 - let foundUnknownExtension = false - let foundEndingIndex = 0 - - const generateParentNodeIssue = (tagEntries, startingIndex, endingIndex) => { - return [ - hedTag, - [ - generateIssue( - 'invalidParentNode', - hedString, - { - parentTag: - tagEntries.length > 1 - ? tagEntries.map((tagEntry) => { - return tagEntry.longTag - }) - : tagEntries[0].longTag, - }, - [startingIndex + offset, endingIndex + offset], - ), - ], - ] - } - - for (const tag of splitTag) { - if (endingIndex !== 0) { - endingIndex++ - } - const startingIndex = endingIndex - endingIndex += tag.length - - const tagEntries = castArray(mapping.shortToTags.get(tag)) - - if (foundUnknownExtension) { - if (mapping.shortToTags.has(tag)) { - return generateParentNodeIssue(tagEntries, startingIndex, endingIndex) - } else { - continue - } - } - if (!mapping.shortToTags.has(tag)) { - if (foundTagEntry === null) { - return [hedTag, [generateIssue('invalidTag', hedString, {}, [startingIndex + offset, endingIndex + offset])]] - } - - foundUnknownExtension = true - continue - } - - let tagFound = false - for (const tagEntry of tagEntries) { - const tagString = tagEntry.longFormattedTag - const mainHedPortion = cleanedTag.slice(0, endingIndex) - - if (tagString.endsWith(mainHedPortion)) { - tagFound = true - foundEndingIndex = endingIndex - foundTagEntry = tagEntry - if (tagEntry.takesValue) { - takesValueTag = true - } - break - } - } - if (!tagFound && !takesValueTag) { - return generateParentNodeIssue(tagEntries, startingIndex, endingIndex) - } - } - - const remainder = hedTag.slice(foundEndingIndex) - const longTagString = foundTagEntry.longTag + remainder - return [longTagString, []] -} - -/** - * Convert a HED tag to short form. - * - * @param {Schema} schema The schema object containing a short-to-long mapping. - * @param {string} hedTag The HED tag to convert. - * @param {string} hedString The full HED string (for error messages). - * @param {number} offset The offset of this tag within the HED string. - * @returns {[string, Issue[]]} The short-form tag and any issues. - */ -export const convertTagToShort = function (schema, hedTag, hedString, offset) { - const mapping = schema.mapping - - if (hedTag.startsWith('/')) { - hedTag = hedTag.slice(1) - } - if (hedTag.endsWith('/')) { - hedTag = hedTag.slice(0, -1) - } - - const cleanedTag = hedTag.toLowerCase() - const splitTag = cleanedTag.split('/') - splitTag.reverse() - - /** - * @type {TagEntry} - */ - let foundTagEntry = null - let index = hedTag.length - let lastFoundIndex = index - - for (const tag of splitTag) { - if (mapping.shortToTags.has(tag)) { - foundTagEntry = mapping.shortToTags.get(tag) - lastFoundIndex = index - index -= tag.length - break - } - - lastFoundIndex = index - index -= tag.length - - if (index !== 0) { - index-- - } - } - - if (foundTagEntry === null) { - return [hedTag, [generateIssue('invalidTag', hedString, {}, [index + offset, lastFoundIndex + offset])]] - } - - const mainHedPortion = cleanedTag.slice(0, lastFoundIndex) - const tagString = foundTagEntry.longFormattedTag - if (!tagString.endsWith(mainHedPortion)) { - return [ - hedTag, - [ - generateIssue('invalidParentNode', hedString, { parentTag: foundTagEntry.longTag }, [ - index + offset, - lastFoundIndex + offset, - ]), - ], - ] - } - - const remainder = hedTag.slice(lastFoundIndex) - const shortTagString = foundTagEntry.shortTag + remainder - return [shortTagString, []] -} - -/** - * Convert a partial HED string to long form. - * - * This is for the internal string parsing for the validation side. - * - * @param {Schema} schema The schema object containing a short-to-long mapping. - * @param {string} partialHedString The partial HED string to convert to long form. - * @param {string} fullHedString The full HED string. - * @param {number} offset The offset of the partial HED string within the full string. - * @returns {[string, Issue[]]} The converted string and any issues. - */ -export const convertPartialHedStringToLong = function (schema, partialHedString, fullHedString, offset) { - let issues = [] - - const hedString = removeSlashesAndSpaces(partialHedString) - - if (hedString === '') { - issues.push(generateIssue('emptyTagFound', '')) - return [hedString, issues] - } - - const hedTags = splitHedString(hedString) - let finalString = '' - - for (const [isHedTag, [startPosition, endPosition]] of hedTags) { - const tag = hedString.slice(startPosition, endPosition) - if (isHedTag) { - const [shortTagString, singleError] = convertTagToLong(schema, tag, fullHedString, startPosition + offset) - issues = issues.concat(singleError) - finalString += shortTagString - } else { - finalString += tag - } - } - - return [finalString, issues] -} +import { parseHedString } from '../parser/main' /** * Convert a HED string. * - * @param {Schema} schema The schema object containing a short-to-long mapping. + * @param {Schemas} hedSchemas The HED schema collection. * @param {string} hedString The HED tag to convert. - * @param {function (Schema, string, string, number): [string, Issue[]]} conversionFn The conversion function for a tag. + * @param {boolean} long Whether the tags should be in long form. * @returns {[string, Issue[]]} The converted string and any issues. */ -const convertHedString = function (schema, hedString, conversionFn) { - let issues = [] - - hedString = removeSlashesAndSpaces(hedString) - - if (hedString === '') { - issues.push(generateIssue('emptyTagFound', '')) - return [hedString, issues] - } - - const hedTags = splitHedString(hedString) - let finalString = '' - - for (const [isHedTag, [startPosition, endPosition]] of hedTags) { - const tag = hedString.slice(startPosition, endPosition) - if (isHedTag) { - const [shortTagString, singleError] = conversionFn(schema, tag, hedString, startPosition) - issues = issues.concat(singleError) - finalString += shortTagString - } else { - finalString += tag - } - } - - return [finalString, issues] +const convertHedString = function (hedSchemas, hedString, long) { + const [parsedString, issues] = parseHedString(hedString, hedSchemas) + const flattenedIssues = Object.values(issues).flat() + if (flattenedIssues.some((issue) => issue.level === 'error')) { + return [hedString, flattenedIssues] + } + const convertedString = parsedString.format(long) + return [convertedString, flattenedIssues] } /** @@ -270,10 +24,10 @@ const convertHedString = function (schema, hedString, conversionFn) { * @param {Schemas} schemas The schema container object containing short-to-long mappings. * @param {string} hedString The HED tag to convert. * @returns {[string, Issue[]]} The long-form string and any issues. - * @deprecated + * @deprecated Replaced with {@link ParsedHedString}. Will be removed in version 4.0.0 or earlier. */ export const convertHedStringToLong = function (schemas, hedString) { - return convertHedString(schemas.baseSchema, hedString, convertTagToLong) + return convertHedString(schemas, hedString, true) } /** @@ -282,8 +36,8 @@ export const convertHedStringToLong = function (schemas, hedString) { * @param {Schemas} schemas The schema container object containing short-to-long mappings. * @param {string} hedString The HED tag to convert. * @returns {[string, Issue[]]} The short-form string and any issues. - * @deprecated + * @deprecated Replaced with {@link ParsedHedString}. Will be removed in version 4.0.0 or earlier. */ export const convertHedStringToShort = function (schemas, hedString) { - return convertHedString(schemas.baseSchema, hedString, convertTagToShort) + return convertHedString(schemas, hedString, false) } diff --git a/converter/issues.js b/converter/issues.js deleted file mode 100644 index 736cf17d..00000000 --- a/converter/issues.js +++ /dev/null @@ -1,21 +0,0 @@ -import { generateIssue } from '../common/issues/issues' - -/** - * Generate an issue object for tag conversion. - * - * This is simply a wrapper around the corresponding function in utils/issues.js - * that handles conversion-specific functionality. - * - * @param {string} code The issue code. - * @param {string} hedString The source HED string. - * @param {object} parameters The parameters to the format string. - * @param {number[]} bounds The bounds of the problem tag. - * @returns {Issue} The issue object. - */ -export default function (code, hedString, parameters = {}, bounds = []) { - parameters.tag = hedString.slice(bounds[0], bounds[1]) - parameters.bounds = bounds - const issue = generateIssue(code, parameters) - issue.sourceString = hedString - return issue -} diff --git a/converter/schema.js b/converter/schema.js deleted file mode 100644 index e80bccb0..00000000 --- a/converter/schema.js +++ /dev/null @@ -1,47 +0,0 @@ -import { Mapping, TagEntry } from './types' -import { getTagName } from '../utils/hedStrings' -import { generateIssue, IssueError } from '../common/issues/issues' - -/** - * Build a short-long mapping object from schema XML data. - * - * @param {SchemaEntries} entries The schema XML data. - * @returns {Mapping} The mapping object. - */ -export const buildMappingObject = function (entries) { - /** - * @type {Map} - */ - const shortTagData = new Map() - /** - * @type {Map} - */ - const longTagData = new Map() - /** - * @type {Set} - */ - const takesValueTags = new Set() - /** - * @type {SchemaEntryManager} - */ - const schemaTags = entries.definitions.get('tags') - for (const tag of schemaTags.values()) { - const shortTag = getTagName(tag.name) - const lowercaseShortTag = shortTag.toLowerCase() - if (shortTag === '#') { - takesValueTags.add(getTagName(tag.parent.name).toLowerCase()) - continue - } - const tagObject = new TagEntry(shortTag, tag.name) - longTagData.set(tag.name, tagObject) - if (!shortTagData.has(lowercaseShortTag)) { - shortTagData.set(lowercaseShortTag, tagObject) - } else { - throw new IssueError(generateIssue('duplicateTagsInSchema', {})) - } - } - for (const tag of takesValueTags) { - shortTagData.get(tag).takesValue = true - } - return new Mapping(shortTagData, longTagData) -} diff --git a/converter/splitHedString.js b/converter/splitHedString.js deleted file mode 100644 index c499047d..00000000 --- a/converter/splitHedString.js +++ /dev/null @@ -1,66 +0,0 @@ -const tagDelimiters = new Set([',', '(', ')', '~']) - -/** - * Split a HED string into delimiters and tags. - * - * @param {string} hedString The HED string to split. - * @returns {Array[]} A list of string parts. The boolean is true if the part is - * a tag and false if it is a delimiter. The numbers are the bounds of the part. - */ -export function splitHedString(hedString) { - const resultPositions = [] - let currentSpacing = 0 - let insideDelimiter = true - let startPosition = -1 - let lastEndPosition = 0 - - for (let i = 0; i < hedString.length; i++) { - const character = hedString.charAt(i) - - if (character === ' ') { - currentSpacing++ - continue - } - - if (tagDelimiters.has(character)) { - if (!insideDelimiter) { - insideDelimiter = true - if (startPosition >= 0) { - lastEndPosition = i - currentSpacing - resultPositions.push([true, [startPosition, lastEndPosition]]) - currentSpacing = 0 - startPosition = -1 - } - } - continue - } - - if (insideDelimiter && lastEndPosition >= 0) { - if (lastEndPosition !== i) { - resultPositions.push([false, [lastEndPosition, i]]) - } - lastEndPosition = -1 - } - - currentSpacing = 0 - insideDelimiter = false - if (startPosition < 0) { - startPosition = i - } - } - - if (lastEndPosition >= 0 && hedString.length !== lastEndPosition) { - resultPositions.push([false, [lastEndPosition, hedString.length]]) - } - - if (startPosition >= 0) { - resultPositions.push([true, [startPosition, hedString.length - currentSpacing]]) - if (currentSpacing > 0) { - resultPositions.push([false, [hedString.length - currentSpacing, hedString.length]]) - } - } - - return resultPositions -} - -export default splitHedString diff --git a/converter/types.js b/converter/types.js deleted file mode 100644 index 8ab59a3c..00000000 --- a/converter/types.js +++ /dev/null @@ -1,63 +0,0 @@ -/** - * A tag dictionary entry. - */ -export class TagEntry { - /** - * The short version of the tag. - * @type {string} - */ - shortTag - /** - * The long version of the tag. - * @type {string} - */ - longTag - /** - * The formatted long version of the tag. - * @type {string} - */ - longFormattedTag - /** - * Whether this tag takes a value. - * @type {boolean} - */ - takesValue - - /** - * Constructor. - * @param {string} shortTag The short version of the tag. - * @param {string} longTag The long version of the tag. - */ - constructor(shortTag, longTag) { - this.shortTag = shortTag - this.longTag = longTag - this.longFormattedTag = longTag.toLowerCase() - } -} - -/** - * A short-to-long mapping. - */ -export class Mapping { - /** - * A dictionary mapping short forms to TagEntry instances. - * @type {Map} - */ - shortToTags - /** - * A dictionary mapping long forms to TagEntry instances. - * @type {Map} - */ - longToTags - - /** - * Constructor. - * - * @param {Map} shortToTags A dictionary mapping short forms to TagEntry instances. - * @param {Map} longToTags A dictionary mapping long forms to TagEntry instances. - */ - constructor(shortToTags, longToTags) { - this.shortToTags = shortToTags - this.longToTags = longToTags - } -} diff --git a/package-lock.json b/package-lock.json index c45165ca..e126ed6f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "xml2js": "^0.6.2" }, "devDependencies": { + "@jest/globals": "^29.7.0", "chai": "^4.3.6", "esbuild": "^0.20.2", "esbuild-plugin-globals": "^0.2.0", diff --git a/package.json b/package.json index 59209dae..0b8b475c 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "xml2js": "^0.6.2" }, "devDependencies": { + "@jest/globals": "^29.7.0", "chai": "^4.3.6", "esbuild": "^0.20.2", "esbuild-plugin-globals": "^0.2.0", diff --git a/parser/converter.js b/parser/converter.js new file mode 100644 index 00000000..eafa3f5d --- /dev/null +++ b/parser/converter.js @@ -0,0 +1,151 @@ +import { generateIssue, IssueError } from '../common/issues/issues' +import { getTagSlashIndices } from '../utils/hedStrings' +import { SchemaValueTag } from '../validator/schema/types' + +/** + * Converter from a tas specification to a schema-based tag object. + */ +export default class TagConverter { + /** + * A parsed tag token. + * @type {TagSpec} + */ + tagSpec + /** + * The tag string to convert. + * @type {string} + */ + tagString + /** + * The tag string split by slashes. + * @type {string[]} + */ + tagLevels + /** + * The indices of the tag string's slashes. + * @type {number[]} + */ + tagSlashes + /** + * A HED schema collection. + * @type {Schemas} + */ + hedSchemas + /** + * The entry manager for the tags in the active schema. + * @type {SchemaTagManager} + */ + tagMapping + /** + * The converted tag in the schema. + * @type {SchemaTag} + */ + schemaTag + /** + * The remainder of the tag string. + * @type {string} + */ + remainder + + /** + * Constructor. + * + * @param {TagSpec} tagSpec The tag specification to convert. + * @param {Schemas} hedSchemas The HED schema collection. + */ + constructor(tagSpec, hedSchemas) { + this.hedSchemas = hedSchemas + this.tagMapping = hedSchemas.getSchema(tagSpec.library).entries.tags + this.tagSpec = tagSpec + this.tagString = tagSpec.tag + this.tagLevels = this.tagString.split('/') + this.tagSlashes = getTagSlashIndices(this.tagString) + } + + /** + * Retrieve the {@link SchemaTag} object for a tag specification. + * + * @returns {[SchemaTag, string]} The schema's corresponding tag object and the remainder of the tag string. + */ + convert() { + const firstLevel = this._checkFirstLevel() + if (firstLevel) { + return [firstLevel, ''] + } + + return this._checkLowerLevels() + } + + _checkFirstLevel() { + const firstLevel = this.tagLevels[0].toLowerCase().trimStart() + const schemaTag = this.tagMapping.getEntry(firstLevel) + if (!schemaTag || firstLevel === '' || firstLevel !== firstLevel.trim()) { + throw new IssueError(generateIssue('invalidTag', { tag: this.tagString })) + } + if (this.tagLevels.length === 1) { + return schemaTag + } else { + return undefined + } + } + + _checkLowerLevels() { + let parentTag = this._getSchemaTag(0) + for (let i = 1; i < this.tagLevels.length; i++) { + if (parentTag?.valueTag) { + this._setSchemaTag(parentTag.valueTag, i) + break + } + const childTag = this._validateChildTag(parentTag, i) + if (childTag === undefined) { + this._setSchemaTag(parentTag, i) + } + parentTag = childTag + } + this._setSchemaTag(parentTag, this.tagLevels.length + 1) + return [this.schemaTag, this.remainder] + } + + _validateChildTag(parentTag, i) { + const childTag = this._getSchemaTag(i) + if (this.schemaTag instanceof SchemaValueTag) { + throw new IssueError( + generateIssue('internalConsistencyError', { + message: 'Child tag is a value tag which should have been handled earlier.', + }), + ) + } + if (childTag === undefined && parentTag && !parentTag.hasAttributeName('extensionAllowed')) { + throw new IssueError( + generateIssue('invalidExtension', { + tag: this.tagLevels[i], + parentTag: parentTag.longName, + }), + ) + } + if (childTag !== undefined && (childTag.parent === undefined || childTag.parent !== parentTag)) { + throw new IssueError( + generateIssue('invalidParentNode', { + tag: this.tagLevels[i], + parentTag: childTag.longName, + }), + ) + } + return childTag + } + + _getSchemaTag(i) { + const tagLevel = this.tagLevels[i].toLowerCase() + if (tagLevel === '' || tagLevel !== tagLevel.trim()) { + throw new IssueError(generateIssue('invalidTag', { tag: this.tagString })) + } + return this.tagMapping.getEntry(tagLevel) + } + + _setSchemaTag(schemaTag, i) { + if (this.schemaTag === undefined) { + this.schemaTag = schemaTag + this.remainder = this.tagLevels.slice(i).join('/') + } + } +} diff --git a/parser/parsedHedColumnSplice.js b/parser/parsedHedColumnSplice.js index fefa6132..c67381c4 100644 --- a/parser/parsedHedColumnSplice.js +++ b/parser/parsedHedColumnSplice.js @@ -7,9 +7,11 @@ export class ParsedHedColumnSplice extends ParsedHedSubstring { /** * Nicely format this column splice template. * + * @param {boolean} long Whether the tags should be in long form. * @returns {string} */ - format() { + // eslint-disable-next-line no-unused-vars + format(long = true) { return '{' + this.originalTag + '}' } diff --git a/parser/parsedHedGroup.js b/parser/parsedHedGroup.js index a32a8a6c..8e7a2a58 100644 --- a/parser/parsedHedGroup.js +++ b/parser/parsedHedGroup.js @@ -92,10 +92,11 @@ export class ParsedHedGroup extends ParsedHedSubstring { /** * Nicely format this tag group. * + * @param {boolean} long Whether the tags should be in long form. * @returns {string} */ - format() { - return '(' + this.tags.map((substring) => substring.format()).join(', ') + ')' + format(long = true) { + return '(' + this.tags.map((substring) => substring.format(long)).join(', ') + ')' } /** diff --git a/parser/parsedHedString.js b/parser/parsedHedString.js index 71df9d6b..ad63d65d 100644 --- a/parser/parsedHedString.js +++ b/parser/parsedHedString.js @@ -86,10 +86,11 @@ export class ParsedHedString { /** * Nicely format this HED string. * + * @param {boolean} long Whether the tags should be in long form. * @returns {string} */ - format() { - return this.parseTree.map((substring) => substring.format()).join(', ') + format(long = true) { + return this.parseTree.map((substring) => substring.format(long)).join(', ') } get definitions() { diff --git a/parser/parsedHedSubstring.js b/parser/parsedHedSubstring.js index 636d8ce6..bc8a3aad 100644 --- a/parser/parsedHedSubstring.js +++ b/parser/parsedHedSubstring.js @@ -46,10 +46,11 @@ export class ParsedHedSubstring extends Memoizer { * * This is left blank for the subclasses to override. * + * @param {boolean} long Whether the tags should be in long form. * @returns {string} * @abstract */ - format() {} + format(long = true) {} /** * Override of {@link Object.prototype.toString}. diff --git a/parser/parsedHedTag.js b/parser/parsedHedTag.js index 9e47fffa..cf6ac97d 100644 --- a/parser/parsedHedTag.js +++ b/parser/parsedHedTag.js @@ -1,8 +1,9 @@ import { generateIssue } from '../common/issues/issues' import { Schema } from '../common/schema/types' -import { convertPartialHedStringToLong } from '../converter/converter' import { getTagLevels, replaceTagNameWithPound } from '../utils/hedStrings' import ParsedHedSubstring from './parsedHedSubstring' +import { SchemaValueTag } from '../validator/schema/types' +import TagConverter from './converter' /** * A parsed HED tag. @@ -33,15 +34,13 @@ export class ParsedHedTag extends ParsedHedSubstring { * Constructor. * * @param {string} originalTag The original HED tag. - * @param {string} hedString The original HED string. * @param {number[]} originalBounds The bounds of the HED tag in the original HED string. - * @param {Schemas} hedSchemas The collection of HED schemas. - * @param {string} schemaName The label of this tag's schema in the dataset's schema spec. */ - constructor(originalTag, hedString, originalBounds, hedSchemas, schemaName = '') { + constructor(originalTag, originalBounds) { super(originalTag, originalBounds) - this._convertTag(hedString, hedSchemas, schemaName) + this.canonicalTag = this.originalTag + this.conversionIssues = [] this.formattedTag = this._formatTag() } @@ -62,23 +61,12 @@ export class ParsedHedTag extends ParsedHedSubstring { /** * Nicely format this tag. * + * @param {boolean} long Whether the tags should be in long form. * @returns {string} The nicely formatted version of this tag. */ - format() { - return this.toString() - } - - /** - * Convert this tag to long form. - * - * @param {string} hedString The original HED string. - * @param {Schemas} hedSchemas The collection of HED schemas. - * @param {string} schemaName The label of this tag's schema in the dataset's schema spec. - */ // eslint-disable-next-line no-unused-vars - _convertTag(hedString, hedSchemas, schemaName) { - this.canonicalTag = this.originalTag - this.conversionIssues = [] + format(long = true) { + return this.toString() } /** @@ -293,24 +281,49 @@ export class ParsedHedTag extends ParsedHedSubstring { */ export class ParsedHed3Tag extends ParsedHedTag { /** - * Convert this tag to long form. + * The schema's representation of this tag. + * + * @type {SchemaTag} + * @private + */ + _schemaTag + /** + * The remaining part of the tag after the portion actually in the schema. + * + * @type {string} + * @private + */ + _remainder + + /** + * Constructor. * + * @param {TagSpec} tagSpec The token for this tag. + * @param {Schemas} hedSchemas The collection of HED schemas. * @param {string} hedString The original HED string. + */ + constructor(tagSpec, hedSchemas, hedString) { + super(tagSpec.tag, tagSpec.bounds) + + this._convertTag(hedSchemas, hedString, tagSpec) + + this.formattedTag = this._formatTag() + } + + /** + * Convert this tag to long form. + * * @param {Schemas} hedSchemas The collection of HED schemas. - * @param {string} schemaName The label of this tag's schema in the dataset's schema spec. + * @param {string} hedString The original HED string. + * @param {TagSpec} tagSpec The token for this tag. */ - _convertTag(hedString, hedSchemas, schemaName) { + _convertTag(hedSchemas, hedString, tagSpec) { const hed3ValidCharacters = /^[^{}[\]()~,\0\t]+$/ if (!hed3ValidCharacters.test(this.originalTag)) { throw new Error('The parser failed to properly remove an illegal or special character.') } - if (hedSchemas.isSyntaxOnly) { - this.canonicalTag = this.originalTag - this.conversionIssues = [] - return - } - + const schemaName = tagSpec.library this.schema = hedSchemas.getSchema(schemaName) if (this.schema === undefined) { if (schemaName !== '') { @@ -331,23 +344,30 @@ export class ParsedHed3Tag extends ParsedHedTag { return } - const [canonicalTag, conversionIssues] = convertPartialHedStringToLong( - this.schema, - this.originalTag, - hedString, - this.originalBounds[0], - ) - this.canonicalTag = canonicalTag - this.conversionIssues = conversionIssues + try { + const [schemaTag, remainder] = new TagConverter(tagSpec, hedSchemas).convert() + this._schemaTag = schemaTag + this._remainder = remainder + this.canonicalTag = this._schemaTag.longExtend(remainder) + this.conversionIssues = [] + } catch (error) { + this.conversionIssues = [error.issue] + } } /** * Nicely format this tag. * + * @param {boolean} long Whether the tags should be in long form. * @returns {string} The nicely formatted version of this tag. */ - format() { - let tagName = this.schema?.entries.definitions.get('tags').getEntry(this.formattedTag)?.name + format(long = true) { + let tagName + if (long) { + tagName = this._schemaTag?.longExtend(this._remainder) + } else { + tagName = this._schemaTag?.extend(this._remainder) + } if (tagName === undefined) { tagName = this.originalTag } @@ -365,7 +385,7 @@ export class ParsedHed3Tag extends ParsedHedTag { */ get existsInSchema() { return this._memoize('existsInSchema', () => { - return this.schema?.entries.definitions.get('tags').hasEntry(this.formattedTag) + return this.schema?.entries?.tags?.hasLongNameEntry(this.formattedTag) }) } @@ -394,11 +414,7 @@ export class ParsedHed3Tag extends ParsedHedTag { */ get takesValueTag() { return this._memoize('takesValueTag', () => { - if (this.takesValueFormattedTag !== null) { - return this.schema?.entries.definitions.get('tags').getEntry(this.takesValueFormattedTag) - } else { - return null - } + return this._schemaTag }) } @@ -420,10 +436,7 @@ export class ParsedHed3Tag extends ParsedHedTag { */ get hasUnitClass() { return this._memoize('hasUnitClass', () => { - if (!this.schema?.entries.definitions.has('unitClasses')) { - return false - } - if (this.takesValueTag === null) { + if (!this.takesValueTag) { return false } return this.takesValueTag.hasUnitClasses @@ -475,7 +488,7 @@ export class ParsedHed3Tag extends ParsedHedTag { const tagUnitClasses = this.unitClasses const units = new Set() for (const unitClass of tagUnitClasses) { - const unitClassUnits = this.schema?.entries.unitClassMap.getEntry(unitClass.name).units + const unitClassUnits = this.schema?.entries.unitClasses.getEntry(unitClass.name).units for (const unit of unitClassUnits.values()) { units.add(unit) } diff --git a/parser/splitHedString.js b/parser/splitHedString.js index 387377d5..d0d96db0 100644 --- a/parser/splitHedString.js +++ b/parser/splitHedString.js @@ -1,407 +1,23 @@ -import flattenDeep from 'lodash/flattenDeep' - import { ParsedHed3Tag, ParsedHedTag } from './parsedHedTag' import ParsedHedColumnSplice from './parsedHedColumnSplice' import ParsedHedGroup from './parsedHedGroup' import { Schemas } from '../common/schema/types' -import { generateIssue } from '../common/issues/issues' import { recursiveMap } from '../utils/array' -import { replaceTagNameWithPound } from '../utils/hedStrings' import { mergeParsingIssues } from '../utils/hedData' -import { stringIsEmpty } from '../utils/string' import { ParsedHed2Tag } from '../validator/hed2/parser/parsedHed2Tag' - -const openingGroupCharacter = '(' -const closingGroupCharacter = ')' -const openingColumnCharacter = '{' -const closingColumnCharacter = '}' -const commaCharacter = ',' -const colonCharacter = ':' -const slashCharacter = '/' -const invalidCharacters = new Set(['[', ']', '~', '"']) -const invalidCharactersOutsideOfValues = new Set([':']) +import { HedStringTokenizer, ColumnSpliceSpec, TagSpec } from './tokenizer' const generationToClass = [ - ParsedHedTag, - ParsedHedTag, // Generation 1 is not supported by this validator. - ParsedHed2Tag, - ParsedHed3Tag, + (originalTag, hedString, originalBounds, hedSchemas, schemaName, tagSpec) => + new ParsedHedTag(originalTag, originalBounds), + (originalTag, hedString, originalBounds, hedSchemas, schemaName, tagSpec) => + new ParsedHedTag(originalTag, originalBounds), // Generation 1 is not supported by this validator. + (originalTag, hedString, originalBounds, hedSchemas, schemaName, tagSpec) => + new ParsedHed2Tag(originalTag, hedString, originalBounds, hedSchemas, schemaName), + (originalTag, hedString, originalBounds, hedSchemas, schemaName, tagSpec) => + new ParsedHed3Tag(tagSpec, hedSchemas, hedString), ] -/** - * A specification for a tokenized substring. - */ -class SubstringSpec { - /** - * The starting and ending bounds of the substring. - * @type {number[]} - */ - bounds - - constructor(start, end) { - this.bounds = [start, end] - } -} - -/** - * A specification for a tokenized tag. - */ -class TagSpec extends SubstringSpec { - /** - * The tag this spec represents. - * @type {string} - */ - tag - /** - * The schema prefix for this tag, if any. - * @type {string} - */ - library - - constructor(tag, start, end, librarySchema) { - super(start, end) - - this.tag = tag.trim() - this.library = librarySchema - } -} - -/** - * A specification for a tokenized tag group. - */ -class GroupSpec extends SubstringSpec { - /** - * The child group specifications. - * @type {GroupSpec[]} - */ - children - - constructor(start, end) { - super(start, end) - - this.children = [] - } -} - -/** - * A specification for a tokenized column splice template. - */ -class ColumnSpliceSpec extends SubstringSpec { - /** - * The column name this spec refers to. - * @type {string} - */ - columnName - - constructor(name, start, end) { - super(start, end) - - this.columnName = name.trim() - } -} - -/** - * Class for tokenizing HED strings. - */ -class HedStringTokenizer { - /** - * The HED string being parsed. - * @type {string} - */ - hedString - - syntaxIssues - - /** - * The current substring being parsed. - * @type {string} - */ - currentTag - - groupDepth - startingIndex - resetStartingIndex - slashFound - librarySchema - currentGroupStack - parenthesesStack - ignoringCharacters - - constructor(hedString) { - this.hedString = hedString - } - - /** - * Split the HED string into delimiters and tags. - * - * @returns {[TagSpec[], GroupSpec, Object]} The tag specifications, group bounds, and any issues found. - */ - tokenize() { - this.initializeTokenizer() - - for (let i = 0; i < this.hedString.length; i++) { - const character = this.hedString.charAt(i) - this.tokenizeCharacter(i, character) - if (this.resetStartingIndex) { - this.resetStartingIndex = false - this.startingIndex = i + 1 - this.currentTag = '' - } - } - this.pushTag(this.hedString.length) - - if (this.columnSpliceIndex >= 0) { - this.syntaxIssues.push( - generateIssue('unclosedCurlyBrace', { - index: this.columnSpliceIndex, - string: this.hedString, - }), - ) - } - - this.unwindGroupStack() - - const tagSpecs = this.currentGroupStack.pop() - const groupSpecs = this.parenthesesStack.pop() - const issues = { - syntax: this.syntaxIssues, - conversion: [], - } - return [tagSpecs, groupSpecs, issues] - } - - initializeTokenizer() { - this.syntaxIssues = [] - - this.currentTag = '' - this.groupDepth = 0 - this.startingIndex = 0 - this.resetStartingIndex = false - this.slashFound = false - this.librarySchema = '' - this.columnSpliceIndex = -1 - this.currentGroupStack = [[]] - this.parenthesesStack = [new GroupSpec(0, this.hedString.length)] - this.ignoringCharacters = false - } - - tokenizeCharacter(i, character) { - let dispatchTable - if (this.ignoringCharacters) { - dispatchTable = { - [closingGroupCharacter]: (i, character) => { - this.clearTag() - this.closingGroupCharacter(i) - }, - [commaCharacter]: (i, character) => this.clearTag(), - } - } else { - dispatchTable = { - [openingGroupCharacter]: (i, character) => this.openingGroupCharacter(i), - [closingGroupCharacter]: (i, character) => { - this.pushTag(i) - this.closingGroupCharacter(i) - }, - [openingColumnCharacter]: (i, character) => this.openingColumnCharacter(i), - [closingColumnCharacter]: (i, character) => this.closingColumnCharacter(i), - [commaCharacter]: (i, character) => this.pushTag(i), - [colonCharacter]: (i, character) => this.colonCharacter(character), - [slashCharacter]: (i, character) => this.slashCharacter(character), - } - } - const characterHandler = dispatchTable[character] - if (characterHandler) { - characterHandler(i, character) - } else { - this.otherCharacter(character) - } - } - - openingGroupCharacter(i) { - this.currentGroupStack.push([]) - this.parenthesesStack.push(new GroupSpec(i)) - this.resetStartingIndex = true - this.groupDepth++ - } - - closingGroupCharacter(i) { - if (this.groupDepth <= 0) { - this.syntaxIssues.push( - generateIssue('unopenedParenthesis', { - index: i, - string: this.hedString, - }), - ) - return - } - this.closeGroup(i) - } - - openingColumnCharacter(i) { - if (this.currentTag.length > 0) { - this.syntaxIssues.push( - generateIssue('invalidCharacter', { - character: openingColumnCharacter, - index: i, - string: this.hedString, - }), - ) - this.ignoringCharacters = true - return - } - if (this.columnSpliceIndex >= 0) { - this.syntaxIssues.push( - generateIssue('nestedCurlyBrace', { - index: i, - string: this.hedString, - }), - ) - } - this.columnSpliceIndex = i - } - - closingColumnCharacter(i) { - if (this.columnSpliceIndex < 0) { - this.syntaxIssues.push( - generateIssue('unopenedCurlyBrace', { - index: i, - string: this.hedString, - }), - ) - return - } - if (!stringIsEmpty(this.currentTag)) { - this.currentGroupStack[this.groupDepth].push(new ColumnSpliceSpec(this.currentTag, this.startingIndex, i)) - } else { - this.syntaxIssues.push( - generateIssue('emptyCurlyBrace', { - string: this.hedString, - }), - ) - } - this.columnSpliceIndex = -1 - this.resetStartingIndex = true - this.slashFound = false - } - - colonCharacter(character) { - if (!this.slashFound && !this.librarySchema) { - this.librarySchema = this.currentTag - this.resetStartingIndex = true - } else { - this.currentTag += character - } - } - - slashCharacter(character) { - this.slashFound = true - this.currentTag += character - } - - otherCharacter(character) { - if (this.ignoringCharacters) { - return - } - this.currentTag += character - this.resetStartingIndex = stringIsEmpty(this.currentTag) - } - - unwindGroupStack() { - // groupDepth is decremented in closeGroup. - // eslint-disable-next-line no-unmodified-loop-condition - while (this.groupDepth > 0) { - this.syntaxIssues.push( - generateIssue('unclosedParenthesis', { - index: this.parenthesesStack[this.parenthesesStack.length - 1].bounds[0], - string: this.hedString, - }), - ) - this.closeGroup(this.hedString.length) - } - } - - pushTag(i) { - if (!stringIsEmpty(this.currentTag) && this.columnSpliceIndex < 0) { - this.currentGroupStack[this.groupDepth].push( - new TagSpec(this.currentTag, this.startingIndex, i, this.librarySchema), - ) - } - this.resetStartingIndex = true - this.slashFound = false - this.librarySchema = '' - } - - clearTag() { - this.ignoringCharacters = false - this.resetStartingIndex = true - this.slashFound = false - this.librarySchema = '' - } - - closeGroup(i) { - const groupSpec = this.parenthesesStack.pop() - groupSpec.bounds[1] = i + 1 - this.parenthesesStack[this.groupDepth - 1].children.push(groupSpec) - this.currentGroupStack[this.groupDepth - 1].push(this.currentGroupStack.pop()) - this.groupDepth-- - } -} - -/** - * Check the split HED tags for invalid characters - * - * @param {string} hedString The HED string to be split. - * @param {SubstringSpec[]} tagSpecs The tag specifications. - * @returns {Object} Any issues found. - */ -const checkForInvalidCharacters = function (hedString, tagSpecs) { - const syntaxIssues = [] - const flatTagSpecs = flattenDeep(tagSpecs) - - for (const tagSpec of flatTagSpecs) { - if (tagSpec instanceof ColumnSpliceSpec) { - continue - } - const alwaysInvalidIssues = checkTagForInvalidCharacters(hedString, tagSpec, tagSpec.tag, invalidCharacters) - const valueTag = replaceTagNameWithPound(tagSpec.tag) - const outsideValueIssues = checkTagForInvalidCharacters( - hedString, - tagSpec, - valueTag, - invalidCharactersOutsideOfValues, - ) - syntaxIssues.push(...alwaysInvalidIssues, ...outsideValueIssues) - } - - return { syntax: syntaxIssues, conversion: [] } -} - -/** - * Check an individual tag for invalid characters. - * - * @param {string} hedString The HED string to be split. - * @param {TagSpec} tagSpec A tag specification. - * @param {string} tag The tag form to be checked. - * @param {Set} invalidSet The set of invalid characters. - * @returns {Issue[]} Any issues found. - */ -const checkTagForInvalidCharacters = function (hedString, tagSpec, tag, invalidSet) { - const issues = [] - for (let i = 0; i < tag.length; i++) { - const character = tag.charAt(i) - if (invalidSet.has(character)) { - issues.push( - generateIssue('invalidCharacter', { - character: character, - index: tagSpec.bounds[0] + i, - string: hedString, - }), - ) - } - } - return issues -} - /** * Create the parsed HED tag and group objects. * @@ -414,11 +30,18 @@ const checkTagForInvalidCharacters = function (hedString, tagSpec, tag, invalidS const createParsedTags = function (hedString, hedSchemas, tagSpecs, groupSpecs) { const conversionIssues = [] const syntaxIssues = [] - const ParsedHedTagClass = generationToClass[hedSchemas.generation] + const ParsedHedTagConstructor = generationToClass[hedSchemas.generation] const createParsedTag = (tagSpec) => { if (tagSpec instanceof TagSpec) { - const parsedTag = new ParsedHedTagClass(tagSpec.tag, hedString, tagSpec.bounds, hedSchemas, tagSpec.library) + const parsedTag = ParsedHedTagConstructor( + tagSpec.tag, + hedString, + tagSpec.bounds, + hedSchemas, + tagSpec.library, + tagSpec, + ) conversionIssues.push(...parsedTag.conversionIssues) return parsedTag } else if (tagSpec instanceof ColumnSpliceSpec) { @@ -460,13 +83,11 @@ const createParsedTags = function (hedString, hedSchemas, tagSpecs, groupSpecs) * @returns {[ParsedHedSubstring[], Object]} The parsed HED string data and any issues found. */ export default function splitHedString(hedString, hedSchemas) { - const [tagSpecs, groupBounds, splitIssues] = new HedStringTokenizer(hedString).tokenize() - const characterIssues = checkForInvalidCharacters(hedString, tagSpecs) - mergeParsingIssues(splitIssues, characterIssues) - if (splitIssues.syntax.length > 0) { - return [null, splitIssues] + const [tagSpecs, groupBounds, tokenizingIssues] = new HedStringTokenizer(hedString).tokenize() + if (tokenizingIssues.syntax.length > 0) { + return [null, tokenizingIssues] } const [parsedTags, parsingIssues] = createParsedTags(hedString, hedSchemas, tagSpecs, groupBounds) - mergeParsingIssues(splitIssues, parsingIssues) - return [parsedTags, splitIssues] + mergeParsingIssues(tokenizingIssues, parsingIssues) + return [parsedTags, tokenizingIssues] } diff --git a/parser/tokenizer.js b/parser/tokenizer.js new file mode 100644 index 00000000..89ecaab3 --- /dev/null +++ b/parser/tokenizer.js @@ -0,0 +1,375 @@ +import { generateIssue } from '../common/issues/issues' +import { stringIsEmpty } from '../utils/string' +import { replaceTagNameWithPound } from '../utils/hedStrings' + +const openingGroupCharacter = '(' +const closingGroupCharacter = ')' +const openingColumnCharacter = '{' +const closingColumnCharacter = '}' +const commaCharacter = ',' +const colonCharacter = ':' +const slashCharacter = '/' + +const invalidCharacters = new Set(['[', ']', '~', '"']) +const invalidCharactersOutsideOfValues = new Set([':']) + +/** + * A specification for a tokenized substring. + */ +export class SubstringSpec { + /** + * The starting and ending bounds of the substring. + * @type {number[]} + */ + bounds + + constructor(start, end) { + this.bounds = [start, end] + } +} + +/** + * A specification for a tokenized tag. + */ +export class TagSpec extends SubstringSpec { + /** + * The tag this spec represents. + * @type {string} + */ + tag + /** + * The schema prefix for this tag, if any. + * @type {string} + */ + library + + constructor(tag, start, end, librarySchema) { + super(start, end) + + this.tag = tag.trim() + this.library = librarySchema + } +} + +/** + * A specification for a tokenized tag group. + */ +export class GroupSpec extends SubstringSpec { + /** + * The child group specifications. + * @type {GroupSpec[]} + */ + children + + constructor(start, end) { + super(start, end) + + this.children = [] + } +} + +/** + * A specification for a tokenized column splice template. + */ +export class ColumnSpliceSpec extends SubstringSpec { + /** + * The column name this spec refers to. + * @type {string} + */ + columnName + + constructor(name, start, end) { + super(start, end) + + this.columnName = name.trim() + } +} + +/** + * Class for tokenizing HED strings. + */ +export class HedStringTokenizer { + /** + * The HED string being parsed. + * @type {string} + */ + hedString + + syntaxIssues + + /** + * The current substring being parsed. + * @type {string} + */ + currentTag + + /** + * Whether we are currently closing a group. + * @type {boolean} + */ + closingGroup + + groupDepth + startingIndex + resetStartingIndex + slashFound + librarySchema + currentGroupStack + parenthesesStack + ignoringCharacters + + constructor(hedString) { + this.hedString = hedString + } + + /** + * Split the HED string into delimiters and tags. + * + * @returns {[TagSpec[], GroupSpec, Object]} The tag specifications, group bounds, and any issues found. + */ + tokenize() { + this.initializeTokenizer() + + for (let i = 0; i < this.hedString.length; i++) { + const character = this.hedString.charAt(i) + this.tokenizeCharacter(i, character) + if (this.resetStartingIndex) { + this.resetStartingIndex = false + this.startingIndex = i + 1 + this.currentTag = '' + } + } + this.pushTag(this.hedString.length, true) + + if (this.columnSpliceIndex >= 0) { + this._pushSyntaxIssue('unclosedCurlyBrace', this.columnSpliceIndex) + } + + this.unwindGroupStack() + + const tagSpecs = this.currentGroupStack.pop() + const groupSpecs = this.parenthesesStack.pop() + const issues = { + syntax: this.syntaxIssues, + conversion: [], + } + return [tagSpecs, groupSpecs, issues] + } + + initializeTokenizer() { + this.syntaxIssues = [] + + this.currentTag = '' + this.groupDepth = 0 + this.startingIndex = 0 + this.resetStartingIndex = false + this.slashFound = false + this.librarySchema = '' + this.columnSpliceIndex = -1 + this.currentGroupStack = [[]] + this.parenthesesStack = [new GroupSpec(0, this.hedString.length)] + this.ignoringCharacters = false + this.closingGroup = false + } + + tokenizeCharacter(i, character) { + let dispatchTable + if (this.ignoringCharacters) { + dispatchTable = { + [closingGroupCharacter]: (i /* character */) => { + this.clearTag() + this.closingGroupCharacter(i) + }, + [commaCharacter]: (/*i, character */) => this.clearTag(), + } + } else { + dispatchTable = { + [openingGroupCharacter]: (i /* character */) => this.openingGroupCharacter(i), + [closingGroupCharacter]: (i /* character */) => { + this.pushTag(i, false) + this.closingGroupCharacter(i) + }, + [openingColumnCharacter]: (i /* character */) => this.openingColumnCharacter(i), + [closingColumnCharacter]: (i /* character */) => this.closingColumnCharacter(i), + [commaCharacter]: (i /* character */) => this.pushTag(i, false), + [colonCharacter]: (i, character) => this.colonCharacter(character), + [slashCharacter]: (i, character) => this.slashCharacter(character), + } + } + const characterHandler = dispatchTable[character] + if (characterHandler) { + characterHandler(i, character) + } else if (invalidCharacters.has(character)) { + this._pushInvalidCharacterIssue(character, i) + } else { + this.otherCharacter(character) + } + } + + openingGroupCharacter(i) { + this.currentGroupStack.push([]) + this.parenthesesStack.push(new GroupSpec(i)) + this.resetStartingIndex = true + this.groupDepth++ + } + + closingGroupCharacter(i) { + this.closingGroup = true + if (this.groupDepth <= 0) { + this._pushSyntaxIssue('unopenedParenthesis', i) + return + } + this.closeGroup(i) + } + + openingColumnCharacter(i) { + if (this.currentTag.length > 0) { + this._pushInvalidCharacterIssue(openingColumnCharacter, i) + this.ignoringCharacters = true + return + } + if (this.columnSpliceIndex >= 0) { + this._pushSyntaxIssue('nestedCurlyBrace', i) + } + this.columnSpliceIndex = i + } + + closingColumnCharacter(i) { + this.closingGroup = true + if (this.columnSpliceIndex < 0) { + this._pushSyntaxIssue('unopenedCurlyBrace', i) + return + } + if (!stringIsEmpty(this.currentTag)) { + this.currentGroupStack[this.groupDepth].push(new ColumnSpliceSpec(this.currentTag.trim(), this.startingIndex, i)) + } else { + this.syntaxIssues.push( + generateIssue('emptyCurlyBrace', { + string: this.hedString, + }), + ) + } + this.columnSpliceIndex = -1 + this.resetStartingIndex = true + this.slashFound = false + } + + colonCharacter(character) { + if (!this.slashFound && !this.librarySchema) { + this.librarySchema = this.currentTag + this.resetStartingIndex = true + } else { + this.currentTag += character + } + } + + slashCharacter(character) { + this.slashFound = true + this.currentTag += character + } + + otherCharacter(character) { + if (this.ignoringCharacters) { + return + } + this.currentTag += character + this.resetStartingIndex = stringIsEmpty(this.currentTag) + } + + unwindGroupStack() { + // groupDepth is decremented in closeGroup. + // eslint-disable-next-line no-unmodified-loop-condition + while (this.groupDepth > 0) { + this._pushSyntaxIssue('unclosedParenthesis', this.parenthesesStack[this.parenthesesStack.length - 1].bounds[0]) + this.closeGroup(this.hedString.length) + } + } + + /** + * Push a tag to the current group. + * + * @param {number} i The current index. + * @param {boolean} isEndOfString Whether we are at the end of the string. + */ + pushTag(i, isEndOfString) { + if (stringIsEmpty(this.currentTag) && isEndOfString) { + return + } else if (this.closingGroup) { + this.closingGroup = false + } else if (stringIsEmpty(this.currentTag)) { + this.syntaxIssues.push(generateIssue('emptyTagFound', { index: i })) + } else if (this.columnSpliceIndex < 0) { + this._checkValueTagForInvalidCharacters() + this.currentGroupStack[this.groupDepth].push( + new TagSpec(this.currentTag.trim(), this.startingIndex, i, this.librarySchema), + ) + } + this.resetStartingIndex = true + this.slashFound = false + this.librarySchema = '' + } + + clearTag() { + this.ignoringCharacters = false + this.resetStartingIndex = true + this.slashFound = false + this.librarySchema = '' + } + + closeGroup(i) { + const groupSpec = this.parenthesesStack.pop() + groupSpec.bounds[1] = i + 1 + this.parenthesesStack[this.groupDepth - 1].children.push(groupSpec) + this.currentGroupStack[this.groupDepth - 1].push(this.currentGroupStack.pop()) + this.groupDepth-- + } + + /** + * Check an individual tag for invalid characters. + * + * @private + */ + _checkValueTagForInvalidCharacters() { + const formToCheck = replaceTagNameWithPound(this.currentTag) + for (let i = 0; i < formToCheck.length; i++) { + const character = formToCheck.charAt(i) + if (!invalidCharactersOutsideOfValues.has(character)) { + continue + } + this._pushInvalidCharacterIssue(character, this.startingIndex + i) + } + } + + /** + * Push an issue to the syntax issue list. + * + * @param {string} issueCode The internal code of the issue to be pushed. + * @param {number} index The location of the issue. + * @private + */ + _pushSyntaxIssue(issueCode, index) { + this.syntaxIssues.push( + generateIssue(issueCode, { + index: index, + string: this.hedString, + }), + ) + } + + /** + * Push an invalid character issue to the syntax issue list. + * + * @param {string} character The illegal character to be reported. + * @param {number} index The location of the character. + * @private + */ + _pushInvalidCharacterIssue(character, index) { + this.syntaxIssues.push( + generateIssue('invalidCharacter', { + character: character, + index: index, + string: this.hedString, + }), + ) + } +} diff --git a/tests/bids.spec.js b/tests/bids.spec.js index 08fb2a8e..ddd7dfad 100644 --- a/tests/bids.spec.js +++ b/tests/bids.spec.js @@ -1,8 +1,8 @@ import chai from 'chai' const assert = chai.assert +import { beforeAll, describe, it } from '@jest/globals' import cloneDeep from 'lodash/cloneDeep' -import converterGenerateIssue from '../converter/issues' import { generateIssue } from '../common/issues/issues' import { SchemaSpec, SchemasSpec } from '../common/schema/types' import { buildBidsSchemas, parseSchemasSpec } from '../bids/schema' @@ -88,10 +88,7 @@ describe('BIDS datasets', () => { ), ], error_and_good: [ - BidsHedIssue.fromHedIssue( - converterGenerateIssue('invalidTag', 'Confused', {}, [0, 8]), - bidsSidecars[1][1].file, - ), + BidsHedIssue.fromHedIssue(generateIssue('invalidTag', { tag: 'Confused' }), bidsSidecars[1][1].file), ], } return validator(testDatasets, expectedIssues, specs) @@ -160,7 +157,7 @@ describe('BIDS datasets', () => { tag: 'Speed/300 miles', unitClassUnits: legalSpeedUnits.sort().join(','), }) - const converterMaglevError = converterGenerateIssue('invalidTag', 'Maglev', {}, [0, 6]) + const converterMaglevError = generateIssue('invalidTag', { tag: 'Maglev' }) const maglevError = generateIssue('invalidTag', { tag: 'Maglev' }) const maglevWarning = generateIssue('extension', { tag: 'Train/Maglev' }) const expectedIssues = { @@ -192,8 +189,7 @@ describe('BIDS datasets', () => { all_good: [], all_bad: [ // BidsHedIssue.fromHedIssue(generateIssue('invalidTag', { tag: 'Confused' }), badDatasets[0].file), - BidsHedIssue.fromHedIssue(converterGenerateIssue('invalidTag', 'Confused', {}, [0, 8]), badDatasets[0].file), - // BidsHedIssue.fromHedIssue(converterGenerateIssue('invalidTag', 'Confused,Gray', {}, [0, 8]), badDatasets[0].file), + BidsHedIssue.fromHedIssue(generateIssue('invalidTag', { tag: 'Confused' }), badDatasets[0].file), // TODO: Catch warning in sidecar validation /* BidsHedIssue.fromHedIssue( generateIssue('extension', { tag: 'Train/Maglev' }), @@ -473,7 +469,10 @@ describe('BIDS datasets', () => { const expectedIssues = { bad_tsv: [ BidsHedIssue.fromHedIssue( - generateIssue('illegalDefinitionContext', { string: '(Definition/myDef, (Label/Red, Green))', tsvLine: 2 }), + generateIssue('illegalDefinitionContext', { + string: '(Definition/myDef, (Label/Red, Green))', + tsvLine: 2, + }), badTsvDatasets[0].file, ), ], diff --git a/tests/dataset.spec.js b/tests/dataset.spec.js index b20d3b92..82eaa63b 100644 --- a/tests/dataset.spec.js +++ b/tests/dataset.spec.js @@ -1,9 +1,10 @@ import chai from 'chai' const assert = chai.assert +import { beforeAll, describe, it } from '@jest/globals' + import * as hed from '../validator/dataset' import { buildSchemas } from '../validator/schema/init' import { generateIssue as generateValidationIssue } from '../common/issues/issues' -import generateConverterIssue from '../converter/issues' import { SchemaSpec, SchemasSpec } from '../common/schema/types' describe('HED dataset validation', () => { @@ -65,7 +66,7 @@ describe('HED dataset validation', () => { generateValidationIssue('invalidTag', { tag: testDatasets.multipleInvalid[1], }), - generateConverterIssue('invalidTag', testDatasets.multipleInvalid[1], {}, [0, 12]), + generateValidationIssue('invalidTag', { tag: testDatasets.multipleInvalid[1] }), ], } return validator(testDatasets, expectedIssues) @@ -119,7 +120,7 @@ describe('HED dataset validation', () => { generateValidationIssue('invalidTag', { tag: testDatasets.multipleInvalid[1], }), - generateConverterIssue('invalidTag', testDatasets.multipleInvalid[1], {}, [0, 12]), + generateValidationIssue('invalidTag', { tag: testDatasets.multipleInvalid[1] }), ], } return validator(testDatasets, expectedIssues) diff --git a/tests/event.spec.js b/tests/event.spec.js index ca1fdc55..f684db3e 100644 --- a/tests/event.spec.js +++ b/tests/event.spec.js @@ -1,12 +1,13 @@ import chai from 'chai' const assert = chai.assert +import { beforeAll, describe, it } from '@jest/globals' + import * as hed from '../validator/event' import { buildSchemas } from '../validator/schema/init' import { parseHedString } from '../parser/main' import { ParsedHedTag } from '../parser/parsedHedTag' import { HedValidator, Hed2Validator, Hed3Validator } from '../validator/event' import { generateIssue } from '../common/issues/issues' -import converterGenerateIssue from '../converter/issues' import { Schemas, SchemaSpec, SchemasSpec } from '../common/schema/types' describe('HED string and event validation', () => { @@ -373,32 +374,20 @@ describe('HED string and event validation', () => { } const expectedIssues = { red: [ - converterGenerateIssue( - 'invalidParentNode', - testStrings.red, - { - parentTag: 'Attribute/Visual/Color/Red', - }, - [10, 13], - ), + generateIssue('invalidParentNode', { + tag: 'Red', + parentTag: 'Attribute/Visual/Color/Red', + }), ], redAndBlue: [ - converterGenerateIssue( - 'invalidParentNode', - testStrings.redAndBlue, - { - parentTag: 'Attribute/Visual/Color/Red', - }, - [10, 13], - ), - converterGenerateIssue( - 'invalidParentNode', - testStrings.redAndBlue, - { - parentTag: 'Attribute/Visual/Color/Blue', - }, - [25, 29], - ), + generateIssue('invalidParentNode', { + tag: 'Red', + parentTag: 'Attribute/Visual/Color/Red', + }), + generateIssue('invalidParentNode', { + tag: 'Blue', + parentTag: 'Attribute/Visual/Color/Blue', + }), ], } // This is a no-op function since this is checked during string parsing. @@ -963,36 +952,24 @@ describe('HED string and event validation', () => { const testStrings = { // Duration/20 cm is an obviously invalid tag that should not be caught due to the first error. red: 'Property/RGB-red, Duration/20 cm', - redAndBlue: 'Property/RGB-red, Property/RGB-blue, Duration/20 cm', + redAndBlue: 'Property/RGB-red, Property/RGB-blue/Blah, Duration/20 cm', } const expectedIssues = { red: [ - converterGenerateIssue( - 'invalidParentNode', - testStrings.red, - { - parentTag: 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red', - }, - [9, 16], - ), + generateIssue('invalidParentNode', { + tag: 'RGB-red', + parentTag: 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red', + }), ], redAndBlue: [ - converterGenerateIssue( - 'invalidParentNode', - testStrings.redAndBlue, - { - parentTag: 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red', - }, - [9, 16], - ), - converterGenerateIssue( - 'invalidParentNode', - testStrings.redAndBlue, - { - parentTag: 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-blue', - }, - [27, 35], - ), + generateIssue('invalidParentNode', { + tag: 'RGB-red', + parentTag: 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-red', + }), + generateIssue('invalidParentNode', { + tag: 'RGB-blue', + parentTag: 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/RGB-color/RGB-blue', + }), ], } // This is a no-op since short-to-long conversion errors are handled in the string parsing phase. @@ -1101,14 +1078,15 @@ describe('HED string and event validation', () => { takesValue: [], full: [], extensionAllowed: [generateIssue('extension', { tag: testStrings.extensionAllowed })], - leafExtension: [generateIssue('invalidTag', { tag: testStrings.leafExtension })], + leafExtension: [generateIssue('invalidExtension', { tag: 'Something', parentTag: 'Event/Sensory-event' })], nonExtensionAllowed: [ - generateIssue('invalidTag', { - tag: testStrings.nonExtensionAllowed, + generateIssue('invalidExtension', { + tag: 'Nonsense', + parentTag: 'Event', }), ], illegalComma: [ - converterGenerateIssue('invalidTag', testStrings.illegalComma, { tag: 'This' }, [22, 26]), + generateIssue('invalidTag', { tag: 'This/Is/A/Tag' }), /* Intentionally not thrown (validation ends at parsing stage) generateIssue('extraCommaOrInvalid', { previousTag: 'Label/This_is_a_label', @@ -1562,10 +1540,10 @@ describe('HED string and event validation', () => { withUnit: 'Time-value/# ms', child: 'Left-side-of/#', extensionAllowed: 'Human/Driver/#', - invalidParent: 'Event/Nonsense/#', extensionParent: 'Item/TestDef1/#', missingRequiredUnit: 'Time-value/#', - wrongLocation: 'Item/#/Organism', + wrongLocation: 'Item/#/OtherItem', + duplicatePlaceholder: 'Item/#/#', } const expectedIssues = { takesValue: [], @@ -1576,11 +1554,6 @@ describe('HED string and event validation', () => { tag: testStrings.extensionAllowed, }), ], - invalidParent: [ - generateIssue('invalidPlaceholder', { - tag: testStrings.invalidParent, - }), - ], extensionParent: [ generateIssue('invalidPlaceholder', { tag: testStrings.extensionParent, @@ -1588,16 +1561,23 @@ describe('HED string and event validation', () => { ], missingRequiredUnit: [], wrongLocation: [ - converterGenerateIssue( - 'invalidParentNode', - testStrings.wrongLocation, - { parentTag: 'Item/Biological-item/Organism' }, - [7, 15], - ), - /* Intentionally not thrown (validation ends at parsing stage) generateIssue('invalidPlaceholder', { tag: testStrings.wrongLocation, - }), */ + }), + ], + duplicatePlaceholder: [ + generateIssue('invalidPlaceholder', { + tag: testStrings.duplicatePlaceholder, + }), + generateIssue('invalidPlaceholder', { + tag: testStrings.duplicatePlaceholder, + }), + generateIssue('invalidPlaceholder', { + tag: testStrings.duplicatePlaceholder, + }), + generateIssue('invalidTag', { + tag: testStrings.duplicatePlaceholder, + }), ], } return validatorSemantic(testStrings, expectedIssues, true) @@ -1607,10 +1587,11 @@ describe('HED string and event validation', () => { const expectedPlaceholdersTestStrings = { noPlaceholders: 'Car', noPlaceholderGroup: '(Train, Age/15, RGB-red/0.5)', - noPlaceholderDefinitionGroup: '(Definition/SimpleDefinition)', noPlaceholderTagGroupDefinition: '(Definition/TagGroupDefinition, (Square, RGB-blue))', + noPlaceholderDefinitionWithFixedValue: '(Definition/FixedTagGroupDefinition/Test, (Square, RGB-blue))', singlePlaceholder: 'RGB-green/#', definitionPlaceholder: '(Definition/PlaceholderDefinition/#, (RGB-green/#))', + definitionPlaceholderWithFixedValue: '(Definition/FixedPlaceholderDefinition/Test, (RGB-green/#))', definitionPlaceholderWithTag: 'Car, (Definition/PlaceholderWithTagDefinition/#, (RGB-green/#))', singlePlaceholderWithValidDefinitionPlaceholder: 'Time-value/#, (Definition/SinglePlaceholderWithValidPlaceholderDefinition/#, (RGB-green/#))', @@ -1628,10 +1609,11 @@ describe('HED string and event validation', () => { const noExpectedPlaceholdersTestStrings = { noPlaceholders: 'Car', noPlaceholderGroup: '(Train, Age/15, RGB-red/0.5)', - noPlaceholderDefinitionGroup: '(Definition/SimpleDefinition)', noPlaceholderTagGroupDefinition: '(Definition/TagGroupDefinition, (Square, RGB-blue))', + noPlaceholderDefinitionWithFixedValue: '(Definition/FixedTagGroupDefinition/Test, (Square, RGB-blue))', singlePlaceholder: 'RGB-green/#', definitionPlaceholder: '(Definition/PlaceholderDefinition/#, (RGB-green/#))', + definitionPlaceholderWithFixedValue: '(Definition/FixedPlaceholderDefinition/Test, (RGB-green/#))', definitionPlaceholderWithTag: 'Car, (Definition/PlaceholderWithTagDefinition/#, (RGB-green/#))', singlePlaceholderWithValidDefinitionPlaceholder: 'Time-value/#, (Definition/SinglePlaceholderWithValidPlaceholderDefinition/#, (RGB-green/#))', @@ -1667,12 +1649,28 @@ describe('HED string and event validation', () => { string: expectedPlaceholdersTestStrings.noPlaceholderTagGroupDefinition, }), ], + noPlaceholderDefinitionWithFixedValue: [ + generateIssue('missingPlaceholder', { + string: expectedPlaceholdersTestStrings.noPlaceholderDefinitionWithFixedValue, + }), + generateIssue('invalidPlaceholderInDefinition', { + definition: 'FixedTagGroupDefinition', + }), + ], singlePlaceholder: [], definitionPlaceholder: [ generateIssue('missingPlaceholder', { string: expectedPlaceholdersTestStrings.definitionPlaceholder, }), ], + definitionPlaceholderWithFixedValue: [ + generateIssue('missingPlaceholder', { + string: expectedPlaceholdersTestStrings.definitionPlaceholderWithFixedValue, + }), + generateIssue('invalidPlaceholderInDefinition', { + definition: 'FixedPlaceholderDefinition', + }), + ], definitionPlaceholderWithTag: [ generateIssue('missingPlaceholder', { string: expectedPlaceholdersTestStrings.definitionPlaceholderWithTag, @@ -1721,8 +1719,18 @@ describe('HED string and event validation', () => { noPlaceholderGroup: [], noPlaceholderDefinitionGroup: [], noPlaceholderTagGroupDefinition: [], + noPlaceholderDefinitionWithFixedValue: [ + generateIssue('invalidPlaceholderInDefinition', { + definition: 'FixedTagGroupDefinition', + }), + ], singlePlaceholder: [generateIssue('invalidPlaceholder', { tag: 'RGB-green/#' })], definitionPlaceholder: [], + definitionPlaceholderWithFixedValue: [ + generateIssue('invalidPlaceholderInDefinition', { + definition: 'FixedPlaceholderDefinition', + }), + ], definitionPlaceholderWithTag: [], singlePlaceholderWithValidDefinitionPlaceholder: [ generateIssue('invalidPlaceholder', { tag: 'Time-value/#' }), diff --git a/tests/schema.spec.js b/tests/schema.spec.js index 48e5fc47..e094397c 100644 --- a/tests/schema.spec.js +++ b/tests/schema.spec.js @@ -1,5 +1,7 @@ import chai from 'chai' const assert = chai.assert +import { beforeAll, describe, it } from '@jest/globals' + import { generateIssue } from '../common/issues/issues' import { PartneredSchema, SchemaSpec, SchemasSpec } from '../common/schema/types' import { parseSchemaSpec, parseSchemasSpec } from '../bids/schema' @@ -349,24 +351,15 @@ describe('HED schemas', () => { it('should contain all of the tag group tags', async () => { const hedSchemas = await hedSchemaPromise - const tagGroupTags = ['property/organizational-property/def-expand'] - const schemaTagGroupTags = hedSchemas.baseSchema.entries.definitions - .get('tags') - .getEntriesWithBooleanAttribute('tagGroup') + const tagGroupTags = ['def-expand'] + const schemaTagGroupTags = hedSchemas.baseSchema.entries.tags.getEntriesWithBooleanAttribute('tagGroup') assert.hasAllKeys(schemaTagGroupTags, tagGroupTags) }) it('should contain all of the top-level tag group tags', async () => { const hedSchemas = await hedSchemaPromise - const tagGroupTags = [ - 'property/organizational-property/definition', - 'property/organizational-property/event-context', - 'property/data-property/data-marker/temporal-marker/onset', - 'property/data-property/data-marker/temporal-marker/offset', - ] - const schemaTagGroupTags = hedSchemas.baseSchema.entries.definitions - .get('tags') - .getEntriesWithBooleanAttribute('topLevelTagGroup') + const tagGroupTags = ['definition', 'event-context', 'onset', 'offset'] + const schemaTagGroupTags = hedSchemas.baseSchema.entries.tags.getEntriesWithBooleanAttribute('topLevelTagGroup') assert.hasAllKeys(schemaTagGroupTags, tagGroupTags) }) @@ -403,7 +396,7 @@ describe('HED schemas', () => { weightUnits: ['g', 'gram', 'pound', 'lb'], } - const schemaUnitClasses = hedSchemas.baseSchema.entries.definitions.get('unitClasses') + const schemaUnitClasses = hedSchemas.baseSchema.entries.unitClasses for (const [unitClassName, unitClass] of schemaUnitClasses) { const defaultUnit = unitClass.defaultUnit assert.strictEqual( @@ -426,7 +419,7 @@ describe('HED schemas', () => { takesValue: 88, } - const schemaTags = hedSchemas.baseSchema.entries.definitions.get('tags') + const schemaTags = hedSchemas.baseSchema.entries.tags for (const [attribute, count] of Object.entries(expectedAttributeTagCount)) { assert.lengthOf( schemaTags.getEntriesWithBooleanAttribute(attribute), diff --git a/tests/stringParser.spec.js b/tests/stringParser.spec.js index 441fa62d..933d12e7 100644 --- a/tests/stringParser.spec.js +++ b/tests/stringParser.spec.js @@ -1,12 +1,11 @@ import chai from 'chai' const assert = chai.assert +import { beforeAll, describe, it } from '@jest/globals' import { generateIssue } from '../common/issues/issues' import { Schemas, SchemaSpec, SchemasSpec } from '../common/schema/types' -import converterGenerateIssue from '../converter/issues' import { recursiveMap } from '../utils/array' import { parseHedString } from '../parser/main' -import ParsedHedSubstring from '../parser/parsedHedSubstring' import { ParsedHedTag } from '../parser/parsedHedTag' import splitHedString from '../parser/splitHedString' import { buildSchemas } from '../validator/schema/init' @@ -134,18 +133,11 @@ describe('HED string parsing', () => { const [result, issues] = splitHedString(hedString, nullSchema) assert.isEmpty(Object.values(issues).flat(), 'Parsing issues occurred') assert.deepStrictEqual(result, [ - new ParsedHedTag('Event/Category/Sensory-event', 'Event/Category/Sensory-event', [0, 28], nullSchema), + new ParsedHedTag('Event/Category/Sensory-event', [0, 28]), + new ParsedHedTag('Item/Object/Man-made-object/Vehicle/Train', [29, 70]), new ParsedHedTag( - 'Item/Object/Man-made-object/Vehicle/Train', - 'Item/Object/Man-made-object/Vehicle/Train', - [29, 70], - nullSchema, - ), - new ParsedHedTag( - 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Purple-color/Purple', 'Property/Sensory-property/Sensory-attribute/Visual-attribute/Color/CSS-color/Purple-color/Purple', [71, 167], - nullSchema, ), ]) }) @@ -156,47 +148,34 @@ describe('HED string parsing', () => { const [result, issues] = splitHedString(hedString, nullSchema) assert.isEmpty(Object.values(issues).flat(), 'Parsing issues occurred') assert.deepStrictEqual(result, [ - new ParsedHedTag('/Action/Move/Flex', '/Action/Move/Flex', [0, 17], nullSchema), + new ParsedHedTag('/Action/Move/Flex', [0, 17]), new ParsedHedGroup( [ - new ParsedHedTag( - 'Relation/Spatial-relation/Left-side-of', - 'Relation/Spatial-relation/Left-side-of', - [19, 57], - nullSchema, - ), - new ParsedHedTag('/Action/Move/Bend', '/Action/Move/Bend', [58, 75], nullSchema), - new ParsedHedTag('/Upper-extremity/Elbow', '/Upper-extremity/Elbow', [76, 98], nullSchema), + new ParsedHedTag('Relation/Spatial-relation/Left-side-of', [19, 57]), + new ParsedHedTag('/Action/Move/Bend', [58, 75]), + new ParsedHedTag('/Upper-extremity/Elbow', [76, 98]), ], nullSchema, hedString, [18, 99], ), - new ParsedHedTag('/Position/X-position/70 px', '/Position/X-position/70 px', [100, 126], nullSchema), - new ParsedHedTag('/Position/Y-position/23 px', '/Position/Y-position/23 px', [127, 153], nullSchema), + new ParsedHedTag('/Position/X-position/70 px', [100, 126]), + new ParsedHedTag('/Position/Y-position/23 px', [127, 153]), ]) }) it('should not include blanks', () => { const testStrings = { - doubleComma: '/Item/Object/Man-made-object/Vehicle/Car,,/Action/Perform/Operate', trailingBlank: '/Item/Object/Man-made-object/Vehicle/Car, /Action/Perform/Operate,', } const expectedList = [ - new ParsedHedTag( - '/Item/Object/Man-made-object/Vehicle/Car', - '/Item/Object/Man-made-object/Vehicle/Car', - [0, 40], - nullSchema, - ), - new ParsedHedTag('/Action/Perform/Operate', '/Action/Perform/Operate', [42, 65], nullSchema), + new ParsedHedTag('/Item/Object/Man-made-object/Vehicle/Car', [0, 40]), + new ParsedHedTag('/Action/Perform/Operate', [42, 65]), ] const expectedResults = { - doubleComma: expectedList, trailingBlank: expectedList, } const expectedIssues = { - doubleComma: {}, trailingBlank: {}, } validatorWithIssues(testStrings, expectedResults, expectedIssues, (string) => { @@ -238,7 +217,7 @@ describe('HED string parsing', () => { openingAndClosingDoubleQuotedSlash: formattedHedTag, } validatorWithoutIssues(testStrings, expectedResults, (string) => { - const parsedTag = new ParsedHedTag(string, string, [], nullSchema) + const parsedTag = new ParsedHedTag(string, []) return parsedTag.formattedTag }) }) @@ -371,16 +350,14 @@ describe('HED string parsing', () => { simple: {}, groupAndTag: {}, invalidTag: { - conversion: [converterGenerateIssue('invalidTag', testStrings.invalidTag, {}, [0, 10])], + conversion: [generateIssue('invalidTag', { tag: expectedResults.invalidTag[0] })], }, invalidParentNode: { conversion: [ - converterGenerateIssue( - 'invalidParentNode', - testStrings.invalidParentNode, - { parentTag: 'Item/Object/Man-made-object/Vehicle/Train' }, - [4, 9], - ), + generateIssue('invalidParentNode', { + parentTag: 'Item/Object/Man-made-object/Vehicle/Train', + tag: 'Train', + }), ], }, } @@ -412,6 +389,7 @@ describe('HED string parsing', () => { parsedStrings.push(parsedString) issues.push(...Object.values(parsingIssues).flat()) } + assert.isEmpty(issues, 'Parsing issues') const [baseString, refString, correctString] = parsedStrings const replacementMap = new Map([['stim_file', refString]]) const columnSplicer = new ColumnSplicer( @@ -422,9 +400,8 @@ describe('HED string parsing', () => { ) const splicedString = columnSplicer.splice() const splicingIssues = columnSplicer.issues - issues.push(...splicingIssues) assert.strictEqual(splicedString.format(), correctString.format(), 'Full string') - assert.isEmpty(issues, 'Issues') + assert.isEmpty(splicingIssues, 'Splicing issues') }) }) diff --git a/utils/__tests__/array.spec.js b/utils/__tests__/array.spec.js index 99f7bb22..c0b2908a 100644 --- a/utils/__tests__/array.spec.js +++ b/utils/__tests__/array.spec.js @@ -1,5 +1,7 @@ import chai from 'chai' const assert = chai.assert +import { describe, it } from '@jest/globals' + import * as arrayUtils from '../array' describe('Array utility functions', () => { diff --git a/utils/__tests__/hed.spec.js b/utils/__tests__/hed.spec.js index bfa85141..4caf5304 100644 --- a/utils/__tests__/hed.spec.js +++ b/utils/__tests__/hed.spec.js @@ -1,5 +1,7 @@ import chai from 'chai' const assert = chai.assert +import { beforeAll, describe, it } from '@jest/globals' + import * as hed from '../hedStrings' import { SchemaSpec, SchemasSpec } from '../../common/schema/types' import { buildSchemas } from '../../validator/schema/init' diff --git a/utils/__tests__/map.spec.js b/utils/__tests__/map.spec.js index a66fcd44..5fb2c2bf 100644 --- a/utils/__tests__/map.spec.js +++ b/utils/__tests__/map.spec.js @@ -1,5 +1,7 @@ import chai from 'chai' const assert = chai.assert +import { describe, it } from '@jest/globals' + import isEqual from 'lodash/isEqual' import * as mapUtils from '../map' diff --git a/utils/__tests__/string.spec.js b/utils/__tests__/string.spec.js index 32def2ea..e9764e65 100644 --- a/utils/__tests__/string.spec.js +++ b/utils/__tests__/string.spec.js @@ -1,5 +1,7 @@ import chai from 'chai' const assert = chai.assert +import { describe, it } from '@jest/globals' + import * as stringUtils from '../string' describe('String utility functions', () => { diff --git a/utils/hedData.js b/utils/hedData.js index 3d39508c..13835693 100644 --- a/utils/hedData.js +++ b/utils/hedData.js @@ -1,6 +1,7 @@ import lt from 'semver/functions/lt' import { ParsedHed3Tag } from '../parser/parsedHedTag' +import { TagSpec } from '../parser/tokenizer' /** * Determine the HED generation for a base schema version number. @@ -34,7 +35,11 @@ export const mergeParsingIssues = function (previousIssues, currentIssues) { export const getParsedParentTags = function (hedSchemas, shortTag) { const parentTags = new Map() for (const [schemaNickname, schema] of hedSchemas.schemas) { - const parentTag = new ParsedHed3Tag(shortTag, shortTag, [0, shortTag.length - 1], hedSchemas, schemaNickname) + const parentTag = new ParsedHed3Tag( + new TagSpec(shortTag, 0, shortTag.length - 1, schemaNickname), + hedSchemas, + shortTag, + ) parentTags.set(schema, parentTag) parentTag.conversionIssues = parentTag.conversionIssues.filter((issue) => issue.internalCode !== 'invalidTag') } diff --git a/utils/hedStrings.js b/utils/hedStrings.js index f09aaad2..ed926c4c 100644 --- a/utils/hedStrings.js +++ b/utils/hedStrings.js @@ -16,7 +16,7 @@ export const replaceTagNameWithPound = function (formattedTag) { /** * Get the indices of all slashes in a HED tag. */ -const getTagSlashIndices = function (tag) { +export const getTagSlashIndices = function (tag) { const indices = [] let i = -1 while ((i = tag.indexOf('/', i + 1)) >= 0) { diff --git a/validator/event/hed3.js b/validator/event/hed3.js index bc482615..5ea382bb 100644 --- a/validator/event/hed3.js +++ b/validator/event/hed3.js @@ -96,9 +96,9 @@ export class Hed3Validator extends HedValidator { _checkForTagAttribute(attribute, fn) { const schemas = this.hedSchemas.schemas.values() for (const schema of schemas) { - const tags = schema.entries.definitions.get('tags').getEntriesWithBooleanAttribute(attribute) - for (const tag of tags) { - fn(tag.name) + const tags = schema.entries.tags.getEntriesWithBooleanAttribute(attribute) + for (const tag of tags.values()) { + fn(tag.longName) } } } @@ -213,7 +213,8 @@ export class Hed3Validator extends HedValidator { _checkDefinitionPlaceholderStringSyntaxInGroup(tagGroup) { // Count of placeholders within this Definition group. let definitionPlaceholders = 0 - const definitionHasPlaceholder = tagGroup.definitionValue === '#' + const definitionValue = tagGroup.definitionValue + const definitionHasPlaceholder = definitionValue === '#' const definitionName = tagGroup.definitionName for (const tag of tagGroup.tagIterator()) { if (!definitionHasPlaceholder || tag !== tagGroup.definitionTag) { @@ -221,7 +222,7 @@ export class Hed3Validator extends HedValidator { } } const isValid = - (!definitionHasPlaceholder && definitionPlaceholders === 0) || + (definitionValue === '' && definitionPlaceholders === 0) || (definitionHasPlaceholder && definitionPlaceholders === 1) if (!isValid) { this.pushIssue('invalidPlaceholderInDefinition', { diff --git a/validator/hed2/parser/parsedHed2Tag.js b/validator/hed2/parser/parsedHed2Tag.js index e1394b56..7f259929 100644 --- a/validator/hed2/parser/parsedHed2Tag.js +++ b/validator/hed2/parser/parsedHed2Tag.js @@ -5,6 +5,21 @@ import { ParsedHedTag } from '../../../parser/parsedHedTag' * ParsedHedTag class */ export class ParsedHed2Tag extends ParsedHedTag { + /** + * Constructor. + * + * @param {string} originalTag The original HED tag. + * @param {string} hedString The original HED string. + * @param {number[]} originalBounds The bounds of the HED tag in the original HED string. + * @param {Schemas} hedSchemas The collection of HED schemas. + * @param {string} schemaName The label of this tag's schema in the dataset's schema spec. + */ + constructor(originalTag, hedString, originalBounds, hedSchemas, schemaName = '') { + super(originalTag, originalBounds) + + this._convertTag(hedString, hedSchemas, schemaName) + } + /** * Convert this tag to long form. * diff --git a/validator/schema/hed3.js b/validator/schema/hed3.js index 4e08a035..aaece20f 100644 --- a/validator/schema/hed3.js +++ b/validator/schema/hed3.js @@ -1,3 +1,6 @@ +import zip from 'lodash/zip' +import semver from 'semver' + // TODO: Switch require once upstream bugs are fixed. // import xpath from 'xml2js-xpath' // Temporary @@ -5,25 +8,54 @@ import * as xpath from '../../utils/xpath' import { SchemaParser } from './parser' import { + nodeProperty, + SchemaAttribute, + schemaAttributeProperty, SchemaEntries, SchemaEntryManager, - SchemaAttribute, SchemaProperty, SchemaTag, + SchemaTagManager, SchemaUnit, SchemaUnitClass, SchemaUnitModifier, SchemaValueClass, - nodeProperty, - schemaAttributeProperty, + SchemaValueTag, } from './types' import { generateIssue, IssueError } from '../../common/issues/issues' -import { buildMappingObject } from '../../converter/schema' -import semver from 'semver' const lc = (str) => str.toLowerCase() export class Hed3SchemaParser extends SchemaParser { + /** + * @type {Map} + */ + properties + /** + * @type {Map} + */ + attributes + /** + * The schema's value classes. + * @type {SchemaEntryManager} + */ + valueClasses + /** + * The schema's unit classes. + * @type {SchemaEntryManager} + */ + unitClasses + /** + * The schema's unit modifiers. + * @type {SchemaEntryManager} + */ + unitModifiers + /** + * The schema's tags. + * @type {SchemaTagManager} + */ + tags + constructor(rootElement) { super(rootElement) this._versionDefinitions = {} @@ -37,7 +69,6 @@ export class Hed3SchemaParser extends SchemaParser { populateDictionaries() { this.parseProperties() this.parseAttributes() - this.definitions = new Map() this.parseUnitModifiers() this.parseUnitClasses() this.parseTags() @@ -58,10 +89,16 @@ export class Hed3SchemaParser extends SchemaParser { } } + /** + * Retrieve all the tags in the schema. + * + * @param {string} tagElementName The name of the tag element. + * @returns {Map} The tag names and XML elements. + */ getAllTags(tagElementName = 'node') { const tagElements = xpath.find(this.rootElement, '//' + tagElementName) - const tags = tagElements.map((element) => this.getTagPathFromTagElement(element)) - return [tags, tagElements] + const tags = tagElements.map((element) => this.getElementTagName(element)) + return new Map(zip(tagElements, tags)) } // Rewrite starts here. @@ -121,7 +158,7 @@ export class Hed3SchemaParser extends SchemaParser { const booleanAttributes = booleanAttributeDefinitions.get(name) valueClasses.set(name, new SchemaValueClass(name, booleanAttributes, valueAttributes)) } - this.definitions.set('valueClasses', new SchemaEntryManager(valueClasses)) + this.valueClasses = new SchemaEntryManager(valueClasses) } parseUnitModifiers() { @@ -131,7 +168,7 @@ export class Hed3SchemaParser extends SchemaParser { const booleanAttributes = booleanAttributeDefinitions.get(name) unitModifiers.set(name, new SchemaUnitModifier(name, booleanAttributes, valueAttributes)) } - this.definitions.set('unitModifiers', new SchemaEntryManager(unitModifiers)) + this.unitModifiers = new SchemaEntryManager(unitModifiers) } parseUnitClasses() { @@ -143,13 +180,13 @@ export class Hed3SchemaParser extends SchemaParser { const booleanAttributes = booleanAttributeDefinitions.get(name) unitClasses.set(name, new SchemaUnitClass(name, booleanAttributes, valueAttributes, unitClassUnits.get(name))) } - this.definitions.set('unitClasses', new SchemaEntryManager(unitClasses)) + this.unitClasses = new SchemaEntryManager(unitClasses) } parseUnits() { const unitClassUnits = new Map() const unitClassElements = this.getElementsByName('unitClassDefinition') - const unitModifiers = this.definitions.get('unitModifiers') + const unitModifiers = this.unitModifiers for (const element of unitClassElements) { const elementName = this.getElementTagName(element) const units = new Map() @@ -170,28 +207,33 @@ export class Hed3SchemaParser extends SchemaParser { } parseTags() { - const [tags, tagElements] = this.getAllTags() - const lowercaseTags = tags.map(lc) - this.tags = new Set(lowercaseTags) + const tags = this.getAllTags() + const shortTags = new Map() + for (const tagElement of tags.keys()) { + const shortKey = + this.getElementTagName(tagElement) === '#' + ? this.getParentTagName(tagElement) + '-#' + : this.getElementTagName(tagElement) + shortTags.set(tagElement, shortKey) + } const [booleanAttributeDefinitions, valueAttributeDefinitions] = this._parseAttributeElements( - tagElements, - (element) => this.getTagPathFromTagElement(element), + tags.keys(), + (element) => shortTags.get(element), ) const recursiveAttributes = this._getRecursiveAttributes() - const unitClasses = this.definitions.get('unitClasses') const tagUnitClassAttribute = this.attributes.get('unitClass') + const tagTakesValueAttribute = this.attributes.get('takesValue') const tagUnitClassDefinitions = new Map() const recursiveChildren = new Map() - tags.forEach((tagName, index) => { - const tagElement = tagElements[index] + for (const [tagElement, tagName] of shortTags) { const valueAttributes = valueAttributeDefinitions.get(tagName) if (valueAttributes.has(tagUnitClassAttribute)) { tagUnitClassDefinitions.set( tagName, valueAttributes.get(tagUnitClassAttribute).map((unitClassName) => { - return unitClasses.getEntry(unitClassName) + return this.unitClasses.getEntry(unitClassName) }), ) valueAttributes.delete(tagUnitClassAttribute) @@ -203,31 +245,46 @@ export class Hed3SchemaParser extends SchemaParser { } recursiveChildren.set(attribute, children) } - }) + } for (const [attribute, childTagElements] of recursiveChildren) { for (const tagElement of childTagElements) { - const tagName = this.getTagPathFromTagElement(tagElement) + const tagName = this.getElementTagName(tagElement) booleanAttributeDefinitions.get(tagName).add(attribute) } } const tagEntries = new Map() for (const [name, valueAttributes] of valueAttributeDefinitions) { + if (tagEntries.has(name)) { + throw new IssueError(generateIssue('duplicateTagsInSchema', {})) + } const booleanAttributes = booleanAttributeDefinitions.get(name) const unitClasses = tagUnitClassDefinitions.get(name) - tagEntries.set(lc(name), new SchemaTag(name, booleanAttributes, valueAttributes, unitClasses)) + if (booleanAttributes.has(tagTakesValueAttribute)) { + tagEntries.set(lc(name), new SchemaValueTag(name, booleanAttributes, valueAttributes, unitClasses)) + } else { + tagEntries.set(lc(name), new SchemaTag(name, booleanAttributes, valueAttributes, unitClasses)) + } } - for (const tagElement of tagElements) { - const tagName = this.getTagPathFromTagElement(tagElement) - const parentTagName = this.getParentTagPath(tagElement) + for (const tagElement of tags.keys()) { + const tagName = shortTags.get(tagElement) + const parentTagName = shortTags.get(tagElement.$parent) if (parentTagName) { tagEntries.get(lc(tagName))._parent = tagEntries.get(lc(parentTagName)) } + if (this.getElementTagName(tagElement) === '#') { + tagEntries.get(lc(parentTagName))._valueTag = tagEntries.get(lc(tagName)) + } } - this.definitions.set('tags', new SchemaEntryManager(tagEntries)) + const longNameTagEntries = new Map() + for (const tag of tagEntries.values()) { + longNameTagEntries.set(lc(tag.longName), tag) + } + + this.tags = new SchemaTagManager(tagEntries, longNameTagEntries) } _parseDefinitions(category) { @@ -378,28 +435,19 @@ export class Hed3PartneredSchemaMerger { /** * The source schema's tag collection. * - * @return {SchemaEntryManager} + * @return {SchemaTagManager} */ get sourceTags() { - return this.source.entries.definitions.get('tags') + return this.source.entries.tags } /** * The destination schema's tag collection. * - * @return {SchemaEntryManager} + * @return {SchemaTagManager} */ get destinationTags() { - return this.destination.entries.definitions.get('tags') - } - - /** - * The source schema's mapping from long tag names to TagEntry objects. - * - * @return {Map} - */ - get sourceLongToTags() { - return this.source.mapping.longToTags + return this.destination.entries.tags } /** @@ -409,7 +457,6 @@ export class Hed3PartneredSchemaMerger { */ mergeData() { this.mergeTags() - this.destination.mapping = buildMappingObject(this.destination.entries) return this.destination } @@ -433,15 +480,15 @@ export class Hed3PartneredSchemaMerger { return } - const shortName = this.sourceLongToTags.get(tag.name).shortTag - if (this.destination.mapping.shortToTags.has(shortName.toLowerCase())) { + const shortName = tag.name + if (this.destinationTags.hasEntry(shortName.toLowerCase())) { throw new IssueError(generateIssue('lazyPartneredSchemasShareTag', { tag: shortName })) } const rootedTagShortName = tag.getNamedAttributeValue('rooted') if (rootedTagShortName) { const parentTag = tag.parent - if (this.sourceLongToTags.get(parentTag?.name)?.shortTag?.toLowerCase() !== rootedTagShortName?.toLowerCase()) { + if (parentTag?.name?.toLowerCase() !== rootedTagShortName?.toLowerCase()) { throw new Error(`Node ${shortName} is improperly rooted.`) } } @@ -470,12 +517,24 @@ export class Hed3PartneredSchemaMerger { * @type {SchemaUnitClass[]} */ const unitClasses = tag.unitClasses.map( - (unitClass) => this.destination.entries.unitClassMap.getEntry(unitClass.name) ?? unitClass, + (unitClass) => this.destination.entries.unitClasses.getEntry(unitClass.name) ?? unitClass, ) - const newTag = new SchemaTag(tag.name, booleanAttributes, valueAttributes, unitClasses) - newTag._parent = this.destinationTags.getEntry(tag.parent?.name?.toLowerCase()) + let newTag + if (tag instanceof SchemaValueTag) { + newTag = new SchemaValueTag(tag.name, booleanAttributes, valueAttributes, unitClasses) + } else { + newTag = new SchemaTag(tag.name, booleanAttributes, valueAttributes, unitClasses) + } + const destinationParentTag = this.destinationTags.getEntry(tag.parent?.name?.toLowerCase()) + if (destinationParentTag) { + newTag._parent = destinationParentTag + if (newTag instanceof SchemaValueTag) { + newTag.parent._valueTag = newTag + } + } this.destinationTags._definitions.set(newTag.name.toLowerCase(), newTag) + this.destinationTags._definitionsByLongName.set(newTag.longName.toLowerCase(), newTag) } } diff --git a/validator/schema/init.js b/validator/schema/init.js index 57dde6a9..c7f9c170 100644 --- a/validator/schema/init.js +++ b/validator/schema/init.js @@ -3,7 +3,6 @@ import semver from 'semver' import { Schema, Schemas, Hed2Schema, Hed3Schema, SchemasSpec, PartneredSchema } from '../../common/schema/types' import loadSchema from '../../common/schema/loader' -import { buildMappingObject } from '../../converter/schema' import { setParent } from '../../utils/xml2js' import { Hed2SchemaParser } from '../hed2/schema/hed2SchemaParser' @@ -44,8 +43,7 @@ export const buildSchemaAttributesObject = function (xmlData) { const buildSchemaObject = function (xmlData) { const schemaAttributes = buildSchemaAttributesObject(xmlData) if (isHed3Schema(xmlData)) { - const mapping = buildMappingObject(schemaAttributes) - return new Hed3Schema(xmlData, schemaAttributes, mapping) + return new Hed3Schema(xmlData, schemaAttributes) } else { return new Hed2Schema(xmlData, schemaAttributes) } diff --git a/validator/schema/types.js b/validator/schema/types.js index 5131a8f3..83dcefe8 100644 --- a/validator/schema/types.js +++ b/validator/schema/types.js @@ -11,19 +11,34 @@ pluralize.addUncountableRule('hertz') export class SchemaEntries extends Memoizer { /** * The schema's properties. - * @type {SchemaEntryManager} + * @type {SchemaEntryManager} */ properties /** * The schema's attributes. - * @type {SchemaEntryManager} + * @type {SchemaEntryManager} */ attributes /** - * The schema's definitions. - * @type {Map} + * The schema's value classes. + * @type {SchemaEntryManager} */ - definitions + valueClasses + /** + * The schema's unit classes. + * @type {SchemaEntryManager} + */ + unitClasses + /** + * The schema's unit modifiers. + * @type {SchemaEntryManager} + */ + unitModifiers + /** + * The schema's tags. + * @type {SchemaTagManager} + */ + tags /** * Constructor. @@ -33,15 +48,10 @@ export class SchemaEntries extends Memoizer { super() this.properties = new SchemaEntryManager(schemaParser.properties) this.attributes = new SchemaEntryManager(schemaParser.attributes) - this.definitions = schemaParser.definitions - } - - /** - * Get the schema's unit classes. - * @returns {SchemaEntryManager} - */ - get unitClassMap() { - return this.definitions.get('unitClasses') + this.valueClasses = schemaParser.valueClasses + this.unitClasses = schemaParser.unitClasses + this.unitModifiers = schemaParser.unitModifiers + this.tags = schemaParser.tags } /** @@ -50,7 +60,7 @@ export class SchemaEntries extends Memoizer { get allUnits() { return this._memoize('allUnits', () => { const units = [] - for (const unitClass of this.unitClassMap.values()) { + for (const unitClass of this.unitClasses.values()) { const unitClassUnits = unitClass.units units.push(...unitClassUnits) } @@ -63,8 +73,7 @@ export class SchemaEntries extends Memoizer { * @returns {Map} */ get SIUnitModifiers() { - const unitModifiers = this.definitions.get('unitModifiers') - return unitModifiers.getEntriesWithBooleanAttribute('SIUnitModifier') + return this.unitModifiers.getEntriesWithBooleanAttribute('SIUnitModifier') } /** @@ -72,8 +81,7 @@ export class SchemaEntries extends Memoizer { * @returns {Map} */ get SIUnitSymbolModifiers() { - const unitModifiers = this.definitions.get('unitModifiers') - return unitModifiers.getEntriesWithBooleanAttribute('SIUnitSymbolModifier') + return this.unitModifiers.getEntriesWithBooleanAttribute('SIUnitSymbolModifier') } /** @@ -84,10 +92,10 @@ export class SchemaEntries extends Memoizer { * @returns {boolean} Whether this tag has this attribute. */ tagHasAttribute(tag, tagAttribute) { - if (!this.definitions.get('tags').hasEntry(tag)) { + if (!this.tags.hasLongNameEntry(tag)) { return false } - return this.definitions.get('tags').getEntry(tag).hasAttributeName(tagAttribute) + return this.tags.getLongNameEntry(tag).hasAttributeName(tagAttribute) } } @@ -183,7 +191,20 @@ export class SchemaEntryManager extends Memoizer { * @returns {Map} The filtered map. */ filter(fn) { - const pairArray = Array.from(this._definitions.entries()) + return SchemaEntryManager._filterDefinitionMap(this._definitions, fn) + } + + /** + * Filter a definition map. + * + * @template T + * @param {Map} definitionMap The definition map. + * @param {function ([string, T]): boolean} fn The filtering function. + * @returns {Map} The filtered map. + * @protected + */ + static _filterDefinitionMap(definitionMap, fn) { + const pairArray = Array.from(definitionMap.entries()) return new Map(pairArray.filter((entry) => fn(entry))) } @@ -197,10 +218,64 @@ export class SchemaEntryManager extends Memoizer { } } +/** + * A manager of {@link SchemaTag} objects. + * + * @extends {SchemaEntryManager} + */ +export class SchemaTagManager extends SchemaEntryManager { + /** + * The mapping of tags by long name. + * @type {Map} + */ + _definitionsByLongName + + /** + * Constructor. + * + * @param {Map} byShortName The mapping of tags by short name. + * @param {Map} byLongName The mapping of tags by long name. + */ + constructor(byShortName, byLongName) { + super(byShortName) + this._definitionsByLongName = byLongName + } + + /** + * Determine whether the tag with the given name exists. + * + * @param {string} longName The long name of the tag. + * @return {boolean} Whether the tag exists. + */ + hasLongNameEntry(longName) { + return this._definitionsByLongName.has(longName) + } + + /** + * Get the tag with the given name. + * + * @param {string} longName The long name of the tag to retrieve. + * @return {SchemaTag} The tag with that name. + */ + getLongNameEntry(longName) { + return this._definitionsByLongName.get(longName) + } + + /** + * Filter the map underlying this manager using the long name. + * + * @param {function ([string, SchemaTag]): boolean} fn The filtering function. + * @returns {Map} The filtered map. + */ + filterByLongName(fn) { + return SchemaEntryManager._filterDefinitionMap(this._definitionsByLongName, fn) + } +} + /** * SchemaEntry class */ -export class SchemaEntry { +export class SchemaEntry extends Memoizer { /** * The name of this schema entry. * @type {string} @@ -208,6 +283,7 @@ export class SchemaEntry { _name constructor(name) { + super() this._name = name } @@ -599,16 +675,24 @@ export class SchemaValueClass extends SchemaEntryWithAttributes { * A tag in a HED schema. */ export class SchemaTag extends SchemaEntryWithAttributes { + /** + * This tag's parent tag. + * @type {SchemaTag} + * @private + */ + _parent /** * This tag's unit classes. * @type {SchemaUnitClass[]} + * @private */ _unitClasses /** - * This tag's parent tag. - * @type {SchemaTag} + * This tag's value-taking child. + * @type {SchemaValueTag} + * @private */ - _parent + _valueTag /** * Constructor. @@ -637,7 +721,15 @@ export class SchemaTag extends SchemaEntryWithAttributes { * @returns {boolean} */ get hasUnitClasses() { - return this._unitClasses.length !== 0 + return this.unitClasses.length !== 0 + } + + /** + * This tag's value-taking child. + * @returns {SchemaValueTag} + */ + get valueTag() { + return this._valueTag } /** @@ -647,4 +739,90 @@ export class SchemaTag extends SchemaEntryWithAttributes { get parent() { return this._parent } + + /** + * Return all of this tag's ancestors. + * @returns {*[]} + */ + get ancestors() { + return this._memoize('ancestors', () => { + if (this.parent) { + return [this.parent, ...this.parent.ancestors] + } + return [] + }) + } + + /** + * This tag's long name. + * @returns {string} + */ + get longName() { + const nameParts = this.ancestors.map((parentTag) => parentTag.name) + nameParts.reverse().push(this.name) + return nameParts.join('/') + } + + /** + * Extend this tag's short name. + * + * @param {string} extension The extension. + * @returns {string} The extended short string. + */ + extend(extension) { + if (extension) { + return this.name + '/' + extension + } else { + return this.name + } + } + + /** + * Extend this tag's long name. + * + * @param {string} extension The extension. + * @returns {string} The extended long string. + */ + longExtend(extension) { + if (extension) { + return this.longName + '/' + extension + } else { + return this.longName + } + } +} + +/** + * A value-taking tag in a HED schema. + */ +export class SchemaValueTag extends SchemaTag { + /** + * This tag's long name. + * @returns {string} + */ + get longName() { + const nameParts = this.ancestors.map((parentTag) => parentTag.name) + nameParts.reverse().push('#') + return nameParts.join('/') + } + + /** + * Extend this tag's short name. + * + * @param {string} extension The extension. + * @returns {string} The extended short string. + */ + extend(extension) { + return this.parent.extend(extension) + } + + /** + * Extend this tag's long name. + * + * @param {string} extension The extension. + * @returns {string} The extended long string. + */ + longExtend(extension) { + return this.parent.longExtend(extension) + } }