Merge pull request #210 from VisLab/update-tokenizer

Reorganized the tests, eliminating internal log  @happy5214 I'm going to go ahead and merge since this PR is on refactoring the tests as discussed.

.codeclimate.yml

@@ -0,0 +1,83 @@
+version: '2' # required to adjust maintainability checks
+
+checks:
+  argument-count:
+    enabled: true
+    config:
+      threshold: 4
+  complex-logic:
+    enabled: true
+    config:
+      threshold: 4
+  file-lines:
+    enabled: true
+    config:
+      threshold: 300
+  method-complexity:
+    enabled: true
+    config:
+      threshold: 5
+  method-count:
+    enabled: true
+    config:
+      threshold: 20
+  method-lines:
+    enabled: true
+    config:
+      threshold: 50
+  nested-control-flow:
+    enabled: true
+    config:
+      threshold: 4
+  return-statements:
+    enabled: true
+    config:
+      threshold: 4
+  similar-code:
+    enabled: true
+    config:
+      threshold: #language-specific defaults. overrides affect all languages.
+  identical-code:
+    enabled: true
+    config:
+      threshold: #language-specific defaults. overrides affect all languages.
+
+exclude_patterns:
+  - 'config/'
+  - 'db/'
+  - 'dist/'
+  - 'features/'
+  - '**/node_modules/'
+  - 'script/'
+  - '**/spec/'
+  - '**/test/'
+  - '**/tests/'
+  - '**/spec_tests/'
+  - '**/docs/'
+  - 'Tests/'
+  - '**/vendor/'
+  - '**/*_test.go'
+  - '**/*.d.ts'
+  - '**/*__init__.py'
+  - '**/*setup.py'
+  - '**/setup.cfg'
+  - '**/static/'
+  - '**/constants/'
+  - '**/deploy_hed/'
+  - '**/README*'
+  - '**/.gitignore'
+  - '**/LICENSE'
+  - '**/examples/'
+  - '**/*.yml'
+  - '**/*.yaml'
+  - '**/config_template.py'
+  - '**/*.txt'
+  - '**/*.html'
+  - '**/.coveragerc'
+  - '**/*.md'
+  - '**/*.rst'
+  - '**/templates/'
+  - '**/conf.py'
+  - '**/deprecated/'
+  - '**/versioneer.py'
+  - '**/_version.py'

.github/workflows/ci.yml

@@ -47,4 +47,5 @@ jobs:
         env:
           CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
         with:
+          config: .codeclimate.yml
           coverageCommand: ./node_modules/.bin/jest --coverage --testPathIgnorePatterns=spec_tests

parser/tokenizer.js

@@ -141,19 +141,18 @@ export class HedStringTokenizer {
     // Empty strings cannot be tokenized
     if (this.hedString.trim().length === 0) {
       this.pushIssue('emptyTagFound', 0)
-      return [null, null, { syntax: this.issues }]
+      return [[], null, { syntax: this.issues }]
     }
     for (let i = 0; i < this.hedString.length; i++) {
       const character = this.hedString.charAt(i)
       this.handleCharacter(i, character)
-      //this.tokenizeCharacter(i, character)
       if (this.issues.length > 0) {
-        return [null, null, { syntax: this.issues }]
+        return [[], null, { syntax: this.issues }]
       }
     }
     this.finalizeTokenizer()
     if (this.issues.length > 0) {
-      return [null, null, { syntax: this.issues }]
+      return [[], null, { syntax: this.issues }]
     } else {
       return [this.state.currentGroupStack.pop(), this.state.parenthesesStack.pop(), { syntax: [] }]
     }