fix: Added information whether a token is skipped or not (#3)

Author: shirayu

README.md

@@ -20,7 +20,7 @@ Install with [npm](https://www.npmjs.com/):
 
 ### createTokenMatcher(expectedTokens): function
 
-`createTokenMatcher()` return `function(token): { match: boolean, tokens?: Array }`.
+`createTokenMatcher()` return `function(token): { match: boolean, tokens?: Array, skipped? Array }`.
 
 We want to check "名詞かもしれない" contain "かも" token. 
 Write following:
@@ -107,7 +107,7 @@ If want to get matched token, write following:
 ```js
 let resultTokens = [];
 const result = tokens.some(token => {
-    const {match, tokens} = expectToken(token);
+    const {match, tokens, skipped} = expectToken(token);
     resultTokens = tokens;
     return match;
 });

src/morpheme-match.js

@@ -25,31 +25,38 @@ module.exports = function createTokenMatcher(matchedTokens) {
     let currentTokenPosition = 0;
     const tokenCount = matchedTokens.length;
     const matchTokens = [];
-    return (token) => {
+    const matchSkipped = [];
+    return (token, index) => {
         while (currentTokenPosition < tokenCount) {
             const expectedToken = matchedTokens[currentTokenPosition];
             if (matchToken(token, expectedToken)) {
                 matchTokens.push(token);
+                matchSkipped.push(false);
                 currentTokenPosition += 1;
                 break;
             } else if (expectedToken["_skippable"]) {
                 currentTokenPosition += 1;
+                matchSkipped.push(true);
             } else {
                 // reset position
                 matchTokens.length = 0;
+                matchSkipped.length = 0;
                 currentTokenPosition = 0;
                 break;
             }
         }
         // match all tokens
         if (currentTokenPosition === tokenCount) {
             const tokens = matchTokens.slice();
+            const skipped = matchSkipped.slice();
             // match -> reset
             currentTokenPosition = 0;
             matchTokens.length = 0;
+            matchSkipped.length = 0;
             return {
                 match: true,
-                tokens: tokens
+                tokens: tokens,
+                skipped: skipped,
             };
         }
         return {

test/morpheme-match-test.js

@@ -3,7 +3,7 @@
 import expectTokenStream from "../src/morpheme-match";
 const assert = require("power-assert");
 describe("expectTokenStream", function() {
-    it("should return {match, tokens}", function() {
+    it("should return {match, tokens, skipped}", function() {
         // http://localhost:8080/#名詞(かも)しれない
         const expectToken = expectTokenStream([
             {
@@ -28,6 +28,7 @@ describe("expectTokenStream", function() {
             },
             {
                 "surface_form": "しれ",
+                "_skippable": true,
             },
             {
                 "surface_form": "、",
@@ -90,15 +91,18 @@ describe("expectTokenStream", function() {
             }
         ];
         let resultTokens = [];
+        let resultSkipped = [];
         const result = tokens.some(token => {
-            const {match, tokens} = expectToken(token);
+            const {match, tokens, skipped} = expectToken(token);
             if (!match) {
                 assert(tokens === undefined);
             }
             resultTokens = tokens;
+            resultSkipped = skipped;
             return match;
         });
         assert(result);
         assert.equal(resultTokens.length, 2);
+        assert.deepEqual(resultSkipped, [false, true, true, false, true, true]);
     });
 });