diff --git a/README.md b/README.md index 45a9048..9fd3b1d 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,26 @@ const expectToken = createTokenMatcher([ ]); ``` +キー`_skippable`が`true`の場合はマッチしない場合は無視されます。 + + +```js +const expectToken = createTokenMatcher([ + { + "surface_form": "かも", + }, + { + "surface_form": "、", + "_skippable": true, + }, + { + "surface_form": "しれ", + }, +]); +``` + + + ## 関連 - [azu/morpheme-match-all: A wrapper of morpheme-match API. Match all kuromoji's tokens.](https://github.com/azu/morpheme-match-all) diff --git a/src/morpheme-match.js b/src/morpheme-match.js index 4a59e7c..835193c 100644 --- a/src/morpheme-match.js +++ b/src/morpheme-match.js @@ -26,14 +26,20 @@ module.exports = function createTokenMatcher(matchedTokens) { const tokenCount = matchedTokens.length; const matchTokens = []; return (token) => { - const expectedToken = matchedTokens[currentTokenPosition]; - if (matchToken(token, expectedToken)) { - matchTokens.push(token); - currentTokenPosition += 1; - } else { - // reset position - matchTokens.length = 0; - currentTokenPosition = 0; + while (currentTokenPosition < tokenCount) { + const expectedToken = matchedTokens[currentTokenPosition]; + if (matchToken(token, expectedToken)) { + matchTokens.push(token); + currentTokenPosition += 1; + break; + } else if (expectedToken["_skippable"]) { + currentTokenPosition += 1; + } else { + // reset position + matchTokens.length = 0; + currentTokenPosition = 0; + break; + } } // match all tokens if (currentTokenPosition === tokenCount) { @@ -50,4 +56,4 @@ module.exports = function createTokenMatcher(matchedTokens) { match: false }; } -}; \ No newline at end of file +}; diff --git a/test/morpheme-match-test.js b/test/morpheme-match-test.js index 0bf58f2..82e8267 100644 --- a/test/morpheme-match-test.js +++ b/test/morpheme-match-test.js @@ -2,8 +2,8 @@ "use strict"; import expectTokenStream from "../src/morpheme-match"; const assert = require("power-assert"); -describe("expectTokenStream", function () { - it("should return {match, tokens}", function () { +describe("expectTokenStream", function() { + it("should return {match, tokens}", function() { // http://localhost:8080/#名詞(かも)しれない const expectToken = expectTokenStream([ { @@ -17,7 +17,26 @@ describe("expectTokenStream", function () { "basic_form": "かも", "reading": "カモ", "pronunciation": "カモ" - } + }, + { + "surface_form": "、", + "_skippable": true, + }, + { + "surface_form": "、", + "_skippable": true, + }, + { + "surface_form": "しれ", + }, + { + "surface_form": "、", + "_skippable": true, + }, + { + "surface_form": "、", + "_skippable": true, + }, ]); const tokens = [ { @@ -73,13 +92,13 @@ describe("expectTokenStream", function () { let resultTokens = []; const result = tokens.some(token => { const {match, tokens} = expectToken(token); - if(!match) { + if (!match) { assert(tokens === undefined); } resultTokens = tokens; return match; }); assert(result); - assert.equal(resultTokens.length, 1); + assert.equal(resultTokens.length, 2); }); -}); \ No newline at end of file +});