From 7e227110a0ee494f7f30ac3f94ed6c1d822ccc02 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Thu, 28 Dec 2023 12:05:27 -0500 Subject: [PATCH 1/8] Add ReasonRaw --- types/ext/deinflector.d.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/types/ext/deinflector.d.ts b/types/ext/deinflector.d.ts index 5defbf7959..18d0f04a89 100644 --- a/types/ext/deinflector.d.ts +++ b/types/ext/deinflector.d.ts @@ -20,12 +20,14 @@ import type * as TranslationInternal from './translation-internal'; export type ReasonTypeRaw = 'v1' | 'v1d' | 'v1p' | 'v5' | 'vs' | 'vk' | 'vz' | 'adj-i' | 'iru'; export type ReasonsRaw = { - [reason: string]: { - kanaIn: string; - kanaOut: string; - rulesIn: ReasonTypeRaw[]; - rulesOut: ReasonTypeRaw[]; - }[]; + [reason: string]: ReasonRaw[]; +}; + +export type ReasonRaw = { + kanaIn: string; + kanaOut: string; + rulesIn: ReasonTypeRaw[]; + rulesOut: ReasonTypeRaw[]; }; export type ReasonVariant = [ From aee3012e2b8f19f13112f62f1766cea37e9650f4 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Thu, 28 Dec 2023 12:05:43 -0500 Subject: [PATCH 2/8] Set up cycle checks --- test/deinflection-cycles.test.js | 170 +++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 test/deinflection-cycles.test.js diff --git a/test/deinflection-cycles.test.js b/test/deinflection-cycles.test.js new file mode 100644 index 0000000000..8acf211947 --- /dev/null +++ b/test/deinflection-cycles.test.js @@ -0,0 +1,170 @@ +/* + * Copyright (C) 2024 Yomitan Authors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import {readFileSync} from 'fs'; +import {join, dirname as pathDirname} from 'path'; +import {fileURLToPath} from 'url'; +import {parseJson} from '../dev/json.js'; + +class DeinflectionNode { + /** + * @param {string} text + * @param {import('deinflector').ReasonTypeRaw[]} ruleNames + * @param {?RuleNode} ruleNode + * @param {?DeinflectionNode} previous + */ + constructor(text, ruleNames, ruleNode, previous) { + /** @type {string} */ + this.text = text; + /** @type {import('deinflector').ReasonTypeRaw[]} */ + this.ruleNames = ruleNames; + /** @type {?RuleNode} */ + this.ruleNode = ruleNode; + /** @type {?DeinflectionNode} */ + this.previous = previous; + } + + /** + * @param {DeinflectionNode} other + * @returns {boolean} + */ + historyIncludes(other) { + /** @type {?DeinflectionNode} */ + // eslint-disable-next-line @typescript-eslint/no-this-alias + let node = this; + for (; node !== null; node = node.previous) { + if ( + node.ruleNode === other.ruleNode && + node.text === other.text && + arraysAreEqual(node.ruleNames, other.ruleNames) + ) { + return true; + } + } + return false; + } + + /** + * @returns {DeinflectionNode[]} + */ + getHistory() { + /** @type {DeinflectionNode[]} */ + const results = []; + /** @type {?DeinflectionNode} */ + // eslint-disable-next-line @typescript-eslint/no-this-alias + let node = this; + for (; node !== null; node = node.previous) { + results.unshift(node); + } + return results; + } +} + +class RuleNode { + /** + * @param {string} groupName + * @param {import('deinflector').ReasonRaw} rule + */ + constructor(groupName, rule) { + /** @type {string} */ + this.groupName = groupName; + /** @type {import('deinflector').ReasonRaw} */ + this.rule = rule; + } +} + +/** + * @template [T=unknown] + * @param {T[]} rules1 + * @param {T[]} rules2 + * @returns {boolean} + */ +function arraysAreEqual(rules1, rules2) { + if (rules1.length !== rules2.length) { return false; } + for (const rule1 of rules1) { + if (!rules2.includes(rule1)) { return false; } + } + return true; +} + +/** + * @template [T=unknown] + * @param {T[]} rules1 + * @param {T[]} rules2 + * @returns {T[]} + */ +function getIntersection(rules1, rules2) { + return rules1.filter((item) => rules2.includes(item)); +} + +const dirname = pathDirname(fileURLToPath(import.meta.url)); + +/** @type {import('deinflector').ReasonsRaw} */ +const content = parseJson(readFileSync(join(dirname, '../ext/data/deinflect.json'), {encoding: 'utf8'})); + +/** @type {RuleNode[]} */ +const ruleNodes = []; +for (const [groupName, rules] of Object.entries(content)) { + for (const rule of rules) { + ruleNodes.push(new RuleNode(groupName, rule)); + } +} + +// TODO : Change this value +const checkRules = false; +/** @type {DeinflectionNode[]} */ +const deinflectionNodes = []; +for (const ruleNode of ruleNodes) { + deinflectionNodes.push(new DeinflectionNode(`?${ruleNode.rule.kanaIn}`, [], null, null)); +} +for (let i = 0; i < deinflectionNodes.length; ++i) { + const deinflectionNode = deinflectionNodes[i]; + const {text, ruleNames} = deinflectionNode; + for (const ruleNode of ruleNodes) { + const {kanaIn, kanaOut, rulesIn, rulesOut} = ruleNode.rule; + if ( + (checkRules && ruleNames.length !== 0 && getIntersection(ruleNames, rulesIn).length === 0) || + !text.endsWith(kanaIn) || + (text.length - kanaIn.length + kanaOut.length) <= 0 + ) { + continue; + } + + const newDeinflectionNode = new DeinflectionNode( + text.substring(0, text.length - kanaIn.length) + kanaOut, + rulesOut, + ruleNode, + deinflectionNode + ); + + // Cycle check + if (deinflectionNode.historyIncludes(newDeinflectionNode)) { + const stack = []; + for (const item of newDeinflectionNode.getHistory()) { + stack.push( + item.ruleNode === null ? + `${item.text} (start)` : + `${item.text} (${item.ruleNode.groupName}, ${item.ruleNode.rule.rulesIn.join(',')}=>${item.ruleNode.rule.rulesOut.join(',')}, ${item.ruleNode.rule.kanaIn}=>${item.ruleNode.rule.kanaOut})` + ); + } + console.log(`Cycle detected:\n ${stack.join('\n ')}`); + continue; + } + + deinflectionNodes.push(newDeinflectionNode); + } +} From ef255f6e3a629b78e65734cbbb9e239a3619c2ad Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sun, 21 Jan 2024 13:34:32 -0500 Subject: [PATCH 3/8] Use Deinflector.rulesMatch --- test/deinflector.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/deinflector.test.js b/test/deinflector.test.js index 660b909a5c..69495b4c00 100644 --- a/test/deinflector.test.js +++ b/test/deinflector.test.js @@ -38,7 +38,7 @@ function hasTermReasons(deinflector, source, expectedTerm, expectedRule, expecte if (term !== expectedTerm) { continue; } if (typeof expectedRule !== 'undefined') { const expectedFlags = Deinflector.rulesToRuleFlags([expectedRule]); - if (rules !== 0 && (rules & expectedFlags) !== expectedFlags) { continue; } + if (!Deinflector.rulesMatch(rules, expectedFlags)) { continue; } } let okay = true; if (typeof expectedReasons !== 'undefined') { From 458253c2cca045c4607a0e272c5db23d5d35a63f Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sun, 21 Jan 2024 13:38:06 -0500 Subject: [PATCH 4/8] Remove checkRules --- test/deinflection-cycles.test.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/deinflection-cycles.test.js b/test/deinflection-cycles.test.js index 8acf211947..17aa2780da 100644 --- a/test/deinflection-cycles.test.js +++ b/test/deinflection-cycles.test.js @@ -124,8 +124,6 @@ for (const [groupName, rules] of Object.entries(content)) { } } -// TODO : Change this value -const checkRules = false; /** @type {DeinflectionNode[]} */ const deinflectionNodes = []; for (const ruleNode of ruleNodes) { @@ -137,7 +135,7 @@ for (let i = 0; i < deinflectionNodes.length; ++i) { for (const ruleNode of ruleNodes) { const {kanaIn, kanaOut, rulesIn, rulesOut} = ruleNode.rule; if ( - (checkRules && ruleNames.length !== 0 && getIntersection(ruleNames, rulesIn).length === 0) || + (ruleNames.length !== 0 && getIntersection(ruleNames, rulesIn).length === 0) || !text.endsWith(kanaIn) || (text.length - kanaIn.length + kanaOut.length) <= 0 ) { From be50ef61bfe49cbaaa0cb8d4007d371bb055899a Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sun, 21 Jan 2024 13:39:39 -0500 Subject: [PATCH 5/8] Use Deinflector.rulesMatch --- test/deinflection-cycles.test.js | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/test/deinflection-cycles.test.js b/test/deinflection-cycles.test.js index 17aa2780da..e0faecc080 100644 --- a/test/deinflection-cycles.test.js +++ b/test/deinflection-cycles.test.js @@ -19,6 +19,7 @@ import {readFileSync} from 'fs'; import {join, dirname as pathDirname} from 'path'; import {fileURLToPath} from 'url'; import {parseJson} from '../dev/json.js'; +import {Deinflector} from '../ext/js/language/deinflector.js'; class DeinflectionNode { /** @@ -101,16 +102,6 @@ function arraysAreEqual(rules1, rules2) { return true; } -/** - * @template [T=unknown] - * @param {T[]} rules1 - * @param {T[]} rules2 - * @returns {T[]} - */ -function getIntersection(rules1, rules2) { - return rules1.filter((item) => rules2.includes(item)); -} - const dirname = pathDirname(fileURLToPath(import.meta.url)); /** @type {import('deinflector').ReasonsRaw} */ @@ -135,7 +126,7 @@ for (let i = 0; i < deinflectionNodes.length; ++i) { for (const ruleNode of ruleNodes) { const {kanaIn, kanaOut, rulesIn, rulesOut} = ruleNode.rule; if ( - (ruleNames.length !== 0 && getIntersection(ruleNames, rulesIn).length === 0) || + !Deinflector.rulesMatch(Deinflector.rulesToRuleFlags(ruleNames), Deinflector.rulesToRuleFlags(rulesIn)) || !text.endsWith(kanaIn) || (text.length - kanaIn.length + kanaOut.length) <= 0 ) { From b9164c13ee61fe96b722e5e64ceed53697a1b23f Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sun, 21 Jan 2024 13:44:52 -0500 Subject: [PATCH 6/8] Convert to test --- test/deinflection-cycles.test.js | 100 ++++++++++++++++--------------- 1 file changed, 53 insertions(+), 47 deletions(-) diff --git a/test/deinflection-cycles.test.js b/test/deinflection-cycles.test.js index e0faecc080..17a273d3e8 100644 --- a/test/deinflection-cycles.test.js +++ b/test/deinflection-cycles.test.js @@ -18,6 +18,7 @@ import {readFileSync} from 'fs'; import {join, dirname as pathDirname} from 'path'; import {fileURLToPath} from 'url'; +import {describe, test} from 'vitest'; import {parseJson} from '../dev/json.js'; import {Deinflector} from '../ext/js/language/deinflector.js'; @@ -102,58 +103,63 @@ function arraysAreEqual(rules1, rules2) { return true; } -const dirname = pathDirname(fileURLToPath(import.meta.url)); +describe('Deinflection data', () => { + test('Check for cycles', ({expect}) => { + const dirname = pathDirname(fileURLToPath(import.meta.url)); -/** @type {import('deinflector').ReasonsRaw} */ -const content = parseJson(readFileSync(join(dirname, '../ext/data/deinflect.json'), {encoding: 'utf8'})); + /** @type {import('deinflector').ReasonsRaw} */ + const content = parseJson(readFileSync(join(dirname, '../ext/data/deinflect.json'), {encoding: 'utf8'})); -/** @type {RuleNode[]} */ -const ruleNodes = []; -for (const [groupName, rules] of Object.entries(content)) { - for (const rule of rules) { - ruleNodes.push(new RuleNode(groupName, rule)); - } -} - -/** @type {DeinflectionNode[]} */ -const deinflectionNodes = []; -for (const ruleNode of ruleNodes) { - deinflectionNodes.push(new DeinflectionNode(`?${ruleNode.rule.kanaIn}`, [], null, null)); -} -for (let i = 0; i < deinflectionNodes.length; ++i) { - const deinflectionNode = deinflectionNodes[i]; - const {text, ruleNames} = deinflectionNode; - for (const ruleNode of ruleNodes) { - const {kanaIn, kanaOut, rulesIn, rulesOut} = ruleNode.rule; - if ( - !Deinflector.rulesMatch(Deinflector.rulesToRuleFlags(ruleNames), Deinflector.rulesToRuleFlags(rulesIn)) || - !text.endsWith(kanaIn) || - (text.length - kanaIn.length + kanaOut.length) <= 0 - ) { - continue; + /** @type {RuleNode[]} */ + const ruleNodes = []; + for (const [groupName, rules] of Object.entries(content)) { + for (const rule of rules) { + ruleNodes.push(new RuleNode(groupName, rule)); + } } - const newDeinflectionNode = new DeinflectionNode( - text.substring(0, text.length - kanaIn.length) + kanaOut, - rulesOut, - ruleNode, - deinflectionNode - ); + /** @type {DeinflectionNode[]} */ + const deinflectionNodes = []; + for (const ruleNode of ruleNodes) { + deinflectionNodes.push(new DeinflectionNode(`?${ruleNode.rule.kanaIn}`, [], null, null)); + } + for (let i = 0; i < deinflectionNodes.length; ++i) { + const deinflectionNode = deinflectionNodes[i]; + const {text, ruleNames} = deinflectionNode; + for (const ruleNode of ruleNodes) { + const {kanaIn, kanaOut, rulesIn, rulesOut} = ruleNode.rule; + if ( + !Deinflector.rulesMatch(Deinflector.rulesToRuleFlags(ruleNames), Deinflector.rulesToRuleFlags(rulesIn)) || + !text.endsWith(kanaIn) || + (text.length - kanaIn.length + kanaOut.length) <= 0 + ) { + continue; + } - // Cycle check - if (deinflectionNode.historyIncludes(newDeinflectionNode)) { - const stack = []; - for (const item of newDeinflectionNode.getHistory()) { - stack.push( - item.ruleNode === null ? - `${item.text} (start)` : - `${item.text} (${item.ruleNode.groupName}, ${item.ruleNode.rule.rulesIn.join(',')}=>${item.ruleNode.rule.rulesOut.join(',')}, ${item.ruleNode.rule.kanaIn}=>${item.ruleNode.rule.kanaOut})` + const newDeinflectionNode = new DeinflectionNode( + text.substring(0, text.length - kanaIn.length) + kanaOut, + rulesOut, + ruleNode, + deinflectionNode ); + + // Cycle check + if (deinflectionNode.historyIncludes(newDeinflectionNode)) { + const stack = []; + for (const item of newDeinflectionNode.getHistory()) { + stack.push( + item.ruleNode === null ? + `${item.text} (start)` : + `${item.text} (${item.ruleNode.groupName}, ${item.ruleNode.rule.rulesIn.join(',')}=>${item.ruleNode.rule.rulesOut.join(',')}, ${item.ruleNode.rule.kanaIn}=>${item.ruleNode.rule.kanaOut})` + ); + } + const message = `Cycle detected:\n ${stack.join('\n ')}`; + expect.soft(true, message).toEqual(false); + continue; + } + + deinflectionNodes.push(newDeinflectionNode); } - console.log(`Cycle detected:\n ${stack.join('\n ')}`); - continue; } - - deinflectionNodes.push(newDeinflectionNode); - } -} + }); +}); \ No newline at end of file From 3ebc724711a7041d3bec6a189d56c779b489898e Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sun, 21 Jan 2024 13:48:35 -0500 Subject: [PATCH 7/8] Rename --- test/deinflection-cycles.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/deinflection-cycles.test.js b/test/deinflection-cycles.test.js index 17a273d3e8..fe697f4d0a 100644 --- a/test/deinflection-cycles.test.js +++ b/test/deinflection-cycles.test.js @@ -108,11 +108,11 @@ describe('Deinflection data', () => { const dirname = pathDirname(fileURLToPath(import.meta.url)); /** @type {import('deinflector').ReasonsRaw} */ - const content = parseJson(readFileSync(join(dirname, '../ext/data/deinflect.json'), {encoding: 'utf8'})); + const deinflectionReasons = parseJson(readFileSync(join(dirname, '../ext/data/deinflect.json'), {encoding: 'utf8'})); /** @type {RuleNode[]} */ const ruleNodes = []; - for (const [groupName, rules] of Object.entries(content)) { + for (const [groupName, rules] of Object.entries(deinflectionReasons)) { for (const rule of rules) { ruleNodes.push(new RuleNode(groupName, rule)); } From 8cec32a86a302cf8e269faf464495c0c001e2c09 Mon Sep 17 00:00:00 2001 From: toasted-nutbread Date: Sun, 21 Jan 2024 13:50:17 -0500 Subject: [PATCH 8/8] Rename --- test/deinflection-cycles.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/deinflection-cycles.test.js b/test/deinflection-cycles.test.js index fe697f4d0a..a010d7a363 100644 --- a/test/deinflection-cycles.test.js +++ b/test/deinflection-cycles.test.js @@ -112,8 +112,8 @@ describe('Deinflection data', () => { /** @type {RuleNode[]} */ const ruleNodes = []; - for (const [groupName, rules] of Object.entries(deinflectionReasons)) { - for (const rule of rules) { + for (const [groupName, reasonInfo] of Object.entries(deinflectionReasons)) { + for (const rule of reasonInfo) { ruleNodes.push(new RuleNode(groupName, rule)); } }