add translator text transformation tests

.eslintrc.json

@@ -611,7 +611,9 @@
             "files": [
                 "ext/js/core.js",
                 "ext/js/core/extension-error.js",
-                "ext/js/**/sandbox/**/*.js"
+                "ext/js/**/sandbox/**/*.js",
+                "ext/js/language/japanese.js",
+                "ext/js/language/japanese-wanakana.js"
             ],
             "env": {
                 "webextensions": false

ext/js/background/backend.js

@@ -16,7 +16,6 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import * as wanakana from '../../lib/wanakana.js';
 import {AccessibilityController} from '../accessibility/accessibility-controller.js';
 import {AnkiConnect} from '../comm/anki-connect.js';
 import {ClipboardMonitor} from '../comm/clipboard-monitor.js';
@@ -34,7 +33,7 @@ import {ArrayBufferUtil} from '../data/sandbox/array-buffer-util.js';
 import {DictionaryDatabase} from '../dictionary/dictionary-database.js';
 import {Environment} from '../extension/environment.js';
 import {ObjectPropertyAccessor} from '../general/object-property-accessor.js';
-import {JapaneseUtil} from '../language/sandbox/japanese-util.js';
+import {distributeFuriganaInflected, isCodePointJapanese, isStringPartiallyJapanese, convertKatakanaToHiragana as jpConvertKatakanaToHiragana} from '../language/japanese.js';
 import {Translator} from '../language/translator.js';
 import {AudioDownloader} from '../media/audio-downloader.js';
 import {MediaUtil} from '../media/media-util.js';
@@ -54,8 +53,6 @@ export class Backend {
     constructor(webExtension) {
         /** @type {import('../extension/web-extension.js').WebExtension} */
         this._webExtension = webExtension;
-        /** @type {JapaneseUtil} */
-        this._japaneseUtil = new JapaneseUtil(wanakana);
         /** @type {Environment} */
         this._environment = new Environment();
         /** @type {AnkiConnect} */
@@ -70,7 +67,6 @@ export class Backend {
             this._dictionaryDatabase = new DictionaryDatabase();
             /** @type {Translator|TranslatorProxy} */
             this._translator = new Translator({
-                japaneseUtil: this._japaneseUtil,
                 database: this._dictionaryDatabase
             });
             /** @type {ClipboardReader|ClipboardReaderProxy} */
@@ -93,7 +89,6 @@ export class Backend {
 
         /** @type {ClipboardMonitor} */
         this._clipboardMonitor = new ClipboardMonitor({
-            japaneseUtil: this._japaneseUtil,
             clipboardReader: this._clipboardReader
         });
         /** @type {?import('settings').Options} */
@@ -108,7 +103,6 @@ export class Backend {
         this._requestBuilder = new RequestBuilder();
         /** @type {AudioDownloader} */
         this._audioDownloader = new AudioDownloader({
-            japaneseUtil: this._japaneseUtil,
             requestBuilder: this._requestBuilder
         });
         /** @type {OptionsUtil} */
@@ -852,7 +846,7 @@ export class Backend {
 
     /** @type {import('api').ApiHandler<'textHasJapaneseCharacters'>} */
     _onApiTextHasJapaneseCharacters({text}) {
-        return this._japaneseUtil.isStringPartiallyJapanese(text);
+        return isStringPartiallyJapanese(text);
     }
 
     /** @type {import('api').ApiHandler<'getTermFrequencies'>} */
@@ -1376,7 +1370,6 @@ export class Backend {
      * @returns {Promise<import('api').ParseTextLine[]>}
      */
     async _textParseScanning(text, scanLength, optionsContext) {
-        const jp = this._japaneseUtil;
         /** @type {import('translator').FindTermsMode} */
         const mode = 'simple';
         const options = this._getProfileOptions(optionsContext, false);
@@ -1398,13 +1391,13 @@ export class Backend {
             if (
                 dictionaryEntries.length > 0 &&
                 originalTextLength > 0 &&
-                (originalTextLength !== character.length || jp.isCodePointJapanese(codePoint))
+                (originalTextLength !== character.length || isCodePointJapanese(codePoint))
             ) {
                 previousUngroupedSegment = null;
                 const {headwords: [{term, reading}]} = dictionaryEntries[0];
                 const source = text.substring(i, i + originalTextLength);
                 const textSegments = [];
-                for (const {text: text2, reading: reading2} of jp.distributeFuriganaInflected(term, reading, source)) {
+                for (const {text: text2, reading: reading2} of distributeFuriganaInflected(term, reading, source)) {
                     textSegments.push({text: text2, reading: reading2});
                 }
                 results.push(textSegments);
@@ -1427,8 +1420,6 @@ export class Backend {
      * @returns {Promise<import('backend').MecabParseResults>}
      */
     async _textParseMecab(text) {
-        const jp = this._japaneseUtil;
-
         let parseTextResults;
         try {
             parseTextResults = await this._mecab.parseText(text);
@@ -1444,9 +1435,9 @@ export class Backend {
             for (const line of lines) {
                 for (const {term, reading, source} of line) {
                     const termParts = [];
-                    for (const {text: text2, reading: reading2} of jp.distributeFuriganaInflected(
+                    for (const {text: text2, reading: reading2} of distributeFuriganaInflected(
                         term.length > 0 ? term : source,
-                        jp.convertKatakanaToHiragana(reading),
+                        jpConvertKatakanaToHiragana(reading),
                         source
                     )) {
                         termParts.push({text: text2, reading: reading2});

ext/js/background/offscreen.js

@@ -16,12 +16,10 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import * as wanakana from '../../lib/wanakana.js';
 import {ClipboardReader} from '../comm/clipboard-reader.js';
 import {createApiMap, invokeApiMapHandler} from '../core/api-map.js';
 import {ArrayBufferUtil} from '../data/sandbox/array-buffer-util.js';
 import {DictionaryDatabase} from '../dictionary/dictionary-database.js';
-import {JapaneseUtil} from '../language/sandbox/japanese-util.js';
 import {Translator} from '../language/translator.js';
 
 /**
@@ -33,13 +31,10 @@ export class Offscreen {
      * Creates a new instance.
      */
     constructor() {
-        /** @type {JapaneseUtil} */
-        this._japaneseUtil = new JapaneseUtil(wanakana);
         /** @type {DictionaryDatabase} */
         this._dictionaryDatabase = new DictionaryDatabase();
         /** @type {Translator} */
         this._translator = new Translator({
-            japaneseUtil: this._japaneseUtil,
             database: this._dictionaryDatabase
         });
         /** @type {ClipboardReader} */

ext/js/comm/clipboard-monitor.js

@@ -17,18 +17,17 @@
  */
 
 import {EventDispatcher} from '../core/event-dispatcher.js';
+import {isStringPartiallyJapanese} from '../language/japanese.js';
 
 /**
  * @augments EventDispatcher<import('clipboard-monitor').Events>
  */
 export class ClipboardMonitor extends EventDispatcher {
     /**
-     * @param {{japaneseUtil: import('../language/sandbox/japanese-util.js').JapaneseUtil, clipboardReader: import('clipboard-monitor').ClipboardReaderLike}} details
+     * @param {{clipboardReader: import('clipboard-monitor').ClipboardReaderLike}} details
      */
-    constructor({japaneseUtil, clipboardReader}) {
+    constructor({clipboardReader}) {
         super();
-        /** @type {import('../language/sandbox/japanese-util.js').JapaneseUtil} */
-        this._japaneseUtil = japaneseUtil;
         /** @type {import('clipboard-monitor').ClipboardReaderLike} */
         this._clipboardReader = clipboardReader;
         /** @type {?import('core').Timeout} */
@@ -72,7 +71,7 @@ export class ClipboardMonitor extends EventDispatcher {
                 text !== this._previousText
             ) {
                 this._previousText = text;
-                if (canChange && this._japaneseUtil.isStringPartiallyJapanese(text)) {
+                if (canChange && isStringPartiallyJapanese(text)) {
                     this.trigger('change', {text});
                 }
             }

ext/js/data/anki-note-builder.js

@@ -16,20 +16,18 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import {deferPromise} from '../core/utilities.js';
 import {ExtensionError} from '../core/extension-error.js';
+import {deferPromise} from '../core/utilities.js';
+import {convertHiraganaToKatakana, convertKatakanaToHiragana} from '../language/japanese.js';
 import {yomitan} from '../yomitan.js';
 import {AnkiUtil} from './anki-util.js';
 
 export class AnkiNoteBuilder {
     /**
      * Initiate an instance of AnkiNoteBuilder.
-     * @param {import('../language/sandbox/japanese-util.js').JapaneseUtil} japaneseUtil
      * @param {import('../templates/template-renderer-proxy.js').TemplateRendererProxy|import('../templates/sandbox/template-renderer.js').TemplateRenderer} templateRenderer
      */
-    constructor(japaneseUtil, templateRenderer) {
-        /** @type {import('../language/sandbox/japanese-util.js').JapaneseUtil} */
-        this._japaneseUtil = japaneseUtil;
+    constructor(templateRenderer) {
         /** @type {RegExp} */
         this._markerPattern = AnkiUtil.cloneFieldMarkerPattern(true);
         /** @type {import('../templates/template-renderer-proxy.js').TemplateRendererProxy|import('../templates/sandbox/template-renderer.js').TemplateRenderer} */
@@ -530,9 +528,9 @@ export class AnkiNoteBuilder {
     _convertReading(reading, readingMode) {
         switch (readingMode) {
             case 'hiragana':
-                return this._japaneseUtil.convertKatakanaToHiragana(reading);
+                return convertKatakanaToHiragana(reading);
             case 'katakana':
-                return this._japaneseUtil.convertHiraganaToKatakana(reading);
+                return convertHiraganaToKatakana(reading);
             default:
                 return reading;
         }

ext/js/data/sandbox/anki-note-data-creator.js

@@ -17,21 +17,13 @@
  */
 
 import {DictionaryDataUtil} from '../../dictionary/dictionary-data-util.js';
+import {distributeFurigana} from '../../language/japanese.js';
 
 /**
  * This class is used to convert the internal dictionary entry format to the
  * format used by Anki, for backwards compatibility.
  */
 export class AnkiNoteDataCreator {
-    /**
-     * Creates a new instance.
-     * @param {import('../../language/sandbox/japanese-util.js').JapaneseUtil} japaneseUtil An instance of `JapaneseUtil`.
-     */
-    constructor(japaneseUtil) {
-        /** @type {import('../../language/sandbox/japanese-util.js').JapaneseUtil} */
-        this._japaneseUtil = japaneseUtil;
-    }
-
     /**
      * Creates a compatibility representation of the specified data.
      * @param {string} marker The marker that is being used for template rendering.
@@ -860,7 +852,7 @@ export class AnkiNoteDataCreator {
     _getTermHeadwordFuriganaSegments(term, reading) {
         /** @type {import('anki-templates').FuriganaSegment[]} */
         const result = [];
-        for (const {text, reading: reading2} of this._japaneseUtil.distributeFurigana(term, reading)) {
+        for (const {text, reading: reading2} of distributeFurigana(term, reading)) {
             result.push({text, furigana: reading2});
         }
         return result;

ext/js/display/display-anki.js

@@ -30,9 +30,8 @@ export class DisplayAnki {
     /**
      * @param {import('./display.js').Display} display
      * @param {import('./display-audio.js').DisplayAudio} displayAudio
-     * @param {import('../language/sandbox/japanese-util.js').JapaneseUtil} japaneseUtil
      */
-    constructor(display, displayAudio, japaneseUtil) {
+    constructor(display, displayAudio) {
         /** @type {import('./display.js').Display} */
         this._display = display;
         /** @type {import('./display-audio.js').DisplayAudio} */
@@ -42,7 +41,7 @@ export class DisplayAnki {
         /** @type {?string} */
         this._ankiFieldTemplatesDefault = null;
         /** @type {AnkiNoteBuilder} */
-        this._ankiNoteBuilder = new AnkiNoteBuilder(japaneseUtil, new TemplateRendererProxy());
+        this._ankiNoteBuilder = new AnkiNoteBuilder(new TemplateRendererProxy());
         /** @type {?import('./display-notification.js').DisplayNotification} */
         this._errorNotification = null;
         /** @type {?EventListenerCollection} */

ext/js/display/display-generator.js

@@ -16,10 +16,11 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import {isObject} from '../core/utilities.js';
 import {ExtensionError} from '../core/extension-error.js';
+import {isObject} from '../core/utilities.js';
 import {DictionaryDataUtil} from '../dictionary/dictionary-data-util.js';
 import {HtmlTemplateCollection} from '../dom/html-template-collection.js';
+import {distributeFurigana, getKanaMorae, getPitchCategory, isCodePointKanji, isStringPartiallyJapanese} from '../language/japanese.js';
 import {yomitan} from '../yomitan.js';
 import {PronunciationGenerator} from './sandbox/pronunciation-generator.js';
 import {StructuredContentGenerator} from './sandbox/structured-content-generator.js';
@@ -28,19 +29,17 @@ export class DisplayGenerator {
     /**
      * @param {import('display').DisplayGeneratorConstructorDetails} details
      */
-    constructor({japaneseUtil, contentManager, hotkeyHelpController = null}) {
-        /** @type {import('../language/sandbox/japanese-util.js').JapaneseUtil} */
-        this._japaneseUtil = japaneseUtil;
+    constructor({contentManager, hotkeyHelpController = null}) {
         /** @type {import('./display-content-manager.js').DisplayContentManager} */
         this._contentManager = contentManager;
         /** @type {?import('../input/hotkey-help-controller.js').HotkeyHelpController} */
         this._hotkeyHelpController = hotkeyHelpController;
         /** @type {HtmlTemplateCollection} */
         this._templates = new HtmlTemplateCollection();
         /** @type {StructuredContentGenerator} */
-        this._structuredContentGenerator = new StructuredContentGenerator(this._contentManager, japaneseUtil, document);
+        this._structuredContentGenerator = new StructuredContentGenerator(this._contentManager, document);
         /** @type {PronunciationGenerator} */
-        this._pronunciationGenerator = new PronunciationGenerator(japaneseUtil);
+        this._pronunciationGenerator = new PronunciationGenerator();
     }
 
     /** */
@@ -725,11 +724,9 @@ export class DisplayGenerator {
      * @returns {HTMLElement}
      */
     _createPronunciationPitchAccent(pitchAccent, details) {
-        const jp = this._japaneseUtil;
-
         const {position, nasalPositions, devoicePositions, tags} = pitchAccent;
         const {reading, exclusiveTerms, exclusiveReadings} = details;
-        const morae = jp.getKanaMorae(reading);
+        const morae = getKanaMorae(reading);
 
         const node = this._instantiate('pronunciation');
 
@@ -912,10 +909,9 @@ export class DisplayGenerator {
      * @param {string} text
      */
     _appendKanjiLinks(container, text) {
-        const jp = this._japaneseUtil;
         let part = '';
         for (const c of text) {
-            if (jp.isCodePointKanji(/** @type {number} */ (c.codePointAt(0)))) {
+            if (isCodePointKanji(/** @type {number} */ (c.codePointAt(0)))) {
                 if (part.length > 0) {
                     container.appendChild(document.createTextNode(part));
                     part = '';
@@ -969,7 +965,7 @@ export class DisplayGenerator {
      */
     _appendFurigana(container, term, reading, addText) {
         container.lang = 'ja';
-        const segments = this._japaneseUtil.distributeFurigana(term, reading);
+        const segments = distributeFurigana(term, reading);
         for (const {text, reading: furigana} of segments) {
             if (furigana) {
                 const ruby = document.createElement('ruby');
@@ -1000,7 +996,7 @@ export class DisplayGenerator {
     _setTextContent(node, value, language) {
         if (typeof language === 'string') {
             node.lang = language;
-        } else if (this._japaneseUtil.isStringPartiallyJapanese(value)) {
+        } else if (isStringPartiallyJapanese(value)) {
             node.lang = 'ja';
         }
 
@@ -1017,7 +1013,7 @@ export class DisplayGenerator {
         // cause the text to not copy correctly.
         if (typeof language === 'string') {
             node.lang = language;
-        } else if (this._japaneseUtil.isStringPartiallyJapanese(value)) {
+        } else if (isStringPartiallyJapanese(value)) {
             node.lang = 'ja';
         }
 
@@ -1051,7 +1047,7 @@ export class DisplayGenerator {
             if (termPronunciation.headwordIndex !== headwordIndex) { continue; }
             for (const pronunciation of termPronunciation.pronunciations) {
                 if (pronunciation.type !== 'pitch-accent') { continue; }
-                const category = this._japaneseUtil.getPitchCategory(reading, pronunciation.position, isVerbOrAdjective);
+                const category = getPitchCategory(reading, pronunciation.position, isVerbOrAdjective);
                 if (category !== null) {
                     categories.add(category);
                 }