add translator text transformation tests (#565)

* Copy functions from JapaneseUtil * Remove JapaneseUtil * Update usages of JapaneseUtil functions * add tests * Copy functions from JapaneseUtil * Remove JapaneseUtil * Update usages of JapaneseUtil functions --------- Co-authored-by: toasted-nutbread <toasted-nutbread@users.noreply.github.com> Co-authored-by: Darius Jahandarie <djahandarie@gmail.com>
yomidevs · Jan 28, 2024 · 309d0b4 · 309d0b4
1 parent 60835f2
commit 309d0b4
Show file tree

Hide file tree

Showing 6 changed files with 3,930 additions and 4 deletions.
diff --git a/test/data/anki-note-builder-test-results.json b/test/data/anki-note-builder-test-results.json
@@ -2903,5 +2903,314 @@
         "url": "<a href=\"url:\">url:</a>"
       }
     ]
+  },
+  {
+    "name": "Test text transformations - convertNumericCharacters",
+    "results": [
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "39",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "３９",
+        "frequencies": "",
+        "furigana": "<ruby>３９<rt>さんきゅう</rt></ruby>",
+        "furigana-plain": "３９[さんきゅう]",
+        "glossary": "<div style=\"text-align: left;\"><i>(Test Dictionary 2)</i> sankyuu definition</div>",
+        "glossary-brief": "<div style=\"text-align: left;\">sankyuu definition</div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\">sankyuu definition</div>",
+        "part-of-speech": "Unknown",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "さんきゅう",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefix39cloze-suffix",
+        "sentence-furigana": "cloze-prefix39cloze-suffix",
+        "tags": "",
+        "url": "<a href=\"url:\">url:</a>"
+      }
+    ]
+  },
+  {
+    "name": "Test text transformations - convertAlphabeticCharacters",
+    "results": [
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "utsu",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "打つ",
+        "frequencies": "<ul style=\"text-align: left;\"><li>Test Dictionary 2: 2</li><li>Test Dictionary 2: 6</li><li>Test Dictionary 2: 10</li><li>Test Dictionary 2: sixteen</li><li>Test Dictionary 2: twenty-two (22)</li><li>Test Dictionary 2: 28</li></ul>",
+        "furigana": "<ruby>打<rt>う</rt></ruby>つ",
+        "furigana-plain": "打[う]つ",
+        "glossary": "<div style=\"text-align: left;\"><i>(vt, Test Dictionary 2)</i> <ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "glossary-brief": "<div style=\"text-align: left;\"><ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\"><i>(vt)</i> <ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "part-of-speech": "Godan verb",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "うつ",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefixutsucloze-suffix",
+        "sentence-furigana": "cloze-prefixutsucloze-suffix",
+        "tags": "vt",
+        "url": "<a href=\"url:\">url:</a>"
+      },
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "utsu",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "打つ",
+        "frequencies": "<ul style=\"text-align: left;\"><li>Test Dictionary 2: 2</li><li>Test Dictionary 2: 6</li><li>Test Dictionary 2: 10</li><li>Test Dictionary 2: sixteen</li><li>Test Dictionary 2: twenty-two (22)</li><li>Test Dictionary 2: 28</li></ul>",
+        "furigana": "<ruby>打<rt>う</rt></ruby>つ",
+        "furigana-plain": "打[う]つ",
+        "glossary": "<div style=\"text-align: left;\"><i>(vt, Test Dictionary 2)</i> <ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "glossary-brief": "<div style=\"text-align: left;\"><ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\"><i>(vt)</i> <ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "part-of-speech": "Godan verb",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "うつ",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefixutsucloze-suffix",
+        "sentence-furigana": "cloze-prefixutsucloze-suffix",
+        "tags": "vt",
+        "url": "<a href=\"url:\">url:</a>"
+      }
+    ]
+  },
+  {
+    "name": "Test text transformations - convertKatakanaToHiragana",
+    "results": [
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "ウツ",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "打つ",
+        "frequencies": "<ul style=\"text-align: left;\"><li>Test Dictionary 2: 2</li><li>Test Dictionary 2: 6</li><li>Test Dictionary 2: 10</li><li>Test Dictionary 2: sixteen</li><li>Test Dictionary 2: twenty-two (22)</li><li>Test Dictionary 2: 28</li></ul>",
+        "furigana": "<ruby>打<rt>う</rt></ruby>つ",
+        "furigana-plain": "打[う]つ",
+        "glossary": "<div style=\"text-align: left;\"><i>(vt, Test Dictionary 2)</i> <ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "glossary-brief": "<div style=\"text-align: left;\"><ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\"><i>(vt)</i> <ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "part-of-speech": "Godan verb",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "うつ",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefixウツcloze-suffix",
+        "sentence-furigana": "cloze-prefixウツcloze-suffix",
+        "tags": "vt",
+        "url": "<a href=\"url:\">url:</a>"
+      },
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "ウツ",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "打つ",
+        "frequencies": "<ul style=\"text-align: left;\"><li>Test Dictionary 2: 2</li><li>Test Dictionary 2: 6</li><li>Test Dictionary 2: 10</li><li>Test Dictionary 2: sixteen</li><li>Test Dictionary 2: twenty-two (22)</li><li>Test Dictionary 2: 28</li></ul>",
+        "furigana": "<ruby>打<rt>う</rt></ruby>つ",
+        "furigana-plain": "打[う]つ",
+        "glossary": "<div style=\"text-align: left;\"><i>(vt, Test Dictionary 2)</i> <ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "glossary-brief": "<div style=\"text-align: left;\"><ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\"><i>(vt)</i> <ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "part-of-speech": "Godan verb",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "うつ",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefixウツcloze-suffix",
+        "sentence-furigana": "cloze-prefixウツcloze-suffix",
+        "tags": "vt",
+        "url": "<a href=\"url:\">url:</a>"
+      }
+    ]
+  },
+  {
+    "name": "Test text transformations - convertHiraganaToKatakana",
+    "results": [
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "てきすと",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "テキスト",
+        "frequencies": "",
+        "furigana": "テキスト",
+        "furigana-plain": "テキスト",
+        "glossary": "<div style=\"text-align: left;\"><i>(n, Test Dictionary 2)</i> <ul><li>text definition 1</li><li>text definition 2</li></ul></div>",
+        "glossary-brief": "<div style=\"text-align: left;\"><ul><li>text definition 1</li><li>text definition 2</li></ul></div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\"><i>(n)</i> <ul><li>text definition 1</li><li>text definition 2</li></ul></div>",
+        "part-of-speech": "Noun",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "テキスト",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefixてきすとcloze-suffix",
+        "sentence-furigana": "cloze-prefixてきすとcloze-suffix",
+        "tags": "n",
+        "url": "<a href=\"url:\">url:</a>"
+      }
+    ]
+  },
+  {
+    "name": "Test text transformations - convertHalfWidthCharacters, convertKatakanaToHiragana",
+    "results": [
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "ｳﾂ",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "打つ",
+        "frequencies": "<ul style=\"text-align: left;\"><li>Test Dictionary 2: 2</li><li>Test Dictionary 2: 6</li><li>Test Dictionary 2: 10</li><li>Test Dictionary 2: sixteen</li><li>Test Dictionary 2: twenty-two (22)</li><li>Test Dictionary 2: 28</li></ul>",
+        "furigana": "<ruby>打<rt>う</rt></ruby>つ",
+        "furigana-plain": "打[う]つ",
+        "glossary": "<div style=\"text-align: left;\"><i>(vt, Test Dictionary 2)</i> <ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "glossary-brief": "<div style=\"text-align: left;\"><ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\"><i>(vt)</i> <ul><li>utsu definition 1</li><li>utsu definition 2</li></ul></div>",
+        "part-of-speech": "Godan verb",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "うつ",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefixｳﾂcloze-suffix",
+        "sentence-furigana": "cloze-prefixｳﾂcloze-suffix",
+        "tags": "vt",
+        "url": "<a href=\"url:\">url:</a>"
+      },
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "ｳﾂ",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "打つ",
+        "frequencies": "<ul style=\"text-align: left;\"><li>Test Dictionary 2: 2</li><li>Test Dictionary 2: 6</li><li>Test Dictionary 2: 10</li><li>Test Dictionary 2: sixteen</li><li>Test Dictionary 2: twenty-two (22)</li><li>Test Dictionary 2: 28</li></ul>",
+        "furigana": "<ruby>打<rt>う</rt></ruby>つ",
+        "furigana-plain": "打[う]つ",
+        "glossary": "<div style=\"text-align: left;\"><i>(vt, Test Dictionary 2)</i> <ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "glossary-brief": "<div style=\"text-align: left;\"><ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\"><i>(vt)</i> <ul><li>utsu definition 3</li><li>utsu definition 4</li></ul></div>",
+        "part-of-speech": "Godan verb",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "うつ",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefixｳﾂcloze-suffix",
+        "sentence-furigana": "cloze-prefixｳﾂcloze-suffix",
+        "tags": "vt",
+        "url": "<a href=\"url:\">url:</a>"
+      }
+    ]
+  },
+  {
+    "name": "Test text transformations - collapseEmphaticSequences",
+    "results": [
+      {
+        "audio": "",
+        "clipboard-image": "",
+        "clipboard-text": "",
+        "cloze-body": "すっっごーーい",
+        "cloze-prefix": "cloze-prefix",
+        "cloze-suffix": "cloze-suffix",
+        "conjugation": "",
+        "dictionary": "Test Dictionary 2",
+        "document-title": "title",
+        "expression": "凄い",
+        "frequencies": "",
+        "furigana": "<ruby>凄<rt>すご</rt></ruby>い",
+        "furigana-plain": "凄[すご]い",
+        "glossary": "<div style=\"text-align: left;\"><i>(adj-i, Test Dictionary 2)</i> sugoi definition</div>",
+        "glossary-brief": "<div style=\"text-align: left;\">sugoi definition</div>",
+        "glossary-no-dictionary": "<div style=\"text-align: left;\"><i>(adj-i)</i> sugoi definition</div>",
+        "part-of-speech": "I-adjective",
+        "pitch-accents": "No pitch accent data",
+        "pitch-accent-graphs": "No pitch accent data",
+        "pitch-accent-positions": "No pitch accent data",
+        "phonetic-transcriptions": "",
+        "reading": "すごい",
+        "screenshot": "",
+        "search-query": "fullQuery",
+        "selection-text": "",
+        "sentence": "cloze-prefixすっっごーーいcloze-suffix",
+        "sentence-furigana": "cloze-prefixすっっごーーいcloze-suffix",
+        "tags": "adj-i",
+        "url": "<a href=\"url:\">url:</a>"
+      }
+    ]
   }
 ]
diff --git a/test/data/database-test-cases.json b/test/data/database-test-cases.json
@@ -27,7 +27,7 @@
         "ipa": 1
       },
       "terms": {
-        "total": 25
+        "total": 27
       }
     }
   },
@@ -36,7 +36,7 @@
       {
         "kanji": 2,
         "kanjiMeta": 6,
-        "terms": 25,
+        "terms": 27,
         "termMeta": 39,
         "tagMeta": 15,
         "media": 6
@@ -45,7 +45,7 @@
     "total": {
       "kanji": 2,
       "kanjiMeta": 6,
-      "terms": 25,
+      "terms": 27,
       "termMeta": 39,
       "tagMeta": 15,
       "media": 6

diff --git a/test/data/dictionaries/valid-dictionary1/term_bank_1.json b/test/data/dictionaries/valid-dictionary1/term_bank_1.json
@@ -339,5 +339,7 @@
         101, "P E1"
     ],
     ["のたまう", "のたまう", "v5", "v5", 1, ["notamau definition"], 15, ""],
-    ["のたもうた", "のたもうた", "", "", 1, [["のたまう", ["past"]]], 16, ""]
+    ["のたもうた", "のたもうた", "", "", 1, [["のたまう", ["past"]]], 16, ""],
+    ["３９", "さんきゅう", "", "", 1, ["sankyuu definition"], 17, ""],
+    ["凄い", "すごい", "adj-i", "adj-i", 1, ["sugoi definition"], 18, ""]
 ]