Update to ZH normalizer regex (#1453)

* Update chinese-reading-normalizer.test.js 3 symbols added: ' ’ - Signed-off-by: shiki-tm <36088384+shiki-tm@users.noreply.github.com> * Update chinese.js 3 symbols added: '’- Signed-off-by: shiki-tm <36088384+shiki-tm@users.noreply.github.com> * Update regex pattern for chinese.js Signed-off-by: shiki-tm <36088384+shiki-tm@users.noreply.github.com> * revert change * readd separators, escape quote --------- Signed-off-by: shiki-tm <36088384+shiki-tm@users.noreply.github.com> Co-authored-by: Cashew <tungbinem@gmail.com>
yomidevs · Oct 13, 2024 · f25775c · f25775c
1 parent e0d381b
commit f25775c
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/ext/js/language/zh/chinese.js b/ext/js/language/zh/chinese.js
@@ -71,5 +71,5 @@ export function isCodePointChinese(codePoint) {
 
 /** @type {import('language').ReadingNormalizer} */
 export function normalizePinyin(str) {
-    return str.normalize('NFC').toLowerCase().replace(/[\s・:]|\/\//g, '');
+    return str.normalize('NFC').toLowerCase().replace(/[\s・:'’-]|\/\//g, '');
 }
diff --git a/test/language/chinese-reading-normalizer.test.js b/test/language/chinese-reading-normalizer.test.js
@@ -25,6 +25,9 @@ const tests = [
     ['wán:zhěng', 'wánzhěng'],
     ['fān・yì', 'fānyì'],
     ['fān//yì', 'fānyì'],
+    ['fān’yì', 'fānyì'],
+    ['fān\'yì', 'fānyì'],
+    ['fān-yì', 'fānyì'],
 ];
 
 describe('Normalize Pinyin', () => {