fix(tn): 300w张 50000票 (#156)

wenet-e2e · Nov 15, 2023 · 3d3cb80 · 3d3cb80
1 parent e91e6f8
commit 3d3cb80
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 2 deletions.
diff --git a/tn/chinese/data/measure/units_zh.tsv b/tn/chinese/data/measure/units_zh.tsv
@@ -7,6 +7,7 @@
 平方
 立方
 公里
+票
 多
 余
 几

diff --git a/tn/chinese/rules/measure.py b/tn/chinese/rules/measure.py
@@ -16,7 +16,7 @@
 from tn.processor import Processor
 
 from pynini import accep, cross, string_file
-from pynini.lib.pynutil import delete, insert
+from pynini.lib.pynutil import delete, insert, add_weight
 
 
 class Measure(Processor):
@@ -29,7 +29,8 @@ def __init__(self):
     def build_tagger(self):
         units_en = string_file('tn/chinese/data/measure/units_en.tsv')
         units_zh = string_file('tn/chinese/data/measure/units_zh.tsv')
-        units = units_en | units_zh
+        units = add_weight((cross("k", "千") | cross("w", "万")), 0.1).ques + \
+            (units_en | units_zh)
         rmspace = delete(' ').ques
         to = cross('-', '到') | cross('~', '到') | accep('到')
 

diff --git a/tn/chinese/test/data/normalizer.txt b/tn/chinese/test/data/normalizer.txt
@@ -40,3 +40,5 @@ B2B => B to B
 给12315打个电话 => 给幺二三幺五打个电话
 人均200以内 => 人均两百以内
 当场票数≥100万 => 当场票数大于等于一百万
+独得300w张 => 独得三百万张
+面积是10km² => 面积是十平方千米
-Original file line number
+Diff line change
@@ Expand Up / @@ -7,6 +7,7 @@ @@
     平方
     立方
     公里
+    票
     多
     余
     几
@@ Expand Down @@