fix breaking change in nltk

holunda-io · Oct 28, 2024 · c1bddde · c1bddde
1 parent 34fafc0
commit c1bddde
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/bpm_ai_inference/translation/easy_nmt/easy_nmt.py b/bpm_ai_inference/translation/easy_nmt/easy_nmt.py
@@ -407,9 +407,9 @@ def sentence_splitting(self, text: str, lang: str = None):
             sentences = list(re.findall(u'[^!?。\.]+[!?。\.]*', text, flags=re.U))
         else:
             try:
-                nltk.data.find('tokenizers/punkt')
+                nltk.data.find('tokenizers/punkt_tab')
             except LookupError:
-                nltk.download('punkt')
+                nltk.download('punkt_tab')
 
             sentences = nltk.sent_tokenize(text)