fix sentiment maximum token size

Dadmatech · May 27, 2024 · 74d68f6 · 74d68f6
1 parent ec74904
commit 74d68f6
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/dadmatools/pipeline/language.py b/dadmatools/pipeline/language.py
@@ -156,7 +156,7 @@ def __init__(self, pipelines, lang='persian', cache_dir=None, gpu=True, embeddin
         # sentiment if possible
         if SENT in self.pipelines:
             model_path = "cardiffnlp/twitter-xlm-roberta-base-sentiment"
-            self._sent_model = pipeline("sentiment-analysis", model=model_path, tokenizer=model_path)
+            self._sent_model = pipeline("sentiment-analysis", model=model_path, tokenizer=model_path, max_length=512, truncation=True)
 
             # self._sent_model = {}
             # for lang in self.added_langs:

diff --git a/setup.py b/setup.py
@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="dadmatools",
-    version="2.0.3",
+    version="2.0.4",
     author="Dadmatech AI Company",
     author_email="info@dadmatech.ir",
     description="DadmaTools is a Persian NLP toolkit",