[fix] Add tokenizer.pad_token for CLM

tamohannes · Aug 19, 2024 · a1e3b1e · a1e3b1e
1 parent b89a815
commit a1e3b1e
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/urartu/VERSION b/urartu/VERSION
@@ -1 +1 @@
-2.0.26
+2.0.27
diff --git a/urartu/models/model_causal_language.py b/urartu/models/model_causal_language.py
@@ -31,6 +31,7 @@ def model(self) -> AutoModelForCausalLM:
     def tokenizer(self):
         if self._tokenizer is None:
             self._tokenizer = AutoTokenizer.from_pretrained(self.cfg.name)
+            self._tokenizer.pad_token = self._tokenizer.eos_token
         return self._tokenizer
 
     def generate(self, prompt: str, generate_cfg=None):