Fix missing embed_tokens (#671)

casper-hansen · Dec 6, 2024 · 8c5dcc6 · 8c5dcc6
1 parent f2171f3
commit 8c5dcc6
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 0 deletions.
diff --git a/awq/models/base.py b/awq/models/base.py
@@ -309,6 +309,7 @@ def forward(self, x):
             max_shard_size=shard_size,
             safe_serialization=safetensors,
             force_contiguous=True,
+            shared_tensors_to_discard=self.model._tied_weights_keys,
         )
 
     @classmethod

diff --git a/setup.py b/setup.py
@@ -42,6 +42,7 @@
     "accelerate",
     "datasets>=2.20",
     "zstandard",
+    "huggingface_hub @ git+https://github.com/huggingface/huggingface_hub@fix-discard-shared-tensors",
 ]
 
 setup(