ImperialNLP · 772435284 · May 16, 2022
diff --git a/transformers_code/layers/__pycache__/__init__.cpython-38.pyc b/transformers_code/layers/__pycache__/__init__.cpython-38.pyc
diff --git a/transformers_code/layers/__pycache__/decoder.cpython-38.pyc b/transformers_code/layers/__pycache__/decoder.cpython-38.pyc
diff --git a/transformers_code/layers/__pycache__/decoder_layer.cpython-38.pyc b/transformers_code/layers/__pycache__/decoder_layer.cpython-38.pyc
diff --git a/transformers_code/layers/__pycache__/embed.cpython-38.pyc b/transformers_code/layers/__pycache__/embed.cpython-38.pyc
diff --git a/transformers_code/layers/__pycache__/encoder.cpython-38.pyc b/transformers_code/layers/__pycache__/encoder.cpython-38.pyc
diff --git a/transformers_code/layers/__pycache__/encoder_layer.cpython-38.pyc b/transformers_code/layers/__pycache__/encoder_layer.cpython-38.pyc
diff --git a/transformers_code/layers/__pycache__/mha.cpython-38.pyc b/transformers_code/layers/__pycache__/mha.cpython-38.pyc
diff --git a/transformers_code/layers/__pycache__/positional_encoding.cpython-38.pyc b/transformers_code/layers/__pycache__/positional_encoding.cpython-38.pyc
diff --git a/transformers_code/layers/__pycache__/pwffn.cpython-38.pyc b/transformers_code/layers/__pycache__/pwffn.cpython-38.pyc
diff --git a/transformers_code/layers/__pycache__/residual_layer_norm.cpython-38.pyc b/transformers_code/layers/__pycache__/residual_layer_norm.cpython-38.pyc
diff --git a/transformers_code/layers/__pycache__/transformers.cpython-38.pyc b/transformers_code/layers/__pycache__/transformers.cpython-38.pyc
diff --git a/transformers_code/layers/mha.py b/transformers_code/layers/mha.py
@@ -35,7 +35,7 @@ def scaled_dot_product_attention(self, Q, K, V, mask=None):
         # shape(K, V) = [B x seq_len x D/num_heads]
         # shape(Q, K, V) = [B x num_heads x seq_len x D/num_heads]
 
-        Q_K_matmul = torch.matmul(Q, K.permute(0, 1, 3, 2))
+        Q_K_matmul = torch.matmul(Q, K.permute(0, 2, 1))
         scores = Q_K_matmul / m.sqrt(self.d)
         # shape(scores) = [B x num_heads x seq_len x seq_len]
 

diff --git a/transformers_code_answers/examples/__pycache__/create_mask.cpython-38.pyc b/transformers_code_answers/examples/__pycache__/create_mask.cpython-38.pyc
diff --git a/transformers_code_answers/examples/__pycache__/mha.cpython-38.pyc b/transformers_code_answers/examples/__pycache__/mha.cpython-38.pyc
diff --git a/transformers_code_answers/examples/toy_examples.py b/transformers_code_answers/examples/toy_examples.py
@@ -97,3 +97,5 @@
 # %%
 toy_scores = toy_scores.masked_fill(toy_mask == False, -1)
 print("Toy Scores Masked: \n", toy_scores)
+
+# %%
diff --git a/transformers_code_answers/layers/__pycache__/__init__.cpython-38.pyc b/transformers_code_answers/layers/__pycache__/__init__.cpython-38.pyc
diff --git a/transformers_code_answers/layers/__pycache__/decoder.cpython-38.pyc b/transformers_code_answers/layers/__pycache__/decoder.cpython-38.pyc
diff --git a/transformers_code_answers/layers/__pycache__/decoder_layer.cpython-38.pyc b/transformers_code_answers/layers/__pycache__/decoder_layer.cpython-38.pyc
diff --git a/transformers_code_answers/layers/__pycache__/embed.cpython-38.pyc b/transformers_code_answers/layers/__pycache__/embed.cpython-38.pyc
diff --git a/transformers_code_answers/layers/__pycache__/encoder.cpython-38.pyc b/transformers_code_answers/layers/__pycache__/encoder.cpython-38.pyc
diff --git a/transformers_code_answers/layers/__pycache__/encoder_layer.cpython-38.pyc b/transformers_code_answers/layers/__pycache__/encoder_layer.cpython-38.pyc
diff --git a/transformers_code_answers/layers/__pycache__/mha.cpython-38.pyc b/transformers_code_answers/layers/__pycache__/mha.cpython-38.pyc
diff --git a/transformers_code_answers/layers/__pycache__/positional_encoding.cpython-38.pyc b/transformers_code_answers/layers/__pycache__/positional_encoding.cpython-38.pyc
diff --git a/transformers_code_answers/layers/__pycache__/pwffn.cpython-38.pyc b/transformers_code_answers/layers/__pycache__/pwffn.cpython-38.pyc
diff --git a/transformers_code_answers/layers/__pycache__/residual_layer_norm.cpython-38.pyc b/transformers_code_answers/layers/__pycache__/residual_layer_norm.cpython-38.pyc
diff --git a/transformers_code_answers/layers/__pycache__/transformers.cpython-38.pyc b/transformers_code_answers/layers/__pycache__/transformers.cpython-38.pyc
diff --git a/transformers_code_answers/layers/mha.py b/transformers_code_answers/layers/mha.py
@@ -36,13 +36,16 @@ def scaled_dot_product_attention(self, Q, K, V, mask=None):
         # shape(K, V) = [B x seq_len x D/num_heads]
         # shape(Q, K, V) = [B x num_heads x seq_len x D/num_heads]
 
-
+        
         Q_K_matmul = torch.matmul(Q, K.permute(0, 1, 3, 2))
         scores = Q_K_matmul/m.sqrt(self.d)
         # shape(scores) = [B x num_heads x seq_len x seq_len]
-
-        if mask is not None:
-            scores = scores.masked_fill(mask == False, -1e9)
+        multi_mask = mask.unsqueeze(1)
+        multi_mask = torch.hstack([multi_mask for _ in range(self.num_heads)])
+        #multi_mask = multi_mask.unsqueeze(2)
+        if multi_mask is not None:
+
+            scores = scores.masked_fill(multi_mask == False, -1e9)
 
         attention_weights = F.softmax(scores, dim=-1)
         # shape(attention_weights) = [B x num_heads x seq_len x seq_len]

diff --git a/transformers_code_answers/lightning_logs/version_23/hparams.yaml b/transformers_code_answers/lightning_logs/version_23/hparams.yaml
@@ -0,0 +1 @@
+{}
-Original file line number
+Diff line change
@@ Expand Up / @@ -97,3 +97,5 @@ @@
     # %%
     toy_scores = toy_scores.masked_fill(toy_mask == False, -1)
     print("Toy Scores Masked: \n", toy_scores)
+    # %%