diff --git a/lingvo/core/layers_with_attention.py b/lingvo/core/layers_with_attention.py
index dd2c14d7a..4a343220b 100644
--- a/lingvo/core/layers_with_attention.py
+++ b/lingvo/core/layers_with_attention.py
@@ -1500,7 +1500,6 @@ def FProp(
 
     if p.has_aux_atten:
       assert aux_vecs is not None
-      assert aux_paddings is not None
       with tf.name_scope('aux_atten'):
         atten_vec, atten_prob = self.atten.FProp(theta.atten, atten_vec,
                                                  aux_paddings, aux_vecs,
@@ -1550,7 +1549,6 @@ def ExtendStep(self,
 
     if p.has_aux_atten:
       assert aux_vecs is not None
-      assert aux_paddings is not None
 
     batch_size = py_utils.GetShape(source_vecs)[0]