diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py
index 85d31be09a6c63..1a78d1bfe8e35b 100755
--- a/src/transformers/trainer.py
+++ b/src/transformers/trainer.py
@@ -2313,8 +2313,7 @@ def _inner_training_loop(
                 ):
                     # the `or` condition of `is_last_step_and_steps_less_than_grad_acc` is not covered
                     # in accelerate. So, explicitly enable sync gradients to True in that case.
-                    if is_last_step_and_steps_less_than_grad_acc:
-                        self.accelerator.gradient_state._set_sync_gradients(True)
+                    self.accelerator.gradient_state._set_sync_gradients(True)
 
                     # Gradient clipping
                     if args.max_grad_norm is not None and args.max_grad_norm > 0: