diff --git a/mbridge/core/bridge.py b/mbridge/core/bridge.py
index 6ad5503..085413a 100644
--- a/mbridge/core/bridge.py
+++ b/mbridge/core/bridge.py
@@ -267,6 +267,7 @@ def _save_weights_fast(
     ) -> None:
         if len(glob(os.path.join(weights_path, "*.safetensors"))) > 0:
             raise ValueError(f"The path:{weights_path} should not has safetensors files")
+        torch.distributed.barrier()
 
         def encode_filename(mcore_weight_name, *values):
             return mcore_weight_name + '--' + '--'.join(