From eaf7e0ffaf0c686279ade8b9d7dffb64a425c4cd Mon Sep 17 00:00:00 2001 From: ZhiyuLi-goog Date: Fri, 10 Jan 2025 03:41:39 -0800 Subject: [PATCH] add barrier before RUN_START --- mixture_of_experts_pretraining/mlperf_logging_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mixture_of_experts_pretraining/mlperf_logging_utils.py b/mixture_of_experts_pretraining/mlperf_logging_utils.py index 3d7dd3f6b..ac8bd4de3 100644 --- a/mixture_of_experts_pretraining/mlperf_logging_utils.py +++ b/mixture_of_experts_pretraining/mlperf_logging_utils.py @@ -164,6 +164,8 @@ def on_train_begin(self, args, state, control, **kwargs): ) # device warmup should be done here self.mllogger.end(key=constants.INIT_STOP, value="") + + barrier() self.mllogger.start(constants.RUN_START, value="") self.mllogger.start( constants.BLOCK_START,