fix (#1463)

intelligent-machine-learning · Feb 7, 2025 · 89c310e · 89c310e
1 parent 1c1ac83
commit 89c310e
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/dlrover/python/elastic_agent/torch/training.py b/dlrover/python/elastic_agent/torch/training.py
@@ -885,6 +885,8 @@ def _initialize_workers(self, worker_group, max_errors=3):
                 time.sleep(JobConstant.TRAINING_AGENT_LOOP_DEFAULT_INTERVAL)
                 if time.time() - start_pending > pend_timeout:
                     raise TimeoutError("Timeout to wait for new nodes.")
+            except NodeCheckFailedError as node_check_error:
+                raise node_check_error
             except Exception as e:
                 err_cnt += 1
                 if err_cnt < max_errors: