Skip to content

Commit a217dfb

Browse files
authored
clean gc after ckpt to avoid oom (#28)
1 parent 18db923 commit a217dfb

File tree

1 file changed

+3
-0
lines changed

1 file changed

+3
-0
lines changed

src/zeroband/checkpoint.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
from dataclasses import dataclass
2+
import gc
23
import multiprocessing
34
import os
45
import time
@@ -162,6 +163,8 @@ def save(self, ckpt_path: str, remote_ckpt_path: str | None) -> None:
162163

163164
self._logger.info(f"Saved checkpoint to {ckpt_path} in {time.perf_counter() - time_start} seconds")
164165

166+
gc.collect() # because we are badass engineer
167+
165168
if remote_ckpt_path is not None:
166169
self._async_save_remote(ckpt_path, remote_ckpt_path)
167170

0 commit comments

Comments
 (0)