From 9e715ca11cf19ad30e8384453fa0f480448728df Mon Sep 17 00:00:00 2001 From: Luis Perez Date: Wed, 14 Feb 2024 15:40:50 -0800 Subject: [PATCH] Stop keeping old copies of eventbus files We've never restored from these "backups", and as long as we ensure that the current file points to a fully written file, it's fine to only keep at most 2 files around: the actual current file and a temporary "new" file. To ensure that we're not pointing at a half-written file, we continue using the age-old pattern for atomic file updates: write to another file and swap a symlink once the write is complete --- tron/eventbus.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tron/eventbus.py b/tron/eventbus.py index 8aea37092..c0f0bee7e 100644 --- a/tron/eventbus.py +++ b/tron/eventbus.py @@ -144,9 +144,10 @@ def sync_load_log(self): duration = time.time() - started log.info(f"log read from disk, took {duration:.4}s") - def sync_save_log(self, reason): + def sync_save_log(self, reason: str) -> bool: started = time.time() new_file = os.path.join(self.log_dir, f"{int(started)}.pickle") + previous_file = os.path.realpath(os.path.join(self.log_dir, "current.pickle")) try: with open(new_file, "xb") as f: pickle.dump(self.event_log, f) @@ -165,7 +166,15 @@ def sync_save_log(self, reason): except FileNotFoundError: pass os.symlink(new_file, tmplink) - os.replace(tmplink, self.log_current) + os.replace(src=tmplink, dst=self.log_current) + # once we get here, `self.log_current` is now pointing to `new_file` + # so we can safely delete the previous `self.log_current` target without + # feat of losing data + try: + os.remove(previous_file) + except Exception: + # this shouldn't happen - but we also shouldn't crash if the impossible happens + log.exception(f"unable to delete {previous_file} - continuing anyway.") duration = time.time() - started log.info(f"log dumped to disk because {reason}, took {duration:.4}s")