From 88b16384ce27b39314186690ad23585960383a96 Mon Sep 17 00:00:00 2001 From: "Kevin M. Dean" Date: Mon, 16 Dec 2024 17:26:10 -0600 Subject: [PATCH 1/2] Monitoring Disk Space Implement regular monitoring and reporting of disk space. Also add a try/except statement to catch when the disk space is full. --- .../model/data_sources/bdv_data_source.py | 14 ++++++++---- src/navigate/model/features/image_writer.py | 22 +++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/navigate/model/data_sources/bdv_data_source.py b/src/navigate/model/data_sources/bdv_data_source.py index ff97f2634..22663dd45 100644 --- a/src/navigate/model/data_sources/bdv_data_source.py +++ b/src/navigate/model/data_sources/bdv_data_source.py @@ -180,11 +180,17 @@ def write(self, data: npt.ArrayLike, **kw) -> None: # print(z, dz, dataset_name, self.image[dataset_name].shape, # data[::dx, ::dy].shape) zs = min(z // dz, self.shapes[i, 0] - 1) # TODO: Is this necessary? + try: + # Down-sample in X and Y. + self.image[dataset_name][zs, ...] = data[::dy, ::dx].astype(self.dtype) + if is_kw and (i == 0): + self._views.append(kw) + except OSError as e: + if e.errno == 28: + logger.error("No disk space left on device. Closing the file.") + self.close() + raise "No disk space left on device." - # Down-sample in X and Y. - self.image[dataset_name][zs, ...] = data[::dy, ::dx].astype(self.dtype) - if is_kw and (i == 0): - self._views.append(kw) self._current_frame += 1 # Check if this was the last frame to write diff --git a/src/navigate/model/features/image_writer.py b/src/navigate/model/features/image_writer.py index 1987f2ee5..53e1382ce 100644 --- a/src/navigate/model/features/image_writer.py +++ b/src/navigate/model/features/image_writer.py @@ -147,6 +147,12 @@ def __init__( "y": camera_config.get("flip_y", False), } + #: int: Disk space check interval in seconds. + self.disk_space_check_interval = 60 + + #: int: Minimum disk space required in bytes. + self.min_disk_space = 10 * 1024 * 1024 * 1024 # 10 GB + # initialize saving self.initialize_saving(sub_dir, image_name) @@ -159,8 +165,24 @@ def save_image(self, frame_ids): Index into self.model.data_buffer. """ + last_disk_space_check = time.time() + for idx in frame_ids: + # Check disk space at regular intervals to prevent running out of space + if time.time() - last_disk_space_check > self.disk_space_check_interval: + _, _, free = shutil.disk_usage(self.save_directory) + logger.info(f"Free Disk Space: {free / 1024 / 1024 / 1024} GB") + if free < self.min_disk_space: + logger.warning("Insufficient Disk Space. Acquisition Terminated") + self.close() + self.model.stop_acquisition = True + self.model.event_queue.put( + ("warning", "Insufficient Disk Space. Acquisition Terminated") + ) + return + last_disk_space_check = time.time() + if (idx < 0) or (idx > (self.number_of_frames - 1)): msg = f"Received invalid index {idx}. Skipping this frame." logger.debug(f"Received invalid index: {msg}.") From e0c9340587a3b6b31a3e7e1d54206279a863312d Mon Sep 17 00:00:00 2001 From: Annie Wang Date: Wed, 18 Dec 2024 12:33:34 -0800 Subject: [PATCH 2/2] minor tweaks --- .../model/data_sources/bdv_data_source.py | 2 +- src/navigate/model/features/image_writer.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/navigate/model/data_sources/bdv_data_source.py b/src/navigate/model/data_sources/bdv_data_source.py index 22663dd45..8618678bb 100644 --- a/src/navigate/model/data_sources/bdv_data_source.py +++ b/src/navigate/model/data_sources/bdv_data_source.py @@ -189,7 +189,7 @@ def write(self, data: npt.ArrayLike, **kw) -> None: if e.errno == 28: logger.error("No disk space left on device. Closing the file.") self.close() - raise "No disk space left on device." + raise Exception("No disk space left on device.") self._current_frame += 1 diff --git a/src/navigate/model/features/image_writer.py b/src/navigate/model/features/image_writer.py index 53e1382ce..7c92a3b17 100644 --- a/src/navigate/model/features/image_writer.py +++ b/src/navigate/model/features/image_writer.py @@ -153,6 +153,9 @@ def __init__( #: int: Minimum disk space required in bytes. self.min_disk_space = 10 * 1024 * 1024 * 1024 # 10 GB + #: float: Time of last disk space check + self.last_disk_space_check = 0 + # initialize saving self.initialize_saving(sub_dir, image_name) @@ -165,12 +168,14 @@ def save_image(self, frame_ids): Index into self.model.data_buffer. """ - last_disk_space_check = time.time() - for idx in frame_ids: + if (idx < 0) or (idx > (self.number_of_frames - 1)): + msg = f"Received invalid index {idx}. Skipping this frame." + logger.debug(f"Received invalid index: {msg}.") + continue # Check disk space at regular intervals to prevent running out of space - if time.time() - last_disk_space_check > self.disk_space_check_interval: + if time.time() - self.last_disk_space_check > self.disk_space_check_interval: _, _, free = shutil.disk_usage(self.save_directory) logger.info(f"Free Disk Space: {free / 1024 / 1024 / 1024} GB") if free < self.min_disk_space: @@ -181,12 +186,7 @@ def save_image(self, frame_ids): ("warning", "Insufficient Disk Space. Acquisition Terminated") ) return - last_disk_space_check = time.time() - - if (idx < 0) or (idx > (self.number_of_frames - 1)): - msg = f"Received invalid index {idx}. Skipping this frame." - logger.debug(f"Received invalid index: {msg}.") - continue + self.last_disk_space_check = time.time() # check the saving flag if self.saving_flags: @@ -308,6 +308,7 @@ def calculate_and_check_disk_space(self): Assumes 16-bit image type, without compression.""" + self.last_disk_space_check = time.time() # Return disk usage statistics in bytes _, _, free = shutil.disk_usage(self.save_directory) logger.info(f"Free Disk Space: {free}")