Merge pull request #1048 from TheDeanLab/622-write-fails-when-we-hit-…

…maximum-disk-space

Monitoring Disk Space

src/navigate/model/data_sources/bdv_data_source.py

@@ -180,11 +180,17 @@ def write(self, data: npt.ArrayLike, **kw) -> None:
                 # print(z, dz, dataset_name, self.image[dataset_name].shape,
                 #       data[::dx, ::dy].shape)
                 zs = min(z // dz, self.shapes[i, 0] - 1)  # TODO: Is this necessary?
+                try:
+                    # Down-sample in X and Y.
+                    self.image[dataset_name][zs, ...] = data[::dy, ::dx].astype(self.dtype)
+                    if is_kw and (i == 0):
+                        self._views.append(kw)
+                except OSError as e:
+                    if e.errno == 28:
+                        logger.error("No disk space left on device. Closing the file.")
+                        self.close()
+                        raise Exception("No disk space left on device.")
 
-                # Down-sample in X and Y.
-                self.image[dataset_name][zs, ...] = data[::dy, ::dx].astype(self.dtype)
-                if is_kw and (i == 0):
-                    self._views.append(kw)
         self._current_frame += 1
 
         # Check if this was the last frame to write

src/navigate/model/features/image_writer.py

@@ -147,6 +147,15 @@ def __init__(
             "y": camera_config.get("flip_y", False),
         }
 
+        #: int: Disk space check interval in seconds.
+        self.disk_space_check_interval = 60
+
+        #: int: Minimum disk space required in bytes.
+        self.min_disk_space = 10 * 1024 * 1024 * 1024 # 10 GB
+
+        #: float: Time of last disk space check
+        self.last_disk_space_check = 0
+
         # initialize saving
         self.initialize_saving(sub_dir, image_name)
 
@@ -160,12 +169,25 @@ def save_image(self, frame_ids):
         """
 
         for idx in frame_ids:
-
             if (idx < 0) or (idx > (self.number_of_frames - 1)):
                 msg = f"Received invalid index {idx}. Skipping this frame."
                 logger.debug(f"Received invalid index: {msg}.")
                 continue
 
+            # Check disk space at regular intervals to prevent running out of space
+            if time.time() - self.last_disk_space_check > self.disk_space_check_interval:
+                _, _, free = shutil.disk_usage(self.save_directory)
+                logger.info(f"Free Disk Space: {free / 1024 / 1024 / 1024} GB")
+                if free < self.min_disk_space:
+                    logger.warning("Insufficient Disk Space. Acquisition Terminated")
+                    self.close()
+                    self.model.stop_acquisition = True
+                    self.model.event_queue.put(
+                        ("warning", "Insufficient Disk Space. Acquisition Terminated")
+                    )
+                    return
+                self.last_disk_space_check = time.time()
+
             # check the saving flag
             if self.saving_flags:
                 if not self.saving_flags[idx]:
@@ -286,6 +308,7 @@ def calculate_and_check_disk_space(self):
 
         Assumes 16-bit image type, without compression."""
 
+        self.last_disk_space_check = time.time()
         # Return disk usage statistics in bytes
         _, _, free = shutil.disk_usage(self.save_directory)
         logger.info(f"Free Disk Space: {free}")