INTERNAL: Rationalize band paths and filename functions (still early work) #31

Author: remi-braun

ci/weekly/test_satellites.py

@@ -133,7 +133,7 @@ def check_product_consistency(prod: Product):
     assert isinstance(orbit_dir, OrbitDirection)
 
 
-def check_load(prod: Product, first_band) -> None:
+def check_load(prod: Product, first_band, pixel_size) -> None:
     """
     Check if the loading process
     Args:
@@ -145,33 +145,38 @@ def check_load(prod: Product, first_band) -> None:
 
     # Don't orthorectify with a window to 1000 m
     with tempenv.TemporaryEnvironment({SAR_DEF_PIXEL_SIZE: "0"}):
-        # Load with the raw process
-        band_arr_raw = prod.load(
-            first_band.value,
-            window=Window(col_off=0, row_off=0, width=100, height=100),
-            clean_optical="raw",
-        )[first_band]
-
         # Check that band loaded 2 times gives the same results (disregarding float uncertainties)
         band_arr1 = prod.load(
             first_band,
+            pixel_size=pixel_size,
             window=Window(col_off=0, row_off=0, width=100, height=100),
             clean_optical="nodata",
         )[first_band]
         band_arr2 = prod.load(
             first_band,
+            pixel_size=pixel_size,
             window=Window(col_off=0, row_off=0, width=100, height=100),
         )[first_band]
-
-        np.testing.assert_array_almost_equal(band_arr1, band_arr2)
-
-        # Check dtypes
-        ci.assert_val(band_arr_raw.dtype, np.float32, "band_arr_raw dtype")
         ci.assert_val(band_arr1.dtype, np.float32, "band_arr1 dtype")
         ci.assert_val(band_arr2.dtype, np.float32, "band_arr2 dtype")
+        ci.assert_val(band_arr1.shape, band_arr2.shape, "band_arr2 shape")
+
+        if prod.sensor_type == SensorType.OPTICAL:
+            # Load with the raw process
+            band_arr_raw = prod.load(
+                first_band.value,
+                pixel_size=pixel_size,
+                window=Window(col_off=0, row_off=0, width=100, height=100),
+                clean_optical="raw",
+            )[first_band]
+
+            np.testing.assert_array_almost_equal(band_arr1, band_arr2)
+
+            # Check dtypes
+            ci.assert_val(band_arr_raw.dtype, np.float32, "band_arr_raw dtype")
 
-        # Check shapes between raw and no data cleaning
-        ci.assert_val(band_arr_raw.shape, band_arr1.shape, "band_arr1 shape")
+            # Check shapes between raw and no data cleaning
+            ci.assert_val(band_arr_raw.shape, band_arr1.shape, "band_arr_raw shape")
 
 
 def check_attrs(prod: Product, array: xr.DataArray, long_name) -> None:
@@ -367,7 +372,7 @@ def core(prod_path, possible_bands, debug, **kwargs):
         first_band = stack_bands[0]
 
         # Check that band loaded 2 times gives the same results (disregarding float uncertainties)
-        check_load(prod, first_band)
+        check_load(prod, first_band, pixel_size)
 
         # Check stack
         stack = test_satellites.check_stack(

eoreader/bands/band_names.py

@@ -484,6 +484,26 @@ def is_despeckle(cls, band: "SarBandNames"):
         """
         return "DSPK" in band.name
 
+    @classmethod
+    def is_speckle(cls, band: "SarBandNames"):
+        """
+        Returns True if the band corresponds to a speckle one.
+
+        .. code-block:: python
+
+            >>> SarBandNames.is_despeckle(SarBandNames.VV)
+            True
+            >>> SarBandNames.is_despeckle(SarBandNames.VV_DSPK)
+            False
+
+        Args:
+            band (SarBandNames): Band to test
+
+        Returns:
+            SarBandNames: Despeckled band
+        """
+        return not cls.is_despeckle(band)
+
     @classmethod
     def speckle_list(cls):
         return [band for band in cls if not cls.is_despeckle(band)]

eoreader/products/optical/optical_product.py

@@ -596,10 +596,10 @@ def _load_clouds(
             # First, try to open the cloud band written on disk
             bands_to_load = []
             for band in bands:
-                cloud_path = self.get_band_path(
-                    band, pixel_size, size, writable=False, **kwargs
+                cloud_path, cloud_exists = self._is_existing(
+                    self.get_band_file_name(band, pixel_size, size, **kwargs)
                 )
-                if cloud_path.is_file():
+                if cloud_exists:
                     band_dict[band] = utils.read(cloud_path)
                 else:
                     bands_to_load.append(band)

eoreader/products/optical/s3_slstr_product.py

@@ -632,14 +632,6 @@ def _preprocess(
                     LOGGER.debug(f"Converting {band_str} to reflectance")
                     band_arr = self._rad_2_refl(band_arr, band, suffix)
 
-                    # Debug
-                    # utils.write(
-                    #     band_arr,
-                    #     self._get_band_folder(writable=True).joinpath(
-                    #         f"{self.condensed_name}_{band.name}_rad2refl.tif"
-                    #     ),
-                    # )
-
             # Geocode
             LOGGER.debug(f"Geocoding {pp_name}")
             kwargs.pop("suffix", None)
@@ -675,7 +667,12 @@ def _get_suffix(self, band: Union[str, BandNames] = None, **kwargs) -> str:
         if band is not None:
             # Get the stripe
             if isinstance(band, BandNames):
-                band = self.bands[band].id
+                try:
+                    # Spectral bands
+                    band = self.bands[band].id
+                except KeyError:
+                    # Cloud bands
+                    band = band.value
 
             if band == F1.value:
                 stripe = SlstrStripe.F if self._F1_is_f else SlstrStripe.I
@@ -1136,6 +1133,7 @@ def get_band_path(
         self,
         band: BandNames,
         pixel_size: float = None,
+        size: Union[list, tuple] = None,
         writable: bool = False,
         **kwargs,
     ) -> AnyPathType:

eoreader/products/product.py

@@ -822,7 +822,7 @@ def get_band_file_name(
 
         # Specific if needed
 
-        return f"{self.condensed_name}_{self.bands[band].id}_{res_str.replace('.', '-')}{win_suffix}{self._get_band_file_name_sensor_specific_suffix(band, **kwargs)}.tif"
+        return f"{self.condensed_name}_{to_str(band, as_list=False)}_{res_str.replace('.', '-')}{win_suffix}{self._get_band_file_name_sensor_specific_suffix(band, **kwargs)}.tif"
 
     def _get_band_file_name_sensor_specific_suffix(
         self, band: BandNames, **kwargs
@@ -1393,10 +1393,10 @@ def _load_spectral_indices(
         """
         band_dict = {}
         for idx in index_list:
-            idx_path = self.get_band_path(
-                idx, pixel_size, size, writable=False, **kwargs
+            idx_path, idx_exists = self._is_existing(
+                self.get_band_file_name(idx, pixel_size, size, **kwargs)
             )
-            if idx_path.is_file():
+            if idx_exists:
                 band_dict[idx] = utils.read(idx_path)
             else:
                 idx_arr = compute_index(index=idx, bands=loaded_bands, **kwargs).rename(
@@ -1405,9 +1405,6 @@ def _load_spectral_indices(
                 idx_arr.attrs["long_name"] = idx
 
                 # Write on disk
-                idx_path = self.get_band_path(
-                    idx, pixel_size, size, writable=True, **kwargs
-                )
                 idx_arr = utils.write_path_in_attrs(idx_arr, idx_path)
                 utils.write(idx_arr, idx_path)
                 band_dict[idx] = idx_arr

eoreader/products/sar/cosmo_product.py

@@ -479,7 +479,9 @@ def get_orbit_direction(self) -> OrbitDirection:
 
         return od
 
-    def _pre_process_sar(self, band, pixel_size: float = None, **kwargs) -> AnyPathType:
+    def _pre_process_sar(
+        self, pre_processed_path, band, pixel_size: float = None, **kwargs
+    ) -> AnyPathType:
         """
         Pre-process SAR data (geocoding...)
 
@@ -517,11 +519,11 @@ def _pre_process_sar(self, band, pixel_size: float = None, **kwargs) -> AnyPathT
                     )
             return ortho_path
         elif misc.compare_version(self.get_snap_version(), "11.0.0", ">="):
-            return super()._pre_process_sar(band, pixel_size=pixel_size, **kwargs)
+            return super()._pre_process_sar(pre_processed_path, band, **kwargs)
         else:
             with h5netcdf.File(self._img_path, phony_dims="access") as raw_h5:
                 if self.nof_swaths == 1:
-                    return super()._pre_process_sar(band, pixel_size, **kwargs)
+                    return super()._pre_process_sar(pre_processed_path, band, **kwargs)
                 else:
                     LOGGER.warning(
                         "SNAP (before version 11.0.0) doesn't handle multiswath Cosmo-SkyMed products. This is a workaround. See https://github.com/sertit/eoreader/issues/78"
@@ -581,8 +583,8 @@ def _pre_process_sar(self, band, pixel_size: float = None, **kwargs) -> AnyPathT
                             # Pre-process swath
                             pp_swath_path.append(
                                 super()._pre_process_sar(
+                                    pre_processed_path,
                                     band,
-                                    pixel_size,
                                     prod_path=prod_path,
                                     suffix=group,
                                     **kwargs,