INTERNAL: Rationalize band paths and filename functions (still early work) #31

Author: remi-braun

CHANGES.md

@@ -34,6 +34,7 @@
 - FIX: Fix toa radiance to toa reflectance computation for VHR data
 - FIX: Save Cosmo quicklooks as PNG instead of GeoTiffs as they are not georeferenced
 - FIX: Fix `to_str` function with `as_list=False`
+- INTERNAL: Rationalize band paths and filename functions (still early work) ([#31](https://github.com/sertit/eoreader/issues/31))
 - DOC: Add documentation on how EOReader can improve your data handling ([#108](https://github.com/sertit/eoreader/issues/108))
 - DEPS: Add `ephem` for computing earth-sun distance (according to Maxar's method from docs)
 

ci/weekly/test_satellites.py

@@ -8,6 +8,7 @@
 
 import numpy as np
 import pytest
+import tempenv
 import xarray as xr
 from lxml import etree
 from rasterio.windows import Window
@@ -43,7 +44,7 @@
     VV,
     VV_DSPK,
 )
-from eoreader.env_vars import CI_EOREADER_BAND_FOLDER
+from eoreader.env_vars import CI_EOREADER_BAND_FOLDER, SAR_DEF_PIXEL_SIZE
 from eoreader.keywords import SLSTR_RAD_ADJUST
 from eoreader.products import Product, S2Product, SensorType, SlstrRadAdjust
 from eoreader.products.product import OrbitDirection
@@ -139,33 +140,35 @@ def check_load(prod: Product, first_band) -> None:
     # Check loading 0 bands
     assert len(prod.load([])) == 0
 
-    # Load with the raw process
-    band_arr_raw = prod.load(
-        first_band.value,
-        window=Window(col_off=0, row_off=0, width=100, height=100),
-        clean_optical="raw",
-    )[first_band]
+    # Don't orthorectify with a window to 1000 m
+    with tempenv.TemporaryEnvironment({SAR_DEF_PIXEL_SIZE: "0"}):
+        # Load with the raw process
+        band_arr_raw = prod.load(
+            first_band.value,
+            window=Window(col_off=0, row_off=0, width=100, height=100),
+            clean_optical="raw",
+        )[first_band]
 
-    # Check that band loaded 2 times gives the same results (disregarding float uncertainties)
-    band_arr1 = prod.load(
-        first_band,
-        window=Window(col_off=0, row_off=0, width=100, height=100),
-        clean_optical="nodata",
-    )[first_band]
-    band_arr2 = prod.load(
-        first_band,
-        window=Window(col_off=0, row_off=0, width=100, height=100),
-    )[first_band]
+        # Check that band loaded 2 times gives the same results (disregarding float uncertainties)
+        band_arr1 = prod.load(
+            first_band,
+            window=Window(col_off=0, row_off=0, width=100, height=100),
+            clean_optical="nodata",
+        )[first_band]
+        band_arr2 = prod.load(
+            first_band,
+            window=Window(col_off=0, row_off=0, width=100, height=100),
+        )[first_band]
 
-    np.testing.assert_array_almost_equal(band_arr1, band_arr2)
+        np.testing.assert_array_almost_equal(band_arr1, band_arr2)
 
-    # Check dtypes
-    ci.assert_val(band_arr_raw.dtype, np.float32, "band_arr_raw dtype")
-    ci.assert_val(band_arr1.dtype, np.float32, "band_arr1 dtype")
-    ci.assert_val(band_arr2.dtype, np.float32, "band_arr2 dtype")
+        # Check dtypes
+        ci.assert_val(band_arr_raw.dtype, np.float32, "band_arr_raw dtype")
+        ci.assert_val(band_arr1.dtype, np.float32, "band_arr1 dtype")
+        ci.assert_val(band_arr2.dtype, np.float32, "band_arr2 dtype")
 
-    # Check shapes between raw and no data cleaning
-    ci.assert_val(band_arr_raw.shape, band_arr1.shape, "band_arr1 shape")
+        # Check shapes between raw and no data cleaning
+        ci.assert_val(band_arr_raw.shape, band_arr1.shape, "band_arr1 shape")
 
 
 def check_attrs(prod: Product, array: xr.DataArray, long_name) -> None:

eoreader/products/optical/hls_product.py

@@ -714,9 +714,7 @@ def get_band_paths(
             band_id = self.bands[band].id
 
             # Get clean band path
-            clean_band = self._get_clean_band_path(
-                band, pixel_size=pixel_size, **kwargs
-            )
+            clean_band = self.get_band_path(band, pixel_size=pixel_size, **kwargs)
             if clean_band.is_file():
                 band_paths[band] = clean_band
             else:

eoreader/products/optical/landsat_product.py

@@ -979,9 +979,7 @@ def get_band_paths(
             band_id = self.bands[band].id
 
             # Get clean band path
-            clean_band = self._get_clean_band_path(
-                band, pixel_size=pixel_size, **kwargs
-            )
+            clean_band = self.get_band_path(band, pixel_size=pixel_size, **kwargs)
             if clean_band.is_file():
                 band_paths[band] = clean_band
             else:

eoreader/products/optical/optical_product.py

@@ -317,10 +317,10 @@ def _open_bands(
 
             if not pixel_size:
                 pixel_size = band_arr.rio.resolution()[0]
-            clean_band_path = self._get_clean_band_path(
+            clean_band_path = self.get_band_path(
                 band, pixel_size=pixel_size, writable=True, **kwargs
             )
-            # If raw data, clean it !
+            # If raw data, clean it!
             if AnyPath(band_path).name != clean_band_path.name:
                 # Clean pixels
                 cleaning_method = CleanMethod.from_value(
@@ -596,7 +596,7 @@ def _load_clouds(
             # First, try to open the cloud band written on disk
             bands_to_load = []
             for band in bands:
-                cloud_path = self._construct_band_path(
+                cloud_path = self.get_band_path(
                     band, pixel_size, size, writable=False, **kwargs
                 )
                 if cloud_path.is_file():
@@ -609,7 +609,7 @@ def _load_clouds(
 
             # Write them on disk
             for band_id, band_arr in loaded_bands.items():
-                cloud_path = self._construct_band_path(
+                cloud_path = self.get_band_path(
                     band_id, pixel_size, size, writable=True, **kwargs
                 )
                 band_arr = utils.write_path_in_attrs(band_arr, cloud_path)
@@ -664,7 +664,7 @@ def _load_masks(
             # First, try to open the cloud band written on disk
             bands_to_load = []
             for band in bands:
-                mask_path = self._construct_band_path(
+                mask_path = self.get_band_path(
                     band, pixel_size, size, writable=False, **kwargs
                 )
                 if mask_path.is_file():
@@ -677,7 +677,7 @@ def _load_masks(
 
             # Write them on disk
             for band_id, band_arr in loaded_bands.items():
-                mask_path = self._construct_band_path(
+                mask_path = self.get_band_path(
                     band_id, pixel_size, size, writable=True, **kwargs
                 )
                 band_arr = utils.write_path_in_attrs(band_arr, mask_path)
@@ -719,44 +719,32 @@ def _create_mask(
 
         return mask
 
-    def _get_clean_band_path(
-        self,
-        band: BandNames,
-        pixel_size: float = None,
-        writable: bool = False,
-        **kwargs,
-    ) -> AnyPathType:
+    def _get_band_file_name_sensor_specific_suffix(
+        self, band: BandNames, **kwargs
+    ) -> str:
         """
-        Get clean band path.
-
-        The clean band is the opened band where invalid pixels have been managed.
+        Get the sensor-specific suffix of a band filename.
 
         Args:
             band (BandNames): Wanted band
-            pixel_size (float): Band pixel size in meters
-            writable (bool): True if we want the band folder to be writeable
-            kwargs: Additional arguments
+            **kwargs: Other args
 
         Returns:
-            AnyPathType: Clean band path
+            str: Band filename sensor-specific suffix
         """
-        cleaning_method = CleanMethod.from_value(
-            kwargs.get(CLEAN_OPTICAL, DEF_CLEAN_METHOD)
-        )
-
-        res_str = self._pixel_size_to_str(pixel_size)
+        if is_spectral_band(band):
+            cleaning_method = CleanMethod.from_value(
+                kwargs.get(CLEAN_OPTICAL, DEF_CLEAN_METHOD)
+            )
 
-        # Radiometric processing
-        rad_proc = "" if kwargs.get(TO_REFLECTANCE, True) else "_as_is"
+            # Radiometric processing
+            rad_proc = "" if kwargs.get(TO_REFLECTANCE, True) else "_as_is"
 
-        # Window name
-        win_suffix = utils.get_window_suffix(kwargs.get("window"))
-        if win_suffix:
-            win_suffix += "_"
+            suffix = f"_{cleaning_method.value}{rad_proc}"
+        else:
+            suffix = ""
 
-        return self._get_band_folder(writable).joinpath(
-            f"{self.condensed_name}_{band.name}_{res_str.replace('.', '-')}_{win_suffix}{cleaning_method.value}{rad_proc}.tif",
-        )
+        return suffix
 
     @cache
     def _sun_earth_distance(self) -> float:

eoreader/products/optical/planet_product.py

@@ -428,9 +428,7 @@ def get_band_paths(
         band_path = self._get_stack_path(as_list=False)
         for band in band_list:
             # Get clean band path
-            clean_band = self._get_clean_band_path(
-                band, pixel_size=pixel_size, **kwargs
-            )
+            clean_band = self.get_band_path(band, pixel_size=pixel_size, **kwargs)
             if clean_band.is_file():
                 band_paths[band] = clean_band
             else:

eoreader/products/optical/s2_e84_product.py

@@ -514,9 +514,7 @@ def get_band_paths(
             band_id = self.bands[band].id
 
             # Get clean band path
-            clean_band = self._get_clean_band_path(
-                band, pixel_size=pixel_size, **kwargs
-            )
+            clean_band = self.get_band_path(band, pixel_size=pixel_size, **kwargs)
             if clean_band.is_file():
                 band_paths[band] = clean_band
             else:

eoreader/products/optical/s2_product.py

@@ -668,9 +668,7 @@ def get_band_paths(
         band_paths = {}
         for band in band_list:
             # Get clean band path
-            clean_band = self._get_clean_band_path(
-                band, pixel_size=pixel_size, **kwargs
-            )
+            clean_band = self.get_band_path(band, pixel_size=pixel_size, **kwargs)
             if clean_band.is_file():
                 band_paths[band] = clean_band
             else:
@@ -980,7 +978,7 @@ def _load_masks(
             for band in bands:
                 for associated_band in associated_bands[band]:
                     key = self._get_band_key(band, associated_band, **kwargs)
-                    mask_path = self._construct_band_path(
+                    mask_path = self.get_band_path(
                         key,
                         pixel_size,
                         size,
@@ -1003,7 +1001,7 @@ def _load_masks(
 
             # Write them on disk
             for band_id, band_arr in loaded_bands.items():
-                mask_path = self._construct_band_path(
+                mask_path = self.get_band_path(
                     band_id, pixel_size, size, writable=True, **kwargs
                 )
                 band_arr = utils.write_path_in_attrs(band_arr, mask_path)
@@ -1595,7 +1593,7 @@ def _load_s2_l2a_bands(
             # First, try to open the cloud band written on disk
             bands_to_load = []
             for band in bands:
-                s2_l2a_path = self._construct_band_path(
+                s2_l2a_path = self.get_band_path(
                     band, pixel_size, size, writable=False, **kwargs
                 )
                 if s2_l2a_path.is_file():
@@ -1654,7 +1652,7 @@ def _load_s2_l2a_bands(
 
             # Write them on disk
             for band_id, band_arr in loaded_bands.items():
-                s2_l2a_path = self._construct_band_path(
+                s2_l2a_path = self.get_band_path(
                     band_id, pixel_size, size, writable=True, **kwargs
                 )
                 band_arr = utils.write_path_in_attrs(band_arr, s2_l2a_path)
@@ -2407,9 +2405,7 @@ def get_band_paths(
         band_paths = {}
         for band in band_list:
             # Get clean band path
-            clean_band = self._get_clean_band_path(
-                band, pixel_size=pixel_size, **kwargs
-            )
+            clean_band = self.get_band_path(band, pixel_size=pixel_size, **kwargs)
             if clean_band.is_file():
                 band_paths[band] = clean_band
             else:

eoreader/products/optical/s2_theia_product.py

@@ -327,9 +327,7 @@ def get_band_paths(
         """
         band_paths = {}
         for band in band_list:  # Get clean band path
-            clean_band = self._get_clean_band_path(
-                band, pixel_size=pixel_size, **kwargs
-            )
+            clean_band = self.get_band_path(band, pixel_size=pixel_size, **kwargs)
             if clean_band.is_file():
                 band_paths[band] = clean_band
             else:
@@ -656,7 +654,7 @@ def _load_masks(
             for band in bands:
                 for associated_band in associated_bands[band]:
                     key = self._get_band_key(band, associated_band, **kwargs)
-                    mask_path = self._construct_band_path(
+                    mask_path = self.get_band_path(
                         key,
                         pixel_size,
                         size,
@@ -679,7 +677,7 @@ def _load_masks(
 
             # Write them on disk
             for band_id, band_arr in loaded_bands.items():
-                mask_path = self._construct_band_path(
+                mask_path = self.get_band_path(
                     band_id, pixel_size, size, writable=True, **kwargs
                 )
                 band_arr = utils.write_path_in_attrs(band_arr, mask_path)

eoreader/products/optical/s3_product.py

@@ -423,9 +423,7 @@ def get_band_paths(
         band_paths = {}
         for band in band_list:
             # Get clean band path
-            clean_band = self._get_clean_band_path(
-                band, pixel_size=pixel_size, **kwargs
-            )
+            clean_band = self.get_band_path(band, pixel_size=pixel_size, **kwargs)
             if clean_band.is_file():
                 band_paths[band] = clean_band
             else:

eoreader/products/optical/s3_slstr_product.py

@@ -1132,7 +1132,7 @@ def get_mean_sun_angles(self, view: SlstrView = SlstrView.NADIR) -> (float, floa
 
         return float(sun_az.mean().data) % 360, float(sun_ze.mean().data)
 
-    def _get_clean_band_path(
+    def get_band_path(
         self,
         band: BandNames,
         pixel_size: float = None,
@@ -1153,7 +1153,7 @@ def _get_clean_band_path(
             AnyPathType: Clean band path
         """
 
-        clean_band_path = super()._get_clean_band_path(
+        clean_band_path = super().get_band_path(
             band=band,
             pixel_size=pixel_size,
             writable=writable,

eoreader/products/optical/sv1_product.py

@@ -594,9 +594,7 @@ def get_band_paths(
         band_paths = {}
         for band in band_list:
             # Get clean band path
-            clean_band = self._get_clean_band_path(
-                band, pixel_size=pixel_size, **kwargs
-            )
+            clean_band = self.get_band_path(band, pixel_size=pixel_size, **kwargs)
             if clean_band.is_file():
                 band_paths[band] = clean_band
             else:

eoreader/products/optical/vhr_product.py

@@ -206,9 +206,7 @@ def get_band_paths(
         band_paths = {}
         for band in band_list:
             # Get clean band path
-            clean_band = self._get_clean_band_path(
-                band, pixel_size=pixel_size, **kwargs
-            )
+            clean_band = self.get_band_path(band, pixel_size=pixel_size, **kwargs)
             if clean_band.is_file():
                 band_paths[band] = clean_band
             else: