make the unit test be portable

.gitignore

@@ -129,4 +129,4 @@ tmp/
 /.idea/
 
 dask-worker-space/
-localxradarsat2-config.yaml
+localxradarsat2-config.yaml

pyproject.toml

@@ -15,28 +15,29 @@ fallback_version = "9999"
 
 [project]
 name = "xradarsat2"
-authors = [
-  { name="Yann Reynaud", email="Yann.Reynaud.2@ifremer.fr" },
-]
-license = {text = "MIT"}
+authors = [{ name = "Yann Reynaud", email = "Yann.Reynaud.2@ifremer.fr" }]
+license = { text = "MIT" }
 description = "xarray/dask distributed L1 sar file reader for radarSat2"
 readme = "README.md"
 requires-python = ">=3.9"
 classifiers = [
-    "Programming Language :: Python :: 3",
-    "License :: OSI Approved :: MIT License",
-    "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
 ]
 dependencies = [
-    'xmltodict',
-    'numpy',
-    "xarray>=2024.10.0",
-    'rasterio',
-    'rioxarray',
-    'dask',
-    'affine',
-    'regex',
-    'pyyaml'
+  'xmltodict',
+  'numpy',
+  "xarray>=2024.10.0",
+  'rasterio',
+  'rioxarray',
+  'dask',
+  'affine',
+  'regex',
+  'pyyaml',
+  'fsspec',
+  'aiohttp',
+
 ]
 dynamic = ["version"]
 
@@ -76,4 +77,3 @@ known-third-party = ["xarray", "toolz", "construct"]
 [tool.ruff.lint.flake8-tidy-imports]
 # Disallow all relative imports.
 ban-relative-imports = "all"
-

src/xradarsat2/utils.py

@@ -3,6 +3,10 @@
 import os
 import yaml
 import re
+import aiohttp
+import fsspec
+import warnings
+import zipfile
 
 
 def get_glob(strlist):
@@ -45,3 +49,68 @@ def load_config():
     stream = open(config_path, "r")
     conf = yaml.load(stream, Loader=yaml.CLoader)
     return conf
+
+def get_test_file(fname):
+    """
+    get test file from  https://cyclobs.ifremer.fr/static/sarwing_datarmor/xsardata/
+    file is unzipped and extracted to `config['data_dir']`
+
+    Parameters
+    ----------
+    fname: str
+        file name to get (without '.zip' extension)
+
+    Returns
+    -------
+    str
+        path to file, relative to `config['data_dir']`
+
+    """
+    config = {'data_dir': '/tmp'}
+
+    def url_get(url, cache_dir=os.path.join(config['data_dir'], 'fsspec_cache')):
+        """
+        Get fil from url, using caching.
+
+        Parameters
+        ----------
+        url: str
+        cache_dir: str
+            Cache dir to use. default to `os.path.join(config['data_dir'], 'fsspec_cache')`
+
+        Raises
+        ------
+        FileNotFoundError
+
+        Returns
+        -------
+        filename: str
+            The local file name
+
+        Notes
+        -----
+        Due to fsspec, the returned filename won't match the remote one.
+        """
+
+        if '://' in url:
+            with fsspec.open(
+                    'filecache::%s' % url,
+                    https={'client_kwargs': {'timeout': aiohttp.ClientTimeout(total=3600)}},
+                    filecache={'cache_storage': os.path.join(os.path.join(config['data_dir'], 'fsspec_cache'))}
+            ) as f:
+                fname = f.name
+        else:
+            fname = url
+
+        return fname
+
+    res_path = config['data_dir']
+    base_url = 'https://cyclobs.ifremer.fr/static/sarwing_datarmor/xsardata'
+    file_url = '%s/%s.zip' % (base_url, fname)
+    if not os.path.exists(os.path.join(res_path, fname)):
+        warnings.warn("Downloading %s" % file_url)
+        local_file = url_get(file_url)
+        warnings.warn("Unzipping %s" % os.path.join(res_path, fname))
+        with zipfile.ZipFile(local_file, 'r') as zip_ref:
+            zip_ref.extractall(res_path)
+    return os.path.join(res_path, fname)

src/xradarsat2/xradarsat2-config.yaml

@@ -1 +1 @@
-folder_path: ./2021/137/RS2_OK129673_PK1136693_DK1093025_SCWA_20210517_010235_VV_VH_SGF
+folder_path: RS2_OK135107_PK1187782_DK1151894_SCWA_20220407_182127_VV_VH_SGF

test/test_opening_datatree_radarsat2.py

@@ -1,7 +1,8 @@
-from xradarsat2.utils import load_config
-import xradarsat2
-import time
 import logging
+import time
+
+import xradarsat2
+from xradarsat2.utils import load_config,get_test_file
 
 logging.basicConfig(level=logging.DEBUG)
 logging.debug("start opening RadarSAT-2 product")
@@ -11,13 +12,14 @@
 t0 = time.time()
 conf = load_config()
 folder_path = conf["folder_path"]
-dt = xradarsat2.rs2_reader(folder_path)
+rs2_product_path = get_test_file(conf['folder_path'])
+dt = xradarsat2.rs2_reader(rs2_product_path)
 elapse_t = time.time() - t0
 
 print(type(dt), dt)
 print("out of the reader")
 print(dt)
-print("time to read the SAFE through nfs: %1.2f sec" % elapse_t)
+print(f"time to read the SAFE through nfs: {elapse_t:1.2f} sec")
 dt = xradarsat2.load_digital_number(
     dt, chunks={"pol": "VV", "line": 6000, "sample": 8000}
 )