From 87ca92e9e417b4de19d6773066f2f08bd047016e Mon Sep 17 00:00:00 2001 From: grouazel Date: Wed, 27 Nov 2024 12:22:53 +0100 Subject: [PATCH] make the unit test be portable --- .gitignore | 2 +- pyproject.toml | 34 ++++++------ src/xradarsat2/utils.py | 69 +++++++++++++++++++++++++ src/xradarsat2/xradarsat2-config.yaml | 2 +- test/test_opening_datatree_radarsat2.py | 12 +++-- 5 files changed, 95 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index f41feca..133e187 100644 --- a/.gitignore +++ b/.gitignore @@ -129,4 +129,4 @@ tmp/ /.idea/ dask-worker-space/ -localxradarsat2-config.yaml \ No newline at end of file +localxradarsat2-config.yaml diff --git a/pyproject.toml b/pyproject.toml index 67425a8..1a2c6be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,28 +15,29 @@ fallback_version = "9999" [project] name = "xradarsat2" -authors = [ - { name="Yann Reynaud", email="Yann.Reynaud.2@ifremer.fr" }, -] -license = {text = "MIT"} +authors = [{ name = "Yann Reynaud", email = "Yann.Reynaud.2@ifremer.fr" }] +license = { text = "MIT" } description = "xarray/dask distributed L1 sar file reader for radarSat2" readme = "README.md" requires-python = ">=3.9" classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", ] dependencies = [ - 'xmltodict', - 'numpy', - "xarray>=2024.10.0", - 'rasterio', - 'rioxarray', - 'dask', - 'affine', - 'regex', - 'pyyaml' + 'xmltodict', + 'numpy', + "xarray>=2024.10.0", + 'rasterio', + 'rioxarray', + 'dask', + 'affine', + 'regex', + 'pyyaml', + 'fsspec', + 'aiohttp', + ] dynamic = ["version"] @@ -76,4 +77,3 @@ known-third-party = ["xarray", "toolz", "construct"] [tool.ruff.lint.flake8-tidy-imports] # Disallow all relative imports. ban-relative-imports = "all" - diff --git a/src/xradarsat2/utils.py b/src/xradarsat2/utils.py index bf91277..4defd02 100644 --- a/src/xradarsat2/utils.py +++ b/src/xradarsat2/utils.py @@ -3,6 +3,10 @@ import os import yaml import re +import aiohttp +import fsspec +import warnings +import zipfile def get_glob(strlist): @@ -45,3 +49,68 @@ def load_config(): stream = open(config_path, "r") conf = yaml.load(stream, Loader=yaml.CLoader) return conf + +def get_test_file(fname): + """ + get test file from https://cyclobs.ifremer.fr/static/sarwing_datarmor/xsardata/ + file is unzipped and extracted to `config['data_dir']` + + Parameters + ---------- + fname: str + file name to get (without '.zip' extension) + + Returns + ------- + str + path to file, relative to `config['data_dir']` + + """ + config = {'data_dir': '/tmp'} + + def url_get(url, cache_dir=os.path.join(config['data_dir'], 'fsspec_cache')): + """ + Get fil from url, using caching. + + Parameters + ---------- + url: str + cache_dir: str + Cache dir to use. default to `os.path.join(config['data_dir'], 'fsspec_cache')` + + Raises + ------ + FileNotFoundError + + Returns + ------- + filename: str + The local file name + + Notes + ----- + Due to fsspec, the returned filename won't match the remote one. + """ + + if '://' in url: + with fsspec.open( + 'filecache::%s' % url, + https={'client_kwargs': {'timeout': aiohttp.ClientTimeout(total=3600)}}, + filecache={'cache_storage': os.path.join(os.path.join(config['data_dir'], 'fsspec_cache'))} + ) as f: + fname = f.name + else: + fname = url + + return fname + + res_path = config['data_dir'] + base_url = 'https://cyclobs.ifremer.fr/static/sarwing_datarmor/xsardata' + file_url = '%s/%s.zip' % (base_url, fname) + if not os.path.exists(os.path.join(res_path, fname)): + warnings.warn("Downloading %s" % file_url) + local_file = url_get(file_url) + warnings.warn("Unzipping %s" % os.path.join(res_path, fname)) + with zipfile.ZipFile(local_file, 'r') as zip_ref: + zip_ref.extractall(res_path) + return os.path.join(res_path, fname) \ No newline at end of file diff --git a/src/xradarsat2/xradarsat2-config.yaml b/src/xradarsat2/xradarsat2-config.yaml index 4e19cc0..06bc2b1 100644 --- a/src/xradarsat2/xradarsat2-config.yaml +++ b/src/xradarsat2/xradarsat2-config.yaml @@ -1 +1 @@ -folder_path: ./2021/137/RS2_OK129673_PK1136693_DK1093025_SCWA_20210517_010235_VV_VH_SGF +folder_path: RS2_OK135107_PK1187782_DK1151894_SCWA_20220407_182127_VV_VH_SGF diff --git a/test/test_opening_datatree_radarsat2.py b/test/test_opening_datatree_radarsat2.py index 88111a8..a0df6cc 100644 --- a/test/test_opening_datatree_radarsat2.py +++ b/test/test_opening_datatree_radarsat2.py @@ -1,7 +1,8 @@ -from xradarsat2.utils import load_config -import xradarsat2 -import time import logging +import time + +import xradarsat2 +from xradarsat2.utils import load_config,get_test_file logging.basicConfig(level=logging.DEBUG) logging.debug("start opening RadarSAT-2 product") @@ -11,13 +12,14 @@ t0 = time.time() conf = load_config() folder_path = conf["folder_path"] -dt = xradarsat2.rs2_reader(folder_path) +rs2_product_path = get_test_file(conf['folder_path']) +dt = xradarsat2.rs2_reader(rs2_product_path) elapse_t = time.time() - t0 print(type(dt), dt) print("out of the reader") print(dt) -print("time to read the SAFE through nfs: %1.2f sec" % elapse_t) +print(f"time to read the SAFE through nfs: {elapse_t:1.2f} sec") dt = xradarsat2.load_digital_number( dt, chunks={"pol": "VV", "line": 6000, "sample": 8000} )