From 5cfbc1dc479d6a26dae57b0bf6069eb0d7cc8b07 Mon Sep 17 00:00:00 2001 From: Emanuel Schmid <51439563+emanuel-schmid@users.noreply.github.com> Date: Thu, 3 Oct 2024 16:23:31 +0200 Subject: [PATCH] Load IBTrACs file from archive in case (#956) * hazard.test: fall back for missing IBTraCs file * typo --- climada/hazard/test/__init__.py | 26 +++++++++++++++++++++ climada/hazard/test/test_tc_tracks.py | 18 +++++++++++++- climada/hazard/test/test_tc_tracks_synth.py | 13 +++++++++++ climada/hazard/test/test_trop_cyclone.py | 8 ++++++- 4 files changed, 63 insertions(+), 2 deletions(-) diff --git a/climada/hazard/test/__init__.py b/climada/hazard/test/__init__.py index 90e5a3fdb9..7bc33d61ff 100755 --- a/climada/hazard/test/__init__.py +++ b/climada/hazard/test/__init__.py @@ -18,3 +18,29 @@ init test """ + +import shutil + +from climada.util.constants import SYSTEM_DIR +from climada.util.api_client import Client +from climada.util.files_handler import download_ftp +from climada.hazard.tc_tracks import IBTRACS_FILE, IBTRACS_URL + + +def download_ibtracs(): + """This makes sure a IBTrACS.ALL.v04r00.nc file is present in SYSTEM_DIR + First, downloading from the original sources is attempted. If that fails an old version + is downloaded from the CLIMADA Data API + """ + if SYSTEM_DIR.joinpath(IBTRACS_FILE).is_file(): + return # Nothing to do + + try: + download_ftp(f'{IBTRACS_URL}/{IBTRACS_FILE}', IBTRACS_FILE) + shutil.move(IBTRACS_FILE, SYSTEM_DIR) + + except ValueError: # plan b: download an old version of that file from the climada api + client = Client() + dsinfo = client.get_dataset_info(name="IBTrACS", version="v04r00", status="external") + [fileinfo] = [fi for fi in dsinfo.files if fi.file_name == 'IBTrACS.ALL.v04r00.nc'] + client._download_file(local_path=SYSTEM_DIR, fileinfo=fileinfo) diff --git a/climada/hazard/test/test_tc_tracks.py b/climada/hazard/test/test_tc_tracks.py index 75daa58c74..f5c64e22b3 100644 --- a/climada/hazard/test/test_tc_tracks.py +++ b/climada/hazard/test/test_tc_tracks.py @@ -19,7 +19,9 @@ Test tc_tracks module. """ +from datetime import datetime as dt import unittest + import xarray as xr import numpy as np import pandas as pd @@ -32,7 +34,8 @@ from climada.util.constants import TC_ANDREW_FL import climada.util.coordinates as u_coord from climada.entity import Exposures -from datetime import datetime as dt +from climada.hazard.test import download_ibtracs + DATA_DIR = CONFIG.hazard.test_data.dir() TEST_TRACK = DATA_DIR.joinpath("trac_brb_test.csv") @@ -50,6 +53,10 @@ class TestIbtracs(unittest.TestCase): """Test reading and model of TC from IBTrACS files""" + @classmethod + def setUpClass(cls): + download_ibtracs() + def test_raw_ibtracs_empty_pass(self): """Test reading empty TC from IBTrACS files""" tc_track = tc.TCTracks.from_ibtracs_netcdf( @@ -290,6 +297,11 @@ def test_ibtracs_additional_variables(self): class TestIO(unittest.TestCase): """Test reading of tracks from files of different formats""" + + @classmethod + def setUpClass(cls): + download_ibtracs() + def test_netcdf_io(self): """Test writting and reading netcdf4 TCTracks instances""" path = DATA_DIR.joinpath("tc_tracks_nc") @@ -556,6 +568,10 @@ def test_to_geodataframe_line(self): class TestFuncs(unittest.TestCase): """Test functions over TC tracks""" + @classmethod + def setUpClass(cls): + download_ibtracs() + def test_get_track_pass(self): """Test get_track.""" tc_track = tc.TCTracks.from_processed_ibtracs_csv(TEST_TRACK_SHORT) diff --git a/climada/hazard/test/test_tc_tracks_synth.py b/climada/hazard/test/test_tc_tracks_synth.py index 6d170cf70b..1b2cca2c69 100644 --- a/climada/hazard/test/test_tc_tracks_synth.py +++ b/climada/hazard/test/test_tc_tracks_synth.py @@ -30,6 +30,8 @@ import climada.hazard.tc_tracks_synth as tc_synth import climada.util.coordinates from climada.util.constants import TC_ANDREW_FL +from climada.hazard.test import download_ibtracs + DATA_DIR = Path(__file__).parent.joinpath('data') TEST_TRACK = DATA_DIR.joinpath("trac_brb_test.csv") @@ -39,7 +41,13 @@ TEST_TRACK_DECAY_PENV_GT_PCEN = DATA_DIR.joinpath('1988021S12080_gen2.nc') TEST_TRACK_DECAY_PENV_GT_PCEN_HIST = DATA_DIR.joinpath('1988021S12080.nc') + class TestDecay(unittest.TestCase): + + @classmethod + def setUpClass(cls): + download_ibtracs() + def test_apply_decay_no_landfall_pass(self): """Test _apply_land_decay with no historical tracks with landfall""" tc_track = tc.TCTracks.from_processed_ibtracs_csv(TEST_TRACK_SHORT) @@ -469,6 +477,11 @@ def test_decay_penv_gt_pcen(self): self.assertTrue(np.all(np.diff(p_env_lf - p_synth_lf) <= 0)) class TestSynth(unittest.TestCase): + + @classmethod + def setUpClass(cls): + download_ibtracs() + def test_angle_funs_pass(self): """Test functions used by random walk code""" self.assertAlmostEqual(tc_synth._get_bearing_angle(np.array([15, 20]), diff --git a/climada/hazard/test/test_trop_cyclone.py b/climada/hazard/test/test_trop_cyclone.py index 41ab3a81fb..3fd4f07eb7 100644 --- a/climada/hazard/test/test_trop_cyclone.py +++ b/climada/hazard/test/test_trop_cyclone.py @@ -34,18 +34,24 @@ from climada.hazard.trop_cyclone.trop_cyclone import ( TropCyclone, ) from climada.hazard.centroids.centr import Centroids +from climada.hazard.test import download_ibtracs + DATA_DIR = Path(hazard_test.__file__).parent.joinpath('data') TEST_TRACK = DATA_DIR.joinpath("trac_brb_test.csv") TEST_TRACK_SHORT = DATA_DIR.joinpath("trac_short_test.csv") - CENTR_TEST_BRB = Centroids.from_hdf5(get_test_file('centr_test_brb', file_format='hdf5')) class TestReader(unittest.TestCase): """Test loading funcions from the TropCyclone class""" + + @classmethod + def setUpClass(cls): + download_ibtracs() + def test_memory_limit(self): """Test from_tracks when memory is (very) limited""" tc_track = TCTracks.from_processed_ibtracs_csv(TEST_TRACK)