diff --git a/ecml_tools/create/functions/__init__.py b/ecml_tools/create/functions/__init__.py index d4fd61c..337d3ba 100644 --- a/ecml_tools/create/functions/__init__.py +++ b/ecml_tools/create/functions/__init__.py @@ -6,83 +6,6 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. # -import os - -from climetlab import load_source - - -class Mockup: - write_directory = None - read_directory = None - - def get_filename(self, r): - import hashlib - - h = hashlib.md5(str(r).encode("utf8")).hexdigest() - return h + ".copy" - - def write(self, ds, *args, **kwargs): - if self.write_directory is None: - return - if not hasattr(ds, "path"): - return - - path = os.path.join(self.write_directory, self.get_filename([args, kwargs])) - print(f"Saving to {path} for {args}, {kwargs}") - import shutil - - shutil.copy(ds.path, path) - - def load_source(self, *args, **kwargs): - if self.read_directory is None: - return None - path = os.path.join(self.read_directory, self.get_filename([args, kwargs])) - if os.path.exists(path): - print(f"Loading path {path} for {args}, {kwargs}") - return load_source("file", path) - elif path.startswith("http"): - import requests - - print(f"Loading url {path} for {args}, {kwargs}") - try: - return load_source("url", path) - except requests.exceptions.HTTPError: - pass - return None - - -MOCKUP = Mockup() - - -def enable_save_mars(d): - MOCKUP.write_directory = d - - -def disable_save_mars(): - MOCKUP.write_directory = None - - -def enable_read_mars(d): - MOCKUP.read_directory = d - - -def disable_read_mars(): - MOCKUP.read_directory = None - - -if os.environ.get("MOCKUP_MARS_SAVE_REQUESTS"): - enable_save_mars(os.environ.get("MOCKUP_MARS_SAVE_REQUESTS")) - -if os.environ.get("MOCKUP_MARS_READ_REQUESTS"): - enable_read_mars(os.environ.get("MOCKUP_MARS_READ_REQUESTS")) - - -def _load_source(*args, **kwargs): - ds = MOCKUP.load_source(*args, **kwargs) - if ds is None: - ds = load_source(*args, **kwargs) - MOCKUP.write(ds, *args, **kwargs) - return ds def assert_is_fieldset(obj): diff --git a/ecml_tools/create/functions/actions/accumulations.py b/ecml_tools/create/functions/actions/accumulations.py index 615f18d..5c6250f 100644 --- a/ecml_tools/create/functions/actions/accumulations.py +++ b/ecml_tools/create/functions/actions/accumulations.py @@ -8,11 +8,11 @@ # from copy import deepcopy +from climetlab import load_source + from ecml_tools.create.functions.actions.mars import factorise_requests from ecml_tools.create.utils import to_datetime_list -from .. import _load_source as load_source - DEBUG = True diff --git a/ecml_tools/create/functions/actions/mars.py b/ecml_tools/create/functions/actions/mars.py index f0e5c52..53c067b 100644 --- a/ecml_tools/create/functions/actions/mars.py +++ b/ecml_tools/create/functions/actions/mars.py @@ -9,12 +9,11 @@ import datetime from copy import deepcopy +from climetlab import load_source from climetlab.utils.availability import Availability from ecml_tools.create.utils import to_datetime_list -from .. import _load_source as load_source - DEBUG = False diff --git a/tests/create/perturbations.yaml b/tests/create/perturbations.yaml index 16bea59..45d6e1e 100644 --- a/tests/create/perturbations.yaml +++ b/tests/create/perturbations.yaml @@ -1,27 +1,25 @@ common: - sfc: &sfc + global: &global class: ea expver: "0001" grid: 20.0/20.0 + sfc: &sfc + <<: *global levtype: sfc param: [2t] #levtype: sfc #param: [10u, 10v, 2d, 2t, lsm, msl, sdor, skt, slor, sp, tcw, z, cp, tp] + acc: &acc + <<: *global + levtype: sfc + param: [tp] pl: &pl - class: ea - expver: "0001" - grid: 20.0/20.0 + <<: *global levtype: pl param: [q] level: [50] #param: [q, t, u, v, w, z] #level: [50, 100, 150, 200, 250, 300, 400, 500, 600, 700, 850, 925, 1000] - acc: &acc - class: ea - expver: "0001" - grid: 20.0/20.0 - levtype: sfc - param: [tp] ensembles: &ensembles stream: enda @@ -31,17 +29,6 @@ common: center: ¢er stream: oper type: an - constants: &constants - param: - - cos_latitude - - cos_longitude - - sin_latitude - - sin_longitude - - cos_julian_day - - cos_local_time - - sin_julian_day - - sin_local_time - - insolation dates: start: 2020-12-30 00:00:00 @@ -63,9 +50,6 @@ data_sources: - accumulations: <<: *ensembles <<: *acc - # - constants: - # <<: *constants - # template: ${data_sources.ensembles.join.0.mars} center: join: - mars: @@ -77,16 +61,26 @@ data_sources: - accumulations: <<: *center <<: *acc - # - constants: - # <<: *constants - # template: ${data_sources.center.join.0.mars} - + input: - ensemble_perturbations: - # the ensemble data which has one additional dimension - ensembles: ${data_sources.ensembles} - # the new center of the data - center: ${data_sources.center} + join: + - ensemble_perturbations: + # the ensemble data which has one additional dimension + ensembles: ${data_sources.ensembles} + # the new center of the data + center: ${data_sources.center} + - constants: + template: ${input.join.0.ensemble_perturbations} + param: + - cos_latitude + - cos_longitude + - sin_latitude + - sin_longitude + - cos_julian_day + - cos_local_time + - sin_julian_day + - sin_local_time + - insolation output: order_by: [valid_datetime, param_level, number] diff --git a/tests/create/test_create.py b/tests/create/test_create.py index f6bbb2c..95a69fd 100755 --- a/tests/create/test_create.py +++ b/tests/create/test_create.py @@ -10,6 +10,7 @@ import numpy as np import pytest +from climetlab import load_source from ecml_tools.create import Creator from ecml_tools.create.functions import enable_read_mars @@ -34,6 +35,80 @@ enable_read_mars("https://object-store.os-api.cci1.ecmwf.int/ml-tests/test-data/anemoi-datasets/create") +class Mockup: + write_directory = None + read_directory = None + + def get_filename(self, r): + import hashlib + + h = hashlib.md5(str(r).encode("utf8")).hexdigest() + return h + ".copy" + + def write(self, ds, *args, **kwargs): + if self.write_directory is None: + return + if not hasattr(ds, "path"): + return + + path = os.path.join(self.write_directory, self.get_filename([args, kwargs])) + print(f"Saving to {path} for {args}, {kwargs}") + import shutil + + shutil.copy(ds.path, path) + + def load_source(self, *args, **kwargs): + if self.read_directory is None: + return None + path = os.path.join(self.read_directory, self.get_filename([args, kwargs])) + if os.path.exists(path): + print(f"Loading path {path} for {args}, {kwargs}") + return load_source("file", path) + elif path.startswith("http"): + import requests + + print(f"Loading url {path} for {args}, {kwargs}") + try: + return load_source("url", path) + except requests.exceptions.HTTPError: + pass + return None + + +MOCKUP = Mockup() + + +def enable_save_mars(d): + MOCKUP.write_directory = d + + +def disable_save_mars(): + MOCKUP.write_directory = None + + +def enable_read_mars(d): + MOCKUP.read_directory = d + + +def disable_read_mars(): + MOCKUP.read_directory = None + + +if os.environ.get("MOCKUP_MARS_SAVE_REQUESTS"): + enable_save_mars(os.environ.get("MOCKUP_MARS_SAVE_REQUESTS")) + +if os.environ.get("MOCKUP_MARS_READ_REQUESTS"): + enable_read_mars(os.environ.get("MOCKUP_MARS_READ_REQUESTS")) + + +def _load_source(*args, **kwargs): + ds = MOCKUP.load_source(*args, **kwargs) + if ds is None: + ds = load_source(*args, **kwargs) + MOCKUP.write(ds, *args, **kwargs) + return ds + + def compare_dot_zattrs(a, b): if isinstance(a, dict): a_keys = list(a.keys())