From aafbd0d90dd4eb899241c5270e0f455c6c0d185f Mon Sep 17 00:00:00 2001 From: Florian Pinault Date: Mon, 21 Aug 2023 06:53:07 +0000 Subject: [PATCH 1/5] fix constants source when input request is a list --- climetlab/sources/constants.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/climetlab/sources/constants.py b/climetlab/sources/constants.py index 4ff99632..04a14bdc 100644 --- a/climetlab/sources/constants.py +++ b/climetlab/sources/constants.py @@ -207,6 +207,9 @@ def make_datetime(date, time): class Constants(FieldSet): @normalize("date", "date-list") def __init__(self, source_or_dataset, request={}, repeat=1, **kwargs): + if isinstance(request, list): + request = {k: k for k in request} + request = dict(**request) request.update(kwargs) From afb8a2ba9ddfb7024151f46e09ddd47abaf30397 Mon Sep 17 00:00:00 2001 From: Florian Pinault Date: Mon, 21 Aug 2023 07:44:23 +0000 Subject: [PATCH 2/5] up --- climetlab/loaders/__init__.py | 4 ++-- climetlab/sources/constants.py | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/climetlab/loaders/__init__.py b/climetlab/loaders/__init__.py index b31ed0c0..c30d3f42 100644 --- a/climetlab/loaders/__init__.py +++ b/climetlab/loaders/__init__.py @@ -351,8 +351,8 @@ def _load(loader, config, append, print_=print, **kwargs): data = cml.load_source("loader", config.input) - if "constants" in config.input and config.input.constants: - data = data + cml.load_source("constants", data, config.input.constants) + # if "constants" in config.input and config.input.constants: + # data = data + cml.load_source("constants", data, config.input.constants) assert len(data) print(f"Done in {seconds(time.time()-start)}, length: {len(data):,}.") diff --git a/climetlab/sources/constants.py b/climetlab/sources/constants.py index 04a14bdc..4ff99632 100644 --- a/climetlab/sources/constants.py +++ b/climetlab/sources/constants.py @@ -207,9 +207,6 @@ def make_datetime(date, time): class Constants(FieldSet): @normalize("date", "date-list") def __init__(self, source_or_dataset, request={}, repeat=1, **kwargs): - if isinstance(request, list): - request = {k: k for k in request} - request = dict(**request) request.update(kwargs) From cc1c04f72f3bd6824a1a9841eb46ff83e21ff6bf Mon Sep 17 00:00:00 2001 From: Matthew Chantry Date: Fri, 25 Aug 2023 15:34:08 +0000 Subject: [PATCH 3/5] TISR change --- climetlab/sources/constants.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/climetlab/sources/constants.py b/climetlab/sources/constants.py index 4ff99632..7335b983 100644 --- a/climetlab/sources/constants.py +++ b/climetlab/sources/constants.py @@ -145,10 +145,11 @@ def toa_incident_solar_radiation(self, date): date = to_datetime(date) result = toa_incident_solar_radiation( - date - datetime.timedelta(hours=1), - date, + date - datetime.timedelta(minutes=30), + date + datetime.timedelta(minutes=30), self.latitude_(), self.longitude_(), + intervals_per_hour=2, ) return result.flatten() From 349a2b77f52965cd3c3ff86ef81bbba854ef7bc3 Mon Sep 17 00:00:00 2001 From: Baudouin Raoult Date: Fri, 25 Aug 2023 17:01:22 +0000 Subject: [PATCH 4/5] update --- climetlab/readers/bzip.py | 45 +++++++++++++++++++++++++++++++ climetlab/readers/grib/codes.py | 1 + climetlab/readers/grib/pytorch.py | 2 +- climetlab/utils/patterns.py | 22 ++++++++++++++- 4 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 climetlab/readers/bzip.py diff --git a/climetlab/readers/bzip.py b/climetlab/readers/bzip.py new file mode 100644 index 00000000..9d68013f --- /dev/null +++ b/climetlab/readers/bzip.py @@ -0,0 +1,45 @@ +# (C) Copyright 2020 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. +# + +import bz2 +import os + +from . import Reader +from . import reader as find_reader + + +class BZReader(Reader): + def __init__(self, source, path): + super().__init__(source, path) + + def uncompress(target, _): + with open(target, "wb") as g: + with bz2.open(path, "rb") as f: + while True: + chunk = f.read(1024 * 1204) + if not chunk: + break + g.write(chunk) + + self.unzipped = self.cache_file( + uncompress, + dict(path=path), + ) + + def mutate(self): + print('mutare', self.source, self.unzipped) + return find_reader(self.source, self.unzipped) + + +def reader(source, path, magic=None, deeper_check=False): + if magic is None: # Bypass check and force + return BZReader(source, path) + + if magic[:3] == b"BZh": + return BZReader(source, path) diff --git a/climetlab/readers/grib/codes.py b/climetlab/readers/grib/codes.py index 6ea20d72..658efd4b 100644 --- a/climetlab/readers/grib/codes.py +++ b/climetlab/readers/grib/codes.py @@ -99,6 +99,7 @@ def get(count): except TypeError: pass + class CodesHandle: def __init__(self, handle, path, offset): self.handle = handle diff --git a/climetlab/readers/grib/pytorch.py b/climetlab/readers/grib/pytorch.py index f0f70d5f..37a59f48 100644 --- a/climetlab/readers/grib/pytorch.py +++ b/climetlab/readers/grib/pytorch.py @@ -11,7 +11,7 @@ import numpy as np -from climetlab.ml.torch import to_pytorch_dataloader +# from climetlab.ml.torch import to_pytorch_dataloader from .tensorflow import default_merger, to_funcs diff --git a/climetlab/utils/patterns.py b/climetlab/utils/patterns.py index da8b3f07..86903411 100644 --- a/climetlab/utils/patterns.py +++ b/climetlab/utils/patterns.py @@ -15,6 +15,7 @@ RE1 = re.compile(r"{([^}]*)}") RE2 = re.compile(r"\(([^}]*)\)") +RE3 = re.compile(r"\(([^}]*)\)") class Any: def substitute(self, value, name): @@ -114,6 +115,22 @@ def substitute(self, params): raise ValueError("Missing parameter '{}'".format(self.name)) return self.kind.substitute(params[self.name], self.name) +FUNCTIONS = dict(lower=lambda s: s.lower()) + +class Function: + def __init__(self, value): + functions = value.split('|') + self.name = functions[0] + self.variable = Variable(functions[0]) + self.functions = functions[1:] + + def substitute(self, params): + value = self.variable.substitute(params) + for f in self.functions: + value = FUNCTIONS[f](value) + return value + + class Pattern: def __init__(self, pattern, ignore_missing_keys=False): @@ -125,7 +142,10 @@ def __init__(self, pattern, ignore_missing_keys=False): if i % 2 == 0: self.pattern.append(Constant(p)) else: - v = Variable(p) + if '|' in p: + v = Function(p) + else: + v = Variable(p) self.variables.append(v) self.pattern.append(v) From 43a364ee20914139e28ae4a3a8c64183fb151b03 Mon Sep 17 00:00:00 2001 From: Florian Pinault Date: Wed, 30 Aug 2023 14:10:18 +0000 Subject: [PATCH 5/5] Bump version 0.16.5 --- climetlab/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/climetlab/version b/climetlab/version index 5f2491c5..19270385 100644 --- a/climetlab/version +++ b/climetlab/version @@ -1 +1 @@ -0.16.4 +0.16.5