Merge branch 'release/0.16.5'

climetlab/loaders/__init__.py

@@ -351,8 +351,8 @@ def _load(loader, config, append, print_=print, **kwargs):
 
     data = cml.load_source("loader", config.input)
 
-    if "constants" in config.input and config.input.constants:
-        data = data + cml.load_source("constants", data, config.input.constants)
+    # if "constants" in config.input and config.input.constants:
+    #    data = data + cml.load_source("constants", data, config.input.constants)
 
     assert len(data)
     print(f"Done in {seconds(time.time()-start)}, length: {len(data):,}.")

climetlab/readers/bzip.py

@@ -0,0 +1,45 @@
+# (C) Copyright 2020 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+import bz2
+import os
+
+from . import Reader
+from . import reader as find_reader
+
+
+class BZReader(Reader):
+    def __init__(self, source, path):
+        super().__init__(source, path)
+
+        def uncompress(target, _):
+            with open(target, "wb") as g:
+                with bz2.open(path, "rb") as f:
+                    while True:
+                        chunk = f.read(1024 * 1204)
+                        if not chunk:
+                            break
+                        g.write(chunk)
+
+        self.unzipped = self.cache_file(
+            uncompress,
+            dict(path=path),
+        )
+
+    def mutate(self):
+        print('mutare', self.source, self.unzipped)
+        return find_reader(self.source, self.unzipped)
+
+
+def reader(source, path, magic=None, deeper_check=False):
+    if magic is None:  # Bypass check and force
+        return BZReader(source, path)
+
+    if magic[:3] == b"BZh":
+        return BZReader(source, path)

climetlab/readers/grib/pytorch.py

@@ -11,7 +11,7 @@
 
 import numpy as np
 
-from climetlab.ml.torch import to_pytorch_dataloader
+# from climetlab.ml.torch import to_pytorch_dataloader
 
 from .tensorflow import default_merger, to_funcs
 

climetlab/sources/constants.py

@@ -145,10 +145,11 @@ def toa_incident_solar_radiation(self, date):
 
         date = to_datetime(date)
         result = toa_incident_solar_radiation(
-            date - datetime.timedelta(hours=1),
-            date,
+            date - datetime.timedelta(minutes=30),
+            date + datetime.timedelta(minutes=30),
             self.latitude_(),
             self.longitude_(),
+            intervals_per_hour=2,
         )
         return result.flatten()
 

climetlab/utils/patterns.py

@@ -15,6 +15,7 @@
 RE1 = re.compile(r"{([^}]*)}")
 RE2 = re.compile(r"\(([^}]*)\)")
 
+RE3 = re.compile(r"\(([^}]*)\)")
 
 class Any:
     def substitute(self, value, name):
@@ -114,6 +115,22 @@ def substitute(self, params):
             raise ValueError("Missing parameter '{}'".format(self.name))
         return self.kind.substitute(params[self.name], self.name)
 
+FUNCTIONS = dict(lower=lambda s: s.lower())
+
+class Function:
+    def __init__(self, value):
+        functions = value.split('|')
+        self.name = functions[0]
+        self.variable = Variable(functions[0])
+        self.functions = functions[1:]
+
+    def substitute(self, params):
+        value = self.variable.substitute(params)
+        for f in self.functions:
+            value = FUNCTIONS[f](value)
+        return value
+
+
 
 class Pattern:
     def __init__(self, pattern, ignore_missing_keys=False):
@@ -125,7 +142,10 @@ def __init__(self, pattern, ignore_missing_keys=False):
             if i % 2 == 0:
                 self.pattern.append(Constant(p))
             else:
-                v = Variable(p)
+                if '|' in p:
+                    v = Function(p)
+                else:
+                    v = Variable(p)
                 self.variables.append(v)
                 self.pattern.append(v)
 

climetlab/version

@@ -1 +1 @@
-0.16.4
+0.16.5