Merge branch 'develop' of github.com:ecmwf/climetlab into develop

.github/workflows/long-test.yml

@@ -77,6 +77,10 @@ jobs:
       - name: Install test tools for notebooks
         run: |
           pip install nbformat nbconvert ipykernel
+
+      - name: Install climetlab[interactive] for the notebooks
+        run: |
+          pip install -e .[interactive]
           pip freeze
 
       - name: Tests notebooks

.github/workflows/test-and-release.yml

@@ -86,6 +86,10 @@ jobs:
       - name: Install test tools for notebooks
         run: |
           pip install nbformat nbconvert ipykernel
+
+      - name: Install climetlab[interactive] for the notebooks
+        run: |
+          pip install -e .[interactive]
           pip freeze
 
       - name: Tests notebooks
@@ -124,6 +128,8 @@ jobs:
   deploy:
     if: startsWith(github.ref, 'refs/tags/')
     # add if owner, etc.
+    # Release should be done using the release.sh script
+    # https://github.com/ecmwf-lab/climetlab-private-tools.git
 
     name: Upload to Pypi and release
     needs: checks
@@ -133,7 +139,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Check version
+      - name: Check that tag version matches climetlab version
         run: |
           tag=${GITHUB_REF#refs/tags/}
           version=$(python setup.py --version)

.gitignore

@@ -157,3 +157,5 @@ logs/
 *.index
 *.data-*
 *.out
+test.*
+test

climetlab/config/units.yaml

@@ -0,0 +1,17 @@
+---
+# This file contains convertion for units that are not cf-compliants
+
+- from: kelvin
+  to: degC
+  scaling: 1
+  offset: -273.15
+
+- from: kelvin
+  to: K
+  scaling: 1
+  offset: 0
+
+- from: Celcius
+  to: degC
+  scaling: 1
+  offset: 0

climetlab/core/caching.py

@@ -604,6 +604,11 @@ def cache_file(
 
                 check_cache_size()
 
+        try:
+            os.unlink(lock)
+        except OSError:
+            pass
+
     return path
 
 

climetlab/core/data.py

@@ -23,7 +23,13 @@
 
 YAML_FILES = None
 
-IGNORE = ["magics.yaml", "colours.yaml", "conventions.yaml", "config.yaml"]
+IGNORE = [
+    "magics.yaml",
+    "colours.yaml",
+    "conventions.yaml",
+    "config.yaml",
+    "units.yaml",
+]
 
 
 def _guess(data, path):
@@ -98,6 +104,8 @@ def _load_yaml_files():
                 try:
                     with open(path) as f:
                         data = yaml.load(f.read(), Loader=yaml.SafeLoader)
+                        if not isinstance(data, dict):
+                            continue
                         name, _ = os.path.splitext(os.path.basename(path))
                         kind = _guess(data, path)
                         collection = YAML_FILES[kind]

climetlab/mockup.py

@@ -0,0 +1,96 @@
+# (C) Copyright 2020 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+from climetlab.sources import Source
+
+
+class TestingMockup:
+    def __init__(self, *args, **kwargs):
+        self.args = args
+        self.kwargs = kwargs
+
+
+class TestingXarrayAttrs(dict):
+    pass
+
+
+class TestingXarrayDims(list):
+    pass
+
+
+class TestingDatasetAsXarray(TestingMockup):
+    def __init__(self, *args, **kwargs):
+        super(TestingDatasetAsXarray, self).__init__(*args, **kwargs)
+        self.attrs = TestingXarrayAttrs()
+        self.dims = TestingXarrayDims()
+
+    # TODO: make this generic
+    def min(self, *args, **kwargs):
+        print(f"xr.min({args}, {kwargs})")
+        return 42.0
+
+    def max(self, *args, **kwargs):
+        print(f"xr.min({args}, {kwargs})")
+        return 42.0
+
+    def map(self, *args, **kwargs):
+        print("xr.map(...)")
+        # print(f'xr.map({args}, {kwargs})')
+        return self
+
+    def sortby(self, *args, **kwargs):
+        print(f"xr.sortby({args}, {kwargs})")
+        return self
+
+    def __getitem__(self, key):
+        print(f"xr.__getitem__({key})")
+        return self
+
+    def __setitem__(self, key, value):
+        print(f"xr.__setitem__({key})=...")
+        # print(f'xr.__setitem__({key})={value}')
+        return self
+
+    def chunk(self, *args, **kwargs):
+        print(f"xr.chunk({args}, {kwargs})")
+        return self
+
+    def astype(self, *args, **kwargs):
+        print(f"xr.astype({args}, {kwargs})")
+        return self
+
+    def to_zarr(self, *args, **kwargs):
+        print(f"xr.to_zarr({args}, {kwargs})")
+        return self
+
+    def __getattr__(self, name):
+        print(f"xr.{name} (unkwown)")
+        return self
+
+
+class DatasetMockup(TestingMockup):
+    def __init__(self, *args, **kwargs):
+        self.args = args
+        self.kwargs = kwargs
+        print(f"Climetlab SourceMockup : args={args}, kwargs={kwargs}")
+        super(SourceMockup, self).__init__(**kwargs)
+
+    def to_xarray(self, *args, **kwargs):
+        return TestingDatasetAsXarray(*self.args, **self.kwargs)
+
+
+class SourceMockup(Source):
+    def __init__(self, *args, **kwargs):
+        self.args = args
+        self.kwargs = kwargs
+        print(f"Climetlab SourceMockup : args={args}, kwargs={kwargs}")
+        super(SourceMockup, self).__init__(**kwargs)
+
+    def to_xarray(self, *args, **kwargs):
+        return TestingDatasetAsXarray(*self.args, **self.kwargs)

climetlab/plotting/drivers/magics/driver.py

@@ -8,9 +8,11 @@
 #
 
 import logging
+import os
 
 import yaml
 
+import climetlab
 from climetlab.core.ipython import SVG, Image
 from climetlab.core.metadata import annotation
 from climetlab.core.temporary import temp_file
@@ -22,6 +24,11 @@
 LOG = logging.getLogger(__name__)
 
 
+os.environ["MAGICS_UNITS_CONVERSIONS"] = os.path.join(
+    os.path.dirname(climetlab.__file__), "config", "units.yaml"
+)
+
+
 class Layer:
     def __init__(self, data):
         self._data = data
@@ -177,7 +184,6 @@ def tidy(x):
 
     def plot_xarray(self, ds, variable: str, dimensions: dict = None):
         tmp = self.temporary_file(".nc")
-        tmp = "tmp.nc"
         field = ds[variable].isel({} if dimensions is None else dimensions)
         dataset = field.to_dataset()
 

climetlab/readers/csv.py

@@ -11,6 +11,7 @@
 import io
 import logging
 import mimetypes
+import os
 import zipfile
 
 from climetlab.wrappers import get_wrapper
@@ -109,6 +110,12 @@ def probe_csv(
 
 
 def is_csv(path, probe_size=4096, compression=None):
+
+    _, extension = os.path.splitext(path)
+
+    if extension in (".xml",):
+        return False
+
     dialect, _ = probe_csv(path, probe_size, compression, for_is_csv=True)
     return dialect is not None
 

climetlab/readers/text.py

@@ -38,6 +38,7 @@ def ignore(self):
     def mutate(self):
         if is_csv(self.path):
             return CSVReader(self.source, self.path)
+
         return self
 
 

climetlab/scripts.py

@@ -0,0 +1,83 @@
+# (C) Copyright 2020 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+import os
+import sys
+from importlib import import_module
+
+import climetlab
+
+
+def check():
+    print(
+        (
+            "This script is experimental to help debugging."
+            "Seeing everything to 'ok' does NOT means that you have the right versions installed."
+        )
+    )
+    print("--------------------")
+
+    print("Checking climetlab installation.")
+    print(f"Climetlab installed in {os.path.dirname(climetlab.__file__)}")
+    print("Checking required compiled dependencies...")
+
+    import ecmwflibs
+
+    versions = ecmwflibs.versions()
+
+    for name in ["eccodes", "magics"]:
+        try:
+            print(
+                f"  {name} from ecwmlibs: ok {versions[name]} ({ecmwflibs.find(name)})"
+            )
+        except Exception as e:  # noqa: F841
+            print(f"  {name} from ecmwflib: Warning: ecmwflibs cannot find {name}")
+
+    for name in ["eccodes", "MagPlus", "netcdf"]:
+        try:
+            import findlibs
+        except Exception as e:  # noqa: F841
+            print(f"    {name} from findlibs: Warning: cannot import findlibs")
+            continue
+        try:
+            print(f"    {name} from findlibs: ({findlibs.find(name)})")
+        except Exception as e:  # noqa: F841
+            print(f"    {name} from findlibs: Warning: findlibs cannot find {name}")
+
+    print("Checking required python dependencies...")
+    for name in ["xarray", "Magics", "eccodes", "ecmwflibs"]:
+        more = ""
+        try:
+            lib = import_module(name)
+        except ImportError:
+            print(f"  Error: cannot import {name}.")
+            continue
+        if name == "eccodes":
+            more = f" (using .lib={lib.lib})"
+        print(f"  {name}: ok {lib.__version__} ({os.path.dirname(lib.__file__)}){more}")
+
+    print("Checking optional dependencies...")
+    for name in ["folium", "pdbufr", "pyodc"]:
+        try:
+            lib = import_module(name)
+            print(f"  {name}: ok {lib.__version__} ({os.path.dirname(lib.__file__)})")
+        except Exception as e:
+            print(e)
+            print(f"  Warning: cannot import {name}. Limited capabilities.")
+
+    # TODO: add more
+    # TODO: automate this from requirements.txt. Create a pip install climetlab[extra] or climetlab-light.
+
+
+def main_climetlab():
+    if sys.argv and sys.argv[1] == "check":
+        check()
+
+
+if __name__ == "__main__":
+    main_climetlab()

climetlab/sources/__init__.py

@@ -107,8 +107,14 @@ def load_remote(self, name):
 class SourceMaker:
     def __call__(self, name, *args, **kwargs):
         loader = SourceLoader()
+
         klass = find_plugin(os.path.dirname(__file__), name, loader)
 
+        if os.environ.get("CLIMETLAB_TESTING_ENABLE_MOCKUP_SOURCE", False):
+            from climetlab.mockup import SourceMockup
+
+            klass = SourceMockup
+
         source = klass(*args, **kwargs)
 
         if getattr(source, "name", None) is None: