Merge branch 'develop' of github.com:ecmwf-lab/ecml-tools into develop

.github/workflows/python-publish.yml

@@ -15,18 +15,13 @@ jobs:
     name: Code QA
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - run: pip install black flake8 isort
-    - run: black --version
-    - run: isort --version
-    - run: flake8 --version
-    - run: isort --check .
-    - run: black --check .
-    - run: flake8 .
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: 3.x
+    - uses: pre-commit/action@v3.0.1
 
   checks:
-    if: ${{ github.event_name == 'release' }}
-
     strategy:
       fail-fast: false
       matrix:
@@ -35,10 +30,9 @@ jobs:
 
     name: Python  ${{ matrix.python-version }} on ${{ matrix.platform }}
     runs-on: ${{ matrix.platform }}
-    needs: quality
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-python@v2
       with:
@@ -54,14 +48,46 @@ jobs:
     - name: Tests
       run: pytest
 
+  notify-failure:
+    if: failure()
+    runs-on: ubuntu-latest
+    needs: [quality, checks]
+    name: Notify failure
+    steps:
+      - uses: jdcargile/ms-teams-notification@v1.4
+        with:
+          github-token: ${{ github.token }}
+          ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI_F }}
+          # notification-summary: ${{ steps.qa.outputs.status }}
+          notification-summary: Build failed on ecml-tools. Please check the logs.
+          notification-color: dc3545
+          timezone: Europe/Paris
+          verbose-logging: true
+
+  notify-success:
+    if: success()
+    runs-on: ubuntu-latest
+    needs: [quality, checks]
+    name: Notify success
+    steps:
+      - uses: jdcargile/ms-teams-notification@v1.4
+        with:
+          github-token: ${{ github.token }}
+          ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI_F }}
+          # notification-summary: ${{ steps.qa.outputs.status }}
+          notification-summary: New commit pushed on ecml-tools. Everything is fine.
+          notification-color: 17a2b8
+          timezone: Europe/Paris
+          verbose-logging: true
+
   deploy:
 
     if: ${{ github.event_name == 'release' }}
     runs-on: ubuntu-latest
-    needs: checks
+    needs: [checks, quality]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: Set up Python
       uses: actions/setup-python@v2

.pre-commit-config.yaml

@@ -1,29 +1,49 @@
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
-    hooks:
-    -   id: check-yaml # Check YAML files for syntax errors
-    -   id: debug-statements # Check for debugger imports and py37+ breakpoint()
-#    -   id: end-of-file-fixer # Ensure files end in a newline
-#    -   id: trailing-whitespace # Trailing whitespace checker
-    -   id: no-commit-to-branch # Prevent committing to main / master
--   repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.1.1
-    hooks:
-      - id: black
-        args: [--line-length=120]
--   repo: https://github.com/pycqa/isort
-    rev: 5.13.2
-    hooks:
-      - id: isort
-        name: isort (python)
-      - id: isort
-        name: isort (cython)
-        types: [cython]
-      - id: isort
-        name: isort (pyi)
-        types: [pyi]
--   repo: https://github.com/pycqa/flake8
-    rev: 7.0.0
-    hooks:
-    -   id: flake8
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.4.0
+  hooks:
+  - id: check-yaml # Check YAML files for syntax errors only
+    args: [--unsafe, --allow-multiple-documents]
+  - id: debug-statements # Check for debugger imports and py37+ breakpoint()
+#  - id: end-of-file-fixer # Ensure files end in a newline
+#  - id: trailing-whitespace # Trailing whitespace checker
+  - id: no-commit-to-branch # Prevent committing to main / master
+  - id: check-added-large-files # Check for large files added to git
+  - id: check-merge-conflict # Check for files that contain merge conflict
+- repo: https://github.com/psf/black-pre-commit-mirror
+  rev: 24.1.1
+  hooks:
+    - id: black
+      args: [--line-length=120]
+- repo: https://github.com/pycqa/isort
+  rev: 5.13.2
+  hooks:
+  - id: isort
+    args:
+    - -l 120
+    - --force-single-line-imports
+    - --profile black
+#- repo: https://github.com/pycqa/flake8
+#  rev: 7.0.0
+#  hooks:
+#  -   id: flake8
+#  #    args: [--exit-zero]
+#  #    verbose: true
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.3.0
+  hooks:
+  - id: ruff
+    args:
+    - --line-length=120
+    - --ignore=E203
+    - --fix
+    - --exit-non-zero-on-fix
+    - --preview 
+    - --exclude
+    - 'dev/*.py'
+#- repo: https://github.com/pre-commit/mirrors-mypy
+#  rev: v0.720
+#  hooks:
+#  -   id: mypy
+#      verbose: true
+#      entry: bash -c 'mypy "$@" || true' --

ecml_tools/__init__.py

@@ -5,4 +5,4 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
-__version__ = "0.5.0"
+__version__ = "0.6.0"

ecml_tools/__main__.py

@@ -21,9 +21,7 @@
 
 
 def main():
-    parser = argparse.ArgumentParser(
-        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
-    )
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
 
     parser.add_argument(
         "--version",

ecml_tools/commands/__init__.py

@@ -25,13 +25,7 @@ def register(here, package, select, fail=None):
         full = os.path.join(here, p)
         if p.startswith("_"):
             continue
-        if not (
-            p.endswith(".py")
-            or (
-                os.path.isdir(full)
-                and os.path.exists(os.path.join(full, "__init__.py"))
-            )
-        ):
+        if not (p.endswith(".py") or (os.path.isdir(full) and os.path.exists(os.path.join(full, "__init__.py")))):
             continue
 
         name, _ = os.path.splitext(p)

ecml_tools/commands/copy.py

@@ -8,7 +8,8 @@
 import logging
 import os
 import sys
-from concurrent.futures import ThreadPoolExecutor, as_completed
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import as_completed
 
 import tqdm
 

ecml_tools/commands/create.py

@@ -8,9 +8,7 @@ class Create(Command):
     timestamp = True
 
     def add_arguments(self, command_parser):
-        command_parser.add_argument(
-            "--overwrite", action="store_true", help="Overwrite existing files"
-        )
+        command_parser.add_argument("--overwrite", action="store_true", help="Overwrite existing files")
         command_parser.add_argument("config", help="Configuration file")
         command_parser.add_argument("path", help="Path to store the created data")
 

ecml_tools/commands/inspect/__init__.py

@@ -31,11 +31,7 @@ def add_arguments(self, command_parser):
     def run(self, args):
         dic = vars(args)
         for path in dic.pop("path"):
-            if (
-                os.path.isdir(path)
-                or path.endswith(".zarr.zip")
-                or path.endswith(".zarr")
-            ):
+            if os.path.isdir(path) or path.endswith(".zarr.zip") or path.endswith(".zarr"):
                 self.inspect_zarr(path=path, **dic)
             else:
                 raise ValueError(f"Unknown file type: {path}")

ecml_tools/commands/inspect/zarr.py

@@ -16,9 +16,14 @@
 import semantic_version
 import tqdm
 
-from ecml_tools.data import open_dataset, open_zarr
-from ecml_tools.utils.humanize import bytes, number, when
-from ecml_tools.utils.text import dotted_line, progress, table
+from ecml_tools.data import open_dataset
+from ecml_tools.data import open_zarr
+from ecml_tools.utils.humanize import bytes
+from ecml_tools.utils.humanize import number
+from ecml_tools.utils.humanize import when
+from ecml_tools.utils.text import dotted_line
+from ecml_tools.utils.text import progress
+from ecml_tools.utils.text import table
 
 LOG = logging.getLogger(__name__)
 
@@ -28,9 +33,7 @@ def compute_directory_size(path):
         return None, None
     size = 0
     n = 0
-    for dirpath, _, filenames in tqdm.tqdm(
-        os.walk(path), desc="Computing size", leave=False
-    ):
+    for dirpath, _, filenames in tqdm.tqdm(os.walk(path), desc="Computing size", leave=False):
         for filename in filenames:
             file_path = os.path.join(dirpath, filename)
             size += os.path.getsize(file_path)
@@ -311,21 +314,15 @@ def progress(self):
         assert build_flags.size == build_lengths.size
 
         latest_write_timestamp = self.zarr.attrs.get("latest_write_timestamp")
-        latest = (
-            datetime.datetime.fromisoformat(latest_write_timestamp)
-            if latest_write_timestamp
-            else None
-        )
+        latest = datetime.datetime.fromisoformat(latest_write_timestamp) if latest_write_timestamp else None
 
         if not all(build_flags):
             if latest:
                 print(f"🪫  Dataset not ready, last update {when(latest)}.")
             else:
                 print("🪫  Dataset not ready.")
             total = sum(build_lengths)
-            built = sum(
-                ln if flag else 0 for ln, flag in zip(build_lengths, build_flags)
-            )
+            built = sum(ln if flag else 0 for ln, flag in zip(build_lengths, build_flags))
             print(
                 "📈 Progress:",
                 progress(built, total, width=50),
@@ -411,9 +408,7 @@ def last_date(self):
         assert isinstance(time, int), (time, type(time))
         if time > 100:
             time = time // 100
-        return datetime.datetime.fromisoformat(monthly["stop"]) + datetime.timedelta(
-            hours=time
-        )
+        return datetime.datetime.fromisoformat(monthly["stop"]) + datetime.timedelta(hours=time)
 
     @property
     def frequency(self):
@@ -472,12 +467,8 @@ def _info(self, verbose, history, statistics, **kwargs):
 
         # for backward compatibility
         if "climetlab" in z.attrs:
-            climetlab_version = (
-                z.attrs["climetlab"].get("versions", {}).get("climetlab", "unkwown")
-            )
-            print(
-                f"climetlab version used to create this zarr: {climetlab_version}. Not supported."
-            )
+            climetlab_version = z.attrs["climetlab"].get("versions", {}).get("climetlab", "unkwown")
+            print(f"climetlab version used to create this zarr: {climetlab_version}. Not supported.")
             return
 
         version = z.attrs.get("version")
@@ -503,12 +494,7 @@ def initialised(self):
                 return datetime.datetime.fromisoformat(record["timestamp"])
 
         # Sometimes the first record is missing
-        timestamps = sorted(
-            [
-                datetime.datetime.fromisoformat(d["timestamp"])
-                for d in self.metadata.get("history", [])
-            ]
-        )
+        timestamps = sorted([datetime.datetime.fromisoformat(d["timestamp"]) for d in self.metadata.get("history", [])])
         if timestamps:
             return timestamps[0]
 

ecml_tools/create/check.py

@@ -153,7 +153,7 @@ class StatisticsValueError(ValueError):
 
 def check_data_values(arr, *, name: str, log=[], allow_nan=False):
     if allow_nan is False:
-        allow_nan = lambda x: False
+        allow_nan = lambda x: False  # noqa: E731
     if allow_nan(name):
         arr = arr[~np.isnan(arr)]