Merge remote-tracking branch 'upstream/master' into hotfix_branch_1.0.3

* upstream/master: Remove __repr__ methods from all neighbourhood plugins (#1648) ENH: Avoiding lazy loading in select command calls (#1617) MOBT-180: Weather symbol speed up (#1638) IM-1621: Make ECC error and warning tests more rigorous (#1641) Make flake8 report that it is okay when running improver-tests. (#1645) Update checksums after updating the title of files in apply-emos-coefficients/sites. (#1640) Fixes bug in spot-extraction for multi-time inputs (#1633) Updates checksums for threshold landmask fix (#1636) Update interpret-metadata (#1632) Weather code tree update (#1635) Fix noise in precip accumulation thresholds (#1627) Expanding on triangle time blending doc strings. (#1630) Better handling and documentation of dependencies (#1589) Add tests (#1626) Enhancements on new regridding code (#1560) Speed up interpolation in ensemble_copula_coupling.ConvertProbabilitiesToPercentiles (#1578) Speed up interpolation in ensemble_copula_coupling.ResamplePercentiles (#1548) Spot-extraction additional coordinates ordering fix (#1610)
metoppv · Jan 18, 2022 · 040aca9 · 040aca9
2 parents 00af0b1 + 217acad
commit 040aca9
Show file tree

Hide file tree

Showing 68 changed files with 1,988 additions and 681 deletions.
diff --git a/.github/workflows/tests.yml → .github/workflows/ci.yml b/.github/workflows/tests.yml → .github/workflows/ci.yml
@@ -1,13 +1,16 @@
-name: Tests
+name: CI Tests
 
-on: [pull_request, push]
+on:
+  pull_request:
+  push:
+  workflow_dispatch:
 jobs:
   Sphinx-Pytest-Coverage:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        env: [py37_iris30, py38_iris30]
+        env: [environment_a, environment_b, conda-forge]
     steps:
     - uses: actions/checkout@v2
     - uses: actions/cache@v2
@@ -16,45 +19,54 @@ jobs:
         # Increase this value to reset cache
         CACHE_NUMBER: 2
       with:
-        path: /usr/share/miniconda/envs/improver_${{ matrix.env }}
-        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }}
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
     - name: conda env update
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda install -c conda-forge mamba
-        mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }}
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
     - name: conda info
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         conda info
         conda list
-    - name: sphinx-build
+    - name: sphinx documentation
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         make -C doc html SPHINXOPTS="-W --keep-going"
-    - name: pytest unit-tests & cov-report
+    - name: pytest without coverage
+      if: matrix.env != 'environment_a'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
+        pytest
+    - name: pytest with coverage
+      if: matrix.env == 'environment_a'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
         pytest --cov=improver --cov-report xml:coverage.xml
-    - name: codacy-coverage
-      if: env.CODACY_PROJECT_TOKEN
+    - name: codacy upload
+      if: env.CODACY_PROJECT_TOKEN && matrix.env == 'environment_a'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         python-codacy-coverage -v -r coverage.xml
       env:
         CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}
-    - uses: codecov/codecov-action@v1
+    - name: codecov upload
+      uses: codecov/codecov-action@v1
+      if: matrix.env == 'environment_a'
   Codestyle-and-flake8:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        env: [py37_iris30]
+        env: [environment_a]
     steps:
     - uses: actions/checkout@v2
     - uses: actions/cache@v2
@@ -63,41 +75,41 @@ jobs:
         # Increase this value to reset cache
         CACHE_NUMBER: 2
       with:
-        path: /usr/share/miniconda/envs/improver_${{ matrix.env }}
-        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }}
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
     - name: conda env update
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda install -c conda-forge mamba
-        mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }}
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
     - name: conda info
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         conda info
         conda list
     - name: isort
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         isort --check-only .
     - name: black
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         black --check .
     - name: flake8
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         flake8 improver improver_tests
   Safety-Bandit:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        env: [py37_iris30, py38_iris30]
+        env: [environment_a, environment_b, conda-forge]
     steps:
     - uses: actions/checkout@v2
     - uses: actions/cache@v2
@@ -106,60 +118,27 @@ jobs:
         # Increase this value to reset cache
         CACHE_NUMBER: 2
       with:
-        path: /usr/share/miniconda/envs/improver_${{ matrix.env }}
-        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }}
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
     - name: conda env update
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda install -c conda-forge mamba
-        mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }}
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
     - name: conda info
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         conda info
         conda list
     - name: safety
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         safety check || true
     - name: bandit
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         bandit -r improver
-  Type-checking:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        env: [py37_iris30]
-    steps:
-    - uses: actions/checkout@v2
-    - uses: actions/cache@v2
-      id: cache
-      env:
-        # Increase this value to reset cache
-        CACHE_NUMBER: 2
-      with:
-        path: /usr/share/miniconda/envs/improver_${{ matrix.env }}
-        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }}
-    - name: conda env update
-      if: steps.cache.outputs.cache-hit != 'true'
-      run: |
-        source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda install -c conda-forge mamba
-        mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }}
-    - name: conda info
-      run: |
-        source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
-        conda info
-        conda list
-    - name: mypy
-      run: |
-        source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
-        mypy improver || true
diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml
@@ -0,0 +1,137 @@
+name: Scheduled Tests
+
+on:
+  schedule:
+    - cron: '7 4 * * *'
+  workflow_dispatch:
+jobs:
+  Sphinx-Pytest-Coverage:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        env: [latest]
+    if: github.repository_owner == 'metoppv'
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      id: cache
+      env:
+        # Increase this value to reset cache
+        CACHE_NUMBER: 2
+      with:
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
+    - name: conda env update
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda install -c conda-forge mamba
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
+    - name: conda info
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        conda info
+        conda list
+    - name: sphinx documentation
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        make -C doc html SPHINXOPTS="-W --keep-going"
+    - name: pytest without coverage
+      if: matrix.env != 'environment_a'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        pytest
+    - name: pytest with coverage
+      if: matrix.env == 'environment_a'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        pytest --cov=improver --cov-report xml:coverage.xml
+    - name: codacy upload
+      if: env.CODACY_PROJECT_TOKEN && matrix.env == 'environment_a'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        python-codacy-coverage -v -r coverage.xml
+      env:
+        CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}
+    - name: codecov upload
+      uses: codecov/codecov-action@v1
+      if: matrix.env == 'environment_a'
+  Safety-Bandit:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        env: [latest]
+    if: github.repository_owner == 'metoppv'
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      id: cache
+      env:
+        # Increase this value to reset cache
+        CACHE_NUMBER: 2
+      with:
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
+    - name: conda env update
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda install -c conda-forge mamba
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
+    - name: conda info
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        conda info
+        conda list
+    - name: safety
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        safety check || true
+    - name: bandit
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        bandit -r improver
+  Type-checking:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        env: [latest]
+    if: github.repository_owner == 'metoppv'
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      id: cache
+      env:
+        # Increase this value to reset cache
+        CACHE_NUMBER: 2
+      with:
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
+    - name: conda env update
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda install -c conda-forge mamba
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
+    - name: conda info
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        conda info
+        conda list
+    - name: mypy
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        mypy improver || true
diff --git a/bin/improver-tests b/bin/improver-tests
@@ -68,6 +68,7 @@ function improver_test_isort {
 
 function improver_test_flake8 {
     ${FLAKE8:-flake8} $FILES_TO_TEST
+    echo_ok "flake8"
 }
 
 function improver_test_doc {

diff --git a/codecov.yml b/codecov.yml
@@ -8,3 +8,4 @@ coverage:
           informational: true
 ignore:
   - "improver/cli"  # ignore folders and all its contents
+  - "improver/ensemble_copula_coupling/numba_utilities.py"  # numba not installed in CI environment