diff --git a/.github/workflows/tests.yml b/.github/workflows/ci.yml similarity index 53% rename from .github/workflows/tests.yml rename to .github/workflows/ci.yml index 1e0de81ce2..cbedcb778f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/ci.yml @@ -1,13 +1,16 @@ -name: Tests +name: CI Tests -on: [pull_request, push] +on: + pull_request: + push: + workflow_dispatch: jobs: Sphinx-Pytest-Coverage: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - env: [py37_iris30, py38_iris30] + env: [environment_a, environment_b, conda-forge] steps: - uses: actions/checkout@v2 - uses: actions/cache@v2 @@ -16,45 +19,54 @@ jobs: # Increase this value to reset cache CACHE_NUMBER: 2 with: - path: /usr/share/miniconda/envs/improver_${{ matrix.env }} - key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }} + path: /usr/share/miniconda/envs/im${{ matrix.env }} + key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }} - name: conda env update if: steps.cache.outputs.cache-hit != 'true' run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' conda install -c conda-forge mamba - mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }} + mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }} - name: conda info run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} conda info conda list - - name: sphinx-build + - name: sphinx documentation run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} make -C doc html SPHINXOPTS="-W --keep-going" - - name: pytest unit-tests & cov-report + - name: pytest without coverage + if: matrix.env != 'environment_a' run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} + pytest + - name: pytest with coverage + if: matrix.env == 'environment_a' + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} pytest --cov=improver --cov-report xml:coverage.xml - - name: codacy-coverage - if: env.CODACY_PROJECT_TOKEN + - name: codacy upload + if: env.CODACY_PROJECT_TOKEN && matrix.env == 'environment_a' run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} python-codacy-coverage -v -r coverage.xml env: CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }} - - uses: codecov/codecov-action@v1 + - name: codecov upload + uses: codecov/codecov-action@v1 + if: matrix.env == 'environment_a' Codestyle-and-flake8: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - env: [py37_iris30] + env: [environment_a] steps: - uses: actions/checkout@v2 - uses: actions/cache@v2 @@ -63,41 +75,41 @@ jobs: # Increase this value to reset cache CACHE_NUMBER: 2 with: - path: /usr/share/miniconda/envs/improver_${{ matrix.env }} - key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }} + path: /usr/share/miniconda/envs/im${{ matrix.env }} + key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }} - name: conda env update if: steps.cache.outputs.cache-hit != 'true' run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' conda install -c conda-forge mamba - mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }} + mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }} - name: conda info run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} conda info conda list - name: isort run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} isort --check-only . - name: black run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} black --check . - name: flake8 run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} flake8 improver improver_tests Safety-Bandit: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - env: [py37_iris30, py38_iris30] + env: [environment_a, environment_b, conda-forge] steps: - uses: actions/checkout@v2 - uses: actions/cache@v2 @@ -106,60 +118,27 @@ jobs: # Increase this value to reset cache CACHE_NUMBER: 2 with: - path: /usr/share/miniconda/envs/improver_${{ matrix.env }} - key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }} + path: /usr/share/miniconda/envs/im${{ matrix.env }} + key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }} - name: conda env update if: steps.cache.outputs.cache-hit != 'true' run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' conda install -c conda-forge mamba - mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }} + mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }} - name: conda info run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} conda info conda list - name: safety run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} safety check || true - name: bandit run: | source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} + conda activate im${{ matrix.env }} bandit -r improver - Type-checking: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - env: [py37_iris30] - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2 - id: cache - env: - # Increase this value to reset cache - CACHE_NUMBER: 2 - with: - path: /usr/share/miniconda/envs/improver_${{ matrix.env }} - key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }} - - name: conda env update - if: steps.cache.outputs.cache-hit != 'true' - run: | - source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda install -c conda-forge mamba - mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }} - - name: conda info - run: | - source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} - conda info - conda list - - name: mypy - run: | - source '/usr/share/miniconda/etc/profile.d/conda.sh' - conda activate improver_${{ matrix.env }} - mypy improver || true diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml new file mode 100644 index 0000000000..dcd2c412ca --- /dev/null +++ b/.github/workflows/scheduled.yml @@ -0,0 +1,137 @@ +name: Scheduled Tests + +on: + schedule: + - cron: '7 4 * * *' + workflow_dispatch: +jobs: + Sphinx-Pytest-Coverage: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + env: [latest] + if: github.repository_owner == 'metoppv' + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + id: cache + env: + # Increase this value to reset cache + CACHE_NUMBER: 2 + with: + path: /usr/share/miniconda/envs/im${{ matrix.env }} + key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }} + - name: conda env update + if: steps.cache.outputs.cache-hit != 'true' + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda install -c conda-forge mamba + mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }} + - name: conda info + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + conda info + conda list + - name: sphinx documentation + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + make -C doc html SPHINXOPTS="-W --keep-going" + - name: pytest without coverage + if: matrix.env != 'environment_a' + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + pytest + - name: pytest with coverage + if: matrix.env == 'environment_a' + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + pytest --cov=improver --cov-report xml:coverage.xml + - name: codacy upload + if: env.CODACY_PROJECT_TOKEN && matrix.env == 'environment_a' + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + python-codacy-coverage -v -r coverage.xml + env: + CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }} + - name: codecov upload + uses: codecov/codecov-action@v1 + if: matrix.env == 'environment_a' + Safety-Bandit: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + env: [latest] + if: github.repository_owner == 'metoppv' + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + id: cache + env: + # Increase this value to reset cache + CACHE_NUMBER: 2 + with: + path: /usr/share/miniconda/envs/im${{ matrix.env }} + key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }} + - name: conda env update + if: steps.cache.outputs.cache-hit != 'true' + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda install -c conda-forge mamba + mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }} + - name: conda info + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + conda info + conda list + - name: safety + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + safety check || true + - name: bandit + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + bandit -r improver + Type-checking: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + env: [latest] + if: github.repository_owner == 'metoppv' + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + id: cache + env: + # Increase this value to reset cache + CACHE_NUMBER: 2 + with: + path: /usr/share/miniconda/envs/im${{ matrix.env }} + key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }} + - name: conda env update + if: steps.cache.outputs.cache-hit != 'true' + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda install -c conda-forge mamba + mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }} + - name: conda info + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + conda info + conda list + - name: mypy + run: | + source '/usr/share/miniconda/etc/profile.d/conda.sh' + conda activate im${{ matrix.env }} + mypy improver || true diff --git a/doc/source/Dependencies.rst b/doc/source/Dependencies.rst new file mode 100644 index 0000000000..9a5a63d4c6 --- /dev/null +++ b/doc/source/Dependencies.rst @@ -0,0 +1,162 @@ +Dependencies +================ + +.. contents:: Contents + :depth: 2 + + +IMPROVER builds on the functionality provided by a range of open source +libraries. + +Some of these libraries are widely used throughout the IMPROVER code, so are +considered as required for IMPROVER as a whole. + +Other libraries are only used in specific parts of IMPROVER. +These libraries are optional for IMPROVER as a whole, but are required to use +related parts of IMPROVER. +Optional installation of these libraries allows for smaller sized installations +of IMPROVER and reduces conda environment dependency solving difficulties. + +Required +----------------- + +cartopy +~~~~~~~~~~~~~~~~~ +Cartopy is used for grid projections and coordinate transformations. + +https://scitools.org.uk/cartopy/docs/stable/ + + +cftime +~~~~~~~~~~~~~~~~~ +cftime provides functions for handling time in NetCDF files according to the +Climate and Forecast (CF) conventions. + +https://unidata.github.io/cftime/ + + +cf-units +~~~~~~~~~~~~~~~~~ +cf-units provides units conversion following the Climate and Forecast (CF) +conventions. + +https://cf-units.readthedocs.io/en/stable/ + + +Clize +~~~~~~~~~~~~~~~~~ +Clize automatically generates command line interfaces (CLI) from Python function +signatures. + +https://clize.readthedocs.io/en/stable/ + + +Dask +~~~~~~~~~~~~~~~~~ +Dask lazy-loaded arrays are used (often via Iris) to reduce the amount of data +loaded into memory at once. + +https://dask.org/ + + +Iris +~~~~~~~~~~~~~~~~~ +Iris cubes are used as a primary data structure throughout the IMPROVER code. + +https://scitools-iris.readthedocs.io/en/stable/ + + +NetCDF4 +~~~~~~~~~~~~~~~~~ +Python library for reading and writing NetCDF data files via Iris. + +https://unidata.github.io/netcdf4-python/ + + +Numpy +~~~~~~~~~~~~~~~~~ +Multidimensional numerical array library, used as the basis for Iris cubes and +dask arrays. + +https://numpy.org/doc/stable/ + + +Scipy +~~~~~~~~~~~~~~~~~ +Scientific python library, used for a variety of statistical, image processing, +interpolation and spatial functions. + +https://docs.scipy.org/doc/scipy/reference/ + + +Sigtools +~~~~~~~~~~~~~~~~~ +Sigtools provides introspection tools for function signatures. +Sigtools is required by clize, so this dependency is needed anyway +despite minor usage in IMPROVER. + +https://sigtools.readthedocs.io/en/stable/ + + +Sphinx +~~~~~~~~~~~~~~~~~ +Sphinx is a documentation library for Python. IMPROVER requires it at runtime +to generate help strings from CLI function docstrings via clize. + +https://www.sphinx-doc.org/en/master/ + + +Optional dependencies +--------------------- + +python-stratify +~~~~~~~~~~~~~~~~~~ +Vectorised (cython) interpolation, particularly for vertical levels of the +atmosphere. + +https://github.com/SciTools/python-stratify + +Required for CLIs: ``interpolate-using-difference``, ``phase-change-level`` + +statsmodels +~~~~~~~~~~~~~~~~~~ +Estimation of statistical models, used for +:doc:`EMOS `. + +https://www.statsmodels.org/stable/ + +Required for CLIs: ``estimate-emos-coefficients`` + +numba +~~~~~~~~~~~~~~~~~~ +JIT compiler for numerical Python code, used for better computational performance. + +https://numba.readthedocs.io/en/stable/ + +Required for CLIs: ``generate-timezone-mask-ancillary`` + +PySTEPS +~~~~~~~~~~~~~~~~~~ +Probabilistic nowcasting of radar precipitation fields, used for nowcasting. + +https://pysteps.github.io/ + +Required for CLIs: ``nowcast-accumulate``, ``nowcast-extrapolate``, +``nowcast-optical-flow-from-winds`` + +pytz +~~~~~~~~~~~~~~~~~ +Timezone database for Python. + +https://pythonhosted.org/pytz/ + +Required for CLIs: ``generate-timezone-mask-ancillary`` + +timezonefinder +~~~~~~~~~~~~~~~~~~ +Lookup of timezone using geographic coordinates, used to generate timezone +grids. + +https://timezonefinder.readthedocs.io/en/stable/ + +Required for CLIs: ``generate-timezone-mask-ancillary`` diff --git a/doc/source/Running-at-your-site.rst b/doc/source/Running-at-your-site.rst index 91b8c6fd50..ac7cea27a1 100644 --- a/doc/source/Running-at-your-site.rst +++ b/doc/source/Running-at-your-site.rst @@ -22,13 +22,10 @@ or ``python setup.py install`` (``python setup.py develop``). Note that ``pip install`` will not work in an empty environment due to problems with installation of the dependency ``iris`` via pip. -Required dependencies are listed in -`environment_py37_iris30.yml `_, -and -`environment_py38_iris30.yml `_, -depending on your version requirements. Both environment files are used -to run the test suite on Github actions, so these conda environments -file should stay up to date with any dependency changes. +Example environments are included in the repository ``envs`` directory. +These environment files are used to run the test suite on Github actions, +so they should stay up to date with any dependency changes. See also +documentation about :doc:`use of dependencies in IMPROVER `. Alternatively, you can manually 'install' by downloading the code and putting the IMPROVER ``bin/`` directory in your PATH. diff --git a/doc/source/index.rst b/doc/source/index.rst index a09e7f34cc..85e24660ab 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -15,6 +15,7 @@ Guidance-for-Reviewing-Code How-to-implement-a-command-line-utility Running-at-your-site + Dependencies Ticket-Creation-and-Definition-of-Ready .. toctree:: diff --git a/envs/conda-forge.yml b/envs/conda-forge.yml new file mode 100644 index 0000000000..e1d8793fed --- /dev/null +++ b/envs/conda-forge.yml @@ -0,0 +1,28 @@ +# This environment is intended to be used for the conda-forge improver-feedstock +# dependencies in https://github.com/conda-forge/improver-feedstock +# If this file is changed, recipe/meta.yaml in the improver-feedstock repository +# should also be updated. +# This environment should not include optional dependencies. +# This environment should pin versions so that all unit tests pass. +name: improver_conda_forge +channels: + - conda-forge +dependencies: + - python>=3.6 + # Included in improver-feedstock requirements + - cartopy<0.20 + - cftime<1.5 + - cf-units=2.1.5 + - clize + - dask + - iris>=3.0,<3.1 + - netCDF4 + - numpy<1.21 + - scipy + - sigtools + - sphinx + # Additional libraries to run tests, not included in improver-feedstock + - bandit + - filelock + - pytest + - sphinx-autodoc-typehints diff --git a/envs/environment_py37_iris30.yml b/envs/environment_a.yml similarity index 51% rename from envs/environment_py37_iris30.yml rename to envs/environment_a.yml index e6f0e0a5d4..fff6eaf147 100644 --- a/envs/environment_py37_iris30.yml +++ b/envs/environment_a.yml @@ -1,32 +1,29 @@ -# Use this file to create a conda environment using: -# conda env create --file envs/environment_py37_iris30.yml - -name: improver_py37_iris30 +# This environment is a representative example for usage of IMPROVER +name: improver_a channels: - conda-forge dependencies: - python=3.7 # Required - cartopy=0.19 - - cftime<1.5 + - cftime=1.2.1 - cf-units=2.1.5 - - clize - - dask - - iris=3.0 - - netCDF4 - - numpy<1.21 - - pandas - - python-dateutil - - python-stratify + - clize=4.1.1 + - dask=2021.8.1 + - iris=3.0.3 + - netCDF4=1.4.1 + - numpy=1.20.2 + - pandas=1.2.4 - pytz=2020.5 - - scipy=1.6 - - sigtools - - sphinx + - scipy=1.6.2 + - sigtools=2.0.2 + - sphinx=4.0.1 # Optional - - fastparquet - - numba + - fastparquet=0.7.1 + - statsmodels=0.12.2 + - numba=0.53.1 - pysteps=1.4.1 - - statsmodels + - python-stratify=0.1.1 - timezonefinder=4.1.0 # Development - astroid @@ -35,7 +32,7 @@ dependencies: - codacy-coverage - filelock - flake8 - - isort=5.* + - isort=5.7.0 - mock - mypy - pytest diff --git a/envs/environment_b.yml b/envs/environment_b.yml new file mode 100644 index 0000000000..82057101bf --- /dev/null +++ b/envs/environment_b.yml @@ -0,0 +1,41 @@ +# This environment is a representative example for usage of IMPROVER +# It does not include dependencies for the following IMPROVER CLIs: +# estimate-emos-coefficients, generate-timezone-mask-ancillary, +# nowcast-accumulate, nowcast-extrapolate and nowcast-optical-flow-from-winds +# See doc/source/Dependencies.rst for more information +name: improver_b +channels: + - conda-forge +dependencies: + - python=3.8 + # Required + - cartopy=0.19 + - cftime=1.2.1 + - cf-units=2.1.5 + - clize=4.2.0 + - dask=2021.8.1 + - iris=3.0.3 + - netCDF4=1.5.7 + - numpy=1.20 + - pandas=1.3.4 + - pytz=2020.5 + - scipy=1.6.2 + - sigtools=2.0.3 + - sphinx=4.0.2 + # Optional + - numba=0.53.1 + - python-stratify=0.2.post0 + # Development + - astroid + - bandit + - black=19.10b0 + - filelock + - flake8 + - isort=5 + - mock + - mypy + - pytest + - pytest-cov + - pytest-xdist + - safety + - sphinx-autodoc-typehints diff --git a/envs/environment_py38_iris30.yml b/envs/environment_py38_iris30.yml deleted file mode 100644 index 413f894951..0000000000 --- a/envs/environment_py38_iris30.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Use this file to create a conda environment using: -# conda env create --file envs/environment_py38_iris30.yml - -name: improver_py38_iris30 -channels: - - conda-forge -dependencies: - - python=3.8 - # Required - - cartopy=0.19 - - cftime<1.5 - - cf-units=2.1.5 - - clize - - dask - - iris=3.0 - - netCDF4 - - numpy<1.21 - - pandas - - python-dateutil - - python-stratify - - pytz=2020.5 - - scipy=1.6 - - sigtools - - sphinx - # Optional - # Development - - astroid - - bandit - - black=19.10b0 - - codacy-coverage - - filelock - - flake8 - - isort=5.* - - mock - - mypy - - pytest - - pytest-cov - - pytest-xdist - - safety - - sphinx-autodoc-typehints diff --git a/envs/latest.yml b/envs/latest.yml new file mode 100644 index 0000000000..ae82212e3f --- /dev/null +++ b/envs/latest.yml @@ -0,0 +1,44 @@ +# This environment is intended to provide all optional dependencies and +# use as-recent-as-possible versions with minimal pinning. +# Acceptance test failures due to different output values are OK. +# Unit test failures are not OK - pinning should be used where needed to +# make the unit tests pass. +name: improver_latest +channels: + - conda-forge +dependencies: + - python=3 + # Required + - cartopy<0.20 + - cftime<1.5 + - cf-units=2.1.5 + - clize + - dask + - iris>=3.0 + - netCDF4 + - numpy<1.21 + - pytz + - scipy + - sigtools + - sphinx + # Optional + - python-stratify + - statsmodels + - numba + - pysteps + - timezonefinder + # Development + - astroid + - bandit + - black + - codacy-coverage + - filelock + - flake8 + - isort + - mock + - mypy + - pytest + - pytest-cov + - pytest-xdist + - safety + - sphinx-autodoc-typehints diff --git a/improver/metadata/amend.py b/improver/metadata/amend.py index 9d6a916187..28a3a30a9e 100644 --- a/improver/metadata/amend.py +++ b/improver/metadata/amend.py @@ -29,10 +29,9 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. """Module containing utilities for modifying cube metadata""" -from datetime import datetime +from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Union -from dateutil import tz from iris.cube import Cube, CubeList from improver.metadata.constants.mo_attributes import ( @@ -94,8 +93,9 @@ def set_history_attribute(cube: Cube, value: str, append: bool = False) -> None: If True, add to the existing history rather than replacing the existing attribute. Default is False. """ - tzinfo = tz.tzoffset("Z", 0) - timestamp = datetime.strftime(datetime.now(tzinfo), "%Y-%m-%dT%H:%M:%S%Z") + timestamp = datetime.strftime( + datetime.now(timezone(timedelta(0), name="Z")), "%Y-%m-%dT%H:%M:%S%Z" + ) new_history = "{}: {}".format(timestamp, value) if append and "history" in cube.attributes.keys(): cube.attributes["history"] += "; {}".format(new_history) diff --git a/improver/psychrometric_calculations/psychrometric_calculations.py b/improver/psychrometric_calculations/psychrometric_calculations.py index e7ada36ec6..99d221c101 100644 --- a/improver/psychrometric_calculations/psychrometric_calculations.py +++ b/improver/psychrometric_calculations/psychrometric_calculations.py @@ -38,7 +38,6 @@ from cf_units import Unit from iris.cube import Cube, CubeList from numpy import ndarray -from stratify import interpolate import improver.constants as consts from improver import BasePlugin @@ -587,6 +586,8 @@ def find_falling_level( Returns: Phase change level data asl. """ + from stratify import interpolate + # Create cube of heights above sea level for each height in # the wet bulb integral cube. asl = wb_int_data.copy() diff --git a/improver_tests/acceptance/test_checksums.py b/improver_tests/acceptance/test_checksums.py index 8cd01038c9..d08e8603cd 100644 --- a/improver_tests/acceptance/test_checksums.py +++ b/improver_tests/acceptance/test_checksums.py @@ -131,7 +131,10 @@ def test_checksums_sorted(): This test doesn't depend on having the acceptance test data available, so can run with the unit tests. """ - csum_paths = [str(path) for path in acc.acceptance_checksums().keys()] + try: + csum_paths = [str(path) for path in acc.acceptance_checksums().keys()] + except FileNotFoundError: + pytest.skip("no checksum file, likely due to package being installed") with temporary_sort_locale("C") as strcoll: csum_paths_sorted = sorted(csum_paths, key=functools.cmp_to_key(strcoll)) assert csum_paths == csum_paths_sorted diff --git a/improver_tests/acceptance/test_interpolate_using_difference.py b/improver_tests/acceptance/test_interpolate_using_difference.py index 2cea0b8ea3..fd51706ec2 100644 --- a/improver_tests/acceptance/test_interpolate_using_difference.py +++ b/improver_tests/acceptance/test_interpolate_using_difference.py @@ -85,6 +85,7 @@ def test_filling_with_nearest_use(tmp_path): """Test filling masked areas using difference interpolation, in this case with a hole in the corner of the data that requires use of nearest neighbour interpolation.""" + pytest.importorskip("stratify") kgo_dir = acc.kgo_root() / f"{CLI}/basic" kgo_path = kgo_dir / "sleet_rain_nearest_filled_kgo.nc" output_path = tmp_path / "output.nc" diff --git a/improver_tests/acceptance/test_phase_change_level.py b/improver_tests/acceptance/test_phase_change_level.py index 13a986f08d..035c721282 100644 --- a/improver_tests/acceptance/test_phase_change_level.py +++ b/improver_tests/acceptance/test_phase_change_level.py @@ -55,6 +55,7 @@ def test_phase_change(tmp_path, phase_type, kgo_name, horiz_interp): snow/sleet level sleet/rain level leaving below orography points unfilled. """ + pytest.importorskip("stratify") kgo_dir = acc.kgo_root() / f"{CLI}/basic" kgo_name = "{}_kgo.nc".format(kgo_name) kgo_path = kgo_dir / kgo_name diff --git a/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py b/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py index 2a02423fa4..14aabf638c 100644 --- a/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py +++ b/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py @@ -89,11 +89,24 @@ def global_grid_fixture_360() -> Cube: return cube +@pytest.fixture(name="europe_grid") +def europe_grid_fixture() -> Cube: + data = np.zeros((10, 10), dtype=np.float32) + cube = set_up_variable_cube( + data, + name="template", + grid_spacing=1, + domain_corner=(45, -5), + attributes=GLOBAL_ATTRIBUTES, + ) + return cube + + @pytest.fixture(name="uk_grid") def uk_grid_fixture() -> Cube: """UK grid template""" - data = np.zeros((21, 22), dtype=np.float32) + data = np.zeros((11, 11), dtype=np.float32) cube = set_up_variable_cube( data, name="template", @@ -160,7 +173,7 @@ def test__get_coordinate_pairs(request, grid_fixture): expected_data = { "global_grid": [[-90.0, -180.0], [-90.0, -80.0], [90.0, 180.0]], "global_grid_360": [[-90.0, -180.0], [-90.0, -80.0], [90.0, 180.0]], - "uk_grid": [[44.517, -17.117], [45.548, -4.913], [62.026, 14.410]], + "uk_grid": [[44.517, -17.117], [45.548, -4.913], [54.263, -5.401]], } grid = request.getfixturevalue(grid_fixture) @@ -245,7 +258,7 @@ def test__create_template_cube(request, grid_fixture, include_dst): expected = { "global_grid": {"shape": (19, 37), "attributes": GLOBAL_ATTRIBUTES}, - "uk_grid": {"shape": (21, 22), "attributes": UK_ATTRIBUTES}, + "uk_grid": {"shape": (11, 11), "attributes": UK_ATTRIBUTES}, } # Set expected includes_daylight_savings attribute @@ -309,50 +322,98 @@ def test__group_timezones_empty_group(timezone_mask): # Expected data for process tests - -# ungrouped -GLOBAL_GRID = {"shape": (27, 19, 37), "min": -12 * 3600, "max": 14 * 3600} -# grouped -GLOBAL_GRID_GR = {"shape": (2, 19, 37), "min": -6 * 3600, "max": 6 * 3600} -UK_GRID_GR = {"shape": (2, 21, 22), "min": -6 * 3600, "max": 6 * 3600} - EXPECTED_TIME = {None: 1510286400, "20200716T1500Z": 1594911600} +GROUPED_MIN_MAX = {"min": -6 * 3600, "max": 6 * 3600} EXPECTED = { "ungrouped": { - "global": { - None: {**GLOBAL_GRID, "data": np.array([1, 1, 1, 1, 1, 0, 1, 1, 1, 1])}, - "20200716T1500Z": {**GLOBAL_GRID, "data": np.ones([10])}, - "indices": (12, 2), - }, "uk": { None: { - "shape": (4, 21, 22), - "min": -2 * 3600, + "shape": (3, 11, 11), + "min": -1 * 3600, + "max": 1 * 3600, + "data": np.array( + [ + [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0], + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + ] + ), + }, + "20200716T1500Z": { + "shape": (4, 11, 11), + "min": -1 * 3600, + "max": 2 * 3600, + "data": np.array( + [ + [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0], + [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1], + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + ], + ), + }, + }, + "europe": { + None: { + "shape": (2, 10, 10), + "min": 0, "max": 1 * 3600, - "data": np.array([1, 1, 0, 0, 0, 1]), + "data": np.array( + [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]] + ), }, "20200716T1500Z": { - "shape": (5, 21, 22), - "min": -2 * 3600, + "shape": (3, 10, 10), + "min": 0, "max": 2 * 3600, - "data": np.array([1, 1, 1, 1, 0, 1]), + "data": np.array( + [ + [1, 0, 1, 1, 1, 1, 0, 0, 0, 0], + [0, 1, 0, 0, 0, 0, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + ] + ), }, - "indices": (2, 10), }, }, "grouped": { - "global": { - None: {**GLOBAL_GRID_GR, "data": np.array([0, 0, 0, 0, 0, 0, 1, 1, 1, 1])}, + "uk": { + None: { + "shape": (2, 11, 11), + **GROUPED_MIN_MAX, + "data": np.array( + [ + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + ] + ), + }, "20200716T1500Z": { - **GLOBAL_GRID_GR, - "data": np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]), + "shape": (2, 11, 11), + **GROUPED_MIN_MAX, + "data": np.array( + [ + [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0], + [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1], + ], + ), }, - "indices": (0, 2), }, - "uk": { - None: {**UK_GRID_GR, "data": np.array([0, 0, 0, 0, 0, 1])}, - "20200716T1500Z": {**UK_GRID_GR, "data": np.array([0, 0, 1, 1, 0, 1])}, - "indices": (0, 9), + "europe": { + None: { + "shape": (2, 10, 10), + **GROUPED_MIN_MAX, + "data": np.array( + [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]] + ), + }, + "20200716T1500Z": { + "shape": (2, 10, 10), + **GROUPED_MIN_MAX, + "data": np.array( + [[1, 0, 1, 1, 1, 1, 0, 0, 0, 0], [0, 1, 0, 0, 0, 0, 1, 1, 1, 1]] + ), + }, }, }, } @@ -360,7 +421,7 @@ def test__group_timezones_empty_group(timezone_mask): @pytest.mark.parametrize("grouping", ["ungrouped", "grouped"]) @pytest.mark.parametrize("time", [None, "20200716T1500Z"]) -@pytest.mark.parametrize("grid_fixture", ["global_grid", "global_grid_360", "uk_grid"]) +@pytest.mark.parametrize("grid_fixture", ["uk_grid", "europe_grid"]) def test_process(request, grid_fixture, time, grouping): """Test that the process method returns cubes that take the expected form for different grids and different dates. @@ -376,7 +437,6 @@ def test_process(request, grid_fixture, time, grouping): expected = EXPECTED[grouping][domain][time] expected_time = EXPECTED_TIME[time] - index = EXPECTED[grouping][domain]["indices"] grid = request.getfixturevalue(grid_fixture) @@ -393,5 +453,14 @@ def test_process(request, grid_fixture, time, grouping): assert ( result.coord("UTC_offset").bounds.dtype == TIME_COORDS["UTC_offset"].dtype ) + # slice the first spatial dimension to moderate size of expected arrays + assert_array_equal(result.data[:, 9, :], expected["data"]) - assert_array_equal(result.data[index][::4], expected["data"]) + # check each spatial location in the UTC_offset dimension + zone_count = np.count_nonzero(result.data, axis=0) + if grouping == "grouped": + # grouped outputs have a single UTC_offset with a non-zero entry + assert_array_equal(zone_count, 1) + else: + # ungrouped outputs have a single UTC_offset with a zero entry + assert_array_equal(zone_count, expected["shape"][0] - 1) diff --git a/improver_tests/psychrometric_calculations/test_PhaseChangeLevel.py b/improver_tests/psychrometric_calculations/test_PhaseChangeLevel.py index 237e4b89b7..e061baafb3 100644 --- a/improver_tests/psychrometric_calculations/test_PhaseChangeLevel.py +++ b/improver_tests/psychrometric_calculations/test_PhaseChangeLevel.py @@ -34,6 +34,7 @@ import iris import numpy as np +import pytest from cf_units import Unit from iris.cube import CubeList from iris.tests import IrisTest @@ -109,6 +110,7 @@ class Test_find_falling_level(IrisTest): def setUp(self): """Set up arrays.""" + pytest.importorskip("stratify") self.wb_int_data = np.array( [ [[80.0, 80.0], [70.0, 50.0]], @@ -449,7 +451,7 @@ def setUp(self): """Set up orography and land-sea mask cubes. Also create temperature, pressure, and relative humidity cubes that contain multiple height levels.""" - + pytest.importorskip("stratify") self.setup_cubes_for_process() def setup_cubes_for_process(self, spatial_grid="equalarea"): diff --git a/setup.cfg b/setup.cfg index 73a8026068..74a7564144 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,45 +18,15 @@ packages = find: setup_requires = setuptools >= 38.3.0 setuptools_scm -install_requires = - cartopy - cftime == 1.0.1 - cf_units - clize - dask - netCDF4 - numpy - python-dateutil - pytz - scipy >= 1.3.0, < 1.4.0 - scitools-iris >= 3.0 - sigtools - sphinx - statsmodels - stratify +# Note: no install_requires run-time requirements are included here. +# Requirements are expected to be provided through another method such as conda. +# See envs directory at top level of repository. scripts = bin/improver [options.packages.find] -exclude = improver_tests - -[options.extras_require] -dev = - astroid - bandit - black == 19.10b0 - codacy-coverage - filelock - isort == 5.* - mock - mypy - pytest - pytest-cov - safety - sphinx-autodoc-typehints -full = - numba - pysteps == 1.3.2 - timezonefinder +exclude = + improver_tests + improver_tests.* [flake8] max-line-length = 100