From 709b1e67c171c69cc42f24c47a9be9a499daa5c0 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Tue, 5 Oct 2021 14:56:28 +1100
Subject: [PATCH 01/28] Rename and add environments

---
 .github/workflows/tests.yml                   | 61 ++++++++++---------
 envs/conda-forge.yml                          | 26 ++++++++
 ...ment_py37_iris30.yml => environment_a.yml} |  7 +--
 ...ment_py38_iris30.yml => environment_b.yml} |  9 +--
 envs/latest.yml                               | 45 ++++++++++++++
 5 files changed, 108 insertions(+), 40 deletions(-)
 create mode 100644 envs/conda-forge.yml
 rename envs/{environment_py37_iris30.yml => environment_a.yml} (81%)
 rename envs/{environment_py38_iris30.yml => environment_b.yml} (73%)
 create mode 100644 envs/latest.yml

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 1e0de81ce2..538d01d1d2 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        env: [py37_iris30, py38_iris30]
+        env: [environment_a, environment_b, latest, conda-forge]
     steps:
     - uses: actions/checkout@v2
     - uses: actions/cache@v2
@@ -16,45 +16,46 @@ jobs:
         # Increase this value to reset cache
         CACHE_NUMBER: 2
       with:
-        path: /usr/share/miniconda/envs/improver_${{ matrix.env }}
-        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }}
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
     - name: conda env update
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda install -c conda-forge mamba
-        mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }}
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
     - name: conda info
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         conda info
         conda list
     - name: sphinx-build
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         make -C doc html SPHINXOPTS="-W --keep-going"
     - name: pytest unit-tests & cov-report
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         pytest --cov=improver --cov-report xml:coverage.xml
     - name: codacy-coverage
-      if: env.CODACY_PROJECT_TOKEN
+      if: env.CODACY_PROJECT_TOKEN && matrix.env == 'environment_a'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         python-codacy-coverage -v -r coverage.xml
       env:
         CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}
     - uses: codecov/codecov-action@v1
+      if: matrix.env == 'environment_a'
   Codestyle-and-flake8:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        env: [py37_iris30]
+        env: [environment_a]
     steps:
     - uses: actions/checkout@v2
     - uses: actions/cache@v2
@@ -63,41 +64,41 @@ jobs:
         # Increase this value to reset cache
         CACHE_NUMBER: 2
       with:
-        path: /usr/share/miniconda/envs/improver_${{ matrix.env }}
-        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }}
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
     - name: conda env update
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda install -c conda-forge mamba
-        mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }}
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
     - name: conda info
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         conda info
         conda list
     - name: isort
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         isort --check-only .
     - name: black
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         black --check .
     - name: flake8
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         flake8 improver improver_tests
   Safety-Bandit:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        env: [py37_iris30, py38_iris30]
+        env: [environment_a, environment_b]
     steps:
     - uses: actions/checkout@v2
     - uses: actions/cache@v2
@@ -106,36 +107,36 @@ jobs:
         # Increase this value to reset cache
         CACHE_NUMBER: 2
       with:
-        path: /usr/share/miniconda/envs/improver_${{ matrix.env }}
-        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }}
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
     - name: conda env update
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda install -c conda-forge mamba
-        mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }}
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
     - name: conda info
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         conda info
         conda list
     - name: safety
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         safety check || true
     - name: bandit
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         bandit -r improver
   Type-checking:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        env: [py37_iris30]
+        env: [latest]
     steps:
     - uses: actions/checkout@v2
     - uses: actions/cache@v2
@@ -144,22 +145,22 @@ jobs:
         # Increase this value to reset cache
         CACHE_NUMBER: 2
       with:
-        path: /usr/share/miniconda/envs/improver_${{ matrix.env }}
-        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/environment_{0}.yml', matrix.env))) }}
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
     - name: conda env update
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda install -c conda-forge mamba
-        mamba env update -q --file envs/environment_${{ matrix.env }}.yml --name improver_${{ matrix.env }}
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
     - name: conda info
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         conda info
         conda list
     - name: mypy
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate improver_${{ matrix.env }}
+        conda activate im${{ matrix.env }}
         mypy improver || true
diff --git a/envs/conda-forge.yml b/envs/conda-forge.yml
new file mode 100644
index 0000000000..d7ed42dae1
--- /dev/null
+++ b/envs/conda-forge.yml
@@ -0,0 +1,26 @@
+# This environment is intended to match the conda-forge improver-feedstock at
+# https://github.com/conda-forge/improver-feedstock
+# If it is changed, the feedstock repository should also be updated.
+# This environment should avoid dependencies which are optional in the IMPROVER
+# code, but with restricted versions to allow all tests to pass when those
+# dependencies are installed.
+name: improver_latest
+channels:
+  - conda-forge
+dependencies:
+  - python=3
+  # Included in improver-feedstock requirements
+  - cartopy<0.20
+  - cftime<1.5
+  - cf-units=2.1.5
+  - clize
+  - dask
+  - iris>=3.0,<3.1
+  - netCDF4
+  - numpy<1.21
+  - scipy
+  - sigtools
+  - sphinx
+  # Additional libraries to run tests, not included in improver-feedstock
+  - pytest
+  - filelock
diff --git a/envs/environment_py37_iris30.yml b/envs/environment_a.yml
similarity index 81%
rename from envs/environment_py37_iris30.yml
rename to envs/environment_a.yml
index e6f0e0a5d4..2d2cc78148 100644
--- a/envs/environment_py37_iris30.yml
+++ b/envs/environment_a.yml
@@ -1,7 +1,5 @@
-# Use this file to create a conda environment using:
-# conda env create --file envs/environment_py37_iris30.yml
-
-name: improver_py37_iris30
+# This environment is a representative example for usage of IMPROVER
+name: improver_a
 channels:
   - conda-forge
 dependencies:
@@ -24,6 +22,7 @@ dependencies:
   - sphinx
   # Optional
   - fastparquet
+  - statsmodels
   - numba
   - pysteps=1.4.1
   - statsmodels
diff --git a/envs/environment_py38_iris30.yml b/envs/environment_b.yml
similarity index 73%
rename from envs/environment_py38_iris30.yml
rename to envs/environment_b.yml
index 413f894951..ec6e050f72 100644
--- a/envs/environment_py38_iris30.yml
+++ b/envs/environment_b.yml
@@ -1,7 +1,5 @@
-# Use this file to create a conda environment using:
-# conda env create --file envs/environment_py38_iris30.yml
-
-name: improver_py38_iris30
+# This environment is a representative example for usage of IMPROVER
+name: improver_b
 channels:
   - conda-forge
 dependencies:
@@ -27,10 +25,9 @@ dependencies:
   - astroid
   - bandit
   - black=19.10b0
-  - codacy-coverage
   - filelock
   - flake8
-  - isort=5.*
+  - isort=5
   - mock
   - mypy
   - pytest
diff --git a/envs/latest.yml b/envs/latest.yml
new file mode 100644
index 0000000000..a7abb697ba
--- /dev/null
+++ b/envs/latest.yml
@@ -0,0 +1,45 @@
+# This environment is intended to provide all optional dependencies and
+# use as-recent-as-possible versions with only required specification
+# of versions to make the unit test suite pass.
+# Acceptance test failures due to different output values are OK, but failures
+# due to library API mismatches are not.
+name: improver_latest
+channels:
+  - conda-forge
+dependencies:
+  - python=3
+  # Required
+  - cartopy
+  - cftime<1.5
+  - cf-units=2.1.5
+  - clize
+  - dask
+  - iris>=3.0
+  - netCDF4
+  - numpy
+  - python-dateutil
+  - pytz
+  - scipy
+  - sigtools
+  - sphinx
+  # Optional
+  - python-stratify
+  - statsmodels
+  - numba
+  - pysteps
+  - timezonefinder
+  # Development
+  - astroid
+  - bandit
+  - black
+  - codacy-coverage
+  - filelock
+  - flake8
+  - isort
+  - mock
+  - mypy
+  - pytest
+  - pytest-cov
+  - pytest-xdist
+  - safety
+  - sphinx-autodoc-typehints

From 0f3eae934c78af3f12eb617a1e6c1da41fc6ba59 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Mon, 18 Oct 2021 15:47:08 +1100
Subject: [PATCH 02/28] Thin out requirements in setup.cfg, add note

---
 setup.cfg | 36 +++---------------------------------
 1 file changed, 3 insertions(+), 33 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index 73a8026068..f1a90517fb 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -19,45 +19,15 @@ setup_requires =
     setuptools >= 38.3.0
     setuptools_scm
 install_requires =
-    cartopy
-    cftime == 1.0.1
-    cf_units
-    clize
-    dask
-    netCDF4
-    numpy
-    python-dateutil
-    pytz
-    scipy >= 1.3.0, < 1.4.0
+# Note: only minimal requirements are included here for development usage.
+# Requirements are expected to be provided through another method such as conda.
+# See envs directory at top level of repository.
     scitools-iris >= 3.0
-    sigtools
-    sphinx
-    statsmodels
-    stratify
 scripts = bin/improver
 
 [options.packages.find]
 exclude = improver_tests
 
-[options.extras_require]
-dev =
-    astroid
-    bandit
-    black == 19.10b0
-    codacy-coverage
-    filelock
-    isort == 5.*
-    mock
-    mypy
-    pytest
-    pytest-cov
-    safety
-    sphinx-autodoc-typehints
-full =
-    numba
-    pysteps == 1.3.2
-    timezonefinder
-
 [flake8]
 max-line-length = 100
 select = C,E,F,W

From c2cc73be05fed95c54b8b60bdb7cb84e7fb2cb21 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Mon, 18 Oct 2021 15:47:37 +1100
Subject: [PATCH 03/28] Remove improver_tests tests from built package

The previous exclude was not effective - it excluded the top level
directory, but not the subdirectories containing all the test source
code files.
---
 setup.cfg | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index f1a90517fb..6650dc9761 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -26,7 +26,9 @@ install_requires =
 scripts = bin/improver
 
 [options.packages.find]
-exclude = improver_tests
+exclude =
+    improver_tests
+    improver_tests.*
 
 [flake8]
 max-line-length = 100

From 5ef523681c8b4adf97cfb1bb7720d98ed7045fd3 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Tue, 5 Oct 2021 12:41:28 +1100
Subject: [PATCH 04/28] Skip checksum sorted test if file not available

---
 improver_tests/acceptance/test_checksums.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/improver_tests/acceptance/test_checksums.py b/improver_tests/acceptance/test_checksums.py
index 8cd01038c9..21740246cd 100644
--- a/improver_tests/acceptance/test_checksums.py
+++ b/improver_tests/acceptance/test_checksums.py
@@ -131,7 +131,10 @@ def test_checksums_sorted():
     This test doesn't depend on having the acceptance test data available, so
     can run with the unit tests.
     """
-    csum_paths = [str(path) for path in acc.acceptance_checksums().keys()]
+    try:
+        csum_paths = [str(path) for path in acc.acceptance_checksums().keys()]
+    except FileNotFoundError:
+         pytest.skip("no checksum file, likely due to package being installed")
     with temporary_sort_locale("C") as strcoll:
         csum_paths_sorted = sorted(csum_paths, key=functools.cmp_to_key(strcoll))
     assert csum_paths == csum_paths_sorted

From 0cb26a17ee08ec466c18de61507f92ddceb947bc Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Tue, 5 Oct 2021 16:40:17 +1100
Subject: [PATCH 05/28] Pytest skips for stratify

---
 .../acceptance/test_interpolate_using_difference.py           | 1 +
 improver_tests/acceptance/test_phase_change_level.py          | 1 +
 .../psychrometric_calculations/test_PhaseChangeLevel.py       | 4 +++-
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/improver_tests/acceptance/test_interpolate_using_difference.py b/improver_tests/acceptance/test_interpolate_using_difference.py
index 2cea0b8ea3..fd51706ec2 100644
--- a/improver_tests/acceptance/test_interpolate_using_difference.py
+++ b/improver_tests/acceptance/test_interpolate_using_difference.py
@@ -85,6 +85,7 @@ def test_filling_with_nearest_use(tmp_path):
     """Test filling masked areas using difference interpolation, in this case
     with a hole in the corner of the data that requires use of nearest
     neighbour interpolation."""
+    pytest.importorskip("stratify")
     kgo_dir = acc.kgo_root() / f"{CLI}/basic"
     kgo_path = kgo_dir / "sleet_rain_nearest_filled_kgo.nc"
     output_path = tmp_path / "output.nc"
diff --git a/improver_tests/acceptance/test_phase_change_level.py b/improver_tests/acceptance/test_phase_change_level.py
index 13a986f08d..035c721282 100644
--- a/improver_tests/acceptance/test_phase_change_level.py
+++ b/improver_tests/acceptance/test_phase_change_level.py
@@ -55,6 +55,7 @@ def test_phase_change(tmp_path, phase_type, kgo_name, horiz_interp):
         snow/sleet level
         sleet/rain level leaving below orography points unfilled.
     """
+    pytest.importorskip("stratify")
     kgo_dir = acc.kgo_root() / f"{CLI}/basic"
     kgo_name = "{}_kgo.nc".format(kgo_name)
     kgo_path = kgo_dir / kgo_name
diff --git a/improver_tests/psychrometric_calculations/test_PhaseChangeLevel.py b/improver_tests/psychrometric_calculations/test_PhaseChangeLevel.py
index 237e4b89b7..295d7a80f9 100644
--- a/improver_tests/psychrometric_calculations/test_PhaseChangeLevel.py
+++ b/improver_tests/psychrometric_calculations/test_PhaseChangeLevel.py
@@ -37,6 +37,7 @@
 from cf_units import Unit
 from iris.cube import CubeList
 from iris.tests import IrisTest
+import pytest
 
 from improver.psychrometric_calculations.psychrometric_calculations import (
     PhaseChangeLevel,
@@ -109,6 +110,7 @@ class Test_find_falling_level(IrisTest):
 
     def setUp(self):
         """Set up arrays."""
+        pytest.importorskip("stratify")
         self.wb_int_data = np.array(
             [
                 [[80.0, 80.0], [70.0, 50.0]],
@@ -449,7 +451,7 @@ def setUp(self):
         """Set up orography and land-sea mask cubes. Also create temperature,
         pressure, and relative humidity cubes that contain multiple height
         levels."""
-
+        pytest.importorskip("stratify")
         self.setup_cubes_for_process()
 
     def setup_cubes_for_process(self, spatial_grid="equalarea"):

From 0e71185e49649cd1a73a33d9b3658dd118fdb97c Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Mon, 18 Oct 2021 16:12:46 +1100
Subject: [PATCH 06/28] Move stratify import inside function

---
 .../psychrometric_calculations/psychrometric_calculations.py    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/improver/psychrometric_calculations/psychrometric_calculations.py b/improver/psychrometric_calculations/psychrometric_calculations.py
index e7ada36ec6..ab04a8067c 100644
--- a/improver/psychrometric_calculations/psychrometric_calculations.py
+++ b/improver/psychrometric_calculations/psychrometric_calculations.py
@@ -38,7 +38,6 @@
 from cf_units import Unit
 from iris.cube import Cube, CubeList
 from numpy import ndarray
-from stratify import interpolate
 
 import improver.constants as consts
 from improver import BasePlugin
@@ -587,6 +586,7 @@ def find_falling_level(
         Returns:
             Phase change level data asl.
         """
+        from stratify import interpolate
         # Create cube of heights above sea level for each height in
         # the wet bulb integral cube.
         asl = wb_int_data.copy()

From 07a0e97661228e5e95248e17297bb121a403bcd5 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Mon, 18 Oct 2021 16:39:28 +1100
Subject: [PATCH 07/28] Remove duplicated statsmodels

---
 envs/environment_a.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/envs/environment_a.yml b/envs/environment_a.yml
index 2d2cc78148..ecb27a2d4f 100644
--- a/envs/environment_a.yml
+++ b/envs/environment_a.yml
@@ -25,7 +25,6 @@ dependencies:
   - statsmodels
   - numba
   - pysteps=1.4.1
-  - statsmodels
   - timezonefinder=4.1.0
   # Development
   - astroid

From ea9e48ae5047f22380b8b471a32d2a56f2dad450 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Mon, 18 Oct 2021 16:40:15 +1100
Subject: [PATCH 08/28] Better explanation for latest environment

---
 envs/conda-forge.yml                                      | 7 +++----
 envs/latest.yml                                           | 8 ++++----
 .../psychrometric_calculations.py                         | 1 +
 improver_tests/acceptance/test_checksums.py               | 2 +-
 .../psychrometric_calculations/test_PhaseChangeLevel.py   | 2 +-
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/envs/conda-forge.yml b/envs/conda-forge.yml
index d7ed42dae1..0e240ce61c 100644
--- a/envs/conda-forge.yml
+++ b/envs/conda-forge.yml
@@ -1,14 +1,13 @@
 # This environment is intended to match the conda-forge improver-feedstock at
 # https://github.com/conda-forge/improver-feedstock
 # If it is changed, the feedstock repository should also be updated.
-# This environment should avoid dependencies which are optional in the IMPROVER
-# code, but with restricted versions to allow all tests to pass when those
-# dependencies are installed.
+# This environment should not include optional dependencies.
+# This environment should pin versions so that all unit tests pass.
 name: improver_latest
 channels:
   - conda-forge
 dependencies:
-  - python=3
+  - python>=3.6
   # Included in improver-feedstock requirements
   - cartopy<0.20
   - cftime<1.5
diff --git a/envs/latest.yml b/envs/latest.yml
index a7abb697ba..4934e15815 100644
--- a/envs/latest.yml
+++ b/envs/latest.yml
@@ -1,8 +1,8 @@
 # This environment is intended to provide all optional dependencies and
-# use as-recent-as-possible versions with only required specification
-# of versions to make the unit test suite pass.
-# Acceptance test failures due to different output values are OK, but failures
-# due to library API mismatches are not.
+# use as-recent-as-possible versions with minimal pinning.
+# Acceptance test failures due to different output values are OK.
+# Unit test failures are not OK - pinning should be used where needed to
+# make the unit tests pass.
 name: improver_latest
 channels:
   - conda-forge
diff --git a/improver/psychrometric_calculations/psychrometric_calculations.py b/improver/psychrometric_calculations/psychrometric_calculations.py
index ab04a8067c..99d221c101 100644
--- a/improver/psychrometric_calculations/psychrometric_calculations.py
+++ b/improver/psychrometric_calculations/psychrometric_calculations.py
@@ -587,6 +587,7 @@ def find_falling_level(
             Phase change level data asl.
         """
         from stratify import interpolate
+
         # Create cube of heights above sea level for each height in
         # the wet bulb integral cube.
         asl = wb_int_data.copy()
diff --git a/improver_tests/acceptance/test_checksums.py b/improver_tests/acceptance/test_checksums.py
index 21740246cd..d08e8603cd 100644
--- a/improver_tests/acceptance/test_checksums.py
+++ b/improver_tests/acceptance/test_checksums.py
@@ -134,7 +134,7 @@ def test_checksums_sorted():
     try:
         csum_paths = [str(path) for path in acc.acceptance_checksums().keys()]
     except FileNotFoundError:
-         pytest.skip("no checksum file, likely due to package being installed")
+        pytest.skip("no checksum file, likely due to package being installed")
     with temporary_sort_locale("C") as strcoll:
         csum_paths_sorted = sorted(csum_paths, key=functools.cmp_to_key(strcoll))
     assert csum_paths == csum_paths_sorted
diff --git a/improver_tests/psychrometric_calculations/test_PhaseChangeLevel.py b/improver_tests/psychrometric_calculations/test_PhaseChangeLevel.py
index 295d7a80f9..e061baafb3 100644
--- a/improver_tests/psychrometric_calculations/test_PhaseChangeLevel.py
+++ b/improver_tests/psychrometric_calculations/test_PhaseChangeLevel.py
@@ -34,10 +34,10 @@
 
 import iris
 import numpy as np
+import pytest
 from cf_units import Unit
 from iris.cube import CubeList
 from iris.tests import IrisTest
-import pytest
 
 from improver.psychrometric_calculations.psychrometric_calculations import (
     PhaseChangeLevel,

From a76687a4c219e82f22ea02bf83ee446e5c932387 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Mon, 18 Oct 2021 16:48:39 +1100
Subject: [PATCH 09/28] Add sphinx typehints to conda-forge tests section

---
 envs/conda-forge.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/envs/conda-forge.yml b/envs/conda-forge.yml
index 0e240ce61c..e8a561b73e 100644
--- a/envs/conda-forge.yml
+++ b/envs/conda-forge.yml
@@ -21,5 +21,6 @@ dependencies:
   - sigtools
   - sphinx
   # Additional libraries to run tests, not included in improver-feedstock
-  - pytest
   - filelock
+  - pytest
+  - sphinx-autodoc-typehints

From 48ee11446d4c636b1506df3d70f62769c44d7074 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Mon, 18 Oct 2021 16:49:20 +1100
Subject: [PATCH 10/28] Run security checks on all environments

---
 .github/workflows/tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 538d01d1d2..adcad9ccfe 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -98,7 +98,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        env: [environment_a, environment_b]
+        env: [environment_a, environment_b, latest, conda-forge]
     steps:
     - uses: actions/checkout@v2
     - uses: actions/cache@v2

From 1d913fe3e8a376554ac86a20179128fe26b6c530 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Mon, 18 Oct 2021 16:56:36 +1100
Subject: [PATCH 11/28] Separate coverage and no-coverage environments

---
 .github/workflows/tests.yml | 16 ++++++++++++----
 envs/conda-forge.yml        |  1 +
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index adcad9ccfe..2216fa073c 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -30,17 +30,24 @@ jobs:
         conda activate im${{ matrix.env }}
         conda info
         conda list
-    - name: sphinx-build
+    - name: sphinx documentation
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda activate im${{ matrix.env }}
         make -C doc html SPHINXOPTS="-W --keep-going"
-    - name: pytest unit-tests & cov-report
+    - name: pytest without coverage
+      if: matrix.env != 'environment_a'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        pytest
+    - name: pytest with coverage
+      if: matrix.env == 'environment_a'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda activate im${{ matrix.env }}
         pytest --cov=improver --cov-report xml:coverage.xml
-    - name: codacy-coverage
+    - name: codacy upload
       if: env.CODACY_PROJECT_TOKEN && matrix.env == 'environment_a'
       run: |
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
@@ -48,7 +55,8 @@ jobs:
         python-codacy-coverage -v -r coverage.xml
       env:
         CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}
-    - uses: codecov/codecov-action@v1
+    - name: codecov upload
+      uses: codecov/codecov-action@v1
       if: matrix.env == 'environment_a'
   Codestyle-and-flake8:
     runs-on: ubuntu-latest
diff --git a/envs/conda-forge.yml b/envs/conda-forge.yml
index e8a561b73e..4094e7881f 100644
--- a/envs/conda-forge.yml
+++ b/envs/conda-forge.yml
@@ -21,6 +21,7 @@ dependencies:
   - sigtools
   - sphinx
   # Additional libraries to run tests, not included in improver-feedstock
+  - bandit
   - filelock
   - pytest
   - sphinx-autodoc-typehints

From c13391d6a3c5f03b8be07840049177e9e378ce31 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Mon, 18 Oct 2021 17:37:29 +1100
Subject: [PATCH 12/28] Add detailed pinning to A/B environments

---
 envs/environment_a.yml | 32 ++++++++++++++++----------------
 envs/environment_b.yml | 25 +++++++++++++------------
 2 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/envs/environment_a.yml b/envs/environment_a.yml
index ecb27a2d4f..52c319f496 100644
--- a/envs/environment_a.yml
+++ b/envs/environment_a.yml
@@ -6,25 +6,25 @@ dependencies:
   - python=3.7
   # Required
   - cartopy=0.19
-  - cftime<1.5
+  - cftime=1.2.1
   - cf-units=2.1.5
-  - clize
-  - dask
-  - iris=3.0
-  - netCDF4
-  - numpy<1.21
-  - pandas
-  - python-dateutil
-  - python-stratify
+  - clize=4.1.1
+  - dask=2021.8.1
+  - iris=3.0.3
+  - netCDF4=1.4.1
+  - numpy=1.20.2
+  - pandas=1.2.4
+  - python-dateutil=2.8.2
   - pytz=2020.5
-  - scipy=1.6
-  - sigtools
-  - sphinx
+  - scipy=1.6.2
+  - sigtools=2.0.2
+  - sphinx=4.0.1
   # Optional
-  - fastparquet
-  - statsmodels
-  - numba
+  - fastparquet=0.7.1
+  - statsmodels=0.12.2
+  - numba=0.53.1
   - pysteps=1.4.1
+  - python-stratify=0.1.1
   - timezonefinder=4.1.0
   # Development
   - astroid
@@ -33,7 +33,7 @@ dependencies:
   - codacy-coverage
   - filelock
   - flake8
-  - isort=5.*
+  - isort=5.7.0
   - mock
   - mypy
   - pytest
diff --git a/envs/environment_b.yml b/envs/environment_b.yml
index ec6e050f72..3ceff34ffa 100644
--- a/envs/environment_b.yml
+++ b/envs/environment_b.yml
@@ -6,21 +6,22 @@ dependencies:
   - python=3.8
   # Required
   - cartopy=0.19
-  - cftime<1.5
+  - cftime=1.2.1
   - cf-units=2.1.5
-  - clize
-  - dask
-  - iris=3.0
-  - netCDF4
-  - numpy<1.21
-  - pandas
-  - python-dateutil
-  - python-stratify
+  - clize=4.2.0
+  - dask=2021.8.1
+  - iris=3.0.3
+  - netCDF4=1.5.7
+  - numpy=1.20
+  - pandas=1.3.4
+  - python-dateutil=2.8.2
   - pytz=2020.5
-  - scipy=1.6
-  - sigtools
-  - sphinx
+  - scipy=1.6.2
+  - sigtools=2.0.3
+  - sphinx=4.0.2
   # Optional
+  - numba=0.53.1
+  - python-stratify=0.2.post0
   # Development
   - astroid
   - bandit

From 3eb236a33927d0e6f3494d182835fe2185f087c5 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Mon, 18 Oct 2021 17:37:50 +1100
Subject: [PATCH 13/28] Pin numpy/cartopy in latest environment

Unit test failures occur with older versions
---
 envs/latest.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/envs/latest.yml b/envs/latest.yml
index 4934e15815..a0793c0436 100644
--- a/envs/latest.yml
+++ b/envs/latest.yml
@@ -9,14 +9,14 @@ channels:
 dependencies:
   - python=3
   # Required
-  - cartopy
+  - cartopy<0.20
   - cftime<1.5
   - cf-units=2.1.5
   - clize
   - dask
   - iris>=3.0
   - netCDF4
-  - numpy
+  - numpy<1.21
   - python-dateutil
   - pytz
   - scipy

From b83547cdcbdf7452c06b6d3b3c41fa072d1a73b3 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Tue, 19 Oct 2021 12:04:13 +1100
Subject: [PATCH 14/28] Start documentation

---
 doc/source/Dependencies.rst         | 146 ++++++++++++++++++++++++++++
 doc/source/Running-at-your-site.rst |  11 +--
 doc/source/index.rst                |   1 +
 3 files changed, 151 insertions(+), 7 deletions(-)
 create mode 100644 doc/source/Dependencies.rst

diff --git a/doc/source/Dependencies.rst b/doc/source/Dependencies.rst
new file mode 100644
index 0000000000..5bda546b6d
--- /dev/null
+++ b/doc/source/Dependencies.rst
@@ -0,0 +1,146 @@
+Dependencies
+================
+
+.. contents:: Contents
+    :depth: 2
+
+
+IMPROVER uses a range of open source libraries.
+Some of these are widely used throughout the IMPROVER code and are required
+for IMPROVER as a whole.
+
+Other libraries only used in specific parts of IMPROVER are optional.
+This allows for smaller sized installations of IMPROVER.
+Generally, the library must be installed for the specific part of IMPROVER
+to function.
+
+
+Required
+----------
+
+cartopy
+~~~~~~~~~~
+Cartopy is used for grid projections and coordinate transformations.
+https://scitools.org.uk/cartopy/docs/stable/
+
+
+cftime
+~~~~~~~~~~
+cftime provides functions for handling time in NetCDF files according to the
+Climate and Forecast (CF) conventions.
+https://unidata.github.io/cftime/
+
+
+cf-units
+~~~~~~~~~~
+cf-units provides units conversion following the Climate and Forecast (CF)
+conventions.
+https://cf-units.readthedocs.io/en/stable/
+
+
+Clize
+~~~~~~~~~~
+Clize automatically generates command line interfaces (CLI) from Python function
+signatures.
+https://clize.readthedocs.io/en/stable/
+
+
+Dask
+~~~~~~~~~~
+Dask lazy-loaded arrays are used (often via Iris) to reduce the amount of data
+loaded into memory at once.
+https://dask.org/
+
+
+Iris
+~~~~~~~~~~
+Iris cubes are used as a primary data structure throughout the IMPROVER code.
+https://scitools-iris.readthedocs.io/en/stable/
+
+
+NetCDF4
+~~~~~~~~~~
+Python library for reading and writing NetCDF data files via Iris.
+https://unidata.github.io/netcdf4-python/
+
+
+Numpy
+~~~~~~~~~~
+Multidimensional numerical array library, used as the basis for Iris cubes and
+dask arrays.
+https://numpy.org/doc/stable/
+
+
+Scipy
+~~~~~~~~~~
+Scientific python library, used for a variety of statistical, image processing,
+interpolation and spatial functions.
+https://docs.scipy.org/doc/scipy/reference/
+
+
+Sigtools
+~~~~~~~~~~
+Sigtools provides introspection tools for function signatures.
+Sigtools is required by clize, so this dependency is needed anyway
+despite minor usage in IMPROVER.
+https://sigtools.readthedocs.io/en/stable/
+
+
+Sphinx
+~~~~~~~~~~
+Sphinx is a documentation library for Python. IMPROVER requires it at runtime
+to generate help strings from CLI function docstrings via clize.
+https://www.sphinx-doc.org/en/master/
+
+
+Optional dependencies
+----------------
+
+python-stratify
+~~~~~~~~~~~
+Vectorised (cython) interpolation, particularly for vertical levels of the
+atmosphere.
+https://github.com/SciTools/python-stratify
+
+Required for CLIs: ``interpolate-using-difference``, ``phase-change-level``
+
+statsmodels
+~~~~~~~~~~~
+Estimation of statistical models, used in EMOS.
+https://www.statsmodels.org/stable/
+
+Required for CLIs: ``estimate-emos-coefficients``
+
+numba
+~~~~~~~~~~~
+JIT compiler for numerical Python code, used for performance enhancement.
+https://numba.readthedocs.io/en/stable/
+
+Required for CLIs: ``generate-timezone-mask-ancillary``
+
+pysteps
+~~~~~~~~~~~
+https://pysteps.github.io/
+
+Required for CLIs: ``nowcast-accumulate``, ``nowcast-extrapolate``,
+``nowcast-optical-flow-from-winds``
+
+python-dateutil
+~~~~~~~~~~
+https://dateutil.readthedocs.io/en/stable/
+
+Required for CLIs: FIXME
+
+pytz
+~~~~~~~~~~
+Timezone database for Python.
+https://pythonhosted.org/pytz/
+
+Required for CLIs: ``generate-timezone-mask-ancillary``
+
+timezonefinder
+~~~~~~~~~~~
+https://timezonefinder.readthedocs.io/en/stable/
+
+
+Required for CLIs: ``generate-timezone-mask-ancillary``
diff --git a/doc/source/Running-at-your-site.rst b/doc/source/Running-at-your-site.rst
index 91b8c6fd50..0d9a4f0645 100644
--- a/doc/source/Running-at-your-site.rst
+++ b/doc/source/Running-at-your-site.rst
@@ -22,13 +22,10 @@ or ``python setup.py install`` (``python setup.py develop``). Note that
 ``pip install`` will not work in an empty environment due to problems
 with installation of the dependency ``iris`` via pip.
 
-Required dependencies are listed in
-`environment_py37_iris30.yml <https://github.com/metoppv/improver/blob/master/envs/environment_py37_iris30.yml>`_,
-and
-`environment_py38_iris30.yml <https://github.com/metoppv/improver/blob/master/envs/environment_py38_iris30.yml>`_,
-depending on your version requirements. Both environment files are used
-to run the test suite on Github actions, so these conda environments
-file should stay up to date with any dependency changes.
+Example environments are included in the repository ``envs`` directory.
+These environment files are used to run the test suite on Github actions,
+so they should stay up to date with any dependency changes. See also
+documentation explaining use of dependencies.
 
 Alternatively, you can manually 'install' by downloading the code and
 putting the IMPROVER ``bin/`` directory in your PATH.
diff --git a/doc/source/index.rst b/doc/source/index.rst
index a09e7f34cc..85e24660ab 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -15,6 +15,7 @@
    Guidance-for-Reviewing-Code
    How-to-implement-a-command-line-utility
    Running-at-your-site
+   Dependencies
    Ticket-Creation-and-Definition-of-Ready
 
 .. toctree::

From 27abdcd81c0f8fdcfd17acee839c3199f139ecf7 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Tue, 19 Oct 2021 12:08:35 +1100
Subject: [PATCH 15/28] Remove dateutil dependency

---
 doc/source/Dependencies.rst | 6 ------
 envs/environment_a.yml      | 1 -
 envs/environment_b.yml      | 1 -
 envs/latest.yml             | 1 -
 improver/metadata/amend.py  | 8 ++++----
 5 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/doc/source/Dependencies.rst b/doc/source/Dependencies.rst
index 5bda546b6d..8aca6591ef 100644
--- a/doc/source/Dependencies.rst
+++ b/doc/source/Dependencies.rst
@@ -125,12 +125,6 @@ https://pysteps.github.io/
 Required for CLIs: ``nowcast-accumulate``, ``nowcast-extrapolate``,
 ``nowcast-optical-flow-from-winds``
 
-python-dateutil
-~~~~~~~~~~
-https://dateutil.readthedocs.io/en/stable/
-
-Required for CLIs: FIXME
-
 pytz
 ~~~~~~~~~~
 Timezone database for Python.
diff --git a/envs/environment_a.yml b/envs/environment_a.yml
index 52c319f496..fff6eaf147 100644
--- a/envs/environment_a.yml
+++ b/envs/environment_a.yml
@@ -14,7 +14,6 @@ dependencies:
   - netCDF4=1.4.1
   - numpy=1.20.2
   - pandas=1.2.4
-  - python-dateutil=2.8.2
   - pytz=2020.5
   - scipy=1.6.2
   - sigtools=2.0.2
diff --git a/envs/environment_b.yml b/envs/environment_b.yml
index 3ceff34ffa..c72606aa9c 100644
--- a/envs/environment_b.yml
+++ b/envs/environment_b.yml
@@ -14,7 +14,6 @@ dependencies:
   - netCDF4=1.5.7
   - numpy=1.20
   - pandas=1.3.4
-  - python-dateutil=2.8.2
   - pytz=2020.5
   - scipy=1.6.2
   - sigtools=2.0.3
diff --git a/envs/latest.yml b/envs/latest.yml
index a0793c0436..ae82212e3f 100644
--- a/envs/latest.yml
+++ b/envs/latest.yml
@@ -17,7 +17,6 @@ dependencies:
   - iris>=3.0
   - netCDF4
   - numpy<1.21
-  - python-dateutil
   - pytz
   - scipy
   - sigtools
diff --git a/improver/metadata/amend.py b/improver/metadata/amend.py
index 9d6a916187..28a3a30a9e 100644
--- a/improver/metadata/amend.py
+++ b/improver/metadata/amend.py
@@ -29,10 +29,9 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 """Module containing utilities for modifying cube metadata"""
-from datetime import datetime
+from datetime import datetime, timedelta, timezone
 from typing import Any, Dict, List, Union
 
-from dateutil import tz
 from iris.cube import Cube, CubeList
 
 from improver.metadata.constants.mo_attributes import (
@@ -94,8 +93,9 @@ def set_history_attribute(cube: Cube, value: str, append: bool = False) -> None:
             If True, add to the existing history rather than replacing the
             existing attribute.  Default is False.
     """
-    tzinfo = tz.tzoffset("Z", 0)
-    timestamp = datetime.strftime(datetime.now(tzinfo), "%Y-%m-%dT%H:%M:%S%Z")
+    timestamp = datetime.strftime(
+        datetime.now(timezone(timedelta(0), name="Z")), "%Y-%m-%dT%H:%M:%S%Z"
+    )
     new_history = "{}: {}".format(timestamp, value)
     if append and "history" in cube.attributes.keys():
         cube.attributes["history"] += "; {}".format(new_history)

From 0a99d6a7affdd38b8ec35aa8ec357d5870907e13 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Tue, 19 Oct 2021 12:32:19 +1100
Subject: [PATCH 16/28] More documentation

---
 doc/source/Dependencies.rst         | 82 ++++++++++++++++++-----------
 doc/source/Running-at-your-site.rst |  2 +-
 2 files changed, 53 insertions(+), 31 deletions(-)

diff --git a/doc/source/Dependencies.rst b/doc/source/Dependencies.rst
index 8aca6591ef..9a5a63d4c6 100644
--- a/doc/source/Dependencies.rst
+++ b/doc/source/Dependencies.rst
@@ -5,136 +5,158 @@ Dependencies
     :depth: 2
 
 
-IMPROVER uses a range of open source libraries.
-Some of these are widely used throughout the IMPROVER code and are required
-for IMPROVER as a whole.
+IMPROVER builds on the functionality provided by a range of open source
+libraries.
 
-Other libraries only used in specific parts of IMPROVER are optional.
-This allows for smaller sized installations of IMPROVER.
-Generally, the library must be installed for the specific part of IMPROVER
-to function.
+Some of these libraries are widely used throughout the IMPROVER code, so are
+considered as required for IMPROVER as a whole.
 
+Other libraries are only used in specific parts of IMPROVER.
+These libraries are optional for IMPROVER as a whole, but are required to use
+related parts of IMPROVER.
+Optional installation of these libraries allows for smaller sized installations
+of IMPROVER and reduces conda environment dependency solving difficulties.
 
 Required
-----------
+-----------------
 
 cartopy
-~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
 Cartopy is used for grid projections and coordinate transformations.
+
 https://scitools.org.uk/cartopy/docs/stable/
 
 
 cftime
-~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
 cftime provides functions for handling time in NetCDF files according to the
 Climate and Forecast (CF) conventions.
+
 https://unidata.github.io/cftime/
 
 
 cf-units
-~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
 cf-units provides units conversion following the Climate and Forecast (CF)
 conventions.
+
 https://cf-units.readthedocs.io/en/stable/
 
 
 Clize
-~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
 Clize automatically generates command line interfaces (CLI) from Python function
 signatures.
+
 https://clize.readthedocs.io/en/stable/
 
 
 Dask
-~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
 Dask lazy-loaded arrays are used (often via Iris) to reduce the amount of data
 loaded into memory at once.
+
 https://dask.org/
 
 
 Iris
-~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
 Iris cubes are used as a primary data structure throughout the IMPROVER code.
+
 https://scitools-iris.readthedocs.io/en/stable/
 
 
 NetCDF4
-~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
 Python library for reading and writing NetCDF data files via Iris.
+
 https://unidata.github.io/netcdf4-python/
 
 
 Numpy
-~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
 Multidimensional numerical array library, used as the basis for Iris cubes and
 dask arrays.
+
 https://numpy.org/doc/stable/
 
 
 Scipy
-~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
 Scientific python library, used for a variety of statistical, image processing,
 interpolation and spatial functions.
+
 https://docs.scipy.org/doc/scipy/reference/
 
 
 Sigtools
-~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
 Sigtools provides introspection tools for function signatures.
 Sigtools is required by clize, so this dependency is needed anyway
 despite minor usage in IMPROVER.
+
 https://sigtools.readthedocs.io/en/stable/
 
 
 Sphinx
-~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
 Sphinx is a documentation library for Python. IMPROVER requires it at runtime
 to generate help strings from CLI function docstrings via clize.
+
 https://www.sphinx-doc.org/en/master/
 
 
 Optional dependencies
-----------------
+---------------------
 
 python-stratify
-~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~
 Vectorised (cython) interpolation, particularly for vertical levels of the
 atmosphere.
+
 https://github.com/SciTools/python-stratify
 
 Required for CLIs: ``interpolate-using-difference``, ``phase-change-level``
 
 statsmodels
-~~~~~~~~~~~
-Estimation of statistical models, used in EMOS.
+~~~~~~~~~~~~~~~~~~
+Estimation of statistical models, used for
+:doc:`EMOS <improver.calibration.ensemble_calibration>`.
+
 https://www.statsmodels.org/stable/
 
 Required for CLIs: ``estimate-emos-coefficients``
 
 numba
-~~~~~~~~~~~
-JIT compiler for numerical Python code, used for performance enhancement.
+~~~~~~~~~~~~~~~~~~
+JIT compiler for numerical Python code, used for better computational performance.
+
 https://numba.readthedocs.io/en/stable/
 
 Required for CLIs: ``generate-timezone-mask-ancillary``
 
-pysteps
-~~~~~~~~~~~
+PySTEPS
+~~~~~~~~~~~~~~~~~~
+Probabilistic nowcasting of radar precipitation fields, used for nowcasting.
+
 https://pysteps.github.io/
 
 Required for CLIs: ``nowcast-accumulate``, ``nowcast-extrapolate``,
 ``nowcast-optical-flow-from-winds``
 
 pytz
-~~~~~~~~~~
+~~~~~~~~~~~~~~~~~
 Timezone database for Python.
+
 https://pythonhosted.org/pytz/
 
 Required for CLIs: ``generate-timezone-mask-ancillary``
 
 timezonefinder
-~~~~~~~~~~~
-https://timezonefinder.readthedocs.io/en/stable/
+~~~~~~~~~~~~~~~~~~
+Lookup of timezone using geographic coordinates, used to generate timezone
+grids.
 
+https://timezonefinder.readthedocs.io/en/stable/
 
 Required for CLIs: ``generate-timezone-mask-ancillary``
diff --git a/doc/source/Running-at-your-site.rst b/doc/source/Running-at-your-site.rst
index 0d9a4f0645..ac7cea27a1 100644
--- a/doc/source/Running-at-your-site.rst
+++ b/doc/source/Running-at-your-site.rst
@@ -25,7 +25,7 @@ with installation of the dependency ``iris`` via pip.
 Example environments are included in the repository ``envs`` directory.
 These environment files are used to run the test suite on Github actions,
 so they should stay up to date with any dependency changes. See also
-documentation explaining use of dependencies.
+documentation about :doc:`use of dependencies in IMPROVER <Dependencies>`.
 
 Alternatively, you can manually 'install' by downloading the code and
 putting the IMPROVER ``bin/`` directory in your PATH.

From 1d4c031c052d42ba4751167596f05eded0aecdb0 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Tue, 19 Oct 2021 16:22:51 +1100
Subject: [PATCH 17/28] Remove timezone database dependence in unit test

---
 .../generate_ancillaries/test_GenerateTimezoneMask.py    | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py b/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py
index 2a02423fa4..2d2f8b7c97 100644
--- a/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py
+++ b/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py
@@ -365,9 +365,9 @@ def test_process(request, grid_fixture, time, grouping):
     """Test that the process method returns cubes that take the expected form
     for different grids and different dates.
 
-    The output data is primarily checked in the acceptance tests as a reasonably
-    large number of data points are required to reliably check it. Here we check
-    only a small sample."""
+    The output data is checked in the acceptance tests as a large number of
+    data points are required to reliably check it and the output is highly
+    subject to changes based on the version of the timezone database in use."""
 
     domain = grid_fixture.split("_")[0]
     groupings = None
@@ -376,7 +376,6 @@ def test_process(request, grid_fixture, time, grouping):
 
     expected = EXPECTED[grouping][domain][time]
     expected_time = EXPECTED_TIME[time]
-    index = EXPECTED[grouping][domain]["indices"]
 
     grid = request.getfixturevalue(grid_fixture)
 
@@ -393,5 +392,3 @@ def test_process(request, grid_fixture, time, grouping):
         assert (
             result.coord("UTC_offset").bounds.dtype == TIME_COORDS["UTC_offset"].dtype
         )
-
-    assert_array_equal(result.data[index][::4], expected["data"])

From df67dcd2203e9d29d77608ffd725e7507a81b64c Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Wed, 20 Oct 2021 12:24:03 +1100
Subject: [PATCH 18/28] Fix duplicated name in conda-forge environment

---
 envs/conda-forge.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/envs/conda-forge.yml b/envs/conda-forge.yml
index 4094e7881f..0e4edc4a25 100644
--- a/envs/conda-forge.yml
+++ b/envs/conda-forge.yml
@@ -3,7 +3,7 @@
 # If it is changed, the feedstock repository should also be updated.
 # This environment should not include optional dependencies.
 # This environment should pin versions so that all unit tests pass.
-name: improver_latest
+name: improver_conda_forge
 channels:
   - conda-forge
 dependencies:

From be5ebe9038dea98ac4d112b776e1d8236abf821e Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Wed, 20 Oct 2021 19:44:10 +1100
Subject: [PATCH 19/28] Add note that not all CLIs are available with
 environment_b

---
 envs/environment_b.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/envs/environment_b.yml b/envs/environment_b.yml
index c72606aa9c..82057101bf 100644
--- a/envs/environment_b.yml
+++ b/envs/environment_b.yml
@@ -1,4 +1,8 @@
 # This environment is a representative example for usage of IMPROVER
+# It does not include dependencies for the following IMPROVER CLIs:
+# estimate-emos-coefficients, generate-timezone-mask-ancillary,
+# nowcast-accumulate, nowcast-extrapolate and nowcast-optical-flow-from-winds
+# See doc/source/Dependencies.rst for more information
 name: improver_b
 channels:
   - conda-forge

From 9c8b238adfaa9411ee6ab7b28ff19dbc4181c635 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Mon, 8 Nov 2021 15:36:53 +1100
Subject: [PATCH 20/28] Remove leftover expected data from test_process

---
 .../test_GenerateTimezoneMask.py              | 25 +++++--------------
 1 file changed, 6 insertions(+), 19 deletions(-)

diff --git a/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py b/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py
index 2d2f8b7c97..7739242232 100644
--- a/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py
+++ b/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py
@@ -320,40 +320,27 @@ def test__group_timezones_empty_group(timezone_mask):
 EXPECTED = {
     "ungrouped": {
         "global": {
-            None: {**GLOBAL_GRID, "data": np.array([1, 1, 1, 1, 1, 0, 1, 1, 1, 1])},
-            "20200716T1500Z": {**GLOBAL_GRID, "data": np.ones([10])},
+            None: GLOBAL_GRID,
+            "20200716T1500Z": GLOBAL_GRID,
             "indices": (12, 2),
         },
         "uk": {
-            None: {
-                "shape": (4, 21, 22),
-                "min": -2 * 3600,
-                "max": 1 * 3600,
-                "data": np.array([1, 1, 0, 0, 0, 1]),
-            },
+            None: {"shape": (4, 21, 22), "min": -2 * 3600, "max": 1 * 3600,},
             "20200716T1500Z": {
                 "shape": (5, 21, 22),
                 "min": -2 * 3600,
                 "max": 2 * 3600,
-                "data": np.array([1, 1, 1, 1, 0, 1]),
             },
             "indices": (2, 10),
         },
     },
     "grouped": {
         "global": {
-            None: {**GLOBAL_GRID_GR, "data": np.array([0, 0, 0, 0, 0, 0, 1, 1, 1, 1])},
-            "20200716T1500Z": {
-                **GLOBAL_GRID_GR,
-                "data": np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]),
-            },
+            None: GLOBAL_GRID_GR,
+            "20200716T1500Z": GLOBAL_GRID_GR,
             "indices": (0, 2),
         },
-        "uk": {
-            None: {**UK_GRID_GR, "data": np.array([0, 0, 0, 0, 0, 1])},
-            "20200716T1500Z": {**UK_GRID_GR, "data": np.array([0, 0, 1, 1, 0, 1])},
-            "indices": (0, 9),
-        },
+        "uk": {None: UK_GRID_GR, "20200716T1500Z": UK_GRID_GR, "indices": (0, 9),},
     },
 }
 

From f80c4792387b84b25177dfb784cd4b39c7e95d7e Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Mon, 8 Nov 2021 16:17:55 +1100
Subject: [PATCH 21/28] Fix flake8

---
 .../generate_ancillaries/test_GenerateTimezoneMask.py         | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py b/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py
index 7739242232..12efba4756 100644
--- a/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py
+++ b/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py
@@ -325,7 +325,7 @@ def test__group_timezones_empty_group(timezone_mask):
             "indices": (12, 2),
         },
         "uk": {
-            None: {"shape": (4, 21, 22), "min": -2 * 3600, "max": 1 * 3600,},
+            None: {"shape": (4, 21, 22), "min": -2 * 3600, "max": 1 * 3600},
             "20200716T1500Z": {
                 "shape": (5, 21, 22),
                 "min": -2 * 3600,
@@ -340,7 +340,7 @@ def test__group_timezones_empty_group(timezone_mask):
             "20200716T1500Z": GLOBAL_GRID_GR,
             "indices": (0, 2),
         },
-        "uk": {None: UK_GRID_GR, "20200716T1500Z": UK_GRID_GR, "indices": (0, 9),},
+        "uk": {None: UK_GRID_GR, "20200716T1500Z": UK_GRID_GR, "indices": (0, 9)},
     },
 }
 

From 3fd5cc581e5e6cfb3ed3c65feaac2f1b0e590b85 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Mon, 22 Nov 2021 18:27:16 +1100
Subject: [PATCH 22/28] Remove not-really-working install_requires section

---
 setup.cfg | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index 6650dc9761..74a7564144 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -18,11 +18,9 @@ packages = find:
 setup_requires =
     setuptools >= 38.3.0
     setuptools_scm
-install_requires =
-# Note: only minimal requirements are included here for development usage.
+# Note: no install_requires run-time requirements are included here.
 # Requirements are expected to be provided through another method such as conda.
 # See envs directory at top level of repository.
-    scitools-iris >= 3.0
 scripts = bin/improver
 
 [options.packages.find]

From 1774aea8b7dc3f72dfd66c6403dc58ed9ccb9160 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Mon, 22 Nov 2021 18:30:03 +1100
Subject: [PATCH 23/28] Clarify conda-forge environment comment

---
 envs/conda-forge.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/envs/conda-forge.yml b/envs/conda-forge.yml
index 0e4edc4a25..e1d8793fed 100644
--- a/envs/conda-forge.yml
+++ b/envs/conda-forge.yml
@@ -1,6 +1,7 @@
-# This environment is intended to match the conda-forge improver-feedstock at
-# https://github.com/conda-forge/improver-feedstock
-# If it is changed, the feedstock repository should also be updated.
+# This environment is intended to be used for the conda-forge improver-feedstock
+# dependencies in https://github.com/conda-forge/improver-feedstock
+# If this file is changed, recipe/meta.yaml in the improver-feedstock repository
+# should also be updated.
 # This environment should not include optional dependencies.
 # This environment should pin versions so that all unit tests pass.
 name: improver_conda_forge

From 3c9a87506c3a79b5e379c875df25cd69012c736e Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Mon, 22 Nov 2021 19:00:24 +1100
Subject: [PATCH 24/28] Move latest environment tests to schedule

---
 .github/workflows/{tests.yml => ci.yml} |  39 +------
 .github/workflows/scheduled.yml         | 135 ++++++++++++++++++++++++
 2 files changed, 138 insertions(+), 36 deletions(-)
 rename .github/workflows/{tests.yml => ci.yml} (79%)
 create mode 100644 .github/workflows/scheduled.yml

diff --git a/.github/workflows/tests.yml b/.github/workflows/ci.yml
similarity index 79%
rename from .github/workflows/tests.yml
rename to .github/workflows/ci.yml
index 2216fa073c..7a363d3a2e 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/ci.yml
@@ -1,4 +1,4 @@
-name: Tests
+name: CI Tests
 
 on: [pull_request, push]
 jobs:
@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        env: [environment_a, environment_b, latest, conda-forge]
+        env: [environment_a, environment_b, conda-forge]
     steps:
     - uses: actions/checkout@v2
     - uses: actions/cache@v2
@@ -106,7 +106,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        env: [environment_a, environment_b, latest, conda-forge]
+        env: [environment_a, environment_b, conda-forge]
     steps:
     - uses: actions/checkout@v2
     - uses: actions/cache@v2
@@ -139,36 +139,3 @@ jobs:
         source '/usr/share/miniconda/etc/profile.d/conda.sh'
         conda activate im${{ matrix.env }}
         bandit -r improver
-  Type-checking:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        env: [latest]
-    steps:
-    - uses: actions/checkout@v2
-    - uses: actions/cache@v2
-      id: cache
-      env:
-        # Increase this value to reset cache
-        CACHE_NUMBER: 2
-      with:
-        path: /usr/share/miniconda/envs/im${{ matrix.env }}
-        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
-    - name: conda env update
-      if: steps.cache.outputs.cache-hit != 'true'
-      run: |
-        source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda install -c conda-forge mamba
-        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
-    - name: conda info
-      run: |
-        source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate im${{ matrix.env }}
-        conda info
-        conda list
-    - name: mypy
-      run: |
-        source '/usr/share/miniconda/etc/profile.d/conda.sh'
-        conda activate im${{ matrix.env }}
-        mypy improver || true
diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml
new file mode 100644
index 0000000000..dbc3328faf
--- /dev/null
+++ b/.github/workflows/scheduled.yml
@@ -0,0 +1,135 @@
+name: Nightly tests
+
+on: schedule
+  - cron: '3 * * * *'
+jobs:
+  Sphinx-Pytest-Coverage:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        env: [latest]
+    if: secrets.SCHEDULED_JOBS == 'yes'
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      id: cache
+      env:
+        # Increase this value to reset cache
+        CACHE_NUMBER: 2
+      with:
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
+    - name: conda env update
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda install -c conda-forge mamba
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
+    - name: conda info
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        conda info
+        conda list
+    - name: sphinx documentation
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        make -C doc html SPHINXOPTS="-W --keep-going"
+    - name: pytest without coverage
+      if: matrix.env != 'environment_a'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        pytest
+    - name: pytest with coverage
+      if: matrix.env == 'environment_a'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        pytest --cov=improver --cov-report xml:coverage.xml
+    - name: codacy upload
+      if: env.CODACY_PROJECT_TOKEN && matrix.env == 'environment_a'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        python-codacy-coverage -v -r coverage.xml
+      env:
+        CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}
+    - name: codecov upload
+      uses: codecov/codecov-action@v1
+      if: matrix.env == 'environment_a'
+  Safety-Bandit:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        env: [latest]
+    if: secrets.SCHEDULED_JOBS == 'yes'
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      id: cache
+      env:
+        # Increase this value to reset cache
+        CACHE_NUMBER: 2
+      with:
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
+    - name: conda env update
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda install -c conda-forge mamba
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
+    - name: conda info
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        conda info
+        conda list
+    - name: safety
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        safety check || true
+    - name: bandit
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        bandit -r improver
+  Type-checking:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        env: [latest]
+    if: secrets.SCHEDULED_JOBS == 'yes'
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      id: cache
+      env:
+        # Increase this value to reset cache
+        CACHE_NUMBER: 2
+      with:
+        path: /usr/share/miniconda/envs/im${{ matrix.env }}
+        key: ${{ format('{0}-conda-improver-{1}-{2}-{3}', runner.os, env.CACHE_NUMBER, matrix.env, hashFiles(format('envs/{0}.yml', matrix.env))) }}
+    - name: conda env update
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda install -c conda-forge mamba
+        mamba env update -q --file envs/${{ matrix.env }}.yml --name im${{ matrix.env }}
+    - name: conda info
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        conda info
+        conda list
+    - name: mypy
+      run: |
+        source '/usr/share/miniconda/etc/profile.d/conda.sh'
+        conda activate im${{ matrix.env }}
+        mypy improver || true

From 5f1b2e2472d2396678b7a8f32ecdccf8f000050d Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Mon, 22 Nov 2021 19:01:25 +1100
Subject: [PATCH 25/28] Fix actions YAML

---
 .github/workflows/scheduled.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml
index dbc3328faf..ef956e30d7 100644
--- a/.github/workflows/scheduled.yml
+++ b/.github/workflows/scheduled.yml
@@ -1,7 +1,8 @@
 name: Nightly tests
 
-on: schedule
-  - cron: '3 * * * *'
+on:
+  schedule:
+    - cron: '3 * * * *'
 jobs:
   Sphinx-Pytest-Coverage:
     runs-on: ubuntu-latest

From daea7aa426c80466edd586a1bb8b73bb6fd50bf7 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Mon, 22 Nov 2021 19:05:44 +1100
Subject: [PATCH 26/28] Reschedule

---
 .github/workflows/scheduled.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml
index ef956e30d7..ec31b4b190 100644
--- a/.github/workflows/scheduled.yml
+++ b/.github/workflows/scheduled.yml
@@ -2,7 +2,7 @@ name: Nightly tests
 
 on:
   schedule:
-    - cron: '3 * * * *'
+    - cron: '10 * * * *'
 jobs:
   Sphinx-Pytest-Coverage:
     runs-on: ubuntu-latest

From 196c83ea71338be49111d043852a3519087c71d8 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Tue, 23 Nov 2021 17:26:01 +1100
Subject: [PATCH 27/28] Fix scheduling, add manual running, metoppv-only

---
 .github/workflows/ci.yml        |  5 ++++-
 .github/workflows/scheduled.yml | 11 ++++++-----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7a363d3a2e..cbedcb778f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,6 +1,9 @@
 name: CI Tests
 
-on: [pull_request, push]
+on:
+  pull_request:
+  push:
+  workflow_dispatch:
 jobs:
   Sphinx-Pytest-Coverage:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml
index ec31b4b190..dcd2c412ca 100644
--- a/.github/workflows/scheduled.yml
+++ b/.github/workflows/scheduled.yml
@@ -1,8 +1,9 @@
-name: Nightly tests
+name: Scheduled Tests
 
 on:
   schedule:
-    - cron: '10 * * * *'
+    - cron: '7 4 * * *'
+  workflow_dispatch:
 jobs:
   Sphinx-Pytest-Coverage:
     runs-on: ubuntu-latest
@@ -10,7 +11,7 @@ jobs:
       fail-fast: false
       matrix:
         env: [latest]
-    if: secrets.SCHEDULED_JOBS == 'yes'
+    if: github.repository_owner == 'metoppv'
     steps:
     - uses: actions/checkout@v2
     - uses: actions/cache@v2
@@ -67,7 +68,7 @@ jobs:
       fail-fast: false
       matrix:
         env: [latest]
-    if: secrets.SCHEDULED_JOBS == 'yes'
+    if: github.repository_owner == 'metoppv'
     steps:
     - uses: actions/checkout@v2
     - uses: actions/cache@v2
@@ -106,7 +107,7 @@ jobs:
       fail-fast: false
       matrix:
         env: [latest]
-    if: secrets.SCHEDULED_JOBS == 'yes'
+    if: github.repository_owner == 'metoppv'
     steps:
     - uses: actions/checkout@v2
     - uses: actions/cache@v2

From c2c7591b7e952aa936ccabcec334ed95922b4c45 Mon Sep 17 00:00:00 2001
From: Tom Gale <tom.gale@bom.gov.au>
Date: Wed, 24 Nov 2021 18:38:07 +1100
Subject: [PATCH 28/28] Add timezone mask test on europe lat/lon domain

---
 .../test_GenerateTimezoneMask.py              | 141 ++++++++++++++----
 1 file changed, 113 insertions(+), 28 deletions(-)

diff --git a/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py b/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py
index 12efba4756..14aabf638c 100644
--- a/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py
+++ b/improver_tests/generate_ancillaries/test_GenerateTimezoneMask.py
@@ -89,11 +89,24 @@ def global_grid_fixture_360() -> Cube:
     return cube
 
 
+@pytest.fixture(name="europe_grid")
+def europe_grid_fixture() -> Cube:
+    data = np.zeros((10, 10), dtype=np.float32)
+    cube = set_up_variable_cube(
+        data,
+        name="template",
+        grid_spacing=1,
+        domain_corner=(45, -5),
+        attributes=GLOBAL_ATTRIBUTES,
+    )
+    return cube
+
+
 @pytest.fixture(name="uk_grid")
 def uk_grid_fixture() -> Cube:
     """UK grid template"""
 
-    data = np.zeros((21, 22), dtype=np.float32)
+    data = np.zeros((11, 11), dtype=np.float32)
     cube = set_up_variable_cube(
         data,
         name="template",
@@ -160,7 +173,7 @@ def test__get_coordinate_pairs(request, grid_fixture):
     expected_data = {
         "global_grid": [[-90.0, -180.0], [-90.0, -80.0], [90.0, 180.0]],
         "global_grid_360": [[-90.0, -180.0], [-90.0, -80.0], [90.0, 180.0]],
-        "uk_grid": [[44.517, -17.117], [45.548, -4.913], [62.026, 14.410]],
+        "uk_grid": [[44.517, -17.117], [45.548, -4.913], [54.263, -5.401]],
     }
 
     grid = request.getfixturevalue(grid_fixture)
@@ -245,7 +258,7 @@ def test__create_template_cube(request, grid_fixture, include_dst):
 
     expected = {
         "global_grid": {"shape": (19, 37), "attributes": GLOBAL_ATTRIBUTES},
-        "uk_grid": {"shape": (21, 22), "attributes": UK_ATTRIBUTES},
+        "uk_grid": {"shape": (11, 11), "attributes": UK_ATTRIBUTES},
     }
 
     # Set expected includes_daylight_savings attribute
@@ -309,52 +322,113 @@ def test__group_timezones_empty_group(timezone_mask):
 
 
 # Expected data for process tests
-
-# ungrouped
-GLOBAL_GRID = {"shape": (27, 19, 37), "min": -12 * 3600, "max": 14 * 3600}
-# grouped
-GLOBAL_GRID_GR = {"shape": (2, 19, 37), "min": -6 * 3600, "max": 6 * 3600}
-UK_GRID_GR = {"shape": (2, 21, 22), "min": -6 * 3600, "max": 6 * 3600}
-
 EXPECTED_TIME = {None: 1510286400, "20200716T1500Z": 1594911600}
+GROUPED_MIN_MAX = {"min": -6 * 3600, "max": 6 * 3600}
 EXPECTED = {
     "ungrouped": {
-        "global": {
-            None: GLOBAL_GRID,
-            "20200716T1500Z": GLOBAL_GRID,
-            "indices": (12, 2),
-        },
         "uk": {
-            None: {"shape": (4, 21, 22), "min": -2 * 3600, "max": 1 * 3600},
+            None: {
+                "shape": (3, 11, 11),
+                "min": -1 * 3600,
+                "max": 1 * 3600,
+                "data": np.array(
+                    [
+                        [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1],
+                        [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0],
+                        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
+                    ]
+                ),
+            },
+            "20200716T1500Z": {
+                "shape": (4, 11, 11),
+                "min": -1 * 3600,
+                "max": 2 * 3600,
+                "data": np.array(
+                    [
+                        [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1],
+                        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0],
+                        [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1],
+                        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
+                    ],
+                ),
+            },
+        },
+        "europe": {
+            None: {
+                "shape": (2, 10, 10),
+                "min": 0,
+                "max": 1 * 3600,
+                "data": np.array(
+                    [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
+                ),
+            },
             "20200716T1500Z": {
-                "shape": (5, 21, 22),
-                "min": -2 * 3600,
+                "shape": (3, 10, 10),
+                "min": 0,
                 "max": 2 * 3600,
+                "data": np.array(
+                    [
+                        [1, 0, 1, 1, 1, 1, 0, 0, 0, 0],
+                        [0, 1, 0, 0, 0, 0, 1, 1, 1, 1],
+                        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
+                    ]
+                ),
             },
-            "indices": (2, 10),
         },
     },
     "grouped": {
-        "global": {
-            None: GLOBAL_GRID_GR,
-            "20200716T1500Z": GLOBAL_GRID_GR,
-            "indices": (0, 2),
+        "uk": {
+            None: {
+                "shape": (2, 11, 11),
+                **GROUPED_MIN_MAX,
+                "data": np.array(
+                    [
+                        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+                        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
+                    ]
+                ),
+            },
+            "20200716T1500Z": {
+                "shape": (2, 11, 11),
+                **GROUPED_MIN_MAX,
+                "data": np.array(
+                    [
+                        [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0],
+                        [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1],
+                    ],
+                ),
+            },
+        },
+        "europe": {
+            None: {
+                "shape": (2, 10, 10),
+                **GROUPED_MIN_MAX,
+                "data": np.array(
+                    [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
+                ),
+            },
+            "20200716T1500Z": {
+                "shape": (2, 10, 10),
+                **GROUPED_MIN_MAX,
+                "data": np.array(
+                    [[1, 0, 1, 1, 1, 1, 0, 0, 0, 0], [0, 1, 0, 0, 0, 0, 1, 1, 1, 1]]
+                ),
+            },
         },
-        "uk": {None: UK_GRID_GR, "20200716T1500Z": UK_GRID_GR, "indices": (0, 9)},
     },
 }
 
 
 @pytest.mark.parametrize("grouping", ["ungrouped", "grouped"])
 @pytest.mark.parametrize("time", [None, "20200716T1500Z"])
-@pytest.mark.parametrize("grid_fixture", ["global_grid", "global_grid_360", "uk_grid"])
+@pytest.mark.parametrize("grid_fixture", ["uk_grid", "europe_grid"])
 def test_process(request, grid_fixture, time, grouping):
     """Test that the process method returns cubes that take the expected form
     for different grids and different dates.
 
-    The output data is checked in the acceptance tests as a large number of
-    data points are required to reliably check it and the output is highly
-    subject to changes based on the version of the timezone database in use."""
+    The output data is primarily checked in the acceptance tests as a reasonably
+    large number of data points are required to reliably check it. Here we check
+    only a small sample."""
 
     domain = grid_fixture.split("_")[0]
     groupings = None
@@ -379,3 +453,14 @@ def test_process(request, grid_fixture, time, grouping):
         assert (
             result.coord("UTC_offset").bounds.dtype == TIME_COORDS["UTC_offset"].dtype
         )
+    # slice the first spatial dimension to moderate size of expected arrays
+    assert_array_equal(result.data[:, 9, :], expected["data"])
+
+    # check each spatial location in the UTC_offset dimension
+    zone_count = np.count_nonzero(result.data, axis=0)
+    if grouping == "grouped":
+        # grouped outputs have a single UTC_offset with a non-zero entry
+        assert_array_equal(zone_count, 1)
+    else:
+        # ungrouped outputs have a single UTC_offset with a zero entry
+        assert_array_equal(zone_count, expected["shape"][0] - 1)