Merge branch 'refs/heads/master' into pjordan-make-gradient-consider-…

…distance

* refs/heads/master:
  Height of max (metoppv#2020)
  Migrate CLI functionality: simple_bias_correction (metoppv#2018)
  BUG: Fix Scheduled Tests Sphinx-Pytest-Coverage (metoppv#2021)
  Plugin discovery (metoppv#2009)
  Adds masked_add to cube combiner (metoppv#2015)
  Apply mask to cube (metoppv#2014)
  Make difference module handle circular cubes (metoppv#2016)

# Conflicts:
#	improver/utilities/spatial.py
#	improver_tests/utilities/spatial/test_DifferenceBetweenAdjacentGridSquares.py

.mailmap

@@ -29,6 +29,7 @@ Meabh NicGuidhir <meabh.nicguidhir@metoffice.gov.uk> <meabh.nicguidhir@metoffice
 Neil Crosswaite <neil.crosswaite@metoffice.gov.uk> <43375279+neilCrosswaite@users.noreply.github.com>
 Paul Abernethy <paul.abernethy@metoffice.gov.uk> <paul.abernethy@metoffice.gov.uk>
 Peter Jordan <peter.jordan@metoffice.gov.uk> <52462411+mo-peterjordan@users.noreply.github.com>
+Sam Griffiths <sam.griffiths@metoffice.gov.uk> <122271903+SamGriffithsMO@users.noreply.github.com>
 Shafiat Dewan <87321907+ShafiatDewan@users.noreply.github.com> <87321907+ShafiatDewan@users.noreply.github.com>
 Shubhendra Singh Chauhan <withshubh@gmail.com> <withshubh@gmail.com>
 Simon Jackson <simon.jackson@metoffice.gov.uk> <simon.jackson@metoffice.gov.uk>

CONTRIBUTING.md

@@ -49,6 +49,7 @@ below:
  - Zhiliang Fan (Bureau of Meteorology, Australia)
  - Ben Fitzpatrick (Met Office, UK)
  - Tom Gale (Bureau of Meteorology, Australia)
+ - Sam Griffiths (Met Office, UK)
  - Ben Hooper (Met Office, UK)
  - Aaron Hopkinson (Met Office, UK)
  - Kathryn Howard (Met Office, UK)

envs/latest.yml

@@ -45,3 +45,6 @@ dependencies:
   - sphinx-autodoc-typehints
   - sphinx_rtd_theme
   - threadpoolctl
+  # Pinned dependencies of dependencies
+  - pillow<=10.0.1  # https://github.com/metoppv/improver/issues/2010
+  - pandas<=2.0.0  # https://github.com/metoppv/improver/issues/2010

improver/api/__init__.py

@@ -0,0 +1,132 @@
+# (C) Crown copyright, Met Office. All rights reserved.
+#
+# This file is part of IMPROVER and is released under a BSD 3-Clause license.
+# See LICENSE in the root of the repository for full licensing details.
+# flake8: noqa
+"""
+This module contains the plugins for the IMPROVER project.  This aids in discoverability
+by making them available to a single flat namespace.  This also protects end-users from
+changes in structure to IMPROVER impacting their use of the plugins.
+"""
+from importlib import import_module
+
+# alphabetically sorted IMPROVER plugin lookup
+PROCESSING_MODULES = {
+    "Accumulation": "improver.nowcasting.accumulation",
+    "AdjustLandSeaPoints": "improver.regrid.landsea",
+    "AdvectField": "improver.nowcasting.forecasting",
+    "AggregateReliabilityCalibrationTables": "improver.calibration.reliability_calibration",
+    "apply_mask": "improver.utilities.mask",
+    "ApplyBiasCorrection": "improver.calibration.simple_bias_correction",
+    "ApplyDecisionTree": "improver.categorical.decision_tree",
+    "ApplyDzRescaling": "improver.calibration.dz_rescaling",
+    "ApplyEMOS": "improver.calibration.ensemble_calibration",
+    "ApplyGriddedLapseRate": "improver.lapse_rate",
+    "ApplyNeighbourhoodProcessingWithAMask": "improver.nbhood.use_nbhood",
+    "ApplyOrographicEnhancement": "improver.nowcasting.utilities",
+    "ApplyRainForestsCalibration": "improver.calibration.rainforest_calibration",
+    "ApplyReliabilityCalibration": "improver.calibration.reliability_calibration",
+    "BaseNeighbourhoodProcessing": "improver.nbhood.nbhood",
+    "CalculateForecastBias": "improver.calibration.simple_bias_correction",
+    "CalibratedForecastDistributionParameters": "improver.calibration.ensemble_calibration",
+    "ChooseDefaultWeightsLinear": "improver.blending.weights",
+    "ChooseDefaultWeightsNonLinear": "improver.blending.weights",
+    "ChooseDefaultWeightsTriangular": "improver.blending.weights",
+    "ChooseWeightsLinear": "improver.blending.weights",
+    "CloudCondensationLevel": "improver.psychrometric_calculations.cloud_condensation_level",
+    "CloudTopTemperature": "improver.psychrometric_calculations.cloud_top_temperature",
+    "Combine": "improver.cube_combiner",
+    "ConstructReliabilityCalibrationTables": "improver.calibration.reliability_calibration",
+    "ContinuousRankedProbabilityScoreMinimisers": "improver.calibration.ensemble_calibration",
+    "ConvectionRatioFromComponents": "improver.precipitation_type.convection",
+    "ConvertProbabilitiesToPercentiles": "improver.ensemble_copula_coupling.ensemble_copula_coupling",
+    "CopyAttributes": "improver.utilities.copy_attributes",
+    "CorrectLandSeaMask": "improver.generate_ancillaries.generate_ancillary",
+    "CreateExtrapolationForecast": "improver.nowcasting.forecasting",
+    "CubeCombiner": "improver.cube_combiner",
+    "DayNightMask": "improver.utilities.solar",
+    "DifferenceBetweenAdjacentGridSquares": "improver.utilities.spatial",
+    "EnforceConsistentForecasts": "improver.utilities.forecast_reference_enforcement",
+    "EnsembleReordering": "improver.ensemble_copula_coupling.ensemble_copula_coupling",
+    "EstimateCoefficientsForEnsembleCalibration": "improver.calibration.ensemble_calibration",
+    "EstimateDzRescaling": "improver.calibration.dz_rescaling",
+    "ExpectedValue": "improver.expected_value",
+    "ExtendRadarMask": "improver.nowcasting.utilities",
+    "ExtractLevel": "improver.utilities.cube_extraction",
+    "ExtractSubCube": "improver.utilities.cube_extraction",
+    "FieldTexture": "improver.utilities.textural",
+    "FillRadarHoles": "improver.nowcasting.utilities",
+    "FreezingRain": "improver.precipitation_type.freezing_rain",
+    "FrictionVelocity": "improver.wind_calculations.wind_downscaling",
+    "GenerateClearskySolarRadiation": "improver.generate_ancillaries.generate_derived_solar_fields",
+    "GenerateOrographyBandAncils": "improver.generate_ancillaries.generate_ancillary",
+    "GenerateSolarTime": "improver.generate_ancillaries.generate_derived_solar_fields",
+    "GenerateTimeLaggedEnsemble": "improver.utilities.time_lagging",
+    "GenerateTopographicZoneWeights": "improver.generate_ancillaries.generate_topographic_zone_weights",
+    "GradientBetweenAdjacentGridSquares": "improver.utilities.spatial",
+    "HailFraction": "improver.precipitation_type.hail_fraction",
+    "HailSize": "improver.psychrometric_calculations.hail_size",
+    "HumidityMixingRatio": "improver.psychrometric_calculations.psychrometric_calculations",
+    "Integration": "improver.utilities.mathematical_operations",
+    "InterpolateUsingDifference": "improver.utilities.interpolation",
+    "LapseRate": "improver.lapse_rate",
+    "LightningFromCapePrecip": "improver.lightning",
+    "LightningMultivariateProbability_USAF2024": "improver.lightning",
+    "ManipulateReliabilityTable": "improver.calibration.reliability_calibration",
+    "MaxInTimeWindow": "improver.cube_combiner",
+    "MergeCubes": "improver.utilities.cube_manipulation",
+    "MergeCubesForWeightedBlending": "improver.blending.weighted_blend",
+    "MetaCloudCondensationLevel": "improver.psychrometric_calculations.cloud_condensation_level",
+    "MetaNeighbourhood": "improver.nbhood.nbhood",
+    "MetaWetBulbFreezingLevel": "improver.psychrometric_calculations.wet_bulb_temperature",
+    "ModalCategory": "improver.categorical.modal_code",
+    "NeighbourSelection": "improver.spotdata.neighbour_finding",
+    "NowcastLightning": "improver.nowcasting.lightning",
+    "OccurrenceBetweenThresholds": "improver.between_thresholds",
+    "OccurrenceWithinVicinity": "improver.utilities.spatial",
+    "OpticalFlow": "improver.nowcasting.optical_flow",
+    "OrographicEnhancement": "improver.orographic_enhancement",
+    "OrographicSmoothingCoefficients": "improver.generate_ancillaries.generate_orographic_smoothing_coefficients",
+    "PercentileConverter": "improver.percentile",
+    "PhaseChangeLevel": "improver.psychrometric_calculations.psychrometric_calculations",
+    "PostProcessingPlugin": "improver.__init__",
+    "PrecipPhaseProbability": "improver.psychrometric_calculations.precip_phase_probability",
+    "PystepsExtrapolate": "improver.nowcasting.pysteps_advection",
+    "RebadgePercentilesAsRealizations": "improver.ensemble_copula_coupling.ensemble_copula_coupling",
+    "RebadgeRealizationsAsPercentiles": "improver.ensemble_copula_coupling.ensemble_copula_coupling",
+    "RecursiveFilter": "improver.nbhood.recursive_filter",
+    "RegridLandSea": "improver.regrid.landsea",
+    "RegridWithLandSeaMask": "improver.regrid.landsea2",
+    "ResamplePercentiles": "improver.ensemble_copula_coupling.ensemble_copula_coupling",
+    "ResolveWindComponents": "improver.wind_calculations.wind_components",
+    "RoughnessCorrection": "improver.wind_calculations.wind_downscaling",
+    "SaturatedVapourPressureTable": "improver.generate_ancillaries.generate_svp_table",
+    "ShowerConditionProbability": "improver.precipitation_type.shower_condition_probability",
+    "SignificantPhaseMask": "improver.psychrometric_calculations.significant_phase_mask",
+    "SnowFraction": "improver.precipitation_type.snow_fraction",
+    "SnowSplitter": "improver.precipitation_type.snow_splitter",
+    "SpatiallyVaryingWeightsFromMask": "improver.blending.spatial_weights",
+    "SpotExtraction": "improver.spotdata.spot_extraction",
+    "SpotHeightAdjustment": "improver.spotdata.height_adjustment",
+    "SpotLapseRateAdjust": "improver.spotdata.apply_lapse_rate",
+    "SpotManipulation": "improver.spotdata.spot_manipulation",
+    "StandardiseMetadata": "improver.standardise",
+    "TemporalInterpolation": "improver.utilities.temporal_interpolation",
+    "Threshold": "improver.threshold",
+    "TriangularWeightedBlendAcrossAdjacentPoints": "improver.blending.blend_across_adjacent_points",
+    "VerticalUpdraught": "improver.wind_calculations.vertical_updraught",
+    "VisibilityCombineCloudBase": "improver.visibility.visibility_combine_cloud_base",
+    "WeightAndBlend": "improver.blending.calculate_weights_and_blend",
+    "WeightedBlendAcrossWholeDimension": "improver.blending.weighted_blend",
+    "WetBulbTemperature": "improver.psychrometric_calculations.wet_bulb_temperature",
+    "WetBulbTemperatureIntegral": "improver.psychrometric_calculations.wet_bulb_temperature",
+    "WindDirection": "improver.wind_calculations.wind_direction",
+    "WindGustDiagnostic": "improver.wind_calculations.wind_gust_diagnostic",
+}
+
+
+def __getattr__(name):
+    if name.startswith("__") and name.endswith("__"):
+        raise AttributeError(f"{name} is not a valid attribute")
+    mod = import_module(PROCESSING_MODULES[name])
+    return getattr(mod, name)

improver/calibration/simple_bias_correction.py

@@ -4,14 +4,16 @@
 # See LICENSE in the root of the repository for full licensing details.
 """Simple bias correction plugins."""
 
-from typing import Dict, Optional
+import warnings
+from typing import Dict, Optional, Union
 
 import iris
 import numpy.ma as ma
 from iris.cube import Cube, CubeList
 from numpy import ndarray
 
 from improver import BasePlugin
+from improver.calibration import add_warning_comment, split_forecasts_and_bias_files
 from improver.calibration.utilities import (
     check_forecast_consistency,
     create_unified_frt_coord,
@@ -23,6 +25,7 @@
     create_new_diagnostic_cube,
     generate_mandatory_attributes,
 )
+from improver.utilities.common_input_handle import as_cubelist
 from improver.utilities.cube_manipulation import (
     clip_cube_data,
     collapsed,
@@ -248,11 +251,59 @@ class ApplyBiasCorrection(BasePlugin):
     the specified bias values.
     """
 
-    def __init__(self):
+    def __init__(
+        self,
+        lower_bound: Optional[float] = None,
+        upper_bound: Optional[float] = None,
+        fill_masked_bias_values: bool = False,
+    ):
         """
         Initialise class for applying simple bias correction.
+
+        Args:
+            lower_bound:
+                A lower bound below which all values will be remapped to
+                after the bias correction step.
+            upper_bound:
+                An upper bound above which all values will be remapped to
+                after the bias correction step.
+            fill_masked_bias_values:
+                Flag to specify whether masked areas in the bias data
+                should be filled to an appropriate fill value.
+        """
+        self._correction_method = apply_additive_correction
+        self._lower_bound = lower_bound
+        self._upper_bound = upper_bound
+        self._fill_masked_bias_values = fill_masked_bias_values
+
+    def _split_forecasts_and_bias(self, cubes: CubeList):
+        """
+        Wrapper for the split_forecasts_and_bias_files function.
+
+        Args:
+            cubes:
+                Cubelist containing the input forecast and bias cubes.
+
+        Return:
+            - Cube containing the forecast data to be bias-corrected.
+            - Cubelist containing the bias data to use in bias-correction.
+              Or None if no bias data is provided.
         """
-        self.correction_method = apply_additive_correction
+        forecast_cube, bias_cubes = split_forecasts_and_bias_files(cubes)
+
+        # Check whether bias data supplied, if not then return unadjusted input cube.
+        # This behaviour is to allow spin-up of the bias-correction terms.
+        if not bias_cubes:
+            msg = (
+                "There are no forecast_error (bias) cubes provided for calibration. "
+                "The uncalibrated forecast will be returned."
+            )
+            warnings.warn(msg)
+            forecast_cube = add_warning_comment(forecast_cube)
+            return forecast_cube, None
+        else:
+            bias_cubes = as_cubelist(bias_cubes)
+            return forecast_cube, bias_cubes
 
     def _get_mean_bias(self, bias_values: CubeList) -> Cube:
         """
@@ -302,6 +353,11 @@ def _check_forecast_bias_consistent(
         """Check that forecast and bias values are defined over the same
         valid-hour and forecast-period.
 
+        Checks that between the bias_data Cubes there is a common hour value for the
+        forecast_reference_time and single coordinate value for forecast_period. Then check
+        forecast Cube contains the same hour value for the forecast_reference_time and the
+        same forecast_period coordinate value.
+
         Args:
             forecast:
                 Cube containing forecast data to be bias-corrected.
@@ -339,16 +395,8 @@ def _check_forecast_bias_consistent(
                 "Forecast period differ between forecast and bias datasets."
             )
 
-    def process(
-        self,
-        forecast: Cube,
-        bias: CubeList,
-        lower_bound: Optional[float] = None,
-        upper_bound: Optional[float] = None,
-        fill_masked_bias_values: Optional[bool] = False,
-    ) -> Cube:
-        """
-        Apply bias correction using the specified bias values.
+    def process(self, *cubes: Union[Cube, CubeList],) -> Cube:
+        """Split then apply bias correction using the specified bias values.
 
         Where the bias data is defined point-by-point, the bias-correction will also
         be applied in this way enabling a form of statistical downscaling where coherent
@@ -362,36 +410,37 @@ def process(
         filled using an appropriate fill value to leave the forecast data unchanged
         in the masked areas.
 
+        If no bias correction is provided, then the forecast is returned, unaltered.
+
         Args:
-            forecast:
-                The cube to which bias correction is to be applied.
-            bias:
-                The cubelist containing the bias values for which to use in
-                the bias correction.
-            lower_bound:
-                A lower bound below which all values will be remapped to
-                after the bias correction step.
-            upper_bound:
-                An upper bound above which all values will be remapped to
-                after the bias correction step.
-            fill_masked_bias_values:
-                Flag to specify whether masked areas in the bias data
-                should be filled to an appropriate fill value.
+            cubes:
+                A list of cubes containing:
+                - A Cube containing the forecast to be calibrated. The input format is expected
+                to be realizations.
+                - A cube or cubelist containing forecast bias data over a specified
+                set of forecast reference times. If a list of cubes is passed in, each cube
+                should represent the forecast error for a single forecast reference time; the
+                mean value will then be evaluated over the forecast_reference_time coordinate.
 
         Returns:
             Bias corrected forecast cube.
         """
-        self._check_forecast_bias_consistent(forecast, bias)
-        bias = self._get_mean_bias(bias)
+        cubes = as_cubelist(*cubes)
+        forecast, bias_cubes = self._split_forecasts_and_bias(cubes)
+        if bias_cubes is None:
+            return forecast
+
+        self._check_forecast_bias_consistent(forecast, bias_cubes)
+        bias = self._get_mean_bias(bias_cubes)
 
         corrected_forecast = forecast.copy()
-        corrected_forecast.data = self.correction_method(
-            forecast, bias, fill_masked_bias_values
+        corrected_forecast.data = self._correction_method(
+            forecast, bias, self._fill_masked_bias_values
         )
 
-        if (lower_bound is not None) or (upper_bound is not None):
+        if (self._lower_bound is not None) or (self._upper_bound is not None):
             corrected_forecast = clip_cube_data(
-                corrected_forecast, lower_bound, upper_bound
+                corrected_forecast, self._lower_bound, self._upper_bound
             )
 
         return corrected_forecast

improver/cli/apply_bias_correction.py

@@ -12,7 +12,7 @@
 @cli.clizefy
 @cli.with_output
 def process(
-    *input_cubes: cli.inputcube,
+    *cubes: cli.inputcube,
     lower_bound: float = None,
     upper_bound: float = None,
     fill_masked_bias_data: bool = False,
@@ -32,7 +32,7 @@ def process(
     sensible post-bias correction.
 
     Args:
-        input_cubes (iris.cube.Cube or list of iris.cube.Cube):
+        cubes (iris.cube.Cube or list of iris.cube.Cube):
             A list of cubes containing:
             - A Cube containing the forecast to be calibrated. The input format is expected
             to be realizations.
@@ -52,28 +52,8 @@ def process(
         iris.cube.Cube:
             Forecast cube with bias correction applied on a per member basis.
     """
-    import warnings
-
-    import iris
-
-    from improver.calibration import add_warning_comment, split_forecasts_and_bias_files
     from improver.calibration.simple_bias_correction import ApplyBiasCorrection
 
-    forecast_cube, bias_cubes = split_forecasts_and_bias_files(input_cubes)
-
-    # Check whether bias data supplied, if not then return unadjusted input cube.
-    # This behaviour is to allow spin-up of the bias-correction terms.
-    if not bias_cubes:
-        msg = (
-            "There are no forecast_error (bias) cubes provided for calibration. "
-            "The uncalibrated forecast will be returned."
-        )
-        warnings.warn(msg)
-        forecast_cube = add_warning_comment(forecast_cube)
-        return forecast_cube
-    else:
-        bias_cubes = iris.cube.CubeList(bias_cubes)
-        plugin = ApplyBiasCorrection()
-        return plugin.process(
-            forecast_cube, bias_cubes, lower_bound, upper_bound, fill_masked_bias_data
-        )
+    return ApplyBiasCorrection(lower_bound, upper_bound, fill_masked_bias_data).process(
+        *cubes
+    )