From 8c65394d934587ad492964ba157d256119dae4e2 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 9 Jun 2025 18:45:19 +0100 Subject: [PATCH 01/59] dev --- cf/field.py | 86 +++++++++++++++++++++++ cf/mixin/fielddomain.py | 149 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 235 insertions(+) diff --git a/cf/field.py b/cf/field.py index c8b2bc9a48..b59cf2a862 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4885,6 +4885,92 @@ def bin( return out + @_inplace_enabled(default=False) + def healpix_refinement_level(self, level=None, inplace=False): + """TODOHEALPIX""" + f = _inplace_enabled_define_and_cleanup(self) + + if not level: + # Level None or 0 + return f + + key, healpix_index =f.coordinate("healpix_index", item=True, defualt=(None, None)) + + axis = f.domain_axis("healpix_index", key=True ) + + crs = f.coordinate_reference("grid_mapping_name:healpix", default=None) + + healpix_index_method = crs.coordinate_conversion.get_property( + "healpix_index_method", None + ) + if healpix_index_method != "nested": + raise ValueError("TODOHEALPIX") + + refinement_level = crs.coordinate_conversion.get_property( + "refinement_level", None + ) + if refinement_level is None: + raise ValueError("TODOHEALPIX") + + if level > 0: + if level > refinement_level: + raise ValueError("TODOHEALPIX") + + level -= refinement_level + elif level < -refinement_level: + raise ValueError("TODOHEALPIX") + + refinement_level = crs.coordinate_conversion.set_property( + "refinement_level", refinement_level + level + ) + N = -level * 4 + + i = f.get_data_axes().index(axis) + f.data.coarsen(np.ma.mean, axes={i: N}, + trim_excess=True, inplace=True) + + # Coarsen 1-d domain ancillary constructs that span the + # HEALPix dimension + domain_ancillaries = f.domain_ancillaries( + filter_by_axis=(axis,), axis_mode="and", todict=True + ) + for key, domain_ancillary in domain_ancillaries.items(): + i = f.get_data_axes(key).index(axis) + domain_ancillary.data.coarsen(np.ma.mean, + axes={i: N}, trim_excess=True, + inplace=True) + + coarsened_metadata = domain_ancillaries + + # Coarsen 1-d cell measure constructs that spanq the HEALPix + # dimension + cell_measures = f.cell_measures( + filter_by_axis=(axis,), axis_mode="and", todict=True + ) + for key, cell_measure in cell_measures.items(): + i = f.get_data_axes(key).index(axis) + cell_measure.data.coarsen(np.sum, + axes={i: N}, trim_excess=True, + inplace=True) + + coarsened_metadata |= cell_measures + + # Remove all other metadata constructs that span the HEALPix + # dimension + for key in f.constructs( + filter_by_axis=(axis,), axis_mode="or", todict=True + ).values(): + if key in coarsened_metadata: + continue + + f.del_construct(key) + + #TODOHEALPIX recreated healpix_idnex coordinates + healpix_index = healpix_index[::N] + f.set_construct(healpix_index, axes=axis, copy=False) + + return f + def histogram(self, digitized): """Return a multi-dimensional histogram of the data. diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index e4522c4b89..87b4788cee 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -2,6 +2,8 @@ from numbers import Integral import numpy as np +import dask.array as da + from cfdm import is_log_level_debug, is_log_level_info from dask.array.slicing import normalize_index from dask.base import is_dask_collection @@ -295,6 +297,12 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): f"non-negative integer. Got {halo!r}" ) + # TODOHEALPIX see if there are any implied 1-d corodinates + try: + self = self.create_1d_coordinates() + except ValueError: + pass + domain_axes = self.domain_axes(todict=True) # Initialise the index for each axis @@ -1872,6 +1880,147 @@ def coordinate_reference_domain_axes(self, identity=None): return set(axes) + @_inplace_enabled(default=False) + def create_1d_coordinates(self, persist=False, key=False, inplace=False): + """TODOHEALPIX. + + .. versionadded:: NEXTVERSION + + :Parameters: + + persist: `bool`, optional + TODOHEALPIX + + key: `bool` + TODOHEALPIX If True, return alongside the field + construct the key identifying the auxiliary coordinate + of the field with the newly-computed vertical + coordinates, as a 2-tuple of field and then key. If + False, the default, then only return the field + construct. + + If no coordinates were computed, `None` will be + returned in the key (second) position of the 2-tuple. + + {{inplace: `bool`, optional}} + + :Returns: + + `{{class}}` or `None` + TODOHEALPIX + + """ + f = _inplace_enabled_define_and_cleanup(self) + + keys = () + + crs = f.coordinate_reference("grid_mapping_name:healpix", default=None) + if crs is None: + if key: + return f, keys + + return f + + # ------------------------------------------------------------ + # Get the HEALPix indices as a Dask array + # ------------------------------------------------------------ + c_key, healpix_index = f.coordinate( + "healpix_index", item=True, default=(None, None) + ) + if c_key is None: + raise ValueError("TODOHEALPIX") + + dx = healpix_index.to_dask_array( + _force_mask_hardness=False, _force_to_memory=False + ) + + # ------------------------------------------------------------ + # Define functions to create latitudes and longitudes from + # HEALPix indices + # ------------------------------------------------------------ + refinement_level = crs.coordinate_conversion.get_property( + "refinement_level", None + ) + if refinement_level is None: + raise ValueError("TODOHEALPIX") + + healpix_index_method = crs.coordinate_conversion.get_property( + "healpix_index_method", None + ) + if healpix_index_method is None: + raise ValueError("TODOHEALPIX") + + if healpix_index_method not in ("nested", "ring"): + raise ValueError("TODOHEALPIX") + + from astropy_healpix import HEALPix + + hp = HEALPix(nside=2**refinement_level, order=healpix_index_method) + + def HEALPix_lon_coordinates(a): + return hp.healpix_to_lonlat(a)[0].degree + + def HEALPix_lat_coordinates(a): + return hp.healpix_to_lonlat(a)[1].degree + + def HEALPix_lon_bounds(a): + return hp.boundaries_lonlat(a)[0].degree + + def HEALPix_lat_bounds(a): + return hp.boundaries_lonlat(a)[1].degree + + # ------------------------------------------------------------ + # Create new latitude and longitude coordinates with bounds + # ------------------------------------------------------------ + meta = np.array((), dtype=np.dtype("float64")) + + # Latitude coordinates + dx = dx.map_blocks(HEALPix_lat_coordinates, meta=meta) + lat = f._AuxiliaryCoordinate( + data=f._Data(dx, "degrees_north", copy=False), + properties={"standard_name": "latitude"}, + copy=False, + ) + + # Longitude coordinates + dx = dx.map_blocks(HEALPix_lon_coordinates, meta=meta) + lon = f._AuxiliaryCoordinate( + data=f._Data(dx, "degrees_east", copy=False), + properties={"standard_name": "longitude"}, + copy=False, + ) + + # Latitude bounds + dx = da.blockwise( + HEALPix_lat_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta + ) + bounds = f._Bounds(data=dx) + lat.set_bounds(bounds) + + # Longitude bounds + dx = da.blockwise( + HEALPix_lon_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta + ) + bounds = f._Bounds(data=dx) + lon.set_bounds(bounds) + + if persist: + lat = lat.persist(inplace=True) + lon = lon.persist(inplace=True) + + # ------------------------------------------------------------ + # Set the new latitude and longitude coordinates + # ------------------------------------------------------------ + axis = f.get_domain_axes(c_key)[0] + lat_key = f.set_construct(lat, axes=axis, copy=False) + lon_key = f.set_construct(lon, axes=axis, copy=False) + keys = (lat_key, lon_key) + + if key: + return f, keys + + return f + def cyclic( self, *identity, iscyclic=True, period=None, config={}, **filter_kwargs ): From 48628677ca10b7c968ce1ddab1847f58eb01bf18 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 10 Jun 2025 11:00:46 +0100 Subject: [PATCH 02/59] dev --- cf/data/data.py | 32 +++++++++++++ cf/field.py | 123 +++++++++++++++++++++++++++++++++++------------- 2 files changed, 121 insertions(+), 34 deletions(-) diff --git a/cf/data/data.py b/cf/data/data.py index 7fcd08024f..5b9849eafb 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1342,6 +1342,38 @@ def ceil(self, inplace=False, i=False): d._set_dask(dx) return d + @_inplace_enabled(default=False) + def coarsen( + self, reduction, axes, trim_excess=False, + inplace=False, + ): + """TODOHEALPIX + + .. versionadded:: NEXTVERSION + + :Parameters: + + TODOHEALPIX + + {{inplace: `bool`, optional}} + + :Returns: + + `Data` or `None` + TODOHEALPIX of the data. If the operation was in-place + then `None` is returned. + + **Examples** + + TODOHEALPIX + + """ + d = _inplace_enabled_define_and_cleanup(self) + dx = d.to_dask_array() + dx = da.coarsen(reduction, axes, trim_excess=trim_excess=False) + d._set_dask(dx) + return d + @_inplace_enabled(default=False) def convolution_filter( self, diff --git a/cf/field.py b/cf/field.py index b59cf2a862..eeb91fdce3 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4887,62 +4887,110 @@ def bin( @_inplace_enabled(default=False) def healpix_refinement_level(self, level=None, inplace=False): - """TODOHEALPIX""" - f = _inplace_enabled_define_and_cleanup(self) + """TODOHEALPIX - if not level: - # Level None or 0 - return f + .. versionadded:: NEXTVERSION + + :Parameters: + + TODOHEALPIX - key, healpix_index =f.coordinate("healpix_index", item=True, defualt=(None, None)) + {{inplace: `bool`, optional}} + + :Returns: + + TODOHEALPIX + + **Examples** - axis = f.domain_axis("healpix_index", key=True ) + TODOHEALPIX - crs = f.coordinate_reference("grid_mapping_name:healpix", default=None) + """ + f = _inplace_enabled_define_and_cleanup(self) + + key, healpix_index = f.coordinate( + "healpix_index", item=True, default=(None, None) + ) + if healpix_index is None: + raise ValueError("TODOHEALPIX") + + # Get the key of the HEALPix domain axis + axis = f.get_data_axes(key)[0] + + cr = f.coordinate_reference("grid_mapping_name:healpix", default=None) + if cr is None: + raise ValueError("TODOHEALPIX") - healpix_index_method = crs.coordinate_conversion.get_property( + healpix_index_method = cr.coordinate_conversion.get_property( "healpix_index_method", None ) + if healpix_index_method is None: + raise ValueError("TODOHEALPIX") + if healpix_index_method != "nested": raise ValueError("TODOHEALPIX") - refinement_level = crs.coordinate_conversion.get_property( + refinement_level = cr.coordinate_conversion.get_property( "refinement_level", None ) if refinement_level is None: raise ValueError("TODOHEALPIX") - - if level > 0: - if level > refinement_level: + + # Parse 'level' + if not level: + # Level None or 0: Keep the current refinement level + return f + + if level >= 0: + if level > refinement_level: raise ValueError("TODOHEALPIX") level -= refinement_level elif level < -refinement_level: raise ValueError("TODOHEALPIX") - refinement_level = crs.coordinate_conversion.set_property( - "refinement_level", refinement_level + level - ) - N = -level * 4 + # Find the coarsening factor + factor = 4 ** -level + + # Create the healpix_index coordinates for the new refinement + # level + healpix = healpix.persist() + if healpix.data[0] % factor: + raise ValueError("TODOHEALPIX non-full first cell") + + new_index = healpix_index[::factor].persist() + if (ttt % factor).any(): + raise ValueError("TODOHEALPIX") + + if not (healpix.data.diff() > 0).all(): + raise ValueError( + "TODOHEALPIX not strictly monotinically increaing" + ) + if not (healpix[factor-1::factor] - new_index == factor -1).all(): + raise ValueError("TODOHEALPIX non-full cells") + + new_index //= factor + + # Coarsen the field data i = f.get_data_axes().index(axis) - f.data.coarsen(np.ma.mean, axes={i: N}, - trim_excess=True, inplace=True) + f.data.coarsen(np.ma.mean, axes={i: factor }, + inplace=True) - # Coarsen 1-d domain ancillary constructs that span the - # HEALPix dimension + # Coarsen domain ancillary constructs that span the HEALPix + # dimension domain_ancillaries = f.domain_ancillaries( filter_by_axis=(axis,), axis_mode="and", todict=True ) for key, domain_ancillary in domain_ancillaries.items(): i = f.get_data_axes(key).index(axis) domain_ancillary.data.coarsen(np.ma.mean, - axes={i: N}, trim_excess=True, + axes={i: factor }, inplace=True) - coarsened_metadata = domain_ancillaries + coarsened_construct_keys = domain_ancillaries - # Coarsen 1-d cell measure constructs that spanq the HEALPix + # Coarsen the cell measure constructs that span the HEALPix # dimension cell_measures = f.cell_measures( filter_by_axis=(axis,), axis_mode="and", todict=True @@ -4950,25 +4998,32 @@ def healpix_refinement_level(self, level=None, inplace=False): for key, cell_measure in cell_measures.items(): i = f.get_data_axes(key).index(axis) cell_measure.data.coarsen(np.sum, - axes={i: N}, trim_excess=True, + axes={i: factor }, inplace=True) - coarsened_metadata |= cell_measures + coarsened_construct_keys |= cell_measures # Remove all other metadata constructs that span the HEALPix - # dimension + # dimension, including the healpix_index coordinate construct. for key in f.constructs( filter_by_axis=(axis,), axis_mode="or", todict=True ).values(): - if key in coarsened_metadata: - continue + if key not in coarsened_construct_keys: + f.del_construct(key) - f.del_construct(key) + # Replace the HEALPix domain axis + domain_axis = f.domain_axis(axis) + domain_axis.set_size(f.data.shape[i]) + f.set_construct(domain_axis, key=axis) - #TODOHEALPIX recreated healpix_idnex coordinates - healpix_index = healpix_index[::N] - f.set_construct(healpix_index, axes=axis, copy=False) - + # Set the healpix_index coordinates for the new refinement + # level + f.set_construct(new_index, axes=axis, copy=False) + + # Set the new refinement level + cr.coordinate_conversion.set_property( + "refinement_level", refinement_level + level + ) return f def histogram(self, digitized): From 861b6aafbe64a17dfeed845cfa4dd9de1e157b01 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 10 Jun 2025 16:30:03 +0100 Subject: [PATCH 03/59] dev --- cf/data/data.py | 9 ++- cf/field.py | 146 +++++++++++++++++++++++----------------- cf/mixin/fielddomain.py | 69 +++++++++++++++++-- 3 files changed, 155 insertions(+), 69 deletions(-) diff --git a/cf/data/data.py b/cf/data/data.py index 5b9849eafb..36d41d9396 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1344,7 +1344,10 @@ def ceil(self, inplace=False, i=False): @_inplace_enabled(default=False) def coarsen( - self, reduction, axes, trim_excess=False, + self, + reduction, + axes, + trim_excess=False, inplace=False, ): """TODOHEALPIX @@ -1354,7 +1357,7 @@ def coarsen( :Parameters: TODOHEALPIX - + {{inplace: `bool`, optional}} :Returns: @@ -1370,7 +1373,7 @@ def coarsen( """ d = _inplace_enabled_define_and_cleanup(self) dx = d.to_dask_array() - dx = da.coarsen(reduction, axes, trim_excess=trim_excess=False) + dx = da.coarsen(reduction, dx, axes, trim_excess=trim_excess) d._set_dask(dx) return d diff --git a/cf/field.py b/cf/field.py index eeb91fdce3..9ac66ef022 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4886,7 +4886,9 @@ def bin( return out @_inplace_enabled(default=False) - def healpix_refinement_level(self, level=None, inplace=False): + def new_healpix_refinement_level( + self, level=None, reduction=np.ma.mean, check_coordinates=True, inplace=False + ): """TODOHEALPIX .. versionadded:: NEXTVERSION @@ -4894,7 +4896,7 @@ def healpix_refinement_level(self, level=None, inplace=False): :Parameters: TODOHEALPIX - + {{inplace: `bool`, optional}} :Returns: @@ -4908,124 +4910,144 @@ def healpix_refinement_level(self, level=None, inplace=False): """ f = _inplace_enabled_define_and_cleanup(self) + # Get the healpix_index coordinates key, healpix_index = f.coordinate( "healpix_index", item=True, default=(None, None) ) if healpix_index is None: - raise ValueError("TODOHEALPIX") + raise ValueError( + "Can't change refinement level: HEALPix index coordinates " + "have not been set" + ) # Get the key of the HEALPix domain axis axis = f.get_data_axes(key)[0] cr = f.coordinate_reference("grid_mapping_name:healpix", default=None) if cr is None: - raise ValueError("TODOHEALPIX") - - healpix_index_method = cr.coordinate_conversion.get_property( - "healpix_index_method", None + raise ValueError( + "Can't change refinement level: HEALPix grid mapping has not " + "been set" + ) + + healpix_order = cr.coordinate_conversion.get_property( + "healpix_order", None ) - if healpix_index_method is None: - raise ValueError("TODOHEALPIX") + if healpix_order is None: + raise ValueError( + "Can't change refinement level: HEALPix order has not " + "been set" + ) - if healpix_index_method != "nested": - raise ValueError("TODOHEALPIX") + if healpix_order != "nested": + raise ValueError( + "HEALPix order must be 'nested' for the refinement " + f"level to be changed. Got: {healpix_order!r}" + ) refinement_level = cr.coordinate_conversion.get_property( "refinement_level", None ) if refinement_level is None: - raise ValueError("TODOHEALPIX") + raise ValueError("HEALPix refinement level has been set") # Parse 'level' if not level: # Level None or 0: Keep the current refinement level return f - if level >= 0: + if level > 0: if level > refinement_level: raise ValueError("TODOHEALPIX") + # Convert 'level' to a negative number level -= refinement_level elif level < -refinement_level: raise ValueError("TODOHEALPIX") - - # Find the coarsening factor - factor = 4 ** -level + + # Find the coarsening factor, i.e. the ratio of the number of + # cells in the original grid to the number of cells in the new + # refinement level. + factor = 4**-level + + if check_coordinates: + data = healpix_index.data.persist() + if data[0] % factor: + raise ValueError("TODOHEALPIX non-full first cell") + + # new_data = data[::factor] + # if (new_data % factor).any(): + # raise ValueError("TODOHEALPIX non-full cells") + + if not (data.diff() > 0).all(): + raise ValueError( + "TODOHEALPIX not strictly monotinically increaing" + ) + + if ( + data[factor - 1 :: factor] - data[::factor] != factor - 1 + ).any(): + raise ValueError("TODOHEALPIX non-full cells") + + del data # Create the healpix_index coordinates for the new refinement # level - healpix = healpix.persist() - if healpix.data[0] % factor: - raise ValueError("TODOHEALPIX non-full first cell") + new_index = healpix_index[::factor] // factor - new_index = healpix_index[::factor].persist() - if (ttt % factor).any(): - raise ValueError("TODOHEALPIX") - - if not (healpix.data.diff() > 0).all(): - raise ValueError( - "TODOHEALPIX not strictly monotinically increaing" - ) - - if not (healpix[factor-1::factor] - new_index == factor -1).all(): - raise ValueError("TODOHEALPIX non-full cells") - - new_index //= factor - # Coarsen the field data i = f.get_data_axes().index(axis) - f.data.coarsen(np.ma.mean, axes={i: factor }, - inplace=True) + f.data.coarsen( + reduction, axes={i: factor}, trim_excess=False, inplace=True + ) # Coarsen domain ancillary constructs that span the HEALPix # dimension - domain_ancillaries = f.domain_ancillaries( - filter_by_axis=(axis,), axis_mode="and", todict=True - ) - for key, domain_ancillary in domain_ancillaries.items(): + for key, domain_ancillary in f.domain_ancillaries( + filter_by_axis=(axis,), axis_mode="and", todict=True + ).items(): i = f.get_data_axes(key).index(axis) - domain_ancillary.data.coarsen(np.ma.mean, - axes={i: factor }, - inplace=True) + domain_ancillary.data.coarsen( + np.ma.mean, axes={i: factor}, trim_excess=False, inplace=True + ) - coarsened_construct_keys = domain_ancillaries - # Coarsen the cell measure constructs that span the HEALPix # dimension - cell_measures = f.cell_measures( + for key, cell_measure in f.cell_measures( filter_by_axis=(axis,), axis_mode="and", todict=True - ) - for key, cell_measure in cell_measures.items(): + ).items(): i = f.get_data_axes(key).index(axis) - cell_measure.data.coarsen(np.sum, - axes={i: factor }, - inplace=True) + cell_measure.data.coarsen( + np.sum, axes={i: factor}, trim_excess=False, inplace=True + ) - coarsened_construct_keys |= cell_measures - # Remove all other metadata constructs that span the HEALPix # dimension, including the healpix_index coordinate construct. - for key in f.constructs( - filter_by_axis=(axis,), axis_mode="or", todict=True - ).values(): - if key not in coarsened_construct_keys: - f.del_construct(key) + for key in ( + f.constructs.filter_by_type("cell_measure", "domain_ancillary") + .filter_by_axis(axis) + .inverse_filter() + .filter_by_data(todict=True) + ): + f.del_construct(key) - # Replace the HEALPix domain axis + # Re-size the HEALPix domain axis domain_axis = f.domain_axis(axis) - domain_axis.set_size(f.data.shape[i]) + domain_axis.set_size(f.shape[i]) f.set_construct(domain_axis, key=axis) # Set the healpix_index coordinates for the new refinement # level - f.set_construct(new_index, axes=axis, copy=False) + f.set_construct( + new_index, axes=axis, copy=False, config={"no-op": True} + ) - # Set the new refinement level + # Set the new refinement level (note that 'level' is negative) cr.coordinate_conversion.set_property( "refinement_level", refinement_level + level ) return f - + def histogram(self, digitized): """Return a multi-dimensional histogram of the data. diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 87b4788cee..57bd6edada 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -1,9 +1,8 @@ import logging from numbers import Integral -import numpy as np import dask.array as da - +import numpy as np from cfdm import is_log_level_debug, is_log_level_info from dask.array.slicing import normalize_index from dask.base import is_dask_collection @@ -298,11 +297,11 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): ) # TODOHEALPIX see if there are any implied 1-d corodinates - try: + try: self = self.create_1d_coordinates() except ValueError: pass - + domain_axes = self.domain_axes(todict=True) # Initialise the index for each axis @@ -2426,6 +2425,68 @@ def get_coordinate_reference( return out + def get_refinement_level(self, default=ValueError()): + """TODOHEALPIX + + .. versionadded:: NEXTVERSION + + :Parameters: + + TODOHEALPIX + + :Returns: + + `int` + TODOHEALPIX + + **Examples** + + TODOHEALPIX + + """ + cr = self.coordinate_reference( + "grid_mapping_name:healpix", default=None + ) + if cr is not None: + refinement_level = cr.coordinate_conversion.get_property( + "refinement_level", None + ) + if refinement_level is not None: + return refinement_level + + return self._default(default, "HEALPix refinement level has been set") + + def get_healpix_order(self, default=ValueError()): + """TODOHEALPIX + + .. versionadded:: NEXTVERSION + + :Parameters: + + TODOHEALPIX + + :Returns: + + `str` + TODOHEALPIX + + **Examples** + + TODOHEALPIX + + """ + cr = self.coordinate_reference( + "grid_mapping_name:healpix", default=None + ) + if cr is not None: + healpix_order= cr.coordinate_conversion.get_property( + "healpix_order", None + ) + if healpix_order is not None: + return healpix_order + + return self._default(default, "HEALPix order has been set") + def iscyclic(self, *identity, **filter_kwargs): """Returns True if the given axis is cyclic. From 93ea1d0117dfe64a7df90fda0446f98ff4c33e6c Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 11 Jun 2025 23:53:09 +0100 Subject: [PATCH 04/59] dev --- cf/field.py | 126 ++++++++++++++++------------ cf/mixin/fielddomain.py | 180 ++++++++++++++++++++++++++-------------- 2 files changed, 191 insertions(+), 115 deletions(-) diff --git a/cf/field.py b/cf/field.py index 9ac66ef022..d7c057ce5c 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4886,8 +4886,12 @@ def bin( return out @_inplace_enabled(default=False) - def new_healpix_refinement_level( - self, level=None, reduction=np.ma.mean, check_coordinates=True, inplace=False + def healpix_decrease_refinement_level( + self, + level=None, + reduction=np.ma.mean, + check_healpix_index=True, + inplace=False, ): """TODOHEALPIX @@ -4905,29 +4909,33 @@ def new_healpix_refinement_level( **Examples** - TODOHEALPIX + >>> TODOHEALPIX """ f = _inplace_enabled_define_and_cleanup(self) - # Get the healpix_index coordinates + # Get the healpix_index oordinates and the key of the HEALPix + # domain axis key, healpix_index = f.coordinate( - "healpix_index", item=True, default=(None, None) + "healpix_index", + filter_by_naxes=(1,), + item=True, + default=(None, None), ) if healpix_index is None: raise ValueError( - "Can't change refinement level: HEALPix index coordinates " + "Can't decrease refinement level: HEALPix index coordinates " "have not been set" ) - # Get the key of the HEALPix domain axis axis = f.get_data_axes(key)[0] + # Parse the HEALPix coordinate reference cr = f.coordinate_reference("grid_mapping_name:healpix", default=None) if cr is None: raise ValueError( - "Can't change refinement level: HEALPix grid mapping has not " - "been set" + "Can't decrease refinement level: HEALPix grid mapping has " + "not been set" ) healpix_order = cr.coordinate_conversion.get_property( @@ -4935,21 +4943,24 @@ def new_healpix_refinement_level( ) if healpix_order is None: raise ValueError( - "Can't change refinement level: HEALPix order has not " + "Can't decrease refinement level: HEALPix order has not " "been set" ) if healpix_order != "nested": raise ValueError( - "HEALPix order must be 'nested' for the refinement " - f"level to be changed. Got: {healpix_order!r}" + "Can't decrease refinement level: Must have a HEALPix order " + f"of 'nested'. Got: {healpix_order!r}" ) refinement_level = cr.coordinate_conversion.get_property( "refinement_level", None ) if refinement_level is None: - raise ValueError("HEALPix refinement level has been set") + raise ValueError( + "Can't decrease refinement level: HEALPix refinement level " + "has been set" + ) # Parse 'level' if not level: @@ -4958,71 +4969,77 @@ def new_healpix_refinement_level( if level > 0: if level > refinement_level: - raise ValueError("TODOHEALPIX") + raise ValueError( + "'level' keyword can't be larger than the current " + f"refinement level {refinement_level}. Got: {level!r}" + ) # Convert 'level' to a negative number level -= refinement_level elif level < -refinement_level: - raise ValueError("TODOHEALPIX") - - # Find the coarsening factor, i.e. the ratio of the number of - # cells in the original grid to the number of cells in the new - # refinement level. - factor = 4**-level + raise ValueError( + "'level' keyword can't be less than minus the current " + f"refinement level {refinement_level}. Got: {level!r}" + ) - if check_coordinates: - data = healpix_index.data.persist() - if data[0] % factor: - raise ValueError("TODOHEALPIX non-full first cell") + # Find the number of cells at the original refinement level + # that are contained in one cell at the new refinement level. + ncells = 4**-level - # new_data = data[::factor] - # if (new_data % factor).any(): - # raise ValueError("TODOHEALPIX non-full cells") + new_refinement_level = refinement_level + level + if check_healpix_index: + data = healpix_index.data.persist() if not (data.diff() > 0).all(): raise ValueError( - "TODOHEALPIX not strictly monotinically increaing" + "Can't decrease refinement level: Nested HEALPix indices " + "are not strictly monotonically increasing" ) - if ( - data[factor - 1 :: factor] - data[::factor] != factor - 1 + if (data[::ncells] % ncells).any() or ( + data[ncells - 1 :: ncells] - data[::ncells] != ncells - 1 ).any(): - raise ValueError("TODOHEALPIX non-full cells") + raise ValueError( + "Can't decrease refinemnent level: At least one cell at " + f"the new refinement level ({new_refinement_level}) " + f"contains fewer than {ncells} cells at the original " + f"refinement level {refinement_level}" + ) del data # Create the healpix_index coordinates for the new refinement # level - new_index = healpix_index[::factor] // factor + new_index = healpix_index[::ncells] // ncells # Coarsen the field data i = f.get_data_axes().index(axis) f.data.coarsen( - reduction, axes={i: factor}, trim_excess=False, inplace=True + reduction, axes={i: ncells}, trim_excess=False, inplace=True ) # Coarsen domain ancillary constructs that span the HEALPix - # dimension + # axis for key, domain_ancillary in f.domain_ancillaries( filter_by_axis=(axis,), axis_mode="and", todict=True ).items(): i = f.get_data_axes(key).index(axis) domain_ancillary.data.coarsen( - np.ma.mean, axes={i: factor}, trim_excess=False, inplace=True + np.ma.mean, axes={i: ncells}, trim_excess=False, inplace=True ) - # Coarsen the cell measure constructs that span the HEALPix - # dimension + # Coarsen cell measure constructs that span the HEALPix axis for key, cell_measure in f.cell_measures( filter_by_axis=(axis,), axis_mode="and", todict=True ).items(): i = f.get_data_axes(key).index(axis) cell_measure.data.coarsen( - np.sum, axes={i: factor}, trim_excess=False, inplace=True + np.sum, axes={i: ncells}, trim_excess=False, inplace=True ) # Remove all other metadata constructs that span the HEALPix - # dimension, including the healpix_index coordinate construct. + # axis, including the original healpix_index coordinate + # construct. for key in ( f.constructs.filter_by_type("cell_measure", "domain_ancillary") .filter_by_axis(axis) @@ -5031,21 +5048,24 @@ def new_healpix_refinement_level( ): f.del_construct(key) - # Re-size the HEALPix domain axis + # Re-size the HEALPix axis domain_axis = f.domain_axis(axis) domain_axis.set_size(f.shape[i]) f.set_construct(domain_axis, key=axis) # Set the healpix_index coordinates for the new refinement # level - f.set_construct( - new_index, axes=axis, copy=False, config={"no-op": True} - ) + if new_index.construct_type == "dimension_coordinate": + # Convert indices to auxiliary coordinates + new_index = f._AuxiliaryCoordinate(source=new_index, copy=False) - # Set the new refinement level (note that 'level' is negative) + f.set_construct(new_index, axes=axis, copy=False) + + # Set the new refinement level cr.coordinate_conversion.set_property( - "refinement_level", refinement_level + level + "refinement_level", new_refinement_level ) + return f def histogram(self, digitized): @@ -7041,7 +7061,7 @@ def collapse( # axes. Also delete the corresponding domain ancillaries. # # This is because missing domain ancillaries in a - # coordinate refernce are assumed to have the value zero, + # coordinate reference are assumed to have the value zero, # which is most likely inapproriate. # -------------------------------------------------------- if remove_vertical_crs: @@ -7098,7 +7118,6 @@ def collapse( # REMOVE all 2+ dimensional auxiliary coordinates # which span this axis - # c = auxiliary_coordinates.filter_by_naxes(gt(1), view=True) c = f.auxiliary_coordinates( filter_by_naxes=(gt(1),), filter_by_axis=(axis,), @@ -7113,14 +7132,13 @@ def collapse( f.del_construct(key) - # REMOVE all 1 dimensional auxiliary coordinates which - # span this axis and have different values in their - # data array and bounds. + # REMOVE all 1-d auxiliary coordinates which span this + # axis and have different values in their data array + # and bounds. # - # KEEP, after changing their data arrays, all - # one-dimensional auxiliary coordinates which span - # this axis and have the same values in their data - # array and bounds. + # KEEP, after changing their data arrays, all 1-d + # auxiliary coordinates which span this axis and have + # the same values in their data array and bounds. c = f.auxiliary_coordinates( filter_by_axis=(axis,), axis_mode="exact", todict=True ) diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 57bd6edada..d89d14df78 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -296,11 +296,9 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): f"non-negative integer. Got {halo!r}" ) - # TODOHEALPIX see if there are any implied 1-d corodinates - try: - self = self.create_1d_coordinates() - except ValueError: - pass + # Create any implied 1-d latitude and longitude coordinates + # (e.g. implied by HEALPix indices) + self = self.create_1d_latlon_coordinates() domain_axes = self.domain_axes(todict=True) @@ -1879,8 +1877,34 @@ def coordinate_reference_domain_axes(self, identity=None): return set(axes) + def to_ugrid(self): + """TODOHEALPIX""" + if not self.is_healpix: + raise ValueError("TODOHEALPIX") + + f, (lat_key, lon_key) = self.create_1d_latlon_coordinates(key=True) + + lat = f.construct(lat_key) + lon = f.construct(lon_key) + + bounds_lat = lat.bounds.to_dask_array() + bounds_lon = lon.bounds.to_dask_array() + + _, lat_indices = np.unique(bounds_lat, return_inverse=True) + _, lon_indices = np.unique(bounds_lon, return_inverse=True) + + nodes = lat_indices* lat_indices.size + lon_indices + nodes = nodes.reshape(bounds_lat.shape) + + domain_topology = f._DomainToplogy(data=f._Data(nodes)) + domain_topology.set_cell('face') + + axis = f.get_data_axes(keys[0])[0] + f.set_construct(domain_topology, axes=axis, copy=False) + + @_inplace_enabled(default=False) - def create_1d_coordinates(self, persist=False, key=False, inplace=False): + def create_1d_latlon_coordinates(self, persist=False, key=False, inplace=False): """TODOHEALPIX. .. versionadded:: NEXTVERSION @@ -1911,96 +1935,126 @@ def create_1d_coordinates(self, persist=False, key=False, inplace=False): """ f = _inplace_enabled_define_and_cleanup(self) - keys = () - crs = f.coordinate_reference("grid_mapping_name:healpix", default=None) if crs is None: if key: - return f, keys + return f, () return f - # ------------------------------------------------------------ - # Get the HEALPix indices as a Dask array - # ------------------------------------------------------------ - c_key, healpix_index = f.coordinate( - "healpix_index", item=True, default=(None, None) - ) - if c_key is None: - raise ValueError("TODOHEALPIX") - - dx = healpix_index.to_dask_array( - _force_mask_hardness=False, _force_to_memory=False - ) - - # ------------------------------------------------------------ - # Define functions to create latitudes and longitudes from - # HEALPix indices - # ------------------------------------------------------------ refinement_level = crs.coordinate_conversion.get_property( "refinement_level", None ) if refinement_level is None: - raise ValueError("TODOHEALPIX") - - healpix_index_method = crs.coordinate_conversion.get_property( - "healpix_index_method", None - ) - if healpix_index_method is None: - raise ValueError("TODOHEALPIX") - - if healpix_index_method not in ("nested", "ring"): - raise ValueError("TODOHEALPIX") + if key: + return f, () - from astropy_healpix import HEALPix + return f - hp = HEALPix(nside=2**refinement_level, order=healpix_index_method) + healpix_order = crs.coordinate_conversion.get_property( + "healpix_order", None + ) + if healpix_order not in ("nested", "ring"): + if key: + return f, () - def HEALPix_lon_coordinates(a): - return hp.healpix_to_lonlat(a)[0].degree + return f - def HEALPix_lat_coordinates(a): - return hp.healpix_to_lonlat(a)[1].degree + # Get the healpix_index oordinates and the key of the HEALPix + # domain axis + c_key, healpix_index = f.coordinate( + "healpix_index", + filter_by_naxes=(1,), + item=True, + default=(None, None), + ) + if healpix_index is None: + if key: + return f, () - def HEALPix_lon_bounds(a): - return hp.boundaries_lonlat(a)[0].degree + return f + + axis = f.get_data_axes(key)[0] - def HEALPix_lat_bounds(a): - return hp.boundaries_lonlat(a)[1].degree + # ------------------------------------------------------------ + # Define functions to create latitudes and longitudes from + # HEALPix indices + # ------------------------------------------------------------ + import healpix +# from astropy_healpix import HEALPix + + nside = healpix.order2nside(refinement_level) +# hp = HEALPix(nside=nside, order=healpix_order) + nested = healpix_order=="nested" + + coord_func = partial(healpix.pix2ang, nside=nside, nest=nested, lonlat=True) + if nested: + bounds_func = healpix._chp.nest2ang_uv + else: + bounds_func = healpix._chp.ring2ang_uv + + bounds_func = partial(bounds_func , nside) + + def HEALPix_coordinates(a, lat=False, lon=False): + if lat: + pos = 1 + elif lon: + pos = 0 + + coord_func(ipix=a)[pos] + +# return hp.healpix_to_lonlat(a)[1].degree + + def HEALPix_bounds(a, lat=False, lon=False): + # Keep an eye on https://github.com/ntessore/healpix/issues/66 + if lat: + pos = 1 + elif lon: + pos = 0 + + bounds = np.empty((a.size, 4), dtype='float64') + + for i, (u, v) in enumerate((1, 0), (1,1), (0, 1), (0,0)): + b = bounds_func(a, u, v)[pos] + np.rad2deg(b, out=b) + bounds[:,i] = b + + return bounds # ------------------------------------------------------------ # Create new latitude and longitude coordinates with bounds # ------------------------------------------------------------ - meta = np.array((), dtype=np.dtype("float64")) + dx = healpix_index.to_dask_array() + meta = np.array((), dtype="float64") # Latitude coordinates - dx = dx.map_blocks(HEALPix_lat_coordinates, meta=meta) + dy = dx.map_blocks(HEALPix_coordinates, meta=meta, lat=True ) lat = f._AuxiliaryCoordinate( - data=f._Data(dx, "degrees_north", copy=False), + data=f._Data(dy, "degrees_north", copy=False), properties={"standard_name": "latitude"}, copy=False, ) # Longitude coordinates - dx = dx.map_blocks(HEALPix_lon_coordinates, meta=meta) + dy = dx.map_blocks(HEALPix_coordinates, meta=meta, lon=True ) lon = f._AuxiliaryCoordinate( - data=f._Data(dx, "degrees_east", copy=False), + data=f._Data(dy, "degrees_east", copy=False), properties={"standard_name": "longitude"}, copy=False, ) # Latitude bounds - dx = da.blockwise( - HEALPix_lat_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta + dy = da.blockwise( + HEALPix_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta, lat=True ) ) - bounds = f._Bounds(data=dx) + bounds = f._Bounds(data=dy) lat.set_bounds(bounds) # Longitude bounds - dx = da.blockwise( - HEALPix_lon_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta + dy = da.blockwise( + HEALPix_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta, lon=True ) ) - bounds = f._Bounds(data=dx) + bounds = f._Bounds(data=dy) lon.set_bounds(bounds) if persist: @@ -2010,13 +2064,11 @@ def HEALPix_lat_bounds(a): # ------------------------------------------------------------ # Set the new latitude and longitude coordinates # ------------------------------------------------------------ - axis = f.get_domain_axes(c_key)[0] lat_key = f.set_construct(lat, axes=axis, copy=False) lon_key = f.set_construct(lon, axes=axis, copy=False) - keys = (lat_key, lon_key) if key: - return f, keys + return f, (lat_key, lon_key) return f @@ -2479,7 +2531,7 @@ def get_healpix_order(self, default=ValueError()): "grid_mapping_name:healpix", default=None ) if cr is not None: - healpix_order= cr.coordinate_conversion.get_property( + healpix_order = cr.coordinate_conversion.get_property( "healpix_order", None ) if healpix_order is not None: @@ -2548,6 +2600,8 @@ def is_discrete_axis(self, *identity, **filter_kwargs): * An axis spanned by the domain topology construct of an unstructured grid. + * A HEALPix axis. + * The axis with geometry cells. .. versionaddedd:: 3.16.3 @@ -2617,6 +2671,10 @@ def is_discrete_axis(self, *identity, **filter_kwargs): ): return True + # HEALPix + if f.coordinate("healpix_index", filter_by_axis=(axis,), axis_mode="exact", default=None): + return True + # Geometries for aux in self.auxiliary_coordinates( filter_by_axis=(axis,), axis_mode="exact", todict=True From 9524af8bc50049ec70f4a0b824745d7e73080a7e Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 12 Jun 2025 12:51:31 +0100 Subject: [PATCH 05/59] dev --- cf/mixin/fielddomain.py | 344 +++++++++++++++++++++------------------- cf/weights.py | 208 ++++++++++++++++++++++++ 2 files changed, 392 insertions(+), 160 deletions(-) diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index d89d14df78..0f0917f66c 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -297,7 +297,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): ) # Create any implied 1-d latitude and longitude coordinates - # (e.g. implied by HEALPix indices) + # (e.g. as implied by HEALPix indices) self = self.create_1d_latlon_coordinates() domain_axes = self.domain_axes(todict=True) @@ -1877,53 +1877,85 @@ def coordinate_reference_domain_axes(self, identity=None): return set(axes) - def to_ugrid(self): - """TODOHEALPIX""" - if not self.is_healpix: + @_inplace_enabled(default=False) + def healpix_to_ugrid(self, inplace=False): + """TODOHEALPIX""" + axis = self.healpix_axis() + if axis is None: raise ValueError("TODOHEALPIX") - f, (lat_key, lon_key) = self.create_1d_latlon_coordinates(key=True) - - lat = f.construct(lat_key) - lon = f.construct(lon_key) + f = _inplace_enabled_define_and_cleanup(self) + + f.create_1d_latlon_coordinates(('HEALPix',) inplace=True) - bounds_lat = lat.bounds.to_dask_array() - bounds_lon = lon.bounds.to_dask_array() + x = f.auxiliary_coordinate( + "Y", filter_by_axis=(axis,), axis_mode="exact", default=None + ) + y = f.auxiliary_coordinate( + "X", filter_by_axis=(axis,), axis_mode="exact", default=None + ) + if x is None: + raise ValueError("TODOHEALPIX") - _, lat_indices = np.unique(bounds_lat, return_inverse=True) - _, lon_indices = np.unique(bounds_lon, return_inverse=True) + if y is None: + raise ValueError("TODOHEALPIX") + + bounds_y = y.get_bounds(None) + bounds_x = x.get_bounds(None) + if bounds_y is None: + raise ValueError("TODOHEALPIX") - nodes = lat_indices* lat_indices.size + lon_indices - nodes = nodes.reshape(bounds_lat.shape) + if bounds_x is None: + raise ValueError("TODOHEALPIX") + + # Create an identifer for each unique node location + bounds_y = bounds_y.to_dask_array() + bounds_x = bounds_x.to_dask_array() + + _, y_indices = np.unique(bounds_y, return_inverse=True) + _, x_indices = np.unique(bounds_x, return_inverse=True) + nodes = y_indices * y_indices.size + x_indices + + # Create a Domain Topology construct domain_topology = f._DomainToplogy(data=f._Data(nodes)) domain_topology.set_cell('face') - - axis = f.get_data_axes(keys[0])[0] + domain_topology.set_property( + "long_name", "UGRID topology derived from HEALPix" + ) f.set_construct(domain_topology, axes=axis, copy=False) - + + # Remove the HEALPix grid mapping and index coordinates + # TODOHEALPIX - make new grid mapping for generic grid mapping + # parametes? + f.del_construct("grid_mapping_name:healpix") + f.del_construct(key) + + return f + + def healpix_axis(self): + """TODOHEALPIX. + + .. versionadded:: NEXTVERSION + + """ + key = f.coordinate( + "healpix_index", filter_by_naxes=(1,), key=True, default=None + ) + if key is None: + return + + return self.get_data_axes(key)[0] @_inplace_enabled(default=False) - def create_1d_latlon_coordinates(self, persist=False, key=False, inplace=False): + def create_1d_latlon_coordinates(self, grid_type=None, inplace=False): """TODOHEALPIX. .. versionadded:: NEXTVERSION :Parameters: - persist: `bool`, optional - TODOHEALPIX - - key: `bool` - TODOHEALPIX If True, return alongside the field - construct the key identifying the auxiliary coordinate - of the field with the newly-computed vertical - coordinates, as a 2-tuple of field and then key. If - False, the default, then only return the field - construct. - - If no coordinates were computed, `None` will be - returned in the key (second) position of the 2-tuple. + TODOHEALPIX {{inplace: `bool`, optional}} @@ -1935,140 +1967,132 @@ def create_1d_latlon_coordinates(self, persist=False, key=False, inplace=False): """ f = _inplace_enabled_define_and_cleanup(self) - crs = f.coordinate_reference("grid_mapping_name:healpix", default=None) - if crs is None: - if key: - return f, () - - return f - - refinement_level = crs.coordinate_conversion.get_property( - "refinement_level", None - ) - if refinement_level is None: - if key: - return f, () - - return f - - healpix_order = crs.coordinate_conversion.get_property( - "healpix_order", None - ) - if healpix_order not in ("nested", "ring"): - if key: - return f, () - - return f - - # Get the healpix_index oordinates and the key of the HEALPix - # domain axis - c_key, healpix_index = f.coordinate( - "healpix_index", - filter_by_naxes=(1,), - item=True, - default=(None, None), - ) - if healpix_index is None: - if key: - return f, () - - return f - - axis = f.get_data_axes(key)[0] + if grid_type is None: + grid_type = ("HEALPix",) # ------------------------------------------------------------ - # Define functions to create latitudes and longitudes from - # HEALPix indices + # HEALPix # ------------------------------------------------------------ - import healpix -# from astropy_healpix import HEALPix - - nside = healpix.order2nside(refinement_level) -# hp = HEALPix(nside=nside, order=healpix_order) - nested = healpix_order=="nested" - - coord_func = partial(healpix.pix2ang, nside=nside, nest=nested, lonlat=True) - if nested: - bounds_func = healpix._chp.nest2ang_uv - else: - bounds_func = healpix._chp.ring2ang_uv - - bounds_func = partial(bounds_func , nside) - - def HEALPix_coordinates(a, lat=False, lon=False): - if lat: - pos = 1 - elif lon: - pos = 0 - - coord_func(ipix=a)[pos] - -# return hp.healpix_to_lonlat(a)[1].degree - - def HEALPix_bounds(a, lat=False, lon=False): - # Keep an eye on https://github.com/ntessore/healpix/issues/66 - if lat: - pos = 1 - elif lon: - pos = 0 - - bounds = np.empty((a.size, 4), dtype='float64') + if "HEALPix" in grid_type: + cr = f.coordinate_reference( + "grid_mapping_name:healpix", default=None + ) + if cr is None: + # No healpix grid mapping + return f + + parameters = cr.coordinate_conversion.parameters() + refinement_level = parameters.get("refinement_level") + if refinement_level is None: + # No refinement_level + return f + + healpix_order = parameters.get("healpix_order") + if healpix_order not in ("nested", "ring"): + # No suitable healpix_order + return f + + c_key, healpix_index = f.coordinate( + "healpix_index", + filter_by_naxes=(1,), + item=True, + default=(None, None), + ) + if healpix_index is None: + # No healpix_index coordinates + return f - for i, (u, v) in enumerate((1, 0), (1,1), (0, 1), (0,0)): - b = bounds_func(a, u, v)[pos] - np.rad2deg(b, out=b) - bounds[:,i] = b - - return bounds - - # ------------------------------------------------------------ - # Create new latitude and longitude coordinates with bounds - # ------------------------------------------------------------ - dx = healpix_index.to_dask_array() - meta = np.array((), dtype="float64") - - # Latitude coordinates - dy = dx.map_blocks(HEALPix_coordinates, meta=meta, lat=True ) - lat = f._AuxiliaryCoordinate( - data=f._Data(dy, "degrees_north", copy=False), - properties={"standard_name": "latitude"}, - copy=False, - ) - - # Longitude coordinates - dy = dx.map_blocks(HEALPix_coordinates, meta=meta, lon=True ) - lon = f._AuxiliaryCoordinate( - data=f._Data(dy, "degrees_east", copy=False), - properties={"standard_name": "longitude"}, - copy=False, - ) - - # Latitude bounds - dy = da.blockwise( - HEALPix_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta, lat=True ) - ) - bounds = f._Bounds(data=dy) - lat.set_bounds(bounds) - - # Longitude bounds - dy = da.blockwise( - HEALPix_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta, lon=True ) - ) - bounds = f._Bounds(data=dy) - lon.set_bounds(bounds) - - if persist: - lat = lat.persist(inplace=True) - lon = lon.persist(inplace=True) - - # ------------------------------------------------------------ - # Set the new latitude and longitude coordinates - # ------------------------------------------------------------ - lat_key = f.set_construct(lat, axes=axis, copy=False) - lon_key = f.set_construct(lon, axes=axis, copy=False) + if f.coordinates('X', 'Y', filter_by_axis=(axis,), axis_mode="exact", todict=True): + # X and/or Y coordinates already exist, so don't create any. + return f + + # Get the HEALPix axis + axis = f.get_data_axes(key)[0] + + # Define functions to create latitudes and longitudes from + # HEALPix indices + import healpix + + nside = healpix.order2nside(refinement_level) + nested = healpix_order=="nested" + + coord_func = partial(healpix.pix2ang, nside=nside, nest=nested, lonlat=True) + if nested: + bounds_func = healpix._chp.nest2ang_uv + else: + bounds_func = healpix._chp.ring2ang_uv + + bounds_func = partial(bounds_func , nside) + + def HEALPix_coordinates(a, lat=False, lon=False): + if lat: + pos = 1 + elif lon: + pos = 0 + + coord_func(ipix=a)[pos] + + def HEALPix_bounds(a, lat=False, lon=False): + # Keep an eye on + # https://github.com/ntessore/healpix/issues/66 + if lat: + pos = 1 + elif lon: + pos = 0 + + bounds = np.empty((a.size, 4), dtype='float64') + + # Vertex position u, v + # --------------- ---- + # right 1, 0 + # top 1, 1 + # left 0, 1 + # bottom 0, 0 + for i, (u, v) in enumerate((1, 0), (1,1), (0, 1), (0,0)): + b = bounds_func(a, u, v)[pos] + np.rad2deg(b, out=b) + bounds[:,i] = b + + return bounds + + # Create new latitude and longitude coordinates with bounds + dx = healpix_index.to_dask_array() + meta = np.array((), dtype="float64") + + # Latitude coordinates + dy = dx.map_blocks(HEALPix_coordinates, meta=meta, lat=True ) + lat = f._AuxiliaryCoordinate( + data=f._Data(dy, "degrees_north", copy=False), + properties={"standard_name": "latitude"}, + copy=False, + ) + + # Longitude coordinates + dy = dx.map_blocks(HEALPix_coordinates, meta=meta, lon=True ) + lon = f._AuxiliaryCoordinate( + data=f._Data(dy, "degrees_east", copy=False), + properties={"standard_name": "longitude"}, + copy=False, + ) + + # Latitude bounds + dy = da.blockwise( + HEALPix_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta, lat=True ) + ) + bounds = f._Bounds(data=dy) + lat.set_bounds(bounds) + + # Longitude bounds + dy = da.blockwise( + HEALPix_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta, lon=True ) + ) + bounds = f._Bounds(data=dy) + lon.set_bounds(bounds) - if key: - return f, (lat_key, lon_key) + # Set the new latitude and longitude coordinates + lat_key = f.set_construct(lat, axes=axis, copy=False) + lon_key = f.set_construct(lon, axes=axis, copy=False) + cr.set_coordinates((lat_key ,lon_key)) return f diff --git a/cf/weights.py b/cf/weights.py index 24a63aa8f7..9b40c3f02c 100644 --- a/cf/weights.py +++ b/cf/weights.py @@ -1912,3 +1912,211 @@ def _spherical_polygon_areas(cls, f, x, y, N, interior_rings=None): areas = interior_angles.sum(-1, squeeze=True) - (N - 2) * pi areas.override_units(Units("m2"), inplace=True) return areas + + @classmethod + def healpix_area( + cls, + f, + domain_axis, + weights, + weights_axes, + auto=False, + measure=False, + radius=None, + return_areas=False, + methods=False, + ): + """Creates area weights for polygon geometry cells.TODOHEALPIX + + .. versionadded:: NEXTVERSION + + :Parameters: + + f: `Field` + The field for which the weights are being created. + + domain_axis: `str` or `None` + If set to a domain axis identifier + (e.g. ``'domainaxis1'``) then only accept cells that + recognise the given axis. If `None` then the cells may + span any axis. + + {{weights weights: `dict`}} + + {{weights weights_axes: `set`}} + + {{weights auto: `bool`, optional}} + + {{weights measure: `bool`, optional}} + + {{radius: optional}} + + {{weights methods: `bool`, optional}} + + :Returns: + + `bool` or `Data` + `True` if weights were created, otherwise `False`. If + *return_areas* is True and weights were created, then + the weights are returned. + + """ + axis = f.healpix_axis() + if axis is None: + if auto: + return False + + if domain_axis is None: + raise ValueError("No polygon cells") + + raise ValueError( + "No polygon cells for " + f"{f.constructs.domain_axis_identity(domain_axis)!r} axis" + ) + + if axis in weights_axes: + if auto: + return False + + raise ValueError( + "Multiple weights specifications for " + f"{f.constructs.domain_axis_identity(axis)!r} axis" + ) + + if not measure: + return False + + if methods: + weights[(axis,)] = "HEALPix equal area" + return True + + cr = f.coordinate_reference( + "grid_mapping_name:healpix", default=None + ) + if auto: + return False + + elif cr is None: + # No healpix grid mapping + return f + + parameters = cr.coordinate_conversion.parameters() + refinement_level = parameters.get("refinement_level") + if refinement_level is None: + # No refinement_level + return f + + if methods: + weights[(axis,)] = "HEALPix equal area" + return True + + if radius is not None: + radius = f.radius(default=radius) + + area = cf.Data.full( + shape, + 4 * np.pi / (12 * (4 ** refinement_level)), + units="1" + ) + area = area * (radius ** 2) + + if return_areas: + return area + + weights[(axis,)] = area + weights_axes.add(axis) + return True + + + + from .units import Units + + axis, aux_X, aux_Y, aux_Z, ugrid = cls._geometry_ugrid_cells( + f, domain_axis, "polygon", auto=auto + ) + + if axis is None: + if auto: + return False + + if domain_axis is None: + raise ValueError("No polygon cells") + + raise ValueError( + "No polygon cells for " + f"{f.constructs.domain_axis_identity(domain_axis)!r} axis" + ) + + if axis in weights_axes: + if auto: + return False + + raise ValueError( + "Multiple weights specifications for " + f"{f.constructs.domain_axis_identity(axis)!r} axis" + ) + + x = aux_X.bounds.data + y = aux_Y.bounds.data + + radians = Units("radians") + metres = Units("metres") + + if x.Units.equivalent(radians) and y.Units.equivalent(radians): + if not great_circle: + raise ValueError( + "Must set great_circle=True to allow the derivation of " + "area weights from spherical polygons composed from " + "great circle segments." + ) + + if methods: + weights[(axis,)] = "area spherical polygon" + return True + + spherical = True + x.Units = radians + elif x.Units.equivalent(metres) and y.Units.equivalent(metres): + if methods: + weights[(axis,)] = "area plane polygon" + return True + + spherical = False + else: + return False + + y.Units = x.Units + x = x.persist() + y = y.persist() + + # Find the number of nodes per polygon + n_nodes = x.count(axis=-1, keepdims=False).array + if (y.count(axis=-1, keepdims=False) != n_nodes).any(): + raise ValueError( + "Can't create area weights for " + f"{f.constructs.domain_axis_identity(axis)!r} axis: " + f"{aux_X!r} and {aux_Y!r} have inconsistent bounds " + "specifications" + ) + + if ugrid: + areas = cls._polygon_area_ugrid(f, x, y, n_nodes, spherical) + else: + areas = cls._polygon_area_geometry( + f, x, y, aux_X, aux_Y, n_nodes, spherical + ) + + del x, y, n_nodes + + if not measure: + areas.override_units(Units("1"), inplace=True) + elif spherical: + areas = cls._spherical_area_measure(f, areas, aux_Z, radius) + + if return_areas: + return areas + + weights[(axis,)] = areas + weights_axes.add(axis) + return True + From 20e08daf7368f8e5bd2137cd2404df72a14769ec Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 13 Jun 2025 08:53:24 +0100 Subject: [PATCH 06/59] dev --- cf/weights.py | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/cf/weights.py b/cf/weights.py index 9b40c3f02c..ec50cc54e4 100644 --- a/cf/weights.py +++ b/cf/weights.py @@ -1962,15 +1962,25 @@ def healpix_area( """ axis = f.healpix_axis() + if axis is None: if auto: return False - + if domain_axis is None: - raise ValueError("No polygon cells") + raise ValueError("No HEALPix cells") + + raise ValueError( + "No HEALPix cells for " + f"{f.constructs.domain_axis_identity(domain_axis)!r} axis" + ) + + if domain_axis is not None and domain_axis != axis: + if auto: + return False raise ValueError( - "No polygon cells for " + "No HEALPix cells for " f"{f.constructs.domain_axis_identity(domain_axis)!r} axis" ) @@ -1984,7 +1994,7 @@ def healpix_area( ) if not measure: - return False + return True # TODOHEALPIX if methods: weights[(axis,)] = "HEALPix equal area" @@ -1993,18 +2003,27 @@ def healpix_area( cr = f.coordinate_reference( "grid_mapping_name:healpix", default=None ) - if auto: - return False - - elif cr is None: + if cr is None: # No healpix grid mapping - return f - + if auto: + return False + + raise ValueError( + "Can't create weights: No HEALPix grid mapping for " + f"{f.constructs.domain_axis_identity(axis)!r} axis" + ) + parameters = cr.coordinate_conversion.parameters() refinement_level = parameters.get("refinement_level") if refinement_level is None: # No refinement_level - return f + if auto: + return False + + raise ValueError( + "Can't create weights: No HEALPix refinement_level for " + f"{f.constructs.domain_axis_identity(axis)!r} axis" + ) if methods: weights[(axis,)] = "HEALPix equal area" From 3bfa054952a977317815eb423d9fdfb99d4b4a5f Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 16 Jun 2025 18:29:48 +0100 Subject: [PATCH 07/59] dev --- cf/dask_utils.py | 178 +++++++++++++++++++++++++++++ cf/field.py | 57 ++++++---- cf/mixin/fielddomain.py | 245 +++++++++++++++++++--------------------- cf/weights.py | 182 +++++++++++++---------------- 4 files changed, 401 insertions(+), 261 deletions(-) create mode 100644 cf/dask_utils.py diff --git a/cf/dask_utils.py b/cf/dask_utils.py new file mode 100644 index 0000000000..6e18acedae --- /dev/null +++ b/cf/dask_utils.py @@ -0,0 +1,178 @@ +# Define functions to create latitudes and longitudes from +# HEALPix indices +import healpix + +def cf_HEALPix_coordinates( + a, + healpix_order, refinement_level=None, lat=False, lon=False +): + """Calculate HEALPix cell coordinates. + + .. versionadded:: NEXTVERSION + + :Parameters: + + a: `numpy.ndarray` + + healpix_order: `str` + One of ``'nested'``, ``'ring'``, or ``'nuniq'``. + + refinement_level: `int` or `None`, optional + For a ``'nested'`` or ``'ring'`` ordered grid, the + refinement level of the grid within the HEALPix hierarchy, + starting at 0 for the base tesselation with 12 cells + globally. Ignored for a ``'nuniq'`` ordered grid. + + lat: `bool`, optional + + lon: `bool`, optional + + :Returns: + + `numpy.ndarray` + + """ + if lat: + pos = 1 + elif lon: + pos = 0 + + if healpix_order == "nuniq": + # nuniq + c = np.empty(a.shape, dtype='float64') + + nsides, a = healpix.uniq2pix(a, nest=False) + nsides, index, inverse = np.unique(nsides, + return_index=True, + return_inverse=True) + for nside, i in zip(nsides, index): + level = np.where(inverse == inverse[i])[0] + c[level] = healpix.pix2ang( + nside=nside, ipix=a[level], nest=False, lonlat=True + )[pos] + else: + # nested or ring + c = healpix.pix2ang( + nside=healpix.order2nside(refinement_level), + ipix=a, + nest=healpix_order=="nested", + lonlat=True + )[pos] + + return c + + +def cf_HEALPix_bounds(a, healpix_order, refinement_level=None, lat=False, lon=False): + """Calculate HEALPix cell bounds. + + .. versionadded:: NEXTVERSION + + :Parameters: + + a: `numpy.ndarray` + + healpix_order: `str` + One of ``'nested'``, ``'ring'``, or ``'nuniq'``. + + refinement_level: `int` or `None`, optional + For a ``'nested'`` or ``'ring'`` ordered grid, the + refinement level of the grid within the HEALPix hierarchy, + starting at 0 for the base tesselation with 12 cells + globally. Ignored for a ``'nuniq'`` ordered grid. + + lat: `bool`, optional + + lon: `bool`, optional + + :Returns: + + `numpy.ndarray` + + """ + # Keep an eye on https://github.com/ntessore/healpix/issues/66 + + if lat: + pos = 1 + elif lon: + pos = 0 + + if healpix_order == "nested": + bounds_func = healpix._chp.nest2ang_uv + else: + bounds_func = healpix._chp.ring2ang_uv + + + # Define the cell vertices in an anticlockwise direction, seen + # from above. + # + # Vertex position vertex (u, v) + # --------------- ------------- + # right (1, 0) + # top (1, 1) + # left (0, 1) + # bottom (0, 0) + vertices = ((1, 0), (1,1), (0, 1), (0,0)) + + b = np.empty((a.size, 4), dtype='float64') + + if healpix_order == "nuniq": + # nuniq + nsides, a = healpix.uniq2pix(a, nest=False) + nsides, index, inverse = np.unique(nsides, + return_index=True, + return_inverse=True) + for nside, i in zip(nsides, index): + level = np.where(inverse == inverse[i])[0] + for j, (u, v) in enumerate(vertices): + b[level, j] = bounds_func(nside, a[level], u, v)[pos] + else: + # nested or ring + nside = healpix.order2nside(refinement_level) + for j, (u, v) in enumerate(vertices): + b[:,j] = bounds_func(nside, a, u, v)[pos] + + # Convert to degrees + np.rad2deg(b, out=b) + + return b + + +def cf_HEALPix_nuniq_weights(a, measure=False, radius=None): + """Calculate HEALPix cell weights for 'nuniq' indices. + + .. versionadded:: NEXTVERSION + + :Parameters: + + a: `numpy.ndarray` + + measure: `bool`, optional + If True then create weights that are actual cell sizes, in + units of the square of the radius units. + + radius: number, optional + The radius of the sphere, in units of length. Must be set + if *measure * is True, otherwise ignored. + + :Returns: + + `numpy.ndarray` + The weights. + + """ + if measure: + f = 4.0 * np.pi * (radius**2) / 12 + else: + f = 1.0 + + nsides = healpix.uniq2pix(a)[0] + nsides, index, inverse = np.unique(nsides, + return_index=True, + return_inverse=True) + + w = np.empty(a.shape, dtype='float64') + + for nside, i in zip(nsides, index): + w = np.where(inverse == inverse[i], f / (nside**2), w) + + return w diff --git a/cf/field.py b/cf/field.py index d7c057ce5c..c4862c0a6f 100644 --- a/cf/field.py +++ b/cf/field.py @@ -3647,7 +3647,20 @@ def weights( ): # Found area weights from UGRID/geometry cells area_weights = True - + elif Weights.healpix_area( + self, + None, + comp, + weights_axes, + measure=measure, + radius=radius, + great_circle=great_circle, + methods=methods, + auto=True, + ): + # Found area weights from HEALPix cells + area_weights = True + if not area_weights: raise ValueError( "Can't create weights: Unable to find cell areas" @@ -4938,9 +4951,8 @@ def healpix_decrease_refinement_level( "not been set" ) - healpix_order = cr.coordinate_conversion.get_property( - "healpix_order", None - ) + parameters = cr.coordinate_conversion.parameters() + healpix_order = parameters.get( "healpix_order") if healpix_order is None: raise ValueError( "Can't decrease refinement level: HEALPix order has not " @@ -4950,16 +4962,14 @@ def healpix_decrease_refinement_level( if healpix_order != "nested": raise ValueError( "Can't decrease refinement level: Must have a HEALPix order " - f"of 'nested'. Got: {healpix_order!r}" + f"of 'nested' for this operation. Got: {healpix_order!r}" ) - refinement_level = cr.coordinate_conversion.get_property( - "refinement_level", None - ) + refinement_level = parameters.get("refinement_level") if refinement_level is None: raise ValueError( "Can't decrease refinement level: HEALPix refinement level " - "has been set" + "has not been set" ) # Parse 'level' @@ -4979,17 +4989,16 @@ def healpix_decrease_refinement_level( elif level < -refinement_level: raise ValueError( "'level' keyword can't be less than minus the current " - f"refinement level {refinement_level}. Got: {level!r}" + f"refinement level of {refinement_level}. Got: {level!r}" ) - # Find the number of cells at the original refinement level - # that are contained in one cell at the new refinement level. + # The number of cells at the original refinement level which are + # contained in one cell at the coarser refinement level ncells = 4**-level new_refinement_level = refinement_level + level if check_healpix_index: - data = healpix_index.data.persist() if not (data.diff() > 0).all(): raise ValueError( "Can't decrease refinement level: Nested HEALPix indices " @@ -5001,25 +5010,23 @@ def healpix_decrease_refinement_level( ).any(): raise ValueError( "Can't decrease refinemnent level: At least one cell at " - f"the new refinement level ({new_refinement_level}) " + f"the coarser refinement level ({new_refinement_level}) " f"contains fewer than {ncells} cells at the original " f"refinement level {refinement_level}" ) - del data - # Create the healpix_index coordinates for the new refinement # level new_index = healpix_index[::ncells] // ncells - # Coarsen the field data + # Coarsen (with 'reduction') the field data i = f.get_data_axes().index(axis) f.data.coarsen( reduction, axes={i: ncells}, trim_excess=False, inplace=True ) - # Coarsen domain ancillary constructs that span the HEALPix - # axis + # Coarsen (with np.ma.mean) the domain ancillary constructs + # that span the HEALPix axis for key, domain_ancillary in f.domain_ancillaries( filter_by_axis=(axis,), axis_mode="and", todict=True ).items(): @@ -5028,7 +5035,8 @@ def healpix_decrease_refinement_level( np.ma.mean, axes={i: ncells}, trim_excess=False, inplace=True ) - # Coarsen cell measure constructs that span the HEALPix axis + # Coarsen (with np.sum) the cell measure constructs that span + # the HEALPix axis for key, cell_measure in f.cell_measures( filter_by_axis=(axis,), axis_mode="and", todict=True ).items(): @@ -5041,17 +5049,16 @@ def healpix_decrease_refinement_level( # axis, including the original healpix_index coordinate # construct. for key in ( - f.constructs.filter_by_type("cell_measure", "domain_ancillary") - .filter_by_axis(axis) - .inverse_filter() - .filter_by_data(todict=True) + f.constructs.filter_by_axis(axis, axis_mode="and") + .filter_by_type("cell_measure", "domain_ancillary") + .inverse_filter(1) + .todict() ): f.del_construct(key) # Re-size the HEALPix axis domain_axis = f.domain_axis(axis) domain_axis.set_size(f.shape[i]) - f.set_construct(domain_axis, key=axis) # Set the healpix_index coordinates for the new refinement # level diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 0f0917f66c..914660591a 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -1885,8 +1885,10 @@ def healpix_to_ugrid(self, inplace=False): raise ValueError("TODOHEALPIX") f = _inplace_enabled_define_and_cleanup(self) - - f.create_1d_latlon_coordinates(('HEALPix',) inplace=True) + + # If lat/lon coordinates do not exist, then derive them from + # the HEALPix indices. + f.create_latlon_coordinates(('HEALPix',) inplace=True) x = f.auxiliary_coordinate( "Y", filter_by_axis=(axis,), axis_mode="exact", default=None @@ -1948,7 +1950,7 @@ def healpix_axis(self): return self.get_data_axes(key)[0] @_inplace_enabled(default=False) - def create_1d_latlon_coordinates(self, grid_type=None, inplace=False): + def create_latlon_coordinates(self, grid_type=None, inplace=False): """TODOHEALPIX. .. versionadded:: NEXTVERSION @@ -1967,13 +1969,10 @@ def create_1d_latlon_coordinates(self, grid_type=None, inplace=False): """ f = _inplace_enabled_define_and_cleanup(self) - if grid_type is None: - grid_type = ("HEALPix",) - # ------------------------------------------------------------ # HEALPix # ------------------------------------------------------------ - if "HEALPix" in grid_type: + if grid_type is None or "HEALPix" in grid_type: cr = f.coordinate_reference( "grid_mapping_name:healpix", default=None ) @@ -1982,16 +1981,16 @@ def create_1d_latlon_coordinates(self, grid_type=None, inplace=False): return f parameters = cr.coordinate_conversion.parameters() - refinement_level = parameters.get("refinement_level") - if refinement_level is None: - # No refinement_level - return f - healpix_order = parameters.get("healpix_order") - if healpix_order not in ("nested", "ring"): - # No suitable healpix_order + if healpix_order not in ("nested", "ring", "nuniq"): + # Bad healpix_order return f - + + refinement_level = parameters.get("refinement_level") + if refinement_level is None and healpix_order in ("nested", "ring"): + # Missing refinement_level + return f + c_key, healpix_index = f.coordinate( "healpix_index", filter_by_naxes=(1,), @@ -1999,68 +1998,32 @@ def create_1d_latlon_coordinates(self, grid_type=None, inplace=False): default=(None, None), ) if healpix_index is None: - # No healpix_index coordinates - return f - - if f.coordinates('X', 'Y', filter_by_axis=(axis,), axis_mode="exact", todict=True): - # X and/or Y coordinates already exist, so don't create any. + # Missing healpix_index coordinates return f - + # Get the HEALPix axis axis = f.get_data_axes(key)[0] + if f.coordinates('X', 'Y', filter_by_axis=(axis,), + axis_mode="exact", todict=True): + # X and/or Y coordinates already exist, so don't create any. + return f + # Define functions to create latitudes and longitudes from # HEALPix indices - import healpix - - nside = healpix.order2nside(refinement_level) - nested = healpix_order=="nested" - - coord_func = partial(healpix.pix2ang, nside=nside, nest=nested, lonlat=True) - if nested: - bounds_func = healpix._chp.nest2ang_uv - else: - bounds_func = healpix._chp.ring2ang_uv - - bounds_func = partial(bounds_func , nside) - - def HEALPix_coordinates(a, lat=False, lon=False): - if lat: - pos = 1 - elif lon: - pos = 0 - - coord_func(ipix=a)[pos] - - def HEALPix_bounds(a, lat=False, lon=False): - # Keep an eye on - # https://github.com/ntessore/healpix/issues/66 - if lat: - pos = 1 - elif lon: - pos = 0 - - bounds = np.empty((a.size, 4), dtype='float64') - - # Vertex position u, v - # --------------- ---- - # right 1, 0 - # top 1, 1 - # left 0, 1 - # bottom 0, 0 - for i, (u, v) in enumerate((1, 0), (1,1), (0, 1), (0,0)): - b = bounds_func(a, u, v)[pos] - np.rad2deg(b, out=b) - bounds[:,i] = b - - return bounds + import ../dask_utils import cf_HEALPix_coordinates, cf_HEALPix_bounds # Create new latitude and longitude coordinates with bounds dx = healpix_index.to_dask_array() meta = np.array((), dtype="float64") # Latitude coordinates - dy = dx.map_blocks(HEALPix_coordinates, meta=meta, lat=True ) + dy = dx.map_blocks( + cf_HEALPix_coordinates, meta=meta, + healpix_order=healpix_order, + refinement_level=refinement_level, + lat=True + ) lat = f._AuxiliaryCoordinate( data=f._Data(dy, "degrees_north", copy=False), properties={"standard_name": "latitude"}, @@ -2068,7 +2031,12 @@ def HEALPix_bounds(a, lat=False, lon=False): ) # Longitude coordinates - dy = dx.map_blocks(HEALPix_coordinates, meta=meta, lon=True ) + dy = dx.map_blocks( + cf_HEALPix_coordinates, meta=meta, + healpix_order=healpix_order, + refinement_level=refinement_level, + lon=True + ) lon = f._AuxiliaryCoordinate( data=f._Data(dy, "degrees_east", copy=False), properties={"standard_name": "longitude"}, @@ -2077,14 +2045,20 @@ def HEALPix_bounds(a, lat=False, lon=False): # Latitude bounds dy = da.blockwise( - HEALPix_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta, lat=True ) + cf_HEALPix_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta, + healpix_order=healpix_order, + refinement_level=refinement_level, + lat=True ) bounds = f._Bounds(data=dy) lat.set_bounds(bounds) # Longitude bounds dy = da.blockwise( - HEALPix_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta, lon=True ) + cf_HEALPix_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta, + healpix_order=healpix_order, + refinement_level=refinement_level, + lon=True ) bounds = f._Bounds(data=dy) lon.set_bounds(bounds) @@ -2094,6 +2068,8 @@ def HEALPix_bounds(a, lat=False, lon=False): lon_key = f.set_construct(lon, axes=axis, copy=False) cr.set_coordinates((lat_key ,lon_key)) + return f + return f def cyclic( @@ -2501,67 +2477,73 @@ def get_coordinate_reference( return out - def get_refinement_level(self, default=ValueError()): - """TODOHEALPIX - - .. versionadded:: NEXTVERSION - - :Parameters: - - TODOHEALPIX - - :Returns: - - `int` - TODOHEALPIX - - **Examples** - - TODOHEALPIX - - """ - cr = self.coordinate_reference( - "grid_mapping_name:healpix", default=None - ) - if cr is not None: - refinement_level = cr.coordinate_conversion.get_property( - "refinement_level", None - ) - if refinement_level is not None: - return refinement_level - - return self._default(default, "HEALPix refinement level has been set") - - def get_healpix_order(self, default=ValueError()): - """TODOHEALPIX - - .. versionadded:: NEXTVERSION - - :Parameters: - - TODOHEALPIX - - :Returns: - - `str` - TODOHEALPIX - - **Examples** - - TODOHEALPIX - - """ - cr = self.coordinate_reference( - "grid_mapping_name:healpix", default=None - ) - if cr is not None: - healpix_order = cr.coordinate_conversion.get_property( - "healpix_order", None - ) - if healpix_order is not None: - return healpix_order - - return self._default(default, "HEALPix order has been set") +# def get_refinement_level(self, default=ValueError()): +# """TODOHEALPIX +# +# .. versionadded:: NEXTVERSION +# +# :Parameters: +# +# TODOHEALPIX +# +# :Returns: +# +# `int` +# TODOHEALPIX +# +# **Examples** +# +# TODOHEALPIX +# +# """ +# cr = self.coordinate_reference( +# "grid_mapping_name:healpix", default=None +# ) +# if cr is not None: +# refinement_level = cr.coordinate_conversion.get_property( +# "refinement_level", None +# ) +# if refinement_level is not None: +# return refinement_level +# +# if default is None: +# return +# +# return self._default(default, "HEALPix refinement level has been set") +# +# def get_healpix_order(self, default=ValueError()): +# """TODOHEALPIX +# +# .. versionadded:: NEXTVERSION +# +# :Parameters: +# +# TODOHEALPIX +# +# :Returns: +# +# `str` +# TODOHEALPIX +# +# **Examples** +# +# TODOHEALPIX +# +# """ +# cr = self.coordinate_reference( +# "grid_mapping_name:healpix", default=None +# ) +# if cr is not None: +# healpix_order = cr.coordinate_conversion.get_property( +# "healpix_order", None +# ) +# if healpix_order is not None: +# return healpix_order +# +# if default is None: +# return +# +# return self._default(default, "HEALPix order has been set") def iscyclic(self, *identity, **filter_kwargs): """Returns True if the given axis is cyclic. @@ -2696,8 +2678,9 @@ def is_discrete_axis(self, *identity, **filter_kwargs): return True # HEALPix - if f.coordinate("healpix_index", filter_by_axis=(axis,), axis_mode="exact", default=None): - return True + if f.coordinate("healpix_index", filter_by_axis=(axis,), + axis_mode="exact", default=None): + return True # Geometries for aux in self.auxiliary_coordinates( diff --git a/cf/weights.py b/cf/weights.py index ec50cc54e4..9b52c42ab4 100644 --- a/cf/weights.py +++ b/cf/weights.py @@ -1968,7 +1968,7 @@ def healpix_area( return False if domain_axis is None: - raise ValueError("No HEALPix cells") + raise ValueError("No HEALPix axis") raise ValueError( "No HEALPix cells for " @@ -1993,13 +1993,6 @@ def healpix_area( f"{f.constructs.domain_axis_identity(axis)!r} axis" ) - if not measure: - return True # TODOHEALPIX - - if methods: - weights[(axis,)] = "HEALPix equal area" - return True - cr = f.coordinate_reference( "grid_mapping_name:healpix", default=None ) @@ -2014,8 +2007,20 @@ def healpix_area( ) parameters = cr.coordinate_conversion.parameters() + + healpix_order = parameters.get("healpix_order") + if healpix_order not in ("nested'", "ring", "nuniq"): + if auto: + return False + + raise ValueError( + "Can't create weights: Invalid HEALPix healpix_order for " + f"{f.constructs.domain_axis_identity(axis)!r} axis: " + f"{healpix_order!r}" + ) + refinement_level = parameters.get("refinement_level") - if refinement_level is None: + if refinement_level is None and healpix_order != "nuniq": # No refinement_level if auto: return False @@ -2025,19 +2030,50 @@ def healpix_area( f"{f.constructs.domain_axis_identity(axis)!r} axis" ) - if methods: - weights[(axis,)] = "HEALPix equal area" - return True - if radius is not None: + if measure and not methods and radius is not None: radius = f.radius(default=radius) + # Weights for "nuniq" ordering + if healpix_order == "nuniq": + if methods: + weights[(axis,)] = "HEALPix Multi-Order Coverage" + return True + + healpix_index = f.coordinate( + "healpix_index", filter_by_axis=(axis,), exact=True + default=None + ) + if healpix_index is None: + if auto: + return False + + raise ValueError( + "Can't create weights: TODOHEALPIX" + ) + + area = self._healpix_nuniq_weights(healpix_index, + measure=measure, radius=radius) + if return_areas: + return area + + weights[(axis,)] = area + weights_axes.add(axis) + return True + + # Weights for "nested" or "ring" ordering + if methods: + return True + + if not measure: + return True + + r2 = radius ** 2 area = cf.Data.full( - shape, - 4 * np.pi / (12 * (4 ** refinement_level)), - units="1" + (f.domain_axis(axis).get_size(),) + 4 * np.pi * float(r2) / (12 * (4 ** refinement_level)), + units=r2.Units ) - area = area * (radius ** 2) if return_areas: return area @@ -2046,96 +2082,32 @@ def healpix_area( weights_axes.add(axis) return True + def _healpix_nuniq_weights(self, + healpix_index, measure=False, radius=None): + """TODOHEALPIX - - from .units import Units - - axis, aux_X, aux_Y, aux_Z, ugrid = cls._geometry_ugrid_cells( - f, domain_axis, "polygon", auto=auto - ) - - if axis is None: - if auto: - return False - - if domain_axis is None: - raise ValueError("No polygon cells") - - raise ValueError( - "No polygon cells for " - f"{f.constructs.domain_axis_identity(domain_axis)!r} axis" - ) - - if axis in weights_axes: - if auto: - return False - - raise ValueError( - "Multiple weights specifications for " - f"{f.constructs.domain_axis_identity(axis)!r} axis" - ) - - x = aux_X.bounds.data - y = aux_Y.bounds.data - - radians = Units("radians") - metres = Units("metres") - - if x.Units.equivalent(radians) and y.Units.equivalent(radians): - if not great_circle: - raise ValueError( - "Must set great_circle=True to allow the derivation of " - "area weights from spherical polygons composed from " - "great circle segments." - ) - - if methods: - weights[(axis,)] = "area spherical polygon" - return True - - spherical = True - x.Units = radians - elif x.Units.equivalent(metres) and y.Units.equivalent(metres): - if methods: - weights[(axis,)] = "area plane polygon" - return True - - spherical = False - else: - return False + .. versionadded:: NEXTVERSION - y.Units = x.Units - x = x.persist() - y = y.persist() + :Parameters: - # Find the number of nodes per polygon - n_nodes = x.count(axis=-1, keepdims=False).array - if (y.count(axis=-1, keepdims=False) != n_nodes).any(): - raise ValueError( - "Can't create area weights for " - f"{f.constructs.domain_axis_identity(axis)!r} axis: " - f"{aux_X!r} and {aux_Y!r} have inconsistent bounds " - "specifications" - ) + TODOHEALPIX + + :Returns: - if ugrid: - areas = cls._polygon_area_ugrid(f, x, y, n_nodes, spherical) + `Data` + TODOHEALPIX + + """ + from dask_utils import cf_HEALPix_nuniq_weights + + dx = healpix_index.to_dask_array() + dx = dx.map_blocks(cf_HEALPix_nuniq_weights, + meta=np.array((), dtype="float64"), + measure=measure, radius=radius) + + if measure: + units = radius.Units ** 2 else: - areas = cls._polygon_area_geometry( - f, x, y, aux_X, aux_Y, n_nodes, spherical - ) - - del x, y, n_nodes - - if not measure: - areas.override_units(Units("1"), inplace=True) - elif spherical: - areas = cls._spherical_area_measure(f, areas, aux_Z, radius) - - if return_areas: - return areas - - weights[(axis,)] = areas - weights_axes.add(axis) - return True - + units = Units("1") + + return Data(dx, units=units, copy=False) From b83bd91102cd98a2e8e6f6c421dbd97f7fce3426 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 17 Jun 2025 22:58:18 +0100 Subject: [PATCH 08/59] dev --- cf/dask_utils.py | 261 +++++++++++++++++--------- cf/domain.py | 2 + cf/field.py | 99 ++++++---- cf/mixin/fielddomain.py | 373 ++++++++++++++++++++++++++------------ cf/regrid/regrid.py | 40 +++- cf/test/test_functions.py | 1 - cf/weights.py | 157 +++++++++------- 7 files changed, 628 insertions(+), 305 deletions(-) diff --git a/cf/dask_utils.py b/cf/dask_utils.py index 6e18acedae..0d34bf5549 100644 --- a/cf/dask_utils.py +++ b/cf/dask_utils.py @@ -1,18 +1,25 @@ -# Define functions to create latitudes and longitudes from -# HEALPix indices +"""Functions intended to be passed to be dask. + +These will typically be functions that operate on dask chunks. For +instance, as would be passed to `dask.array.map_blocks`. + +""" + import healpix +import numpy as np -def cf_HEALPix_coordinates( - a, - healpix_order, refinement_level=None, lat=False, lon=False + +def cf_HEALPix_bounds( + a, healpix_order, refinement_level=None, lat=False, lon=False ): - """Calculate HEALPix cell coordinates. - + """Calculate HEALPix cell bounds. + .. versionadded:: NEXTVERSION :Parameters: a: `numpy.ndarray` + The array of HEALPix indices. healpix_order: `str` One of ``'nested'``, ``'ring'``, or ``'nuniq'``. @@ -20,56 +27,148 @@ def cf_HEALPix_coordinates( refinement_level: `int` or `None`, optional For a ``'nested'`` or ``'ring'`` ordered grid, the refinement level of the grid within the HEALPix hierarchy, - starting at 0 for the base tesselation with 12 cells - globally. Ignored for a ``'nuniq'`` ordered grid. + starting at 0 for the base tesselation with 12 cells. + Ignored for a ``'nuniq'`` ordered grid. lat: `bool`, optional + If True then return latitude bounds. lon: `bool`, optional - + If True then return longitude bounds. + :Returns: `numpy.ndarray` + An array containing the HEALPix cell bounds. + + """ + # Keep an eye on https://github.com/ntessore/healpix/issues/66 + if a.ndim != 1: + raise ValueError( + "Can't calculate HEALPix cell bounds when " + f"healpix_index array has shape {a.shape}" + ) - """ if lat: pos = 1 - elif lon: + elif lon: pos = 0 + if healpix_order == "nested": + bounds_func = healpix._chp.nest2ang_uv + else: + bounds_func = healpix._chp.ring2ang_uv + + # Define the cell vertices in an anticlockwise direction, as seen + # from above. + right = (1, 0) + top = (1, 1) + left = (0, 1) + bottom = (0, 0) + vertices = (right, top, left, bottom) + + b = np.empty((a.size, 4), dtype="float64") + if healpix_order == "nuniq": # nuniq - c = np.empty(a.shape, dtype='float64') - - nsides, a = healpix.uniq2pix(a, nest=False) - nsides, index, inverse = np.unique(nsides, - return_index=True, - return_inverse=True) + nsides, a = healpix.uniq2pix(a, nest=False) + nsides, index, inverse = np.unique( + nsides, return_index=True, return_inverse=True + ) for nside, i in zip(nsides, index): level = np.where(inverse == inverse[i])[0] - c[level] = healpix.pix2ang( - nside=nside, ipix=a[level], nest=False, lonlat=True - )[pos] + for j, (u, v) in enumerate(vertices): + thetaphi = bounds_func(nside, a[level], u, v) + b[level, j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] else: # nested or ring - c = healpix.pix2ang( - nside=healpix.order2nside(refinement_level), - ipix=a, - nest=healpix_order=="nested", - lonlat=True - )[pos] + nside = healpix.order2nside(refinement_level) + for j, (u, v) in enumerate(vertices): + thetaphi = bounds_func(nside, a, u, v) + b[..., j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] - return c - - -def cf_HEALPix_bounds(a, healpix_order, refinement_level=None, lat=False, lon=False): - """Calculate HEALPix cell bounds. + if not pos: + # Longitude bounds + b[np.where(b >= 360)] -= 360.0 + + # Bounds on the north or south pole come out with a longitude + # of NaN, so replace these with a sensible value. + i = np.argwhere(np.isnan(b[:, 1])).flatten() + if i.size: + b[i, 1] = b[i, 3] + + i = np.argwhere(np.isnan(b[:, 3])).flatten() + if i.size: + b[i, 3] = b[i, 1] + + return b + + +def cf_HEALPix_change_order( + a, healpix_order, new_healpix_order, refinement_level +): + """Change the ordering of HEALPix indices. + + .. versionadded:: NEXTVERSION + + :Parameters: + + a: `numpy.ndarray` + The array of HEALPix indices. + + healpix_order: `str` + The original HEALPix order. One of ``'nested'`` or + ``'ring'``. + + new_healpix_order: `str` + The new HEALPix order to change to. One of ``'nested'``, + ``'ring'``, or ``'nuniq'``. + + refinement_level: `int` + The refinement level of the original grid within the + HEALPix hierarchy, starting at 0 for the base tesselation + with 12 cells. + + :Returns: + + `numpy.ndarray` + An array containing the new HEALPix indices. + + """ + nside = healpix.order2nside(refinement_level) + + if healpix_order == "nested": + if new_healpix_order == "ring": + return healpix.nest2ring(nside, a) + + if new_healpix_order == "nuniq": + return healpix._chp.nest2uniq(nside, a) + + elif healpix_order == "ring": + if new_healpix_order == "nested": + return healpix.ring2nest(nside, a) + + if new_healpix_order == "nuniq": + return healpix._chp.ring2uniq(nside, a) + + else: + raise ValueError( + "Can't change HEALPix order: Can only change from HEALPix " + f"order 'nested' or 'ring'. Got {healpix_order!r}" + ) + + +def cf_HEALPix_coordinates( + a, healpix_order, refinement_level=None, lat=False, lon=False +): + """Calculate HEALPix cell coordinates. .. versionadded:: NEXTVERSION - + :Parameters: a: `numpy.ndarray` + The array of HEALPix indices. healpix_order: `str` One of ``'nested'``, ``'ring'``, or ``'nuniq'``. @@ -77,102 +176,94 @@ def cf_HEALPix_bounds(a, healpix_order, refinement_level=None, lat=False, lon=Fa refinement_level: `int` or `None`, optional For a ``'nested'`` or ``'ring'`` ordered grid, the refinement level of the grid within the HEALPix hierarchy, - starting at 0 for the base tesselation with 12 cells - globally. Ignored for a ``'nuniq'`` ordered grid. + starting at 0 for the base tesselation with 12 cells. + Ignored for a ``'nuniq'`` ordered grid. lat: `bool`, optional + If True then return latitude coordinates. lon: `bool`, optional - + If True then return longitude coordinates. + :Returns: `numpy.ndarray` + An array containing the HEALPix cell coordinates. """ - # Keep an eye on https://github.com/ntessore/healpix/issues/66 + if a.ndim != 1: + raise ValueError( + "Can't calculate HEALPix cell coordinates when " + f"healpix_index array has shape {a.shape}" + ) if lat: pos = 1 - elif lon: + elif lon: pos = 0 - if healpix_order == "nested": - bounds_func = healpix._chp.nest2ang_uv - else: - bounds_func = healpix._chp.ring2ang_uv - - - # Define the cell vertices in an anticlockwise direction, seen - # from above. - # - # Vertex position vertex (u, v) - # --------------- ------------- - # right (1, 0) - # top (1, 1) - # left (0, 1) - # bottom (0, 0) - vertices = ((1, 0), (1,1), (0, 1), (0,0)) - - b = np.empty((a.size, 4), dtype='float64') - if healpix_order == "nuniq": - # nuniq - nsides, a = healpix.uniq2pix(a, nest=False) - nsides, index, inverse = np.unique(nsides, - return_index=True, - return_inverse=True) + c = np.empty(a.shape, dtype="float64") + + nest = False + nsides, a = healpix.uniq2pix(a, nest=nest) + nsides, index, inverse = np.unique( + nsides, return_index=True, return_inverse=True + ) for nside, i in zip(nsides, index): level = np.where(inverse == inverse[i])[0] - for j, (u, v) in enumerate(vertices): - b[level, j] = bounds_func(nside, a[level], u, v)[pos] + c[level] = healpix.pix2ang( + nside=nside, ipix=a[level], nest=nest, lonlat=True + )[pos] else: # nested or ring - nside = healpix.order2nside(refinement_level) - for j, (u, v) in enumerate(vertices): - b[:,j] = bounds_func(nside, a, u, v)[pos] - - # Convert to degrees - np.rad2deg(b, out=b) + c = healpix.pix2ang( + nside=healpix.order2nside(refinement_level), + ipix=a, + nest=healpix_order == "nested", + lonlat=True, + )[pos] - return b + return c def cf_HEALPix_nuniq_weights(a, measure=False, radius=None): """Calculate HEALPix cell weights for 'nuniq' indices. .. versionadded:: NEXTVERSION - + :Parameters: a: `numpy.ndarray` + The array of HEALPix indices. measure: `bool`, optional - If True then create weights that are actual cell sizes, in + If True then create weights that are actual cell areas, in units of the square of the radius units. radius: number, optional The radius of the sphere, in units of length. Must be set if *measure * is True, otherwise ignored. - + :Returns: `numpy.ndarray` - The weights. + An array containing the HEALPix cell weights. """ if measure: - f = 4.0 * np.pi * (radius**2) / 12 + x = np.pi * (radius**2) / 3.0 else: - f = 1.0 - + x = 1.0 + nsides = healpix.uniq2pix(a)[0] - nsides, index, inverse = np.unique(nsides, - return_index=True, - return_inverse=True) - - w = np.empty(a.shape, dtype='float64') + nsides, index, inverse = np.unique( + nsides, return_index=True, return_inverse=True + ) + + w = np.empty(a.shape, dtype="float64") for nside, i in zip(nsides, index): - w = np.where(inverse == inverse[i], f / (nside**2), w) - + w = np.where(inverse == inverse[i], x / (nside**2), w) + return w diff --git a/cf/domain.py b/cf/domain.py index 98ca769cdb..78ae41dfc6 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -10,6 +10,7 @@ from .decorators import _inplace_enabled, _inplace_enabled_define_and_cleanup from .dimensioncoordinate import DimensionCoordinate from .domainaxis import DomainAxis +from .domaintopology import DomainTopology from .functions import ( _DEPRECATION_ERROR_ARG, _DEPRECATION_ERROR_METHOD, @@ -83,6 +84,7 @@ def __new__(cls, *args, **kwargs): instance._DomainAxis = DomainAxis instance._DimensionCoordinate = DimensionCoordinate instance._AuxiliaryCoordinate = AuxiliaryCoordinate + instance._DomainTopology = DomainTopology return instance @property diff --git a/cf/field.py b/cf/field.py index c4862c0a6f..bd2f924a70 100644 --- a/cf/field.py +++ b/cf/field.py @@ -18,6 +18,7 @@ Domain, DomainAncillary, DomainAxis, + DomainTopology, FieldList, Flags, Index, @@ -281,6 +282,7 @@ def __new__(cls, *args, **kwargs): instance._Domain = Domain instance._DomainAncillary = DomainAncillary instance._DomainAxis = DomainAxis + instance._DomainTopology = DomainTopology instance._Quantization = Quantization instance._RaggedContiguousArray = RaggedContiguousArray instance._RaggedIndexedArray = RaggedIndexedArray @@ -3654,13 +3656,12 @@ def weights( weights_axes, measure=measure, radius=radius, - great_circle=great_circle, methods=methods, auto=True, ): # Found area weights from HEALPix cells area_weights = True - + if not area_weights: raise ValueError( "Can't create weights: Unable to find cell areas" @@ -3700,6 +3701,18 @@ def weights( ): # Found linear weights from line geometries pass + elif Weights.healpix_area( + self, + da_key, + comp, + weights_axes, + measure=measure, + radius=radius, + methods=methods, + auto=True, + ): + # Found area weights from HEALPix cells + pass else: Weights.linear( self, @@ -4918,11 +4931,12 @@ def healpix_decrease_refinement_level( :Returns: - TODOHEALPIX + `{{class}}` or `None` + TODOHEALPIX **Examples** - >>> TODOHEALPIX + >>> TODOHEALPIX """ f = _inplace_enabled_define_and_cleanup(self) @@ -4937,8 +4951,8 @@ def healpix_decrease_refinement_level( ) if healpix_index is None: raise ValueError( - "Can't decrease refinement level: HEALPix index coordinates " - "have not been set" + "Can't decrease HEALPix refinement level: healpix_index " + "coordinates have not been set" ) axis = f.get_data_axes(key)[0] @@ -4947,28 +4961,29 @@ def healpix_decrease_refinement_level( cr = f.coordinate_reference("grid_mapping_name:healpix", default=None) if cr is None: raise ValueError( - "Can't decrease refinement level: HEALPix grid mapping has " - "not been set" + "Can't decrease HEALPix refinement level: HEALPix " + "grid mapping has not been set" ) - parameters = cr.coordinate_conversion.parameters() - healpix_order = parameters.get( "healpix_order") + parameters = cr.coordinate_conversion.parameters() + healpix_order = parameters.get("healpix_order") if healpix_order is None: raise ValueError( - "Can't decrease refinement level: HEALPix order has not " - "been set" + "Can't decrease HEALPix refinement level: healpix_order has " + "not been set" ) if healpix_order != "nested": raise ValueError( - "Can't decrease refinement level: Must have a HEALPix order " - f"of 'nested' for this operation. Got: {healpix_order!r}" + "Can't decrease HEALPix refinement level: Must have a " + "healpix_order of 'nested' for this operation. " + f"Got: {healpix_order!r}" ) refinement_level = parameters.get("refinement_level") if refinement_level is None: raise ValueError( - "Can't decrease refinement level: HEALPix refinement level " + "Can't decrease HEALPix refinement level: refinement_level " "has not been set" ) @@ -4999,33 +5014,37 @@ def healpix_decrease_refinement_level( new_refinement_level = refinement_level + level if check_healpix_index: - if not (data.diff() > 0).all(): + d = healpix_index.data + if not (d.diff() > 0).all(): raise ValueError( - "Can't decrease refinement level: Nested HEALPix indices " - "are not strictly monotonically increasing" + "Can't decrease HEALPix refinement level: healpix_index " + "cooridnates are not strictly monotonically increasing" ) - if (data[::ncells] % ncells).any() or ( - data[ncells - 1 :: ncells] - data[::ncells] != ncells - 1 + if (d[::ncells] % ncells).any() or ( + d[ncells - 1 :: ncells] - d[::ncells] != ncells - 1 ).any(): raise ValueError( - "Can't decrease refinemnent level: At least one cell at " - f"the coarser refinement level ({new_refinement_level}) " - f"contains fewer than {ncells} cells at the original " - f"refinement level {refinement_level}" + "Can't decrease HEALPix refinement level: At least one " + "cell at the coarser refinement level " + f"({new_refinement_level}) contains fewer than {ncells} " + "cells at the original refinement level " + f"{refinement_level}" ) # Create the healpix_index coordinates for the new refinement # level new_index = healpix_index[::ncells] // ncells - # Coarsen (with 'reduction') the field data + # Coarsen (using 'reduction') the field data. Note that using + # the 'coarsen' technique only works for 'nested' HEALPix + # ordering. i = f.get_data_axes().index(axis) f.data.coarsen( reduction, axes={i: ncells}, trim_excess=False, inplace=True ) - # Coarsen (with np.ma.mean) the domain ancillary constructs + # Coarsen (using np.ma.mean) the domain ancillary constructs # that span the HEALPix axis for key, domain_ancillary in f.domain_ancillaries( filter_by_axis=(axis,), axis_mode="and", todict=True @@ -5035,7 +5054,7 @@ def healpix_decrease_refinement_level( np.ma.mean, axes={i: ncells}, trim_excess=False, inplace=True ) - # Coarsen (with np.sum) the cell measure constructs that span + # Coarsen (using np.sum) the cell measure constructs that span # the HEALPix axis for key, cell_measure in f.cell_measures( filter_by_axis=(axis,), axis_mode="and", todict=True @@ -5050,9 +5069,9 @@ def healpix_decrease_refinement_level( # construct. for key in ( f.constructs.filter_by_axis(axis, axis_mode="and") - .filter_by_type("cell_measure", "domain_ancillary") - .inverse_filter(1) - .todict() + .filter_by_type("cell_measure", "domain_ancillary") + .inverse_filter(1) + .todict() ): f.del_construct(key) @@ -5690,12 +5709,13 @@ def collapse( ``'T'`` for the first collapse, and axes of ``'area'`` for the second collapse. - .. note:: Setting *weights* to `True` is generally a good - way to ensure that all collapses are - appropriately weighted according to the field - construct's metadata. In this case, if it is not - possible to create weights for any axis then an - exception will be raised. + .. note:: Setting *weights* to `True` is generally a + good way to ensure that all collapses are + appropriately weighted according to the + field construct's metadata. In this case, if + it is not possible to create weights for any + of the collapsed axes then an exception will + be raised. However, care needs to be taken if *weights* is `True` when cell volume weights are desired. The @@ -6644,7 +6664,9 @@ def collapse( f = self else: f = self.copy() - + print(f) + f.create_latlon_coordinates(inplace=True) + print(f) # Whether or not to create null bounds for null # collapses. I.e. if the collapse axis has size 1 and no # bounds, whether or not to create upper and lower bounds to @@ -6737,7 +6759,8 @@ def collapse( msg = "Can't find the collapse axis identified by {!r}" for x in iterate_over: - a = self.domain_axis(x, key=True, default=None) + a = f.domain_axis(x, key=True, default=None) + print (a) if a is None: raise ValueError(msg.format(x)) diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 914660591a..5e00b9d479 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -298,7 +298,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): # Create any implied 1-d latitude and longitude coordinates # (e.g. as implied by HEALPix indices) - self = self.create_1d_latlon_coordinates() + self = self.create_latlon_coordinates() domain_axes = self.domain_axes(todict=True) @@ -1877,19 +1877,115 @@ def coordinate_reference_domain_axes(self, identity=None): return set(axes) + @_inplace_enabled(default=False) + def healpix_change_order(self, new_healpix_order, inplace=False): + """Change the ordering of HEALPix indices. + + .. versionadded:: NEXTVERSION + + :Parameters: + + new_healpix_order: `str` + The new HEALPix order to change to. One of + ``'nested'``, ``'ring'``, or ``'nuniq'``. + + {{inplace: `bool`, optional}} + + :Returns: + + `{{class}}` or `None` + TODOHEALPIX An array containing the new HEALPix indices. + + **Examples** + + TODOHEALPIX + + """ + from ..dask_utils import cf_HEALPix_change_order + + f = _inplace_enabled_define_and_cleanup(self) + + if new_healpix_order not in ("nested", "ring", "nuniq"): + raise ValueError( + "Can't change HEALPix order: new_healpix_order must be " + f"'nest', 'ring', or 'nuniq'. Got {new_healpix_order!r}" + ) + + # Parse the HEALPix coordinate reference + cr = f.coordinate_reference("grid_mapping_name:healpix", default=None) + if cr is None: + raise ValueError( + "Can't change HEALPix order: HEALPix grid mapping has " + "not been set" + ) + + parameters = cr.coordinate_conversion.parameters() + + refinement_level = parameters.get("refinement_level") + if refinement_level is None: + raise ValueError( + "Can't change HEALPix order: refinement_level " + "has not been set" + ) + + healpix_order = parameters.get("healpix_order") + if healpix_order is None: + raise ValueError( + "Can't change HEALPix order: healpix_order has not been set" + ) + + if healpix_order not in ("nested", "ring"): + raise ValueError( + "Can't change HEALPix order: Can only change from " + f"healpix_order 'nested' or 'ring'. Got {healpix_order!r}" + ) + + if healpix_order == new_healpix_order: + # No change + return f + + # Get the original HEALPix indices + healpix_index = f.coordinate( + "healpix_index", filter_by_naxes=(1,), default=None + ) + if healpix_index is None: + raise ValueError( + "Can't change HEALPix order: No healpix_index coordinates" + ) + + # Change the HEALPix indices + dx = healpix_index.to_dask_array() + dx = dx.map_blocks( + cf_HEALPix_change_order, + meta=np.array((), dtype="int64"), + healpix_order=healpix_order, + new_healpix_order=new_healpix_order, + refinement_level=refinement_level, + ) + healpix_index.set_data(dx, copy=False) + + # Update the Coordinate Reference + cr.coordinate_conversion.set_parameter( + "healpix_order", new_healpix_order + ) + if new_healpix_order == "nuniq": + cr.coordinate_conversion.del_parameter("refinement_level", None) + + return f + @_inplace_enabled(default=False) def healpix_to_ugrid(self, inplace=False): """TODOHEALPIX""" axis = self.healpix_axis() if axis is None: raise ValueError("TODOHEALPIX") - + f = _inplace_enabled_define_and_cleanup(self) # If lat/lon coordinates do not exist, then derive them from # the HEALPix indices. - f.create_latlon_coordinates(('HEALPix',) inplace=True) - + f.create_latlon_coordinates(("HEALPix",), inplace=True) + x = f.auxiliary_coordinate( "Y", filter_by_axis=(axis,), axis_mode="exact", default=None ) @@ -1901,7 +1997,7 @@ def healpix_to_ugrid(self, inplace=False): if y is None: raise ValueError("TODOHEALPIX") - + bounds_y = y.get_bounds(None) bounds_x = x.get_bounds(None) if bounds_y is None: @@ -1910,45 +2006,72 @@ def healpix_to_ugrid(self, inplace=False): if bounds_x is None: raise ValueError("TODOHEALPIX") - # Create an identifer for each unique node location + # Create a unique identifer for each node location bounds_y = bounds_y.to_dask_array() bounds_x = bounds_x.to_dask_array() _, y_indices = np.unique(bounds_y, return_inverse=True) _, x_indices = np.unique(bounds_x, return_inverse=True) - nodes = y_indices * y_indices.size + x_indices + nodes = y_indices * y_indices.size + x_indices - # Create a Domain Topology construct - domain_topology = f._DomainToplogy(data=f._Data(nodes)) - domain_topology.set_cell('face') + # Create the Domain Topology construct + domain_topology = f._DomainTopology(data=f._Data(nodes)) + domain_topology.set_cell("face") domain_topology.set_property( "long_name", "UGRID topology derived from HEALPix" ) f.set_construct(domain_topology, axes=axis, copy=False) - # Remove the HEALPix grid mapping and index coordinates - # TODOHEALPIX - make new grid mapping for generic grid mapping - # parametes? - f.del_construct("grid_mapping_name:healpix") - f.del_construct(key) + # Upsdate the Coordinate Reference, either by deleting it, or + # converting it to 'latitude_longitude'. + key, cr = f.coordinate_reference( + "grid_mapping_name:healpix", item=True, default=(None, None) + ) + if key is not None: + cc = cr.coordinate_conversion + cc.del_parameter("grid_mapping_name", None) + cc.del_parameter("healpix_order", None) + cc.del_parameter("refinement_level", None) + if cr.coordinate_conversion.parameters(): + # The Coordinate Reference contains generic parameters + # (such as 'earth_radius'), so rename it. + cr.coordinate_conversion.set_parameter( + "grid_mapping_name", "latitude_longitude" + ) + else: + # The Coordinate Reference contains no generic + # parameters + f.del_construct(key) + + # Remove the HEALPix index coordinates + f.del_construct( + "healpix_index", + filter_by_type=( + "auxiliary_coordinate", + "dimension_coordinate", + ), + filter_by_axis=(axis,), + axis_mode="exact", + default=None, + ) return f def healpix_axis(self): """TODOHEALPIX. - + .. versionadded:: NEXTVERSION - + """ - key = f.coordinate( + key = self.coordinate( "healpix_index", filter_by_naxes=(1,), key=True, default=None ) if key is None: return - + return self.get_data_axes(key)[0] - + @_inplace_enabled(default=False) def create_latlon_coordinates(self, grid_type=None, inplace=False): """TODOHEALPIX. @@ -1987,11 +2110,14 @@ def create_latlon_coordinates(self, grid_type=None, inplace=False): return f refinement_level = parameters.get("refinement_level") - if refinement_level is None and healpix_order in ("nested", "ring"): + if refinement_level is None and healpix_order in ( + "nested", + "ring", + ): # Missing refinement_level return f - c_key, healpix_index = f.coordinate( + key, healpix_index = f.coordinate( "healpix_index", filter_by_naxes=(1,), item=True, @@ -2004,61 +2130,78 @@ def create_latlon_coordinates(self, grid_type=None, inplace=False): # Get the HEALPix axis axis = f.get_data_axes(key)[0] - if f.coordinates('X', 'Y', filter_by_axis=(axis,), - axis_mode="exact", todict=True): + if f.coordinates( + "X", + "Y", + filter_by_axis=(axis,), + axis_mode="exact", + todict=True, + ): # X and/or Y coordinates already exist, so don't create any. return f - + # Define functions to create latitudes and longitudes from # HEALPix indices - import ../dask_utils import cf_HEALPix_coordinates, cf_HEALPix_bounds - + from ..dask_utils import cf_HEALPix_bounds, cf_HEALPix_coordinates + # Create new latitude and longitude coordinates with bounds dx = healpix_index.to_dask_array() meta = np.array((), dtype="float64") - + # Latitude coordinates dy = dx.map_blocks( - cf_HEALPix_coordinates, meta=meta, + cf_HEALPix_coordinates, + meta=meta, healpix_order=healpix_order, refinement_level=refinement_level, - lat=True + lat=True, ) lat = f._AuxiliaryCoordinate( data=f._Data(dy, "degrees_north", copy=False), properties={"standard_name": "latitude"}, copy=False, ) - + # Longitude coordinates dy = dx.map_blocks( - cf_HEALPix_coordinates, meta=meta, - healpix_order=healpix_order, + cf_HEALPix_coordinates, + meta=meta, + healpix_order=healpix_order, refinement_level=refinement_level, - lon=True + lon=True, ) lon = f._AuxiliaryCoordinate( data=f._Data(dy, "degrees_east", copy=False), properties={"standard_name": "longitude"}, copy=False, ) - + # Latitude bounds dy = da.blockwise( - cf_HEALPix_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta, - healpix_order=healpix_order, + cf_HEALPix_bounds, + "ij", + dx, + "i", + new_axes={"j": 4}, + meta=meta, + healpix_order=healpix_order, refinement_level=refinement_level, - lat=True + lat=True, ) bounds = f._Bounds(data=dy) lat.set_bounds(bounds) - + # Longitude bounds dy = da.blockwise( - cf_HEALPix_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta, - healpix_order=healpix_order, + cf_HEALPix_bounds, + "ij", + dx, + "i", + new_axes={"j": 4}, + meta=meta, + healpix_order=healpix_order, refinement_level=refinement_level, - lon=True + lon=True, ) bounds = f._Bounds(data=dy) lon.set_bounds(bounds) @@ -2066,7 +2209,7 @@ def create_latlon_coordinates(self, grid_type=None, inplace=False): # Set the new latitude and longitude coordinates lat_key = f.set_construct(lat, axes=axis, copy=False) lon_key = f.set_construct(lon, axes=axis, copy=False) - cr.set_coordinates((lat_key ,lon_key)) + cr.set_coordinates((lat_key, lon_key)) return f @@ -2477,73 +2620,73 @@ def get_coordinate_reference( return out -# def get_refinement_level(self, default=ValueError()): -# """TODOHEALPIX -# -# .. versionadded:: NEXTVERSION -# -# :Parameters: -# -# TODOHEALPIX -# -# :Returns: -# -# `int` -# TODOHEALPIX -# -# **Examples** -# -# TODOHEALPIX -# -# """ -# cr = self.coordinate_reference( -# "grid_mapping_name:healpix", default=None -# ) -# if cr is not None: -# refinement_level = cr.coordinate_conversion.get_property( -# "refinement_level", None -# ) -# if refinement_level is not None: -# return refinement_level -# -# if default is None: -# return -# -# return self._default(default, "HEALPix refinement level has been set") -# -# def get_healpix_order(self, default=ValueError()): -# """TODOHEALPIX -# -# .. versionadded:: NEXTVERSION -# -# :Parameters: -# -# TODOHEALPIX -# -# :Returns: -# -# `str` -# TODOHEALPIX -# -# **Examples** -# -# TODOHEALPIX -# -# """ -# cr = self.coordinate_reference( -# "grid_mapping_name:healpix", default=None -# ) -# if cr is not None: -# healpix_order = cr.coordinate_conversion.get_property( -# "healpix_order", None -# ) -# if healpix_order is not None: -# return healpix_order -# -# if default is None: -# return -# -# return self._default(default, "HEALPix order has been set") + # def get_refinement_level(self, default=ValueError()): + # """TODOHEALPIX + # + # .. versionadded:: NEXTVERSION + # + # :Parameters: + # + # TODOHEALPIX + # + # :Returns: + # + # `int` + # TODOHEALPIX + # + # **Examples** + # + # TODOHEALPIX + # + # """ + # cr = self.coordinate_reference( + # "grid_mapping_name:healpix", default=None + # ) + # if cr is not None: + # refinement_level = cr.coordinate_conversion.get_property( + # "refinement_level", None + # ) + # if refinement_level is not None: + # return refinement_level + # + # if default is None: + # return + # + # return self._default(default, "HEALPix refinement level has been set") + # + # def get_healpix_order(self, default=ValueError()): + # """TODOHEALPIX + # + # .. versionadded:: NEXTVERSION + # + # :Parameters: + # + # TODOHEALPIX + # + # :Returns: + # + # `str` + # TODOHEALPIX + # + # **Examples** + # + # TODOHEALPIX + # + # """ + # cr = self.coordinate_reference( + # "grid_mapping_name:healpix", default=None + # ) + # if cr is not None: + # healpix_order = cr.coordinate_conversion.get_property( + # "healpix_order", None + # ) + # if healpix_order is not None: + # return healpix_order + # + # if default is None: + # return + # + # return self._default(default, "HEALPix order has been set") def iscyclic(self, *identity, **filter_kwargs): """Returns True if the given axis is cyclic. @@ -2678,9 +2821,13 @@ def is_discrete_axis(self, *identity, **filter_kwargs): return True # HEALPix - if f.coordinate("healpix_index", filter_by_axis=(axis,), - axis_mode="exact", default=None): - return True + if self.coordinate( + "healpix_index", + filter_by_axis=(axis,), + axis_mode="exact", + default=None, + ): + return True # Geometries for aux in self.auxiliary_coordinates( diff --git a/cf/regrid/regrid.py b/cf/regrid/regrid.py index e3d4ad79a9..2e2686f33d 100644 --- a/cf/regrid/regrid.py +++ b/cf/regrid/regrid.py @@ -3,6 +3,7 @@ import logging from dataclasses import dataclass, field from datetime import datetime +from pprint import pformat from typing import Any import dask.array as da @@ -496,7 +497,7 @@ def regrid( if is_log_level_debug(logger): logger.debug( - f"Source Grid:\n{src_grid}\n\nDestination Grid:\n{dst_grid}\n" + f"\n{pformat(src_grid)}\n\n{pformat(dst_grid)}\n" ) # pragma: no cover conform_coordinates(src_grid, dst_grid) @@ -720,7 +721,7 @@ def regrid( # ---------------------------------------------------------------- # Insert regridded data into the new field # ---------------------------------------------------------------- - update_data(src, regridded_data, src_grid) + update_data(src, regridded_data, src_grid, dst_grid) if coord_sys == "spherical" and dst_grid.is_grid: # Set the cyclicity of the longitude axis of the new field @@ -1020,6 +1021,8 @@ def get_grid( .. versionadded:: 3.14.0 + :Parameters: + coord_sys: `str` The coordinate system of the source and destination grids. @@ -1032,6 +1035,11 @@ def get_grid( .. versionadded:: 3.16.2 + :Returns: + + `Grid` + The grid definition. + """ if coord_sys == "spherical": return spherical_grid( @@ -1124,6 +1132,13 @@ def spherical_grid( The grid definition. """ + try: + # Try to convert a HEALPix grid to UGRID + f = f.healpix_to_ugrid() + except ValueError: + pass + + print(f) data_axes = f.constructs.data_axes() dim_coords_1d = False @@ -2970,7 +2985,7 @@ def update_non_coordinates(src, dst, src_grid, dst_grid, regrid_operator): src.set_coordinate_reference(ref, parent=dst, strict=True) -def update_data(src, regridded_data, src_grid): +def update_data(src, regridded_data, src_grid, dst_grid): """Insert the regridded field data. .. versionadded:: 3.16.0 @@ -2989,6 +3004,11 @@ def update_data(src, regridded_data, src_grid): src_grid: `Grid` The definition of the source grid. + dst_grid: `Grid` + The definition of the destination grid. + + .. versionadded:: NEXTVERSION + :Returns: `None` @@ -3004,6 +3024,20 @@ def update_data(src, regridded_data, src_grid): index = data_axes.index(src_grid.axis_keys[0]) for axis in src_grid.axis_keys: data_axes.remove(axis) + for key, cm in src.cell_methods(todict=True).items(): + if axis in cm.axes: + # When regridding from one axis to two axes, we + # can safely rename cell method's axis to 'area', + # otherwise we have to delete the cell method to + # allow the domain axis itself to be deleted. + if ( + len(src_grid.axis_keys) == 1 + and len(dst_grid.axis_keys) == 2 + ): + cm.change_axes({axis: "area"}, inplace=True) + else: + src.del_construct(key) + src.del_construct(axis) data_axes[index:index] = src_grid.new_axis_keys diff --git a/cf/test/test_functions.py b/cf/test/test_functions.py index 7ddf71d0e0..1a26a1afdc 100644 --- a/cf/test/test_functions.py +++ b/cf/test/test_functions.py @@ -38,7 +38,6 @@ def test_keyword_deprecation(self): def test_aliases(self): self.assertEqual(cf.log_level(), cf.LOG_LEVEL()) - self.assertEqual(cf.free_memory(), cf.FREE_MEMORY()) self.assertEqual(cf.total_memory(), cf.TOTAL_MEMORY()) self.assertEqual(cf.regrid_logging(), cf.REGRID_LOGGING()) self.assertEqual(cf.relaxed_identities(), cf.RELAXED_IDENTITIES()) diff --git a/cf/weights.py b/cf/weights.py index 9b52c42ab4..43c81e3b1e 100644 --- a/cf/weights.py +++ b/cf/weights.py @@ -1,4 +1,5 @@ import cfdm +import numpy as np from .mixin_container import Container @@ -635,8 +636,6 @@ def _polygon_area_geometry(cls, f, x, y, aux_X, aux_Y, n_nodes, spherical): square metres. """ - import numpy as np - x_units = x.Units y_units = y.Units @@ -798,8 +797,6 @@ def _polygon_area_ugrid(cls, f, x, y, n_nodes, spherical): metres. """ - import numpy as np - y_units = y.Units n_nodes0 = n_nodes.item(0) @@ -1259,7 +1256,6 @@ def _interior_angles(cls, f, lon, lat, interior_rings=None): units of radians. """ - import numpy as np from .query import lt @@ -1962,14 +1958,13 @@ def healpix_area( """ axis = f.healpix_axis() - if axis is None: if auto: return False - + if domain_axis is None: raise ValueError("No HEALPix axis") - + raise ValueError( "No HEALPix cells for " f"{f.constructs.domain_axis_identity(domain_axis)!r} axis" @@ -1993,9 +1988,7 @@ def healpix_area( f"{f.constructs.domain_axis_identity(axis)!r} axis" ) - cr = f.coordinate_reference( - "grid_mapping_name:healpix", default=None - ) + cr = f.coordinate_reference("grid_mapping_name:healpix", default=None) if cr is None: # No healpix grid mapping if auto: @@ -2007,9 +2000,8 @@ def healpix_area( ) parameters = cr.coordinate_conversion.parameters() - healpix_order = parameters.get("healpix_order") - if healpix_order not in ("nested'", "ring", "nuniq"): + if healpix_order not in ("nested", "ring", "nuniq"): if auto: return False @@ -2029,85 +2021,120 @@ def healpix_area( "Can't create weights: No HEALPix refinement_level for " f"{f.constructs.domain_axis_identity(axis)!r} axis" ) - if measure and not methods and radius is not None: radius = f.radius(default=radius) - # Weights for "nuniq" ordering - if healpix_order == "nuniq": + # Create weights for 'nuniq' ordering + if healpix_order == "nuniq": if methods: weights[(axis,)] = "HEALPix Multi-Order Coverage" return True - + healpix_index = f.coordinate( - "healpix_index", filter_by_axis=(axis,), exact=True - default=None + "healpix_index", + filter_by_axis=(axis,), + axis_mode="exact", + default=None, ) if healpix_index is None: - if auto: + if auto: return False - - raise ValueError( - "Can't create weights: TODOHEALPIX" - ) - area = self._healpix_nuniq_weights(healpix_index, - measure=measure, radius=radius) + raise ValueError("Can't create weights: TODOHEALPIX") + + if measure: + units = radius.Units**2 + else: + units = "1" + + from .dask_utils import cf_HEALPix_nuniq_weights + + dx = healpix_index.to_dask_array() + dx = dx.map_blocks( + cf_HEALPix_nuniq_weights, + meta=np.array((), dtype="float64"), + measure=measure, + radius=radius, + ) + area = f._Data(dx, units=units, copy=False) + + # area = cls._healpix_nuniq_weights( + # f, healpix_index, measure=measure, radius=radius + # ) if return_areas: return area - - weights[(axis,)] = area + + weights[(axis,)] = area weights_axes.add(axis) return True - # Weights for "nested" or "ring" ordering + # Still here? Then create weights for 'nested' or 'ring' + # ordering. if methods: + if measure: + weights[(axis,)] = "HEALPix equal area" + return True - + if not measure: + # Non-measure Weights are all equal, so no need to create any. return True - r2 = radius ** 2 - area = cf.Data.full( - (f.domain_axis(axis).get_size(),) - 4 * np.pi * float(r2) / (12 * (4 ** refinement_level)), - units=r2.Units + r2 = radius**2 + area = f._Data.full( + (f.domain_axis(axis).get_size(),), + np.pi * float(r2) / (3.0 * (4**refinement_level)), + units=r2.Units, ) if return_areas: return area - - weights[(axis,)] = area + + weights[(axis,)] = area weights_axes.add(axis) return True - def _healpix_nuniq_weights(self, - healpix_index, measure=False, radius=None): - """TODOHEALPIX - - .. versionadded:: NEXTVERSION - - :Parameters: - TODOHEALPIX - - :Returns: - - `Data` - TODOHEALPIX - - """ - from dask_utils import cf_HEALPix_nuniq_weights - - dx = healpix_index.to_dask_array() - dx = dx.map_blocks(cf_HEALPix_nuniq_weights, - meta=np.array((), dtype="float64"), - measure=measure, radius=radius) - - if measure: - units = radius.Units ** 2 - else: - units = Units("1") - - return Data(dx, units=units, copy=False) +# @classmethod +# def _healpix_nuniq_weights( +# self, f, healpix_index, measure=False, radius=None +# ): +# """TODOHEALPIX +# +# .. versionadded:: NEXTVERSION +# +# :Parameters: +# +# f: `Field` +# The field for which the weights are being created. +# +# healpix_index: `Coordinate` +# TODOHEALPIX +# +# {{weights measure: `bool`, optional}} +# +# {{radius: optional}} +# +# :Returns: +# +# `Data` +# TODOHEALPIX +# +# """ +# from .dask_utils import cf_HEALPix_nuniq_weights +# +# dx = healpix_index.to_dask_array() +# dx = dx.map_blocks( +# cf_HEALPix_nuniq_weights, +# meta=np.array((), dtype="float64"), +# measure=measure, +# radius=radius, +# ) +# +# if measure: +# units = radius.Units**2 +# else: +# units = "1" +# +# return f._Data(dx, units=units, copy=False) From e91d41d1c0cb2d04266b1f04c7774573df294e8f Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 18 Jun 2025 14:38:03 +0100 Subject: [PATCH 09/59] dev --- cf/dask_utils.py | 9 +- cf/data/data.py | 2 +- cf/field.py | 63 +++++++------ cf/mixin/fielddomain.py | 199 ++++++++++++++++++++++++++++------------ cf/regrid/regrid.py | 8 +- 5 files changed, 186 insertions(+), 95 deletions(-) diff --git a/cf/dask_utils.py b/cf/dask_utils.py index 0d34bf5549..30b655cd8b 100644 --- a/cf/dask_utils.py +++ b/cf/dask_utils.py @@ -135,21 +135,20 @@ def cf_HEALPix_change_order( An array containing the new HEALPix indices. """ - nside = healpix.order2nside(refinement_level) if healpix_order == "nested": if new_healpix_order == "ring": - return healpix.nest2ring(nside, a) + return healpix.nest2ring(healpix.order2nside(refinement_level), a) if new_healpix_order == "nuniq": - return healpix._chp.nest2uniq(nside, a) + return healpix._chp.nest2uniq(refinement_level, a) elif healpix_order == "ring": if new_healpix_order == "nested": - return healpix.ring2nest(nside, a) + return healpix.ring2nest(healpix.order2nside(refinement_level), a) if new_healpix_order == "nuniq": - return healpix._chp.ring2uniq(nside, a) + return healpix._chp.ring2uniq(refinement_level, a) else: raise ValueError( diff --git a/cf/data/data.py b/cf/data/data.py index 36d41d9396..b5f9f69701 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1368,7 +1368,7 @@ def coarsen( **Examples** - TODOHEALPIX + >>> TODOHEALPIX """ d = _inplace_enabled_define_and_cleanup(self) diff --git a/cf/field.py b/cf/field.py index bd2f924a70..ab9ff3ab8c 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4914,8 +4914,8 @@ def bin( @_inplace_enabled(default=False) def healpix_decrease_refinement_level( self, - level=None, - reduction=np.ma.mean, + level, + reduction=np.mean, check_healpix_index=True, inplace=False, ): @@ -4927,6 +4927,8 @@ def healpix_decrease_refinement_level( TODOHEALPIX + level: `int` or `None` + {{inplace: `bool`, optional}} :Returns: @@ -4941,9 +4943,16 @@ def healpix_decrease_refinement_level( """ f = _inplace_enabled_define_and_cleanup(self) + # Try to change the order to nested, as that's the only order + # from which we can decrease the refinement level. + try: + f.healpix_change_order("nested", inplace=True) + except ValueError: + pass + # Get the healpix_index oordinates and the key of the HEALPix # domain axis - key, healpix_index = f.coordinate( + hp_key, healpix_index = f.coordinate( "healpix_index", filter_by_naxes=(1,), item=True, @@ -4955,7 +4964,7 @@ def healpix_decrease_refinement_level( "coordinates have not been set" ) - axis = f.get_data_axes(key)[0] + axis = f.get_data_axes(hp_key)[0] # Parse the HEALPix coordinate reference cr = f.coordinate_reference("grid_mapping_name:healpix", default=None) @@ -4976,7 +4985,7 @@ def healpix_decrease_refinement_level( if healpix_order != "nested": raise ValueError( "Can't decrease HEALPix refinement level: Must have a " - "healpix_order of 'nested' for this operation. " + "healpix_order of 'nested' or 'ring' for this operation. " f"Got: {healpix_order!r}" ) @@ -4988,15 +4997,15 @@ def healpix_decrease_refinement_level( ) # Parse 'level' - if not level: - # Level None or 0: Keep the current refinement level + if level is None: + # No change in refinement level return f - if level > 0: + if level >= 0: if level > refinement_level: raise ValueError( "'level' keyword can't be larger than the current " - f"refinement level {refinement_level}. Got: {level!r}" + f"refinement level ({refinement_level}). Got: {level!r}" ) # Convert 'level' to a negative number @@ -5004,21 +5013,25 @@ def healpix_decrease_refinement_level( elif level < -refinement_level: raise ValueError( "'level' keyword can't be less than minus the current " - f"refinement level of {refinement_level}. Got: {level!r}" + f"refinement level ({-refinement_level}). Got: {level!r}" ) + new_refinement_level = refinement_level + level + if new_refinement_level == refinement_level: + # No change in refinement level + return f + # The number of cells at the original refinement level which are # contained in one cell at the coarser refinement level ncells = 4**-level - new_refinement_level = refinement_level + level - if check_healpix_index: d = healpix_index.data if not (d.diff() > 0).all(): raise ValueError( "Can't decrease HEALPix refinement level: healpix_index " - "cooridnates are not strictly monotonically increasing" + "coordinates with 'nested' ordering are not strictly " + "monotonically increasing" ) if (d[::ncells] % ncells).any() or ( @@ -5029,13 +5042,9 @@ def healpix_decrease_refinement_level( "cell at the coarser refinement level " f"({new_refinement_level}) contains fewer than {ncells} " "cells at the original refinement level " - f"{refinement_level}" + f"({refinement_level})" ) - # Create the healpix_index coordinates for the new refinement - # level - new_index = healpix_index[::ncells] // ncells - # Coarsen (using 'reduction') the field data. Note that using # the 'coarsen' technique only works for 'nested' HEALPix # ordering. @@ -5044,14 +5053,14 @@ def healpix_decrease_refinement_level( reduction, axes={i: ncells}, trim_excess=False, inplace=True ) - # Coarsen (using np.ma.mean) the domain ancillary constructs + # Coarsen (using np.mean) the domain ancillary constructs # that span the HEALPix axis for key, domain_ancillary in f.domain_ancillaries( filter_by_axis=(axis,), axis_mode="and", todict=True ).items(): i = f.get_data_axes(key).index(axis) domain_ancillary.data.coarsen( - np.ma.mean, axes={i: ncells}, trim_excess=False, inplace=True + np.mean, axes={i: ncells}, trim_excess=False, inplace=True ) # Coarsen (using np.sum) the cell measure constructs that span @@ -5081,16 +5090,19 @@ def healpix_decrease_refinement_level( # Set the healpix_index coordinates for the new refinement # level + new_index = healpix_index[::ncells] // ncells if new_index.construct_type == "dimension_coordinate": # Convert indices to auxiliary coordinates new_index = f._AuxiliaryCoordinate(source=new_index, copy=False) + hp_key = None - f.set_construct(new_index, axes=axis, copy=False) + new_key = f.set_construct(new_index, axes=axis, key=hp_key, copy=False) # Set the new refinement level - cr.coordinate_conversion.set_property( + cr.coordinate_conversion.set_parameter( "refinement_level", new_refinement_level ) + cr.set_coordinate(new_key) return f @@ -6664,9 +6676,9 @@ def collapse( f = self else: f = self.copy() - print(f) - f.create_latlon_coordinates(inplace=True) - print(f) + + f.create_latlon_coordinates(one_d=True, two_d=False, inplace=True) + # Whether or not to create null bounds for null # collapses. I.e. if the collapse axis has size 1 and no # bounds, whether or not to create upper and lower bounds to @@ -6760,7 +6772,6 @@ def collapse( for x in iterate_over: a = f.domain_axis(x, key=True, default=None) - print (a) if a is None: raise ValueError(msg.format(x)) diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 5e00b9d479..43a8e43163 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -298,7 +298,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): # Create any implied 1-d latitude and longitude coordinates # (e.g. as implied by HEALPix indices) - self = self.create_latlon_coordinates() + self = self.create_latlon_coordinates(one_d=True, two_d=False) domain_axes = self.domain_axes(todict=True) @@ -1877,6 +1877,20 @@ def coordinate_reference_domain_axes(self, identity=None): return set(axes) + def healpix_axis(self): + """TODOHEALPIX. + + .. versionadded:: NEXTVERSION + + """ + key = self.coordinate( + "healpix_index", filter_by_naxes=(1,), key=True, default=None + ) + if key is None: + return + + return self.get_data_axes(key)[0] + @_inplace_enabled(default=False) def healpix_change_order(self, new_healpix_order, inplace=False): """Change the ordering of HEALPix indices. @@ -1945,8 +1959,11 @@ def healpix_change_order(self, new_healpix_order, inplace=False): return f # Get the original HEALPix indices - healpix_index = f.coordinate( - "healpix_index", filter_by_naxes=(1,), default=None + hp_key, healpix_index = f.coordinate( + "healpix_index", + filter_by_naxes=(1,), + item=True, + default=(None, None), ) if healpix_index is None: raise ValueError( @@ -1964,6 +1981,16 @@ def healpix_change_order(self, new_healpix_order, inplace=False): ) healpix_index.set_data(dx, copy=False) + if healpix_index.construct_type == "dimension_coordinate": + # Convert indices to auxiliary coordinates + healpix_index = f._AuxiliaryCoordinate( + source=healpix_index, copy=False + ) + axis = f.get_data_axes(hp_key)[0] + f.del_construct(hp_key) + new_key = f.set_construct(healpix_index, axes=axis, copy=False) + cr.set_coordinate(new_key) + # Update the Coordinate Reference cr.coordinate_conversion.set_parameter( "healpix_order", new_healpix_order @@ -1978,33 +2005,56 @@ def healpix_to_ugrid(self, inplace=False): """TODOHEALPIX""" axis = self.healpix_axis() if axis is None: - raise ValueError("TODOHEALPIX") + raise ValueError( + "Can't convert HEALPix to UGRID: There is no HEALPix domain " + "axis" + ) f = _inplace_enabled_define_and_cleanup(self) # If lat/lon coordinates do not exist, then derive them from # the HEALPix indices. - f.create_latlon_coordinates(("HEALPix",), inplace=True) + f.create_latlon_coordinates(one_d=True, two_d=False, inplace=True) - x = f.auxiliary_coordinate( - "Y", filter_by_axis=(axis,), axis_mode="exact", default=None + x_key, x = f.auxiliary_coordinate( + "Y", + filter_by_axis=(axis,), + axis_mode="exact", + item=True, + default=(None, None), ) - y = f.auxiliary_coordinate( - "X", filter_by_axis=(axis,), axis_mode="exact", default=None + y_key, y = f.auxiliary_coordinate( + "X", + filter_by_axis=(axis,), + axis_mode="exact", + item=True, + default=(None, None), ) if x is None: - raise ValueError("TODOHEALPIX") + raise ValueError( + "Can't convert HEALPix to UGRID: Not able to find nor " + "create longitude coordinates" + ) if y is None: - raise ValueError("TODOHEALPIX") + raise ValueError( + "Can't convert HEALPix to UGRID: Not able to find nor " + "create latitude coordinates" + ) bounds_y = y.get_bounds(None) bounds_x = x.get_bounds(None) if bounds_y is None: - raise ValueError("TODOHEALPIX") + raise ValueError( + "Can't convert HEALPix to UGRID: No latitude coordinate " + "bounds" + ) if bounds_x is None: - raise ValueError("TODOHEALPIX") + raise ValueError( + "Can't convert HEALPix to UGRID: No longitude coordinate " + "bounds" + ) # Create a unique identifer for each node location bounds_y = bounds_y.to_dask_array() @@ -2023,27 +2073,6 @@ def healpix_to_ugrid(self, inplace=False): ) f.set_construct(domain_topology, axes=axis, copy=False) - # Upsdate the Coordinate Reference, either by deleting it, or - # converting it to 'latitude_longitude'. - key, cr = f.coordinate_reference( - "grid_mapping_name:healpix", item=True, default=(None, None) - ) - if key is not None: - cc = cr.coordinate_conversion - cc.del_parameter("grid_mapping_name", None) - cc.del_parameter("healpix_order", None) - cc.del_parameter("refinement_level", None) - if cr.coordinate_conversion.parameters(): - # The Coordinate Reference contains generic parameters - # (such as 'earth_radius'), so rename it. - cr.coordinate_conversion.set_parameter( - "grid_mapping_name", "latitude_longitude" - ) - else: - # The Coordinate Reference contains no generic - # parameters - f.del_construct(key) - # Remove the HEALPix index coordinates f.del_construct( "healpix_index", @@ -2056,24 +2085,38 @@ def healpix_to_ugrid(self, inplace=False): default=None, ) - return f - - def healpix_axis(self): - """TODOHEALPIX. - - .. versionadded:: NEXTVERSION - - """ - key = self.coordinate( - "healpix_index", filter_by_naxes=(1,), key=True, default=None + # Update the Coordinate Reference + cr_key, cr = f.coordinate_reference( + "grid_mapping_name:healpix", item=True, default=(None, None) ) - if key is None: - return + latlon = f.coordinate_reference( + "grid_mapping_name:latitude_longitude", default=None + ) + if latlon is not None: + latlon.set_coordinates((y_key, x_key)) + f.del_construct(cr_key) + elif cr is not None: + cc = cr.coordinate_conversion + cc.del_parameter("grid_mapping_name", None) + cc.del_parameter("healpix_order", None) + cc.del_parameter("refinement_level", None) + if cr.coordinate_conversion.parameters() or cr.datum.parameters(): + # The Coordinate Reference contains generic coordinate + # conversion or datum parameters, so rename it as + # 'latitude_longitude'. + cr.coordinate_conversion.set_parameter( + "grid_mapping_name", "latitude_longitude" + ) + cr.set_coordinates((y_key, x_key)) + else: + # The Coordinate Reference contains no generic + # parameters, so delete it. + f.del_construct(cr_key) - return self.get_data_axes(key)[0] + return f @_inplace_enabled(default=False) - def create_latlon_coordinates(self, grid_type=None, inplace=False): + def create_latlon_coordinates(self, one_d=True, two_d=True, inplace=False): """TODOHEALPIX. .. versionadded:: NEXTVERSION @@ -2092,17 +2135,39 @@ def create_latlon_coordinates(self, grid_type=None, inplace=False): """ f = _inplace_enabled_define_and_cleanup(self) - # ------------------------------------------------------------ - # HEALPix - # ------------------------------------------------------------ - if grid_type is None or "HEALPix" in grid_type: - cr = f.coordinate_reference( - "grid_mapping_name:healpix", default=None - ) - if cr is None: - # No healpix grid mapping - return f + # Get all Coordinate References + identities = { + cr.identity(""): cr + for cr in f.coordinate_references(todict=True).values() + } + if not identities: + return f + + # Keep only grid mappings + identities = { + identity: cr + for identity, cr in identities.items() + if identity.startswith("grid_mapping_name:") + } + if len(identities) > 2: + return f + + # Remove a 'latitude_longitude' grid mapping + latlon_cr = identities.pop( + "grid_mapping_name:latitude_longitude", None + ) + if not identities: + return f + # Still here? Then get the final grid mapping and calulate the + # lat/lon coordinates. + identity, cr = identities.popitem() + + new_coords = False + if one_d and identity == "grid_mapping_name:healpix": + # -------------------------------------------------------- + # HEALPix + # -------------------------------------------------------- parameters = cr.coordinate_conversion.parameters() healpix_order = parameters.get("healpix_order") if healpix_order not in ("nested", "ring", "nuniq"): @@ -2209,9 +2274,23 @@ def create_latlon_coordinates(self, grid_type=None, inplace=False): # Set the new latitude and longitude coordinates lat_key = f.set_construct(lat, axes=axis, copy=False) lon_key = f.set_construct(lon, axes=axis, copy=False) - cr.set_coordinates((lat_key, lon_key)) - return f + new_coords = True + + elif two_d: + # -------------------------------------------------------- + # Plane projection or rotated pole + # -------------------------------------------------------- + pass + + # ------------------------------------------------------------ + # Update the approriate Coordinate Reference + # ------------------------------------------------------------ + if new_coords: + if latlon_cr is not None: + latlon_cr.set_coordinates((lat_key, lon_key)) + else: + cr.set_coordinates((lat_key, lon_key)) return f diff --git a/cf/regrid/regrid.py b/cf/regrid/regrid.py index 2e2686f33d..28a4371d6a 100644 --- a/cf/regrid/regrid.py +++ b/cf/regrid/regrid.py @@ -495,6 +495,8 @@ def regrid( ln_z=ln_z, ) + + if is_log_level_debug(logger): logger.debug( f"\n{pformat(src_grid)}\n\n{pformat(dst_grid)}\n" @@ -1134,11 +1136,11 @@ def spherical_grid( """ try: # Try to convert a HEALPix grid to UGRID - f = f.healpix_to_ugrid() + f.healpix_to_ugrid(inplace=True) except ValueError: - pass + # Create any implied lat/lon coordinates + f.create_latlon_coordinates(inplace=True) - print(f) data_axes = f.constructs.data_axes() dim_coords_1d = False From b47db31348e329dafb2118d81a82154c59fe1afd Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 18 Jun 2025 22:37:01 +0100 Subject: [PATCH 10/59] dev --- cf/dask_utils.py | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/cf/dask_utils.py b/cf/dask_utils.py index 30b655cd8b..1c77a74331 100644 --- a/cf/dask_utils.py +++ b/cf/dask_utils.py @@ -14,6 +14,8 @@ def cf_HEALPix_bounds( ): """Calculate HEALPix cell bounds. + Each cell has four vertices. + .. versionadded:: NEXTVERSION :Parameters: @@ -45,8 +47,8 @@ def cf_HEALPix_bounds( # Keep an eye on https://github.com/ntessore/healpix/issues/66 if a.ndim != 1: raise ValueError( - "Can't calculate HEALPix cell bounds when " - f"healpix_index array has shape {a.shape}" + "Can only calculate HEALPix cell bounds when " + f"healpix_index array has one dimension. Got shape {a.shape}" ) if lat: @@ -59,7 +61,7 @@ def cf_HEALPix_bounds( else: bounds_func = healpix._chp.ring2ang_uv - # Define the cell vertices in an anticlockwise direction, as seen + # Define the cell vertices, in an anticlockwise direction as seen # from above. right = (1, 0) top = (1, 1) @@ -71,12 +73,13 @@ def cf_HEALPix_bounds( if healpix_order == "nuniq": # nuniq - nsides, a = healpix.uniq2pix(a, nest=False) - nsides, index, inverse = np.unique( - nsides, return_index=True, return_inverse=True + orders, a = healpix.uniq2pix(a, nest=False) + orders, index, inverse = np.unique( + orders, return_index=True, return_inverse=True ) - for nside, i in zip(nsides, index): + for order, i in zip(orders, index): level = np.where(inverse == inverse[i])[0] + nside=healpix.order2nside(order) for j, (u, v) in enumerate(vertices): thetaphi = bounds_func(nside, a[level], u, v) b[level, j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] @@ -88,7 +91,7 @@ def cf_HEALPix_bounds( b[..., j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] if not pos: - # Longitude bounds + # Make longitudee bounds in the range [0, 360) b[np.where(b >= 360)] -= 360.0 # Bounds on the north or south pole come out with a longitude @@ -162,6 +165,8 @@ def cf_HEALPix_coordinates( ): """Calculate HEALPix cell coordinates. + THe coordinates are in the centres of the cells. + .. versionadded:: NEXTVERSION :Parameters: @@ -193,7 +198,7 @@ def cf_HEALPix_coordinates( if a.ndim != 1: raise ValueError( "Can't calculate HEALPix cell coordinates when " - f"healpix_index array has shape {a.shape}" + f"healpix_index array has one dimension. Got shape {a.shape}" ) if lat: @@ -205,19 +210,21 @@ def cf_HEALPix_coordinates( c = np.empty(a.shape, dtype="float64") nest = False - nsides, a = healpix.uniq2pix(a, nest=nest) - nsides, index, inverse = np.unique( - nsides, return_index=True, return_inverse=True + orders, a = healpix.uniq2pix(a, nest=nest) + orders, index, inverse = np.unique( + orders, return_index=True, return_inverse=True ) - for nside, i in zip(nsides, index): + for order, i in zip(orders, index): level = np.where(inverse == inverse[i])[0] + nside=healpix.order2nside(order) c[level] = healpix.pix2ang( nside=nside, ipix=a[level], nest=nest, lonlat=True )[pos] else: # nested or ring + nside=healpix.order2nside(refinement_level) c = healpix.pix2ang( - nside=healpix.order2nside(refinement_level), + nside=nside, ipix=a, nest=healpix_order == "nested", lonlat=True, @@ -255,14 +262,15 @@ def cf_HEALPix_nuniq_weights(a, measure=False, radius=None): else: x = 1.0 - nsides = healpix.uniq2pix(a)[0] - nsides, index, inverse = np.unique( - nsides, return_index=True, return_inverse=True + orders = healpix.uniq2pix(a)[0] + orders, index, inverse = np.unique( + orders, return_index=True, return_inverse=True ) w = np.empty(a.shape, dtype="float64") - for nside, i in zip(nsides, index): + for order, i in zip(orders, index): + nside = healpix.order2nside(order) w = np.where(inverse == inverse[i], x / (nside**2), w) return w From 33243456a48489c5abc76330e7d721b4c5d7b47d Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 19 Jun 2025 17:09:37 +0100 Subject: [PATCH 11/59] dev --- cf/dask_utils.py | 10 +++++----- cf/regrid/regrid.py | 2 -- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/cf/dask_utils.py b/cf/dask_utils.py index 1c77a74331..63ab77c283 100644 --- a/cf/dask_utils.py +++ b/cf/dask_utils.py @@ -15,7 +15,7 @@ def cf_HEALPix_bounds( """Calculate HEALPix cell bounds. Each cell has four vertices. - + .. versionadded:: NEXTVERSION :Parameters: @@ -79,7 +79,7 @@ def cf_HEALPix_bounds( ) for order, i in zip(orders, index): level = np.where(inverse == inverse[i])[0] - nside=healpix.order2nside(order) + nside = healpix.order2nside(order) for j, (u, v) in enumerate(vertices): thetaphi = bounds_func(nside, a[level], u, v) b[level, j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] @@ -166,7 +166,7 @@ def cf_HEALPix_coordinates( """Calculate HEALPix cell coordinates. THe coordinates are in the centres of the cells. - + .. versionadded:: NEXTVERSION :Parameters: @@ -216,13 +216,13 @@ def cf_HEALPix_coordinates( ) for order, i in zip(orders, index): level = np.where(inverse == inverse[i])[0] - nside=healpix.order2nside(order) + nside = healpix.order2nside(order) c[level] = healpix.pix2ang( nside=nside, ipix=a[level], nest=nest, lonlat=True )[pos] else: # nested or ring - nside=healpix.order2nside(refinement_level) + nside = healpix.order2nside(refinement_level) c = healpix.pix2ang( nside=nside, ipix=a, diff --git a/cf/regrid/regrid.py b/cf/regrid/regrid.py index 28a4371d6a..c75817a374 100644 --- a/cf/regrid/regrid.py +++ b/cf/regrid/regrid.py @@ -495,8 +495,6 @@ def regrid( ln_z=ln_z, ) - - if is_log_level_debug(logger): logger.debug( f"\n{pformat(src_grid)}\n\n{pformat(dst_grid)}\n" From 27950a66a4905042fed9ce38ad84b587ddd74ce1 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 30 Jun 2025 18:20:22 +0100 Subject: [PATCH 12/59] dev --- Changelog.rst | 19 ++ README.md | 8 +- cf/dask_utils.py | 108 ++++++++-- cf/data/data.py | 65 +++++- cf/field.py | 165 +++++++++----- cf/mixin/fielddomain.py | 408 ++++++++++++++++++++++++----------- cf/regrid/regrid.py | 5 +- cf/test/test_Data.py | 23 ++ cf/test/test_Field.py | 12 +- cf/weights.py | 59 +---- docs/source/installation.rst | 5 + setup.py | 8 +- 12 files changed, 620 insertions(+), 265 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index a1589f3cfb..55f1e3a90a 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -1,3 +1,22 @@ +Version NEXTVERSION +-------------- + +**2025-??-??** + +* New method: `cf.Field.create_latlon_coordinates` + (https://github.com/NCAS-CMS/cf-python/issues/???) +* New HEALPix methods: `cf.Field.healpix_axis`, + `cf.Field.healpix_change_order`, + `cf.Field.healpix_decrease_refinement_level`, + `cf.Field.healpix_to_ugrid` + (https://github.com/NCAS-CMS/cf-python/issues/???) +* New method: `cf.Data.coarsen` + (https://github.com/NCAS-CMS/cf-python/issues/???) +* New optional dependency: ``healpix>=2024.2`` +* Changed dependency: ``cfdm>=1.13.0.0, <1.13.1.0`` + +---- + Version 3.18.0 -------------- diff --git a/README.md b/README.md index ce752fd0af..c851c0ca04 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,7 @@ of its array manipulation and can: * create new field constructs in memory, * write and append field and domain constructs to netCDF datasets on disk, * read, create, and manipulate UGRID mesh topologies, +* read, write, and manipulate HEALPix grids, * read, write, and create coordinates defined by geometry cells, * read netCDF and CDL datasets containing hierarchical groups, * inspect field constructs, @@ -105,11 +106,12 @@ of its array manipulation and can: * manipulate field construct data by arithmetical and trigonometrical operations, * perform weighted statistical collapses on field constructs, - including those with geometry cells and UGRID mesh topologies, + including those with geometry cells, UGRID mesh topologies, and + HEALPix grids, * perform histogram, percentile and binning operations on field constructs, -* regrid structured grid, mesh and DSG field constructs with - (multi-)linear, nearest neighbour, first- and second-order +* regrid structured grid, UGRID, HEALPix, and DSG field constructs + with (multi-)linear, nearest neighbour, first- and second-order conservative and higher order patch recovery methods, including 3-d regridding, * apply convolution filters to field constructs, diff --git a/cf/dask_utils.py b/cf/dask_utils.py index 63ab77c283..f11ddd78dd 100644 --- a/cf/dask_utils.py +++ b/cf/dask_utils.py @@ -5,7 +5,6 @@ """ -import healpix import numpy as np @@ -14,7 +13,10 @@ def cf_HEALPix_bounds( ): """Calculate HEALPix cell bounds. - Each cell has four vertices. + Latitude or longitude locations of the cell vertices are derived + from HEALPix indices. Each cell has four bounds, which are + returned in an anticlockwise direction, as seen from above, + starting with the eastern-most vertex. .. versionadded:: NEXTVERSION @@ -41,13 +43,28 @@ def cf_HEALPix_bounds( :Returns: `numpy.ndarray` - An array containing the HEALPix cell bounds. + A 2-d array containing the HEALPix cell bounds. + + **Examples** + + >>> cf_HEALPix_bounds([0, 1, 2, 3], 'nested', 1, lat=True) + array([[19.47122063, 41.8103149 , 19.47122063, 0. ], + [41.8103149 , 66.44353569, 41.8103149 , 19.47122063], + [41.8103149 , 66.44353569, 41.8103149 , 19.47122063], + [66.44353569, 90. , 66.44353569, 41.8103149 ]]) + >>> cf_HEALPix_bounds([0, 1, 2, 3], 'nested', 1, lon=True) + array([[67.5, 45. , 22.5, 45. ], + [90. , 90. , 45. , 67.5], + [45. , 0. , 0. , 22.5], + [90. , 45. , 0. , 45. ]]) """ + import healpix + # Keep an eye on https://github.com/ntessore/healpix/issues/66 if a.ndim != 1: raise ValueError( - "Can only calculate HEALPix cell bounds when " + "Can only calculate HEALPix cell bounds when the " f"healpix_index array has one dimension. Got shape {a.shape}" ) @@ -61,18 +78,19 @@ def cf_HEALPix_bounds( else: bounds_func = healpix._chp.ring2ang_uv - # Define the cell vertices, in an anticlockwise direction as seen - # from above. + # Define the cell vertices, in an anticlockwise direction, as seen + # from above, starting with the eastern-most vertex. right = (1, 0) top = (1, 1) left = (0, 1) bottom = (0, 0) vertices = (right, top, left, bottom) + # Initialise the output bounds array b = np.empty((a.size, 4), dtype="float64") if healpix_order == "nuniq": - # nuniq + # Create bounds for 'nuniq' cells orders, a = healpix.uniq2pix(a, nest=False) orders, index, inverse = np.unique( orders, return_index=True, return_inverse=True @@ -84,22 +102,28 @@ def cf_HEALPix_bounds( thetaphi = bounds_func(nside, a[level], u, v) b[level, j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] else: - # nested or ring + # Create bounds for 'nested' or 'ring' cells nside = healpix.order2nside(refinement_level) for j, (u, v) in enumerate(vertices): thetaphi = bounds_func(nside, a, u, v) b[..., j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] if not pos: - # Make longitudee bounds in the range [0, 360) - b[np.where(b >= 360)] -= 360.0 - - # Bounds on the north or south pole come out with a longitude - # of NaN, so replace these with a sensible value. + # Ensure that longitude bounds are less than 360 + where_ge_360 = np.where(b >= 360) + if where_ge_360[0].size: + b[where_ge_360] -= 360.0 + + # Bounds on the north (south) pole come out with a longitude + # of NaN, so replace these with a sensible value, i.e. the + # longitude of the southern (northern) vertex. + # + # North pole i = np.argwhere(np.isnan(b[:, 1])).flatten() if i.size: b[i, 1] = b[i, 3] + # South pole i = np.argwhere(np.isnan(b[:, 3])).flatten() if i.size: b[i, 3] = b[i, 1] @@ -112,6 +136,9 @@ def cf_HEALPix_change_order( ): """Change the ordering of HEALPix indices. + Does not change the position of each cell in the array, but + redefines their indices according to the new ordering scheme. + .. versionadded:: NEXTVERSION :Parameters: @@ -137,7 +164,15 @@ def cf_HEALPix_change_order( `numpy.ndarray` An array containing the new HEALPix indices. + **Examples** + + >>> cf_HEALPix_change_order([0, 1, 2, 3], 'nested', 'ring', 1) + array([13, 5, 4, 0]) + >>> cf_HEALPix_change_order([0, 1, 2, 3], 'nuniq', 'ring', 1) + array([16, 17, 18, 19]) + """ + import healpix if healpix_order == "nested": if new_healpix_order == "ring": @@ -165,7 +200,7 @@ def cf_HEALPix_coordinates( ): """Calculate HEALPix cell coordinates. - THe coordinates are in the centres of the cells. + THe coordinates are for the cell centres. .. versionadded:: NEXTVERSION @@ -192,12 +227,21 @@ def cf_HEALPix_coordinates( :Returns: `numpy.ndarray` - An array containing the HEALPix cell coordinates. + A 1-d array containing the HEALPix cell coordinates. + + **Examples** + + >>> cf_HEALPix_coordinates([0, 1, 2, 3], 'nested', 1, lat=True) + array([19.47122063, 41.8103149 , 41.8103149 , 66.44353569]) + >>> cf_HEALPix_coordinates([0, 1, 2, 3], 'nested', 1, lon=True) + array([45. , 67.5, 22.5, 45. ]) """ + import healpix + if a.ndim != 1: raise ValueError( - "Can't calculate HEALPix cell coordinates when " + "Can't calculate HEALPix cell coordinates when the " f"healpix_index array has one dimension. Got shape {a.shape}" ) @@ -207,6 +251,7 @@ def cf_HEALPix_coordinates( pos = 0 if healpix_order == "nuniq": + # Create coordinates for 'nuniq' cells c = np.empty(a.shape, dtype="float64") nest = False @@ -221,27 +266,36 @@ def cf_HEALPix_coordinates( nside=nside, ipix=a[level], nest=nest, lonlat=True )[pos] else: - # nested or ring + # Create coordinates for 'nested' or 'ring' cells + nest = (healpix_order == "nested",) nside = healpix.order2nside(refinement_level) c = healpix.pix2ang( nside=nside, ipix=a, - nest=healpix_order == "nested", + nest=nest, lonlat=True, )[pos] return c -def cf_HEALPix_nuniq_weights(a, measure=False, radius=None): - """Calculate HEALPix cell weights for 'nuniq' indices. +def cf_HEALPix_nuniq_area_weights(a, measure=False, radius=None): + """Calculate HEALPix cell area weights for 'nuniq' indices. + + For mathematical details, see section 4 of: + + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, + et al.. HEALPix: A Framework for High‐Resolution + Discretization and Fast Analysis of Data Distributed on the + Sphere. The Astrophysical Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 .. versionadded:: NEXTVERSION :Parameters: a: `numpy.ndarray` - The array of HEALPix indices. + The array of HEALPix 'nuniq' indices. measure: `bool`, optional If True then create weights that are actual cell areas, in @@ -256,7 +310,18 @@ def cf_HEALPix_nuniq_weights(a, measure=False, radius=None): `numpy.ndarray` An array containing the HEALPix cell weights. + **Examples** + + >>> cf_HEALPix_nuniq_weights([76, 77, 78, 79, 20, 21]) + array([0.0625, 0.0625, 0.0625, 0.0625, 0.25 , 0.25 ]) + >>> cf_HEALPix_nuniq_weights([76, 77, 78, 79, 20, 21], + ... measure=True, radius=6371000) + array([2.65658579e+12, 2.65658579e+12, 2.65658579e+12, 2.65658579e+12, + 1.06263432e+13, 1.06263432e+13]) + """ + import healpix + if measure: x = np.pi * (radius**2) / 3.0 else: @@ -267,6 +332,7 @@ def cf_HEALPix_nuniq_weights(a, measure=False, radius=None): orders, return_index=True, return_inverse=True ) + # Initialise the output weights array w = np.empty(a.shape, dtype="float64") for order, i in zip(orders, index): diff --git a/cf/data/data.py b/cf/data/data.py index b5f9f69701..3f370cf7ff 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1350,28 +1350,83 @@ def coarsen( trim_excess=False, inplace=False, ): - """TODOHEALPIX + """Coarsen the data. + + Coarsen the data by applying the *reduction* function to + combine the elements within fixed-size neighborhoods. .. versionadded:: NEXTVERSION :Parameters: - TODOHEALPIX + reduction: function + The function with which to coarsen the data. + + axes: `dict` + Define how to coarsening neighbourhood for each + axis. A dictionary key is an integer axis position, + with correponding value giving the non-negative + integer size of the coarsening neighbourhood for that + axis. Unspecified axes are not coarsened, which is + equivalent to providing a coarsening neighbourhood of + ``1``. + + *Example:* + Coarsen the axis in position 1 by combining every 4 + elements: ``{1: 4}`` + + *Example:* + Coarsen the axis in position 0 by combining every 3 + elements, and the last axis by combining every 4 + elements: ``{0: 3, -1: 4}`` + + trim_excess: `bool`, optional + If True then do not return a partially-full + neighbourhood at the end of a coarsened axis. If False + (the default) then an exception is raised if there are + any partially-filled neighbourhoods. {{inplace: `bool`, optional}} :Returns: `Data` or `None` - TODOHEALPIX of the data. If the operation was in-place - then `None` is returned. + The coarsened data, or `None` if the operation was + in-place. **Examples** - >>> TODOHEALPIX + >>> import numpy as np + >>> d = cf.Data(np.arange(24).reshape((4, 6))) + >>> print(d.array) + [[ 0 1 2 3 4 5] + [ 6 7 8 9 10 11] + [12 13 14 15 16 17] + [18 19 20 21 22 23]] + >>> e = d.coarsen(np.min, {0: 2, 1: 3}) + >>> print(e.array) + [[ 0 3] + [12 15]] + >>> e = d.coarsen(np.max, {-1: 5}, trim_excess=True) + >>> print(e.array) + [[ 4] + [10] + [16] + [22]] + >>> e = d.coarsen(np.max, {-1: 5}) + ValueError: Coarsening factors {1: 5} do not align with array shape (4, 6). """ d = _inplace_enabled_define_and_cleanup(self) + + # Parse axes + ndim = self.ndim + for k in axes: + if k < -ndim or k > ndim: + raise ValueError("axis {k} is out of bounds for {ndim}-d data") + + axes = {(k + ndim if k < 0 else k): v for k, v in axes.items()} + dx = d.to_dask_array() dx = da.coarsen(reduction, dx, axes, trim_excess=trim_excess) d._set_dask(dx) diff --git a/cf/field.py b/cf/field.py index ab9ff3ab8c..4aaf106f5c 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4911,48 +4911,115 @@ def bin( return out - @_inplace_enabled(default=False) def healpix_decrease_refinement_level( - self, - level, - reduction=np.mean, - check_healpix_index=True, - inplace=False, + self, level, reduction, sort=True, check_healpix_index=True ): - """TODOHEALPIX - + """Decrease the refinement level of a HEALPix grid. + .. versionadded:: NEXTVERSION - :Parameters: + .. seealso:: `healpix_change_order` - TODOHEALPIX + :Parameters: level: `int` or `None` + Specify the new refinement level. If a non-negative + integer then this explicitly defines the new + refinement level. If a negative integer then the new + refinement level is defined by the current refinement + level plus *level*. If `None` then the refinement + level is not changed. - {{inplace: `bool`, optional}} + *Example:* + If the current refinement level is 10 then a new + coarser refinment level of 8 can be specified by + either ``8`` or ``-2``. + + reduction: function + The function used to calculate the values in the new + coarser cells from the original data defined on the + original finer cells. + + *Example:* + For an intensive field quantity (that does not + depend on the size of the cells, such as + "precipitation_flux" with units of kg m-2 s-1), + ``np.mean`` may be appropriate. For an extensive + field quantity (that depends on the size of the + cells, such as "precipitation_amount" with units of + kg m-2), ``np.sum`` may be appropriate. + + sort: `bool`, optional + As a requirement of the coarsening algorithm, the + HEALPix indices are always automatically converted to + 'nested' ordering, and if *sort* is True (the default) + then the HEALPix axis will also be sorted so that + these nested HEALPix indices are monotonically + increasing. Only set to False, which will speed up the + operation, if it is known that the HEALPix indices + expressed in their nested ordering are already + monotonically increasing. + + check_healpix_index: `bool`, optional + As a requirement of the coarsening algorithm, the + HEALPix indices are always automatically converted to + 'nested' ordering, and if *sort* is True then the + HEALPix axis will also be sorted so that these nested + HEALPix indices are monotonically increasing. If + *check_healpix_indices* is True (the default) then it + will be checked that 1) the nested HEALPix indices are + strictly monotonically increasing, and 2) a cell at + the new coarser refinement level contains the maximum + possible number of cells at the original finer + refinement level. + + .. warning:: Only set to False, which will speed up + the operation, if it is known these + conditions are met. If set to False and + any of the conditions is not met then + either an exception will be raised or, + much worse, the operation will complete + and return incorrect results. :Returns: - `{{class}}` or `None` - TODOHEALPIX + `Field` + A new field with a coarsened HEALPix grid. **Examples** - >>> TODOHEALPIX + >>> f = cf.example_field(12) + >>> f + - """ - f = _inplace_enabled_define_and_cleanup(self) + Set the refinement level to 0: + + >>> g = f.healpix_decrease_refinement_level(0, np.mean)g + >>> g + + + Decrease the refinement level by 1: + >>> f.healpix_decrease_refinement_level(-1, np.mean) + + >>> f.array[0, :4].mean() + np.float64(289.15) + >>> g.array[0, 0] + np.float64(289.15) + + """ # Try to change the order to nested, as that's the only order - # from which we can decrease the refinement level. + # from which we can change the refinement level. try: - f.healpix_change_order("nested", inplace=True) - except ValueError: - pass + f = self.healpix_change_order("nested", sort=sort) + except ValueError as error: + raise ValueError( + f"Can't decrease HEALPix refinement level: {error}" + ) - # Get the healpix_index oordinates and the key of the HEALPix + # Get the healpix_index coordinates and the key of the HEALPix # domain axis - hp_key, healpix_index = f.coordinate( + hp_key, healpix_index = f.auxiliary_coordinate( "healpix_index", filter_by_naxes=(1,), item=True, @@ -4960,8 +5027,8 @@ def healpix_decrease_refinement_level( ) if healpix_index is None: raise ValueError( - "Can't decrease HEALPix refinement level: healpix_index " - "coordinates have not been set" + "Can't decrease HEALPix refinement level: There are no " + "healpix_index coordinates" ) axis = f.get_data_axes(hp_key)[0] @@ -4975,20 +5042,6 @@ def healpix_decrease_refinement_level( ) parameters = cr.coordinate_conversion.parameters() - healpix_order = parameters.get("healpix_order") - if healpix_order is None: - raise ValueError( - "Can't decrease HEALPix refinement level: healpix_order has " - "not been set" - ) - - if healpix_order != "nested": - raise ValueError( - "Can't decrease HEALPix refinement level: Must have a " - "healpix_order of 'nested' or 'ring' for this operation. " - f"Got: {healpix_order!r}" - ) - refinement_level = parameters.get("refinement_level") if refinement_level is None: raise ValueError( @@ -5027,11 +5080,11 @@ def healpix_decrease_refinement_level( if check_healpix_index: d = healpix_index.data - if not (d.diff() > 0).all(): + if not sort and not (d.diff() > 0).all(): raise ValueError( - "Can't decrease HEALPix refinement level: healpix_index " - "coordinates with 'nested' ordering are not strictly " - "monotonically increasing" + "Can't decrease HEALPix refinement level: Nested " + "healpix_index coordinates are not strictly " + "monotonically increasing. Consider setting sort=True." ) if (d[::ncells] % ncells).any() or ( @@ -5039,22 +5092,24 @@ def healpix_decrease_refinement_level( ).any(): raise ValueError( "Can't decrease HEALPix refinement level: At least one " - "cell at the coarser refinement level " + "cell at the new coarser refinement level " f"({new_refinement_level}) contains fewer than {ncells} " - "cells at the original refinement level " + "cells at the original finer refinement level " f"({refinement_level})" ) - # Coarsen (using 'reduction') the field data. Note that using - # the 'coarsen' technique only works for 'nested' HEALPix - # ordering. - i = f.get_data_axes().index(axis) + # Coarsen (using the 'reduction' function) the field + # data. Note that using the 'coarsen' technique only works for + # 'nested' HEALPix ordering. + hp_iaxis = f.get_data_axes().index(axis) f.data.coarsen( - reduction, axes={i: ncells}, trim_excess=False, inplace=True + reduction, axes={hp_iaxis: ncells}, trim_excess=False, inplace=True ) - # Coarsen (using np.mean) the domain ancillary constructs - # that span the HEALPix axis + # Coarsen the domain ancillary constructs that span the + # HEALPix axis. We're assuming that domain ancillary data are + # intensive (i.e. do not depend on the size of the cell), so + # we use np.mean for the reduction function. for key, domain_ancillary in f.domain_ancillaries( filter_by_axis=(axis,), axis_mode="and", todict=True ).items(): @@ -5063,8 +5118,10 @@ def healpix_decrease_refinement_level( np.mean, axes={i: ncells}, trim_excess=False, inplace=True ) - # Coarsen (using np.sum) the cell measure constructs that span - # the HEALPix axis + # Coarsen the cell measure constructs that span the HEALPix + # axis. Cell measure data are extensive (i.e. depend on the + # size of the cell), so we use np.sum for the reduction + # function. for key, cell_measure in f.cell_measures( filter_by_axis=(axis,), axis_mode="and", todict=True ).items(): @@ -5086,7 +5143,7 @@ def healpix_decrease_refinement_level( # Re-size the HEALPix axis domain_axis = f.domain_axis(axis) - domain_axis.set_size(f.shape[i]) + domain_axis.set_size(f.shape[hp_iaxis]) # Set the healpix_index coordinates for the new refinement # level diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 43a8e43163..df9a8a5b1f 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -297,7 +297,8 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): ) # Create any implied 1-d latitude and longitude coordinates - # (e.g. as implied by HEALPix indices) + # (e.g. as implied by HEALPix indices). Do not do this + # in-place. self = self.create_latlon_coordinates(one_d=True, two_d=False) domain_axes = self.domain_axes(todict=True) @@ -1877,47 +1878,126 @@ def coordinate_reference_domain_axes(self, identity=None): return set(axes) - def healpix_axis(self): - """TODOHEALPIX. + def healpix_axis(self, default=ValueError()): + """Return the HEALPix axis identifier. .. versionadded:: NEXTVERSION + .. seealso:: `healpix_change_order`, `healpix_to_ugrid` + + :Parameters: + + default: optional + Return the value of the *default* parameter if an + HEALPix axis can not be found. + + {{default Exception}} + + :Returns: + + `str` + The identifier of the HEALPix domain axis construct. + + **Examples** + + >>> f = cf.example_field(12) + >>> print(f) + Field: air_temperature (ncvar%tas) + ---------------------------------- + Data : air_temperature(time(2), healpix_index(48)) K + Cell methods : time(2): mean area: mean + Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian + : height(1) = [1.5] m + Auxiliary coords: healpix_index(healpix_index(48)) = [0, ..., 47] 1 + Coord references: grid_mapping_name:healpix + >>> axis = f.healpix_axis() + >>> axis + 'domainaxis1' + >>> f.constructs.domain_axis_identity(axis) + 'healpix_index' + """ key = self.coordinate( "healpix_index", filter_by_naxes=(1,), key=True, default=None ) if key is None: - return + if default is None: + return + + return self._default(default, "There is no HEALPix axis") return self.get_data_axes(key)[0] - @_inplace_enabled(default=False) - def healpix_change_order(self, new_healpix_order, inplace=False): + def healpix_change_order(self, new_healpix_order, sort=False): """Change the ordering of HEALPix indices. .. versionadded:: NEXTVERSION + .. seealso:: `healpix_axis` + :Parameters: new_healpix_order: `str` - The new HEALPix order to change to. One of - ``'nested'``, ``'ring'``, or ``'nuniq'``. + The new HEALPix order. One of ``'nested'``, + ``'ring'``, or ``'nuniq'``. - {{inplace: `bool`, optional}} + sort: `bool`, optional + If True then sort the HEALPix axis of the output + {{class}} so that the HEALPix indices are + monotonically increasing. If False (the default) then + don't do this. :Returns: - `{{class}}` or `None` - TODOHEALPIX An array containing the new HEALPix indices. + `{{class}}` + The {{class}} with the HEALPix indices redefined for + the new ordering. **Examples** - TODOHEALPIX + >>> f = cf.example_field(12) + >>> print(f) + Field: air_temperature (ncvar%tas) + ---------------------------------- + Data : air_temperature(time(2), healpix_index(48)) K + Cell methods : time(2): mean area: mean + Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian + : height(1) = [1.5] m + Auxiliary coords: healpix_index(healpix_index(48)) = [0, ..., 47] 1 + Coord references: grid_mapping_name:healpix + >>> f.coordinate_reference('grid_mapping_name:healpix').coordinate_conversion.get_parameter('healpix_order') + 'nested' + >>> print(f.coordinate('healpix_index').array) + [ 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47] + + >>> g = f.healpix_change_order('nuniq') + >>> print(g.coordinate('healpix_index').array) + [16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63] + + >>> g = f.healpix_change_order('ring') + >>> print(g.coordinate('healpix_index').array) + [13 5 4 0 15 7 6 1 17 9 8 2 19 11 10 3 28 20 27 12 30 22 21 14 + 32 24 23 16 34 26 25 18 44 37 36 29 45 39 38 31 46 41 40 33 47 43 42 35] + + >>> g = f.healpix_change_order('ring', sort=True) + >>> print(g.coordinate('healpix_index').array) + [ 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47] + >>> h = g.healpix_change_order('nested') + >>> print(h.coordinate('healpix_index').array) + [ 3 7 11 15 2 1 6 5 10 9 14 13 19 0 23 4 27 8 31 12 17 22 21 26 + 25 30 29 18 16 35 20 39 24 43 28 47 34 33 38 37 42 41 46 45 32 36 40 44] + >>> h = g.healpix_change_order('nested', sort=True) + >>> print(h.coordinate('healpix_index').array) + [ 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47] """ from ..dask_utils import cf_HEALPix_change_order - f = _inplace_enabled_define_and_cleanup(self) + f = self.copy() if new_healpix_order not in ("nested", "ring", "nuniq"): raise ValueError( @@ -1929,8 +2009,8 @@ def healpix_change_order(self, new_healpix_order, inplace=False): cr = f.coordinate_reference("grid_mapping_name:healpix", default=None) if cr is None: raise ValueError( - "Can't change HEALPix order: HEALPix grid mapping has " - "not been set" + "Can't change HEALPix order: There is no HEALPix grid " + "mapping coordinate reference" ) parameters = cr.coordinate_conversion.parameters() @@ -1954,11 +2034,8 @@ def healpix_change_order(self, new_healpix_order, inplace=False): f"healpix_order 'nested' or 'ring'. Got {healpix_order!r}" ) - if healpix_order == new_healpix_order: - # No change - return f - - # Get the original HEALPix indices + # Get the original healpix_index coordinates and the key of + # the HEALPix domain axis hp_key, healpix_index = f.coordinate( "healpix_index", filter_by_naxes=(1,), @@ -1967,43 +2044,96 @@ def healpix_change_order(self, new_healpix_order, inplace=False): ) if healpix_index is None: raise ValueError( - "Can't change HEALPix order: No healpix_index coordinates" + "Can't change HEALPix order: There are no " + "healpix_index coordinates" ) - # Change the HEALPix indices - dx = healpix_index.to_dask_array() - dx = dx.map_blocks( - cf_HEALPix_change_order, - meta=np.array((), dtype="int64"), - healpix_order=healpix_order, - new_healpix_order=new_healpix_order, - refinement_level=refinement_level, - ) - healpix_index.set_data(dx, copy=False) + axis = f.get_data_axes(hp_key)[0] + + if healpix_order != new_healpix_order: + # Change the HEALPix indices + dx = healpix_index.to_dask_array() + dx = dx.map_blocks( + cf_HEALPix_change_order, + meta=np.array((), dtype="int64"), + healpix_order=healpix_order, + new_healpix_order=new_healpix_order, + refinement_level=refinement_level, + ) + healpix_index.set_data(dx, copy=False) + + # Update the Coordinate Reference + cr.coordinate_conversion.set_parameter( + "healpix_order", new_healpix_order + ) + if new_healpix_order == "nuniq": + cr.coordinate_conversion.del_parameter( + "refinement_level", None + ) if healpix_index.construct_type == "dimension_coordinate": - # Convert indices to auxiliary coordinates + # Convert healpix_indices to auxiliary coordinates healpix_index = f._AuxiliaryCoordinate( source=healpix_index, copy=False ) - axis = f.get_data_axes(hp_key)[0] f.del_construct(hp_key) - new_key = f.set_construct(healpix_index, axes=axis, copy=False) - cr.set_coordinate(new_key) + hp_key = f.set_construct(healpix_index, axes=axis, copy=False) + cr.set_coordinate(hp_key) - # Update the Coordinate Reference - cr.coordinate_conversion.set_parameter( - "healpix_order", new_healpix_order - ) - if new_healpix_order == "nuniq": - cr.coordinate_conversion.del_parameter("refinement_level", None) + if sort: + # Sort the HEALPix axis so that the HEALPix indices are + # monotonically increasing + d = healpix_index.to_dask_array() + if not (d == da.arange(d.size)).all(): + index = healpix_index.data.compute() + f = f.subspace(**{axis: np.argsort(index)}) return f @_inplace_enabled(default=False) def healpix_to_ugrid(self, inplace=False): - """TODOHEALPIX""" - axis = self.healpix_axis() + """Convert a HEALPix domain to a UGRID domain. + + .. versionadded:: NEXTVERSION + + .. seealso:: `create_latlon_coordinates`, `healpix_axis` + + :Parameters: + + {{inplace: `bool`, optional}} + + :Returns: + + `{{class}}` or `None` + The `{{class}}` converted to UGRID, or `None` if the + operation was in-place. + + **Examples** + + >>> f = cf.example_field(12) + >>> print(f) + Field: air_temperature (ncvar%tas) + ---------------------------------- + Data : air_temperature(time(2), healpix_index(48)) K + Cell methods : time(2): mean area: mean + Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian + : height(1) = [1.5] m + Auxiliary coords: healpix_index(healpix_index(48)) = [0, ..., 47] 1 + Coord references: grid_mapping_name:healpix + >>> print(f.healpix_to_ugrid()) + Field: air_temperature (ncvar%tas) + ---------------------------------- + Data : air_temperature(time(2), ncdim%cell(48)) K + Cell methods : time(2): mean area: mean + Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian + : height(1) = [1.5] m + Auxiliary coords: latitude(ncdim%cell(48)) = [19.47122063449069, ..., -19.47122063449069] degrees_north + : longitude(ncdim%cell(48)) = [45.0, ..., 315.0] degrees_east + Coord references: grid_mapping_name:latitude_longitude + Topologies : cell:face(ncdim%cell(48), 4) = [[965, ..., 3074]] + + """ + axis = self.healpix_axis(None) if axis is None: raise ValueError( "Can't convert HEALPix to UGRID: There is no HEALPix domain " @@ -2056,7 +2186,7 @@ def healpix_to_ugrid(self, inplace=False): "bounds" ) - # Create a unique identifer for each node location + # Create a unique integer identifer for each node location bounds_y = bounds_y.to_dask_array() bounds_x = bounds_x.to_dask_array() @@ -2116,21 +2246,75 @@ def healpix_to_ugrid(self, inplace=False): return f @_inplace_enabled(default=False) - def create_latlon_coordinates(self, one_d=True, two_d=True, inplace=False): - """TODOHEALPIX. + @_manage_log_level_via_verbosity + def create_latlon_coordinates( + self, one_d=True, two_d=True, inplace=False, verbose=None + ): + """Create latitude and longitude coordinates. + + Creates any 1-d or 2-d latitude and longitude coordinate + constructs that are implied by the {{class}}, but are not + atually present as part of the {{class}}'s metadata. Such + coordinates may be created if there is an appropriate + non-latitude_longitude grid mapping coordinate reference + construct. + + When it is not possible to create latitude and longitude + coordinates, the reason why will be reported if *verbose* is + at ``'INFO'`` or higher. .. versionadded:: NEXTVERSION + .. seealso:: `healpix_to_ugrid` + :Parameters: - TODOHEALPIX + one_d: `bool`, optional` + If True (the default) then create 1-d latitude and + longitude coordinates, if possible. Otherwise do not + attempt this. + + two_d: `bool`, optional` + If True (the default) then create 2-d latitude and + longitude coordinates, if possible. Otherwise do not + attempt this. {{inplace: `bool`, optional}} + {{verbose: `int` or `str` or `None`, optional}} + :Returns: `{{class}}` or `None` - TODOHEALPIX + The {{class}} with new latitude and longitude + constructs, if any could be created. If none could be + created then a new, identical field is returned. If + the operation was in-place then `None` is returned. + + **Examples** + + >>> f = cf.example_field(12) + >>> print(f) + Field: air_temperature (ncvar%tas) + ---------------------------------- + Data : air_temperature(time(2), healpix_index(48)) K + Cell methods : time(2): mean area: mean + Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian + : height(1) = [1.5] m + Auxiliary coords: healpix_index(healpix_index(48)) = [0, ..., 47] 1 + Coord references: grid_mapping_name:healpix + >>> g = f.create_latlon_coordinates() + >>> print(g) + Field: air_temperature (ncvar%tas) + ---------------------------------- + Data : air_temperature(time(2), ncdim%cell(48)) K + Cell methods : time(2): mean area: mean + Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian + : height(1) = [1.5] m + Auxiliary coords: healpix_index(ncdim%cell(48)) = [0, ..., 47] 1 + : latitude(ncdim%cell(48)) = [19.47122063449069, ..., -19.47122063449069] degrees_north + : longitude(ncdim%cell(48)) = [45.0, ..., 315.0] degrees_east + Coord references: grid_mapping_name:healpix """ f = _inplace_enabled_define_and_cleanup(self) @@ -2141,37 +2325,62 @@ def create_latlon_coordinates(self, one_d=True, two_d=True, inplace=False): for cr in f.coordinate_references(todict=True).values() } if not identities: + if is_log_level_info(logger): + logger.info( + "Can't create latitude and longitude coordinates: " + "There are no grid mapping coordinate references" + ) # pragma: no cover + return f - # Keep only grid mappings + # Keep only those that are grid mappings identities = { identity: cr for identity, cr in identities.items() if identity.startswith("grid_mapping_name:") } if len(identities) > 2: + if is_log_level_info(logger): + logger.info( + "Can't create latitude and longitude coordinates: There " + "are more than two grid mapping coordinate references" + ) # pragma: no cover + return f - # Remove a 'latitude_longitude' grid mapping + # Remove a 'latitude_longitude' grid mapping, if there is one. latlon_cr = identities.pop( "grid_mapping_name:latitude_longitude", None ) if not identities: + # There is no non-latitude_longitude grid mapping + if is_log_level_info(logger): + logger.info( + "Can't create latitude and longitude coordinates: There " + "is no non-latitude_longitude grid mapping coordinate " + "reference" + ) # pragma: no cover + return f - # Still here? Then get the final grid mapping and calulate the - # lat/lon coordinates. + # Still here? Then get the non-latitude_longitude grid mapping + # and calulate the lat/lon coordinates. identity, cr = identities.popitem() new_coords = False if one_d and identity == "grid_mapping_name:healpix": # -------------------------------------------------------- - # HEALPix + # HEALPix 1-d coordinates # -------------------------------------------------------- parameters = cr.coordinate_conversion.parameters() healpix_order = parameters.get("healpix_order") if healpix_order not in ("nested", "ring", "nuniq"): - # Bad healpix_order + if is_log_level_info(logger): + logger.info( + "Can't create 1-d latitude and longitude coordinates: " + f"Invalid HEALPix order: {healpix_order!r}" + ) # pragma: no cover + return f refinement_level = parameters.get("refinement_level") @@ -2179,9 +2388,15 @@ def create_latlon_coordinates(self, one_d=True, two_d=True, inplace=False): "nested", "ring", ): - # Missing refinement_level + if is_log_level_info(logger): + logger.info( + "Can't create 1-d latitude and longitude coordinates: " + "Missing HEALPix refinemnt level" + ) # pragma: no cover + return f + # The the HEALPix indices key, healpix_index = f.coordinate( "healpix_index", filter_by_naxes=(1,), @@ -2189,7 +2404,12 @@ def create_latlon_coordinates(self, one_d=True, two_d=True, inplace=False): default=(None, None), ) if healpix_index is None: - # Missing healpix_index coordinates + if is_log_level_info(logger): + logger.info( + "Can't create 1-d latitude and longitude coordinates: " + "Missing healpix_index coordinates" + ) # pragma: no cover + return f # Get the HEALPix axis @@ -2202,7 +2422,13 @@ def create_latlon_coordinates(self, one_d=True, two_d=True, inplace=False): axis_mode="exact", todict=True, ): - # X and/or Y coordinates already exist, so don't create any. + if is_log_level_info(logger): + logger.info( + "Can't create 1-d latitude and longitude coordinates: " + "1-d X or Y coordinates already exist for axis " + f"{f.constructs.domain_axis_identity(axis)!r}" + ) # pragma: no cover + return f # Define functions to create latitudes and longitudes from @@ -2699,74 +2925,6 @@ def get_coordinate_reference( return out - # def get_refinement_level(self, default=ValueError()): - # """TODOHEALPIX - # - # .. versionadded:: NEXTVERSION - # - # :Parameters: - # - # TODOHEALPIX - # - # :Returns: - # - # `int` - # TODOHEALPIX - # - # **Examples** - # - # TODOHEALPIX - # - # """ - # cr = self.coordinate_reference( - # "grid_mapping_name:healpix", default=None - # ) - # if cr is not None: - # refinement_level = cr.coordinate_conversion.get_property( - # "refinement_level", None - # ) - # if refinement_level is not None: - # return refinement_level - # - # if default is None: - # return - # - # return self._default(default, "HEALPix refinement level has been set") - # - # def get_healpix_order(self, default=ValueError()): - # """TODOHEALPIX - # - # .. versionadded:: NEXTVERSION - # - # :Parameters: - # - # TODOHEALPIX - # - # :Returns: - # - # `str` - # TODOHEALPIX - # - # **Examples** - # - # TODOHEALPIX - # - # """ - # cr = self.coordinate_reference( - # "grid_mapping_name:healpix", default=None - # ) - # if cr is not None: - # healpix_order = cr.coordinate_conversion.get_property( - # "healpix_order", None - # ) - # if healpix_order is not None: - # return healpix_order - # - # if default is None: - # return - # - # return self._default(default, "HEALPix order has been set") - def iscyclic(self, *identity, **filter_kwargs): """Returns True if the given axis is cyclic. diff --git a/cf/regrid/regrid.py b/cf/regrid/regrid.py index c75817a374..7f928e1aed 100644 --- a/cf/regrid/regrid.py +++ b/cf/regrid/regrid.py @@ -1132,11 +1132,12 @@ def spherical_grid( The grid definition. """ + # Create any implied lat/lon coordinates in-place try: - # Try to convert a HEALPix grid to UGRID + # Try to convert a HEALPix grid to a UGRID grid (which will + # create 1-d lat/lon coordinates) f.healpix_to_ugrid(inplace=True) except ValueError: - # Create any implied lat/lon coordinates f.create_latlon_coordinates(inplace=True) data_axes = f.constructs.data_axes() diff --git a/cf/test/test_Data.py b/cf/test/test_Data.py index 026ad9e2c8..52ea908c1f 100644 --- a/cf/test/test_Data.py +++ b/cf/test/test_Data.py @@ -4684,6 +4684,29 @@ def test_Data_collapse_axes_hdf_chunks(self): self.assertEqual(e._axes, d._axes[1:]) self.assertEqual(e.nc_dataset_chunksizes(), chunks) + def test_Data_coarsen(self): + """Test Data.coarsen.""" + d = cf.Data(np.arange(24).reshape((4, 6))) + a = d.array + + self.assertIsNone(d.coarsen(np.min, axes={}, inplace=True)) + self.assertEqual(d.shape, a.shape) + self.assertTrue((d.array == a).all()) + + e = d.coarsen(np.min, {0: 2, 1: 3}) + self.assertIsInstance(e, cf.Data) + self.assertEqual(e.shape, (2, 2)) + self.assertTrue((e.array == [[0, 3], [12, 15]]).all()) + + # Non-full caorsening neighbourhood with trim_excess=True + e = d.coarsen(np.max, {-1: 5}, trim_excess=True) + self.assertEqual(e.shape, (4, 1)) + self.assertTrue((e.array == [[4], [10], [16], [22]]).all()) + + # Non-full caorsening neighbourhood with trim_excess=False + with self.assertRaises(ValueError): + d.coarsen(np.max, {-1: 5}) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index 03f87c7a46..b432e47327 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -3035,7 +3035,17 @@ def test_Field_filled(self): self.assertEqual(values[0], -999) self.assertEqual(counts[0], 5) - + def test_Field_healpix_axis(self): + """Test Field.healpix_axis.""" + f = cf.example_field(12) + key0 = f.healpix_axis() + key1 = f.auxiliary_coordinate( + "healpix_index", + filter_by_naxes=(1,), + key=True, + default=None) + + self.assertEqual(key0, key1()x if __name__ == "__main__": print("Run date:", datetime.datetime.now()) cf.environment() diff --git a/cf/weights.py b/cf/weights.py index 43c81e3b1e..e87bd24a3c 100644 --- a/cf/weights.py +++ b/cf/weights.py @@ -1922,7 +1922,7 @@ def healpix_area( return_areas=False, methods=False, ): - """Creates area weights for polygon geometry cells.TODOHEALPIX + """Creates area weights for HEALPix cells. .. versionadded:: NEXTVERSION @@ -1957,7 +1957,7 @@ def healpix_area( the weights are returned. """ - axis = f.healpix_axis() + axis = f.healpix_axis(None) if axis is None: if auto: return False @@ -2041,27 +2041,26 @@ def healpix_area( if auto: return False - raise ValueError("Can't create weights: TODOHEALPIX") + raise ValueError( + "Can't create weights: Missing healpix_index coordinates" + ) if measure: units = radius.Units**2 else: units = "1" - from .dask_utils import cf_HEALPix_nuniq_weights + from .dask_utils import cf_HEALPix_nuniq_area_weights dx = healpix_index.to_dask_array() dx = dx.map_blocks( - cf_HEALPix_nuniq_weights, + cf_HEALPix_nuniq_area_weights, meta=np.array((), dtype="float64"), measure=measure, radius=radius, ) area = f._Data(dx, units=units, copy=False) - # area = cls._healpix_nuniq_weights( - # f, healpix_index, measure=measure, radius=radius - # ) if return_areas: return area @@ -2094,47 +2093,3 @@ def healpix_area( weights[(axis,)] = area weights_axes.add(axis) return True - - -# @classmethod -# def _healpix_nuniq_weights( -# self, f, healpix_index, measure=False, radius=None -# ): -# """TODOHEALPIX -# -# .. versionadded:: NEXTVERSION -# -# :Parameters: -# -# f: `Field` -# The field for which the weights are being created. -# -# healpix_index: `Coordinate` -# TODOHEALPIX -# -# {{weights measure: `bool`, optional}} -# -# {{radius: optional}} -# -# :Returns: -# -# `Data` -# TODOHEALPIX -# -# """ -# from .dask_utils import cf_HEALPix_nuniq_weights -# -# dx = healpix_index.to_dask_array() -# dx = dx.map_blocks( -# cf_HEALPix_nuniq_weights, -# meta=np.array((), dtype="float64"), -# measure=measure, -# radius=radius, -# ) -# -# if measure: -# units = radius.Units**2 -# else: -# units = "1" -# -# return f._Data(dx, units=units, copy=False) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index e9e6cf2e15..cb973621c7 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -268,6 +268,11 @@ environments for which these features are not required. must use a copy of the ``pyfive`` branch of the https://github.com/NCAS-CMS/PyActiveStorage repository. +.. rubric:: HEALPix manipulations + +* `healpix `_, version 2024.2 or + newer. + ---- .. _Tests: diff --git a/setup.py b/setup.py index 495dfa08a8..effcf70fd7 100755 --- a/setup.py +++ b/setup.py @@ -185,6 +185,10 @@ def compile(): * write and append field constructs to netCDF datasets on disk, +* read, create, and manipulate UGRID mesh topologies, + +* read, write, and manipulate HEALPix grids, + * read, write, and create coordinates defined by geometry cells, * read netCDF and CDL datasets containing hierarchical groups, @@ -216,8 +220,8 @@ def compile(): * perform histogram, percentile and binning operations on field constructs, -* regrid structured grid, mesh and DSG field constructs with - (multi-)linear, nearest neighbour, first- and second-order +* regrid structured grid, UGRID, HEALPix, and DSG field constructs + with (multi-)linear, nearest neighbour, first- and second-order conservative and higher order patch recovery methods, including 3-d regridding, From 45a715c1d9d67fbac2b6b45e8e12642e123b4873 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 1 Jul 2025 23:12:24 +0100 Subject: [PATCH 13/59] dev --- cf/__init__.py | 2 +- cf/domain.py | 143 +++++++++++++++++++++++++- cf/field.py | 220 ++++++++++++++++++++-------------------- cf/mixin/fielddomain.py | 117 ++++++++++++++++++++- cf/regrid/regrid.py | 95 +++++++++++++---- cf/test/test_Field.py | 104 +++++++++++++++++-- cf/weights.py | 2 +- 7 files changed, 534 insertions(+), 149 deletions(-) diff --git a/cf/__init__.py b/cf/__init__.py index 74aaa73f89..581f044ae9 100644 --- a/cf/__init__.py +++ b/cf/__init__.py @@ -245,7 +245,6 @@ from .tiepointindex import TiePointIndex from .bounds import Bounds -from .domain import Domain from .datum import Datum from .coordinateconversion import CoordinateConversion @@ -264,6 +263,7 @@ from .cellconnectivity import CellConnectivity from .cellmethod import CellMethod from .cellmeasure import CellMeasure +from .domain import Domain from .domainancillary import DomainAncillary from .domainaxis import DomainAxis from .domaintopology import DomainTopology diff --git a/cf/domain.py b/cf/domain.py index 78ae41dfc6..4aef023f03 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -1,11 +1,14 @@ from math import prod +from numbers import Integral import cfdm import numpy as np from . import mixin from .auxiliarycoordinate import AuxiliaryCoordinate +from .bounds import Bounds from .constructs import Constructs +from .coordinatereference import CoordinateReference from .data import Data from .decorators import _inplace_enabled, _inplace_enabled_define_and_cleanup from .dimensioncoordinate import DimensionCoordinate @@ -79,7 +82,9 @@ class Domain(mixin.FieldDomain, mixin.Properties, cfdm.Domain): def __new__(cls, *args, **kwargs): """Creates a new Domain instance.""" instance = super().__new__(cls) + instance._Bounds = Bounds instance._Constructs = Constructs + instance._CoordinateReference = CoordinateReference instance._Data = Data instance._DomainAxis = DomainAxis instance._DimensionCoordinate = DimensionCoordinate @@ -153,12 +158,12 @@ def close(self): @classmethod def create_regular(cls, x_args, y_args, bounds=True): - """ - Create a new domain with the regular longitudes and latitudes. + """Create a new domain with the regular longitudes and latitudes. .. versionadded:: 3.15.1 - .. seealso:: `cf.DimensionCoordinate.create_regular` + .. seealso:: `cf.DimensionCoordinate.create_regular`, + `cf.Domain.create_healpix` :Parameters: @@ -182,7 +187,6 @@ def create_regular(cls, x_args, y_args, bounds=True): **Examples** - >>> import cf >>> domain = cf.Domain.create_regular((-180, 180, 1), (-90, 90, 1)) >>> domain.dump() -------- @@ -259,6 +263,137 @@ def create_regular(cls, x_args, y_args, bounds=True): return domain + @classmethod + def create_healpix( + cls, refinement_level, healpix_order="nested", radius=None + ): + """Create a new global HEALPix domain. + + .. versionadded:: NEXTVERSION + + .. seealso:: `cf.Domain.create_regular` + + :Parameters: + + TODOHEALPIX + + radius: optional + Specify the radius of the latitude-longitude plane + defined in spherical polar coordinates. May be set to + any numeric scalar object, including `numpy` and + `Data` objects. The units of the radius are assumed to + be metres, unless specified by a `Data` object. If the + special value ``'earth'`` is given then the radius + taken as 6371229 metres. If `None` (the default) then + no radius is set. + + *Example:* + Equivalent ways to set a radius of 6371229 metres: + ``6371229``, ``numpy.array(6371229)``, + ``cf.Data(6371229)``, ``cf.Data(6371229, 'm')``, + ``cf.Data(6371.229, 'km')``, ``'earth'``. + + :Returns: + + `Domain` + The newly created HEALPix domain. + + **Examples** + + >>> d = cf.Domain.create_healpix(4) + >>> d.dump() + -------- + Domain: + -------- + Domain Axis: healpix_index(3072) + + Auxiliary coordinate: healpix_index + standard_name = 'healpix_index' + units = '1' + Data(healpix_index(3072)) = [0, ..., 3071] 1 + + Coordinate reference: grid_mapping_name:healpix + Coordinate conversion:grid_mapping_name = healpix + Coordinate conversion:healpix_order = nested + Coordinate conversion:refinement_level = 4 + Auxiliary Coordinate: healpix_index + + >>> d = cf.Domain.create_healpix(8, "nuniq", radius=6371000) + >>> d.dump() + -------- + Domain: + -------- + Domain Axis: healpix_index(786432) + + Auxiliary coordinate: healpix_index + standard_name = 'healpix_index' + units = '1' + Data(healpix_index(786432)) = [262144, ..., 1048575] 1 + + Coordinate reference: grid_mapping_name:healpix + Coordinate conversion:grid_mapping_name = healpix + Coordinate conversion:healpix_order = nuniq + Datum:earth_radius = 6371000.0 + Auxiliary Coordinate: healpix_index + + """ + import dask.array as da + + if not isinstance(refinement_level, Integral) or refinement_level < 0: + raise ValueError( + "'refinement_level' must be a non-negative integer. " + f"Got: {refinement_level!r}" + ) + + nuniq = healpix_order == "nuniq" + if nuniq: + healpix_order = "nested" + elif healpix_order not in ("nested", "ring"): + raise ValueError( + "'healpix_order' must be 'nested', 'ring', or 'nuniq'. " + f"Got: {healpix_order!r}" + ) + + domain = Domain() + + # domain_axis: ncdim%cell + ncells = 12 * (4**refinement_level) + d = domain._DomainAxis(ncells) + d.nc_set_dimension("cell") + axis = domain.set_construct(d, copy=False) + + # auxiliary_coordinate: healpix_index + c = domain._AuxiliaryCoordinate() + c.set_properties({"standard_name": "healpix_index"}) + c.nc_set_variable("healpix_index") + c.set_data(Data(da.arange(ncells), units="1"), copy=False) + key = domain.set_construct(c, axes=axis, copy=False) + + # coordinate_reference: grid_mapping_name:healpix + cr = domain._CoordinateReference() + cr.nc_set_variable("healpix") + cr.set_coordinates({key}) + + if radius is not None: + radius = domain.radius(default=radius) + cr.datum.set_parameter("earth_radius", radius.datum()) + + cr.coordinate_conversion.set_parameters( + { + "grid_mapping_name": "healpix", + "healpix_order": healpix_order, + "refinement_level": refinement_level, + } + ) + + domain.set_construct(cr) + + if nuniq: + # Change from nested to nuniq order + domain = domain.healpix_change_order("nuniq") + + return domain + @_inplace_enabled(default=False) def flip(self, axes=None, inplace=False): """Flip (reverse the direction of) domain axes. diff --git a/cf/field.py b/cf/field.py index 4aaf106f5c..2dd4fc2558 100644 --- a/cf/field.py +++ b/cf/field.py @@ -179,7 +179,7 @@ # -------------------------------------------------------------------- _collapse_ddof_methods = set(("sd", "var")) -_earth_radius = Data(6371229.0, "m") +#_earth_radius = Data(6371229.0, "m") _relational_methods = ( "__eq__", @@ -2629,111 +2629,111 @@ def get_domain(self): return domain - def radius(self, default=None): - """Return the radius of a latitude-longitude plane defined in - spherical polar coordinates. - - The radius is taken from the datums of any coordinate - reference constructs, but if and only if this is not possible - then a default value may be used instead. - - .. versionadded:: 3.0.2 - - .. seealso:: `bin`, `cell_area`, `collapse`, `weights` - - :Parameters: - - default: optional - The radius is taken from the datums of any coordinate - reference constructs, but if and only if this is not - possible then the value set by the *default* parameter - is used. May be set to any numeric scalar object, - including `numpy` and `Data` objects. The units of the - radius are assumed to be metres, unless specified by a - `Data` object. If the special value ``'earth'`` is - given then the default radius taken as 6371229 - metres. If *default* is `None` an exception will be - raised if no unique datum can be found in the - coordinate reference constructs. - - *Parameter example:* - Five equivalent ways to set a default radius of - 6371200 metres: ``6371200``, - ``numpy.array(6371200)``, ``cf.Data(6371200)``, - ``cf.Data(6371200, 'm')``, ``cf.Data(6371.2, - 'km')``. - - :Returns: - - `Data` - The radius of the sphere, in units of metres. - - **Examples** - - >>> f.radius() - - - >>> g.radius() - ValueError: No radius found in coordinate reference constructs and no default provided - >>> g.radius('earth') - - >>> g.radius(1234) - - - """ - radii = [] - for cr in self.coordinate_references(todict=True).values(): - r = cr.datum.get_parameter("earth_radius", None) - if r is not None: - r = Data.asdata(r) - if not r.Units: - r.override_units("m", inplace=True) - - if r.size != 1: - radii.append(r) - continue - - got = False - for _ in radii: - if r == _: - got = True - break - - if not got: - radii.append(r) - - if len(radii) > 1: - raise ValueError( - "Multiple radii found from coordinate reference " - f"constructs: {radii!r}" - ) - - if not radii: - if default is None: - raise ValueError( - "No radius found from coordinate reference constructs " - "and no default provided" - ) - - if isinstance(default, str): - if default != "earth": - raise ValueError( - "The default radius must be numeric, 'earth', " - "or None" - ) - - return _earth_radius.copy() - - r = Data.asdata(default).squeeze() - else: - r = Data.asdata(radii[0]).squeeze() - - if r.size != 1: - raise ValueError(f"Multiple radii: {r!r}") - - r.Units = Units("m") - r.dtype = float - return r +# def radius(self, default=None): +# """Return the radius of a latitude-longitude plane defined in +# spherical polar coordinates. +# +# The radius is taken from the datums of any coordinate +# reference constructs, but if and only if this is not possible +# then a default value may be used instead. +# +# .. versionadded:: 3.0.2 +# +# .. seealso:: `bin`, `cell_area`, `collapse`, `weights` +# +# :Parameters: +# +# default: optional +# The radius is taken from the datums of any coordinate +# reference constructs, but if and only if this is not +# possible then the value set by the *default* parameter +# is used. May be set to any numeric scalar object, +# including `numpy` and `Data` objects. The units of the +# radius are assumed to be metres, unless specified by a +# `Data` object. If the special value ``'earth'`` is +# given then the default radius taken as 6371229 +# metres. If *default* is `None` an exception will be +# raised if no unique datum can be found in the +# coordinate reference constructs. +# +# *Parameter example:* +# Five equivalent ways to set a default radius of +# 6371200 metres: ``6371200``, +# ``numpy.array(6371200)``, ``cf.Data(6371200)``, +# ``cf.Data(6371200, 'm')``, ``cf.Data(6371.2, +# 'km')``. +# +# :Returns: +# +# `Data` +# The radius of the sphere, in units of metres. +# +# **Examples** +# +# >>> f.radius() +# +# +# >>> g.radius() +# ValueError: No radius found in coordinate reference constructs and no default provided +# >>> g.radius('earth') +# +# >>> g.radius(1234) +# +# +# """ +# radii = [] +# for cr in self.coordinate_references(todict=True).values(): +# r = cr.datum.get_parameter("earth_radius", None) +# if r is not None: +# r = Data.asdata(r) +# if not r.Units: +# r.override_units("m", inplace=True) +# +# if r.size != 1: +# radii.append(r) +# continue +# +# got = False +# for _ in radii: +# if r == _: +# got = True +# break +# +# if not got: +# radii.append(r) +# +# if len(radii) > 1: +# raise ValueError( +# "Multiple radii found from coordinate reference " +# f"constructs: {radii!r}" +# ) +# +# if not radii: +# if default is None: +# raise ValueError( +# "No radius found from coordinate reference constructs " +# "and no default provided" +# ) +# +# if isinstance(default, str): +# if default != "earth": +# raise ValueError( +# "The default radius must be numeric, 'earth', " +# "or None" +# ) +# +# return _earth_radius.copy() +# +# r = Data.asdata(default).squeeze() +# else: +# r = Data.asdata(radii[0]).squeeze() +# +# if r.size != 1: +# raise ValueError(f"Multiple radii: {r!r}") +# +# r.Units = Units("m") +# r.dtype = float +# return r def laplacian_xy( self, x_wrap=None, one_sided_at_boundary=False, radius=None @@ -4915,7 +4915,7 @@ def healpix_decrease_refinement_level( self, level, reduction, sort=True, check_healpix_index=True ): """Decrease the refinement level of a HEALPix grid. - + .. versionadded:: NEXTVERSION .. seealso:: `healpix_change_order` @@ -4934,7 +4934,7 @@ def healpix_decrease_refinement_level( If the current refinement level is 10 then a new coarser refinment level of 8 can be specified by either ``8`` or ``-2``. - + reduction: function The function used to calculate the values in the new coarser cells from the original data defined on the @@ -4959,7 +4959,7 @@ def healpix_decrease_refinement_level( operation, if it is known that the HEALPix indices expressed in their nested ordering are already monotonically increasing. - + check_healpix_index: `bool`, optional As a requirement of the coarsening algorithm, the HEALPix indices are always automatically converted to @@ -4993,7 +4993,7 @@ def healpix_decrease_refinement_level( Set the refinement level to 0: - + >>> g = f.healpix_decrease_refinement_level(0, np.mean)g >>> g diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index df9a8a5b1f..5eabc089a4 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -203,7 +203,7 @@ def _equivalent_coordinate_references( return False return True - + def _indices(self, config, data_axes, ancillary_mask, kwargs): """Create indices that define a subspace of the field or domain construct. @@ -2002,7 +2002,7 @@ def healpix_change_order(self, new_healpix_order, sort=False): if new_healpix_order not in ("nested", "ring", "nuniq"): raise ValueError( "Can't change HEALPix order: new_healpix_order must be " - f"'nest', 'ring', or 'nuniq'. Got {new_healpix_order!r}" + f"'nested', 'ring', or 'nuniq'. Got {new_healpix_order!r}" ) # Parse the HEALPix coordinate reference @@ -2082,7 +2082,8 @@ def healpix_change_order(self, new_healpix_order, sort=False): if sort: # Sort the HEALPix axis so that the HEALPix indices are - # monotonically increasing + # monotonically increasing. Test for the common case of + # already-ordered global nested indices. d = healpix_index.to_dask_array() if not (d == da.arange(d.size)).all(): index = healpix_index.data.compute() @@ -3162,6 +3163,112 @@ def _parse_axes(self, axes): return [self.domain_axis(x, key=True) for x in axes] + def radius(self, default=None): + """Return the radius of a latitude-longitude plane defined in + spherical polar coordinates. + + The radius is taken from the datums of any coordinate + reference constructs, but if and only if this is not possible + then a default value may be used instead. + + .. versionadded:: 3.0.2 + + .. seealso:: `bin`, `cell_area`, `collapse`, `weights` + + :Parameters: + + default: optional + The radius is taken from the datums of any coordinate + reference constructs, but if and only if this is not + possible then the value set by the *default* parameter + is used. May be set to any numeric scalar object, + including `numpy` and `Data` objects. The units of the + radius are assumed to be metres, unless specified by a + `Data` object. If the special value ``'earth'`` is + given then the default radius taken as 6371229 + metres. If *default* is `None` an exception will be + raised if no unique datum can be found in the + coordinate reference constructs. + + *Parameter example:* + Five equivalent ways to set a default radius of + 6371200 metres: ``6371200``, + ``numpy.array(6371200)``, ``cf.Data(6371200)``, + ``cf.Data(6371200, 'm')``, ``cf.Data(6371.2, + 'km')``. + + :Returns: + + `Data` + The radius of the sphere, in units of metres. + + **Examples** + + >>> f.radius() + + + >>> g.radius() + ValueError: No radius found in coordinate reference constructs and no default provided + >>> g.radius('earth') + + >>> g.radius(1234) + + + """ + radii = [] + for cr in self.coordinate_references(todict=True).values(): + r = cr.datum.get_parameter("earth_radius", None) + if r is not None: + r = self._Data.asdata(r) + if not r.Units: + r.override_units("m", inplace=True) + + if r.size != 1: + radii.append(r) + continue + + got = False + for _ in radii: + if r == _: + got = True + break + + if not got: + radii.append(r) + + if len(radii) > 1: + raise ValueError( + "Multiple radii found from coordinate reference " + f"constructs: {radii!r}" + ) + + if not radii: + if default is None: + raise ValueError( + "No radius found from coordinate reference constructs " + "and no default provided" + ) + + if isinstance(default, str): + if default != "earth": + raise ValueError( + "The default radius must be numeric, 'earth', " + "or None" + ) + + return self._Data(6371229.0, "m") + + r = self._Data.asdata(default).squeeze() + else: + r = self._Data.asdata(radii[0]).squeeze() + + if r.size != 1: + raise ValueError(f"Multiple radii: {r!r}") + + r.Units = Units("m") + r.dtype = float + return r + def replace_construct( self, *identity, new=None, copy=True, **filter_kwargs ): @@ -3576,7 +3683,9 @@ def set_coordinate_reference( for value in coordinate_reference.coordinates(): if value in coordinates: identity = coordinates[value].identity(strict=strict) - ckeys.append(self.coordinate(identity, key=True, default=None)) + key = self.coordinate(identity, key=True, default=None) + if key is not None: + ckeys.append(key) ref.clear_coordinates() ref.set_coordinates(ckeys) diff --git a/cf/regrid/regrid.py b/cf/regrid/regrid.py index 7f928e1aed..ed89fa1df4 100644 --- a/cf/regrid/regrid.py +++ b/cf/regrid/regrid.py @@ -131,6 +131,10 @@ class Grid: # The integer position in *coords* of a vertical coordinate. If # `None` then there are no vertical coordinates. z_index: Any = None + # TODOHEALPIX + original_domain: Any = None + # Whether or not the original domain is a HEALPix grid + original_healpix: bool = False def regrid( @@ -338,7 +342,7 @@ def regrid( """ if not inplace: src = src.copy() - + spherical = coord_sys == "spherical" cartesian = not spherical @@ -502,11 +506,23 @@ def regrid( conform_coordinates(src_grid, dst_grid) - if method in ("conservative_2nd", "patch"): + if method == "conservative": + if src_grid.original_healpix or dst_grid.original_healpix: + raise ValueError( + f"{method!r} regridding is not available for HEALPix grids" + ) + + elif method in ("conservative_2nd", "patch"): + if method == "conservative_2nd" and (src_grid.original_healpix or dst_grid.original_healpix): + raise ValueError( + f"{method!r} regridding is not available for HEALPix grids" + ) + if not (src_grid.dimensionality >= 2 and dst_grid.dimensionality >= 2): raise ValueError( f"{method!r} regridding is not available for 1-d regridding" ) + elif method in ("nearest_dtos", "nearest_stod"): if not has_coordinate_arrays(src_grid) and not has_coordinate_arrays( dst_grid @@ -660,7 +676,7 @@ def regrid( dst_featureType=dst_grid.featureType, src_z=src_grid.z, dst_z=dst_grid.z, - ln_z=ln_z, + ln_z=ln_z, # TODOHEALPIX ) else: if weights_file is not None: @@ -714,15 +730,15 @@ def regrid( # ---------------------------------------------------------------- # Update the regridded metadata # ---------------------------------------------------------------- - update_non_coordinates(src, dst, src_grid, dst_grid, regrid_operator) + cr_map = update_non_coordinates(src, dst, src_grid, dst_grid, regrid_operator) - update_coordinates(src, dst, src_grid, dst_grid) + update_coordinates(src, dst, src_grid, dst_grid, cr_map) # ---------------------------------------------------------------- # Insert regridded data into the new field # ---------------------------------------------------------------- update_data(src, regridded_data, src_grid, dst_grid) - + if coord_sys == "spherical" and dst_grid.is_grid: # Set the cyclicity of the longitude axis of the new field key, x = src.dimension_coordinate("X", default=(None, None), item=True) @@ -1132,6 +1148,11 @@ def spherical_grid( The grid definition. """ + original_domain = f.copy() + + # Whether or not the original domain is a HEALPix grid + original_healpix = False + # Create any implied lat/lon coordinates in-place try: # Try to convert a HEALPix grid to a UGRID grid (which will @@ -1139,6 +1160,8 @@ def spherical_grid( f.healpix_to_ugrid(inplace=True) except ValueError: f.create_latlon_coordinates(inplace=True) + else: + original_healpix = True data_axes = f.constructs.data_axes() @@ -1442,6 +1465,8 @@ def spherical_grid( z=z, ln_z=ln_z, z_index=z_index, + original_domain=original_domain, + original_healpix=original_healpix, ) set_grid_type(grid) @@ -1459,8 +1484,8 @@ def Cartesian_grid(f, name=None, method=None, axes=None, z=None, ln_z=None): :Parameters: f: `Field` or `Domain` - The field or domain construct from which to get the - coordinates. + The field or domain construct from which to get the + coordinates. name: `str` A name to identify the grid. @@ -1487,6 +1512,8 @@ def Cartesian_grid(f, name=None, method=None, axes=None, z=None, ln_z=None): The grid definition. """ + original_domain = f.copy() + if not axes: if name == "source": raise ValueError( @@ -1670,6 +1697,7 @@ def Cartesian_grid(f, name=None, method=None, axes=None, z=None, ln_z=None): z=z, ln_z=ln_z, z_index=z_index, + original_domain=original_domain ) set_grid_type(grid) @@ -2754,7 +2782,7 @@ def get_mask(f, grid): return mask -def update_coordinates(src, dst, src_grid, dst_grid): +def update_coordinates(src, dst, src_grid, dst_grid, cr_map): """Update the regrid axis coordinates. Replace the existing coordinate constructs that span the regridding @@ -2775,17 +2803,25 @@ def update_coordinates(src, dst, src_grid, dst_grid): dst: `Field` or `Domain` The field or domain containing the destination grid. - src_grid: `Grid` or `Mesh` + src_grid: `Grid` The definition of the source grid. - dst_grid: `Grid` or `Mesh` + dst_grid: `Grid` The definition of the destination grid. + cr_map `dict` + TODOHEALPIX + :Returns: `None` """ + if dst_grid.original_domain is not None: + dst = dst_grid.original_domain + + dst_grid_is_mesh = dst_grid.is_mesh and not dst_grid.original_healpix + src_axis_keys = src_grid.axis_keys dst_axis_keys = dst_grid.axis_keys @@ -2803,7 +2839,7 @@ def update_coordinates(src, dst, src_grid, dst_grid): todict=True, ): src.del_construct(key) - + # Domain axes src_domain_axes = src.domain_axes(todict=True) dst_domain_axes = dst.domain_axes(todict=True) @@ -2840,11 +2876,18 @@ def update_coordinates(src, dst, src_grid, dst_grid): filter_by_axis=dst_axis_keys, axis_mode="subset", todict=True ).items(): axes = [axis_map[axis] for axis in dst_data_axes[key]] - src.set_construct(coord, axes=axes) + coord_key = src.set_construct(coord, axes=axes) + + # Update the coordinates in new source coordinate references + for dst_cr_key, src_cr_key in cr_map.items(): + dst_cr = dst.coordinate_reference(dst_cr_key) + if key in dst_cr.coordinates(): + src_cr = src.coordinate_reference(src_cr_key) + src_cr.set_coordinate(coord_key) # Copy domain topology and cell connectivity constructs from the # destination grid - if dst_grid.is_mesh: + if dst_grid_is_mesh: for key, topology in dst.constructs( filter_by_type=("domain_topology", "cell_connectivity"), filter_by_axis=dst_axis_keys, @@ -2879,9 +2922,13 @@ def update_non_coordinates(src, dst, src_grid, dst_grid, regrid_operator): :Returns: - `None` + `dict` + TODOHEALPIX """ + if dst_grid.original_domain is not None: + dst = dst_grid.original_domain + src_axis_keys = src_grid.axis_keys dst_axis_keys = dst_grid.axis_keys @@ -2974,17 +3021,25 @@ def update_non_coordinates(src, dst, src_grid, dst_grid, regrid_operator): # ---------------------------------------------------------------- # Copy selected coordinate references from the destination grid + # + # Define the mapping of destination corodinate references to + # source coordinate references # ---------------------------------------------------------------- + cr_map = {} + dst_data_axes = dst.constructs.data_axes() - - for ref in dst.coordinate_references(todict=True).values(): + for dst_key, ref in dst.coordinate_references(todict=True).items(): axes = set() for c_key in ref.coordinates(): axes.update(dst_data_axes[c_key]) - + if axes and set(axes).issubset(dst_axis_keys): - src.set_coordinate_reference(ref, parent=dst, strict=True) - + src_cr = ref.copy() + src_cr.clear_coordinates() + src_key = src.set_construct(src_cr) + cr_map[dst_key] = src_key + + return cr_map def update_data(src, regridded_data, src_grid, dst_grid): """Insert the regridded field data. diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index b432e47327..1fdeafaa97 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -78,6 +78,7 @@ class FieldTest(unittest.TestCase): f0 = cf.example_field(0) f1 = cf.example_field(1) + f12 = cf.example_field(12) def test_Field_creation_commands(self): for f in cf.example_fields(): @@ -3037,15 +3038,100 @@ def test_Field_filled(self): def test_Field_healpix_axis(self): """Test Field.healpix_axis.""" - f = cf.example_field(12) - key0 = f.healpix_axis() - key1 = f.auxiliary_coordinate( - "healpix_index", - filter_by_naxes=(1,), - key=True, - default=None) - - self.assertEqual(key0, key1()x + # HEALPix field + f = self.f12 + + key = f.auxiliary_coordinate("healpix_index", key=True) + axis = f.get_data_axes(key)[0] + self.assertEqual(f.healpix_axis(), axis) + + # Non-HEALPix field + self.assertIsNone(self.f0.healpix_axis(None)) + with self.assertRaises(ValueError): + self.f0.healpix_axis() + + def test_Field_healpix_change_order(self): + """Test Field.healpix_change_order.""" + # HEALPix field + f = self.f12 + + g = f.healpix_change_order("ring") + self.assertTrue( + (g.coordinate("healpix_index")[:4].array == [13, 5, 4, 0]).all() + ) + h = g.healpix_change_order("nested") + self.assertTrue( + (h.coordinate("healpix_index")[:4].array == [0, 1, 2, 3]).all() + ) + h = g.healpix_change_order("nuniq") + self.assertTrue( + (h.coordinate("healpix_index")[:4].array == [16, 17, 18, 19]).all() + ) + + g = f.healpix_change_order("ring", sort=True) + self.assertTrue( + (g.coordinate("healpix_index")[:4].array == [0, 1, 2, 3]).all() + ) + h = g.healpix_change_order("nested", sort=False) + self.assertTrue( + (h.coordinate("healpix_index")[:4].array == [3, 7, 11, 15]).all() + ) + h = g.healpix_change_order("nested", sort=True) + self.assertTrue( + (h.coordinate("healpix_index")[:4].array == [0, 1, 2, 3]).all() + ) + + g = f.healpix_change_order("nuniq") + self.assertTrue( + (g.coordinate("healpix_index")[:4].array == [16, 17, 18, 19]).all() + ) + + # Can't change from 'nuniq' + with self.assertRaises(ValueError): + g.healpix_change_order("nested") + + # Non-HEALPix field + with self.assertRaises(ValueError): + self.f0.healpix_change_order("ring") + + def test_Field_healpix_to_ugrid(self): + """Test Field.healpix_to_ugrid.""" + # HEALPix field + f = self.f12.copy() + + u = f.healpix_to_ugrid() + self.assertEqual(len(u.domain_topologies()), 1) + self.assertEqual(len(u.auxiliary_coordinates()), 2) + self.assertTrue( + ( + u.domain_topology()[:4].normalise().array + == [[6, 4, 2, 3], [7, 8, 4, 6], [4, 1, 0, 2], [8, 5, 1, 4]] + ).all() + ) + + self.assertIsNone(f.healpix_to_ugrid(inplace=True)) + self.assertEqual(len(f.domain_topologies()), 1) + + # Non-HEALPix field + with self.assertRaises(ValueError): + self.f0.healpix_to_ugrid() + + def test_Field_create_latlon_coordinates(self): + """Test Field.create_latlon_coordinates.""" + # HEALPix field + f = self.f12.copy() + self.assertEqual(len(f.auxiliary_coordinates()), 1) + self.assertEqual(len(f.auxiliary_coordinates("healpix_index")), 1) + + g = f.create_latlon_coordinates() + self.assertEqual(len(g.auxiliary_coordinates()), 3) + self.assertEqual( + len(g.auxiliary_coordinates("healpix_index", "X", "Y")), 3 + ) + self.assertIsNone(f.create_latlon_coordinates(inplace=True)) + self.assertTrue(f.equals(g)) + + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) cf.environment() diff --git a/cf/weights.py b/cf/weights.py index e87bd24a3c..1601536861 100644 --- a/cf/weights.py +++ b/cf/weights.py @@ -2057,7 +2057,7 @@ def healpix_area( cf_HEALPix_nuniq_area_weights, meta=np.array((), dtype="float64"), measure=measure, - radius=radius, + radius=radius.array, ) area = f._Data(dx, units=units, copy=False) From 4a4863fa69841f1059ad11c5bc072c39328b851a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 2 Jul 2025 17:39:52 +0100 Subject: [PATCH 14/59] dev --- Changelog.rst | 2 +- cf/dask_utils.py | 57 ++--- cf/domain.py | 32 ++- cf/field.py | 212 +++++++++--------- cf/mixin/fielddomain.py | 399 +++++++++++++++++++-------------- cf/regrid/regrid.py | 89 ++++---- cf/regrid/regridoperator.py | 20 +- cf/test/test_Domain.py | 27 +++ cf/test/test_Field.py | 76 ++++++- cf/test/test_RegridOperator.py | 1 + cf/weights.py | 16 +- 11 files changed, 564 insertions(+), 367 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 55f1e3a90a..8e276d9d24 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,7 +8,7 @@ Version NEXTVERSION * New HEALPix methods: `cf.Field.healpix_axis`, `cf.Field.healpix_change_order`, `cf.Field.healpix_decrease_refinement_level`, - `cf.Field.healpix_to_ugrid` + `cf.Field.healpix_to_ugrid`, `cf.Domain.create_healpix` (https://github.com/NCAS-CMS/cf-python/issues/???) * New method: `cf.Data.coarsen` (https://github.com/NCAS-CMS/cf-python/issues/???) diff --git a/cf/dask_utils.py b/cf/dask_utils.py index f11ddd78dd..34fc42fe09 100644 --- a/cf/dask_utils.py +++ b/cf/dask_utils.py @@ -9,12 +9,12 @@ def cf_HEALPix_bounds( - a, healpix_order, refinement_level=None, lat=False, lon=False + a, index_scheme, refinement_level=None, lat=False, lon=False ): """Calculate HEALPix cell bounds. Latitude or longitude locations of the cell vertices are derived - from HEALPix indices. Each cell has four bounds, which are + from HEALPix indices.x Each cell has four bounds, which are returned in an anticlockwise direction, as seen from above, starting with the eastern-most vertex. @@ -25,14 +25,16 @@ def cf_HEALPix_bounds( a: `numpy.ndarray` The array of HEALPix indices. - healpix_order: `str` - One of ``'nested'``, ``'ring'``, or ``'nuniq'``. + index_scheme: `str` + The HEALPix indexing scheme. One of ``'nested'``, + ``'ring'``, or ``'nuniq'``. refinement_level: `int` or `None`, optional For a ``'nested'`` or ``'ring'`` ordered grid, the refinement level of the grid within the HEALPix hierarchy, - starting at 0 for the base tesselation with 12 cells. - Ignored for a ``'nuniq'`` ordered grid. + starting at 0 for the base tesselation with 12 cells. Set + to `None` for a ``'nuniq'`` ordered grid, for which the + refinement level is ignored. lat: `bool`, optional If True then return latitude bounds. @@ -73,7 +75,7 @@ def cf_HEALPix_bounds( elif lon: pos = 0 - if healpix_order == "nested": + if index_scheme == "nested": bounds_func = healpix._chp.nest2ang_uv else: bounds_func = healpix._chp.ring2ang_uv @@ -89,7 +91,7 @@ def cf_HEALPix_bounds( # Initialise the output bounds array b = np.empty((a.size, 4), dtype="float64") - if healpix_order == "nuniq": + if index_scheme == "nuniq": # Create bounds for 'nuniq' cells orders, a = healpix.uniq2pix(a, nest=False) orders, index, inverse = np.unique( @@ -132,7 +134,7 @@ def cf_HEALPix_bounds( def cf_HEALPix_change_order( - a, healpix_order, new_healpix_order, refinement_level + a, index_scheme, new_index_scheme, refinement_level ): """Change the ordering of HEALPix indices. @@ -146,13 +148,13 @@ def cf_HEALPix_change_order( a: `numpy.ndarray` The array of HEALPix indices. - healpix_order: `str` - The original HEALPix order. One of ``'nested'`` or - ``'ring'``. + index_scheme: `str` + The original HEALPix indexing scheme. One of ``'nested'`` + or ``'ring'``. - new_healpix_order: `str` - The new HEALPix order to change to. One of ``'nested'``, - ``'ring'``, or ``'nuniq'``. + new_index_scheme: `str` + The new HEALPix indexing scheme to change to. One of + ``'nested'``, ``'ring'``, or ``'nuniq'``. refinement_level: `int` The refinement level of the original grid within the @@ -174,29 +176,29 @@ def cf_HEALPix_change_order( """ import healpix - if healpix_order == "nested": - if new_healpix_order == "ring": + if index_scheme == "nested": + if new_index_scheme == "ring": return healpix.nest2ring(healpix.order2nside(refinement_level), a) - if new_healpix_order == "nuniq": + if new_index_scheme == "nuniq": return healpix._chp.nest2uniq(refinement_level, a) - elif healpix_order == "ring": - if new_healpix_order == "nested": + elif index_scheme == "ring": + if new_index_scheme == "nested": return healpix.ring2nest(healpix.order2nside(refinement_level), a) - if new_healpix_order == "nuniq": + if new_index_scheme == "nuniq": return healpix._chp.ring2uniq(refinement_level, a) else: raise ValueError( "Can't change HEALPix order: Can only change from HEALPix " - f"order 'nested' or 'ring'. Got {healpix_order!r}" + f"order 'nested' or 'ring'. Got {index_scheme!r}" ) def cf_HEALPix_coordinates( - a, healpix_order, refinement_level=None, lat=False, lon=False + a, index_scheme, refinement_level=None, lat=False, lon=False ): """Calculate HEALPix cell coordinates. @@ -209,8 +211,9 @@ def cf_HEALPix_coordinates( a: `numpy.ndarray` The array of HEALPix indices. - healpix_order: `str` - One of ``'nested'``, ``'ring'``, or ``'nuniq'``. + index_scheme: `str` + The HEALPix indexing scheme. One of ``'nested'``, + ``'ring'``, or ``'nuniq'``. refinement_level: `int` or `None`, optional For a ``'nested'`` or ``'ring'`` ordered grid, the @@ -250,7 +253,7 @@ def cf_HEALPix_coordinates( elif lon: pos = 0 - if healpix_order == "nuniq": + if index_scheme == "nuniq": # Create coordinates for 'nuniq' cells c = np.empty(a.shape, dtype="float64") @@ -267,7 +270,7 @@ def cf_HEALPix_coordinates( )[pos] else: # Create coordinates for 'nested' or 'ring' cells - nest = (healpix_order == "nested",) + nest = (index_scheme == "nested",) nside = healpix.order2nside(refinement_level) c = healpix.pix2ang( nside=nside, diff --git a/cf/domain.py b/cf/domain.py index 4aef023f03..46d8c1b36f 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -265,7 +265,7 @@ def create_regular(cls, x_args, y_args, bounds=True): @classmethod def create_healpix( - cls, refinement_level, healpix_order="nested", radius=None + cls, refinement_level, index_scheme="nested", radius=None ): """Create a new global HEALPix domain. @@ -275,7 +275,15 @@ def create_healpix( :Parameters: - TODOHEALPIX + refinement_level: `int` + The refinement level of the grid within the HEALPix + hierarchy, starting at 0 for the base tesselation with + 12 cells. The number of cells in the global HEALPix + grid is :math:`(12 \times 4^refinement_level)`. + + index_scheme: `str` + The HEALPix indexing scheme. One of ``'nested'`` (the + default), ``'ring'``, or ``'nuniq'``. radius: optional Specify the radius of the latitude-longitude plane @@ -314,7 +322,7 @@ def create_healpix( Coordinate reference: grid_mapping_name:healpix Coordinate conversion:grid_mapping_name = healpix - Coordinate conversion:healpix_order = nested + Coordinate conversion:index_scheme = nested Coordinate conversion:refinement_level = 4 Auxiliary Coordinate: healpix_index @@ -332,7 +340,7 @@ def create_healpix( Coordinate reference: grid_mapping_name:healpix Coordinate conversion:grid_mapping_name = healpix - Coordinate conversion:healpix_order = nuniq + Coordinate conversion:index_scheme = nuniq Datum:earth_radius = 6371000.0 Auxiliary Coordinate: healpix_index @@ -345,19 +353,19 @@ def create_healpix( f"Got: {refinement_level!r}" ) - nuniq = healpix_order == "nuniq" + nuniq = index_scheme == "nuniq" if nuniq: - healpix_order = "nested" - elif healpix_order not in ("nested", "ring"): + index_scheme = "nested" + elif index_scheme not in ("nested", "ring"): raise ValueError( - "'healpix_order' must be 'nested', 'ring', or 'nuniq'. " - f"Got: {healpix_order!r}" + "'index_scheme' must be 'nested', 'ring', or 'nuniq'. " + f"Got: {index_scheme!r}" ) domain = Domain() + ncells = 12 * (4**refinement_level) # domain_axis: ncdim%cell - ncells = 12 * (4**refinement_level) d = domain._DomainAxis(ncells) d.nc_set_dimension("cell") axis = domain.set_construct(d, copy=False) @@ -381,7 +389,7 @@ def create_healpix( cr.coordinate_conversion.set_parameters( { "grid_mapping_name": "healpix", - "healpix_order": healpix_order, + "index_scheme": index_scheme, "refinement_level": refinement_level, } ) @@ -389,7 +397,7 @@ def create_healpix( domain.set_construct(cr) if nuniq: - # Change from nested to nuniq order + # Change from 'nested' to 'nuniq' indexing scheme domain = domain.healpix_change_order("nuniq") return domain diff --git a/cf/field.py b/cf/field.py index 2dd4fc2558..0da76aae34 100644 --- a/cf/field.py +++ b/cf/field.py @@ -179,7 +179,7 @@ # -------------------------------------------------------------------- _collapse_ddof_methods = set(("sd", "var")) -#_earth_radius = Data(6371229.0, "m") +# _earth_radius = Data(6371229.0, "m") _relational_methods = ( "__eq__", @@ -2629,111 +2629,111 @@ def get_domain(self): return domain -# def radius(self, default=None): -# """Return the radius of a latitude-longitude plane defined in -# spherical polar coordinates. -# -# The radius is taken from the datums of any coordinate -# reference constructs, but if and only if this is not possible -# then a default value may be used instead. -# -# .. versionadded:: 3.0.2 -# -# .. seealso:: `bin`, `cell_area`, `collapse`, `weights` -# -# :Parameters: -# -# default: optional -# The radius is taken from the datums of any coordinate -# reference constructs, but if and only if this is not -# possible then the value set by the *default* parameter -# is used. May be set to any numeric scalar object, -# including `numpy` and `Data` objects. The units of the -# radius are assumed to be metres, unless specified by a -# `Data` object. If the special value ``'earth'`` is -# given then the default radius taken as 6371229 -# metres. If *default* is `None` an exception will be -# raised if no unique datum can be found in the -# coordinate reference constructs. -# -# *Parameter example:* -# Five equivalent ways to set a default radius of -# 6371200 metres: ``6371200``, -# ``numpy.array(6371200)``, ``cf.Data(6371200)``, -# ``cf.Data(6371200, 'm')``, ``cf.Data(6371.2, -# 'km')``. -# -# :Returns: -# -# `Data` -# The radius of the sphere, in units of metres. -# -# **Examples** -# -# >>> f.radius() -# -# -# >>> g.radius() -# ValueError: No radius found in coordinate reference constructs and no default provided -# >>> g.radius('earth') -# -# >>> g.radius(1234) -# -# -# """ -# radii = [] -# for cr in self.coordinate_references(todict=True).values(): -# r = cr.datum.get_parameter("earth_radius", None) -# if r is not None: -# r = Data.asdata(r) -# if not r.Units: -# r.override_units("m", inplace=True) -# -# if r.size != 1: -# radii.append(r) -# continue -# -# got = False -# for _ in radii: -# if r == _: -# got = True -# break -# -# if not got: -# radii.append(r) -# -# if len(radii) > 1: -# raise ValueError( -# "Multiple radii found from coordinate reference " -# f"constructs: {radii!r}" -# ) -# -# if not radii: -# if default is None: -# raise ValueError( -# "No radius found from coordinate reference constructs " -# "and no default provided" -# ) -# -# if isinstance(default, str): -# if default != "earth": -# raise ValueError( -# "The default radius must be numeric, 'earth', " -# "or None" -# ) -# -# return _earth_radius.copy() -# -# r = Data.asdata(default).squeeze() -# else: -# r = Data.asdata(radii[0]).squeeze() -# -# if r.size != 1: -# raise ValueError(f"Multiple radii: {r!r}") -# -# r.Units = Units("m") -# r.dtype = float -# return r + # def radius(self, default=None): + # """Return the radius of a latitude-longitude plane defined in + # spherical polar coordinates. + # + # The radius is taken from the datums of any coordinate + # reference constructs, but if and only if this is not possible + # then a default value may be used instead. + # + # .. versionadded:: 3.0.2 + # + # .. seealso:: `bin`, `cell_area`, `collapse`, `weights` + # + # :Parameters: + # + # default: optional + # The radius is taken from the datums of any coordinate + # reference constructs, but if and only if this is not + # possible then the value set by the *default* parameter + # is used. May be set to any numeric scalar object, + # including `numpy` and `Data` objects. The units of the + # radius are assumed to be metres, unless specified by a + # `Data` object. If the special value ``'earth'`` is + # given then the default radius taken as 6371229 + # metres. If *default* is `None` an exception will be + # raised if no unique datum can be found in the + # coordinate reference constructs. + # + # *Parameter example:* + # Five equivalent ways to set a default radius of + # 6371200 metres: ``6371200``, + # ``numpy.array(6371200)``, ``cf.Data(6371200)``, + # ``cf.Data(6371200, 'm')``, ``cf.Data(6371.2, + # 'km')``. + # + # :Returns: + # + # `Data` + # The radius of the sphere, in units of metres. + # + # **Examples** + # + # >>> f.radius() + # + # + # >>> g.radius() + # ValueError: No radius found in coordinate reference constructs and no default provided + # >>> g.radius('earth') + # + # >>> g.radius(1234) + # + # + # """ + # radii = [] + # for cr in self.coordinate_references(todict=True).values(): + # r = cr.datum.get_parameter("earth_radius", None) + # if r is not None: + # r = Data.asdata(r) + # if not r.Units: + # r.override_units("m", inplace=True) + # + # if r.size != 1: + # radii.append(r) + # continue + # + # got = False + # for _ in radii: + # if r == _: + # got = True + # break + # + # if not got: + # radii.append(r) + # + # if len(radii) > 1: + # raise ValueError( + # "Multiple radii found from coordinate reference " + # f"constructs: {radii!r}" + # ) + # + # if not radii: + # if default is None: + # raise ValueError( + # "No radius found from coordinate reference constructs " + # "and no default provided" + # ) + # + # if isinstance(default, str): + # if default != "earth": + # raise ValueError( + # "The default radius must be numeric, 'earth', " + # "or None" + # ) + # + # return _earth_radius.copy() + # + # r = Data.asdata(default).squeeze() + # else: + # r = Data.asdata(radii[0]).squeeze() + # + # if r.size != 1: + # raise ValueError(f"Multiple radii: {r!r}") + # + # r.Units = Units("m") + # r.dtype = float + # return r def laplacian_xy( self, x_wrap=None, one_sided_at_boundary=False, radius=None diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 5eabc089a4..454a20f7a7 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -203,7 +203,7 @@ def _equivalent_coordinate_references( return False return True - + def _indices(self, config, data_axes, ancillary_mask, kwargs): """Create indices that define a subspace of the field or domain construct. @@ -296,10 +296,10 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): f"non-negative integer. Got {halo!r}" ) - # Create any implied 1-d latitude and longitude coordinates - # (e.g. as implied by HEALPix indices). Do not do this - # in-place. - self = self.create_latlon_coordinates(one_d=True, two_d=False) + # Create any implied latitude and longitude coordinates + # (e.g. as implied by a non-latitude_longitude grid mapping + # coordinate reference). Do not do this in-place. + self = self.create_latlon_coordinates() domain_axes = self.domain_axes(todict=True) @@ -375,15 +375,10 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): n_items = len(constructs) n_axes = len(canonical_axes) - if n_items > n_axes: - if n_axes == 1: - a = "axis" - else: - a = "axes" - + if n_axes > 1 and n_items > n_axes: raise ValueError( f"Error: Can't specify {n_items} conditions for " - f"{n_axes} {a}: {points}. Consider applying the " + f"{n_axes} axes: {points}. Consider applying the " "conditions separately." ) @@ -402,150 +397,217 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): # ---------------------------------------------------- # 1-d construct # ---------------------------------------------------- - ind = None - axis = item_axes[0] - item = constructs[0] - value = points[0] - identity = identities[0] + size = domain_axes[axis].get_size() - if debug: - logger.debug( - f" {n_items} 1-d constructs: {constructs!r}\n" - f" item = {item!r}\n" - f" axis = {axis!r}\n" - f" value = {value!r}\n" - f" identity = {identity!r}" - ) # pragma: no cover + ind0 = None + index0 = None - if isinstance(value, (list, slice, tuple, np.ndarray)): - # 1-d CASE 1: Value is already an index, e.g. [0], - # [7,4,2], slice(0,4,2), - # numpy.array([2,4,7]), - # [True,False,True] - if debug: - logger.debug(" 1-d CASE 1:") # pragma: no cover - - index = value - - if envelope or full: - # Set ind - size = domain_axes[axis].get_size() - ind = (np.arange(size)[value],) - # Placeholder which will be overwritten later - index = None - - elif ( - item is not None - and isinstance(value, Query) - and value.operator in ("wi", "wo") - and item.construct_type == "dimension_coordinate" - and self.iscyclic(axis) + for item, value, identity in zip( + constructs, points, identities ): - # 1-d CASE 2: Axis is cyclic and subspace - # criterion is a 'within' or 'without' - # Query instance if debug: - logger.debug(" 1-d CASE 2:") # pragma: no cover - - size = item.size - if item.increasing: - anchor = value.value[0] - else: - anchor = value.value[1] - - item = item.persist() - parameters = {} - item = item.anchor(anchor, parameters=parameters) - n = np.roll(np.arange(size), parameters["shift"], 0) - if value.operator == "wi": - n = n[item == value] - if not n.size: - raise ValueError( - f"No indices found from: {identity}={value!r}" - ) - - start = n[0] - stop = n[-1] + 1 - else: - # "wo" operator - n = n[item == wi(*value.value)] - if n.size == size: - raise ValueError( - f"No indices found from: {identity}={value!r}" - ) + logger.debug( + f" axis = {axis!r}\n" + f" item = {item!r}\n" + f" value = {value!r}\n" + f" identity = {identity!r}" + ) # pragma: no cover - if n.size: - start = n[-1] + 1 - stop = start - n.size + ind = None + + if isinstance(value, (list, slice, tuple, np.ndarray)): + # 1-d CASE 1: Value is already an index, + # e.g. [0], [7,4,2], slice(0,4,2), + # numpy.array([2,4,7]), + # [True,False,True] + if debug: + logger.debug(" 1-d CASE 1:") # pragma: no cover + + index = value + + if envelope or full: + # Set ind + ind = np.zeros((size,), bool) + ind[index] = True + # Placeholder to be overwritten later + index = None + + if n_items > 1 and index is not None: + # Convert 'index' to a boolean array + i = np.zeros((size,), bool) + i[index] = True + index = i + + elif ( + item is not None + and isinstance(value, Query) + and value.operator in ("wi", "wo") + and item.construct_type == "dimension_coordinate" + and self.iscyclic(axis) + ): + # 1-d CASE 2: Axis is cyclic and subspace + # criterion is a 'within' or + # 'without' Query instance + if debug: + logger.debug(" 1-d CASE 2:") # pragma: no cover + + size = item.size + if item.increasing: + anchor = value.value[0] else: - start = size - parameters["shift"] - stop = start + size - if stop > size: - stop -= size - - index = slice(start, stop, 1) - - if full: - # Set ind - try: - index = normalize_slice(index, size, cyclic=True) - except IndexError: - # Index is not a cyclic slice - ind = (np.arange(size)[index],) + anchor = value.value[1] + + item = item.persist() + parameters = {} + item = item.anchor(anchor, parameters=parameters) + n = np.roll(np.arange(size), parameters["shift"], 0) + if value.operator == "wi": + n = n[item == value] + if not n.size: + raise ValueError( + "No indices found from: " + f"{identity}={value!r}" + ) + + start = n[0] + stop = n[-1] + 1 else: - # Index is a cyclic slice - ind = ( - np.arange(size)[ + # "wo" operator + n = n[item == wi(*value.value)] + if n.size == size: + raise ValueError( + "No indices found from: " + f"{identity}={value!r}" + ) + + if n.size: + start = n[-1] + 1 + stop = start - n.size + else: + start = size - parameters["shift"] + stop = start + size + if stop > size: + stop -= size + + index = slice(start, stop, 1) + + if envelope or full: + # Set ind + try: + index = normalize_slice( + index, size, cyclic=True + ) + except IndexError: + # Index is not a cyclic slice + ind = np.zeros((size,), bool) + ind[index] = True + # np.arange(size)[index] + else: + # Index is a cyclic slice + if n_items > 1: + raise ValueError( + "Error: Can't specify multiple " + "conditions for a single axis when " + f"one of those condtions ({value!r}) " + "is effectively a cyclic slice: " + f"{index}. Consider applying the " + "conditions separately." + ) + + ind = np.zeros((size,), bool) + ind[ np.arange( index.start, index.stop, index.step ) - ], - ) + ] = True - # Placeholder which will be overwritten later - index = None + # Placeholder to be overwritten later + index = None - elif item is not None: - # 1-d CASE 3: All other 1-d cases - if debug: - logger.debug(" 1-d CASE 3:") # pragma: no cover + if n_items > 1 and index is not None: + # Convert 'index' to a boolean array + i = np.zeros((size,), bool) + i[index] = True + index = i - index = item == value + elif item is not None: + # 1-d CASE 3: All other 1-d cases + if debug: + logger.debug(" 1-d CASE 3:") # pragma: no cover - # Performance: Convert the 1-d 'index' to a numpy - # array of bool. - # - # This is because Dask can be *very* slow at - # instantiation time when the 'index' is a Dask - # array, in which case contents of 'index' are - # unknown. - index = np.asanyarray(index) + index = item == value - if envelope or full: - # Set ind + # Performance: Convert the 1-d 'index' to a + # numpy array of bool. + # + # This is because Dask can be *very* slow at + # instantiation time when the 'index' is a + # Dask array, in which case contents of + # 'index' are unknown. index = np.asanyarray(index) - if np.ma.isMA(index): - ind = np.ma.where(index) - else: - ind = np.where(index) - # Placeholder which will be overwritten later - index = None + if envelope or full: + # Set ind + index = np.asanyarray(index) + if np.ma.isMA(index): + ind = np.ma.where(index)[0] + else: + ind = np.where(index)[0] + + # Placeholder to be overwritten later + index = None + + if n_items > 1 and ind is not None: + # Convert 'ind' to a boolean array (note + # that 'index' is already a boolean + # array) + i = np.zeros((size,), bool) + i[ind] = True + ind = i + else: - # Convert bool to int, to save memory. - size = domain_axes[axis].get_size() - index = normalize_index(index, (size,))[0] - else: - raise ValueError( - "Could not find a unique construct with identity " - f"{identity!r} from which to infer the indices." - ) + raise ValueError( + "Could not find a unique construct with identity " + f"{identity!r} from which to infer the indices." + ) - if debug: - logger.debug( - f" index = {index}\n ind = {ind}" - ) # pragma: no cover + if debug: + logger.debug( + f" index = {index}\n ind = {ind}" + ) # pragma: no cover + + if n_items > 1: + # Update the 'ind0' and 'index0' boolean + # arrays with the latest 'ind' and 'index' + if ind is not None: + # Note that 'index' must be None when + # 'ind' is not None, so no need to update + # 'index0' in this case. + if ind0 is None: + ind0 = ind + else: + ind0 &= ind + else: + if index0 is None: + index0 = index + else: + index0 &= index + + # Finalise 'ind' and 'index' + if n_items > 1: + if ind0 is not None: + ind = ind0 + + if index0 is not None: + index = index0 + + if ind is not None: + ind = normalize_index(ind, (size,))[0] + ind = (ind,) + + if index is not None and getattr(index, "dtype", None) == bool: + index = normalize_index(index, (size,))[0] # Put the index into the correct place in the list of # indices. @@ -1928,7 +1990,7 @@ def healpix_axis(self, default=ValueError()): return self.get_data_axes(key)[0] - def healpix_change_order(self, new_healpix_order, sort=False): + def healpix_change_order(self, new_index_scheme, sort=False): """Change the ordering of HEALPix indices. .. versionadded:: NEXTVERSION @@ -1937,8 +1999,8 @@ def healpix_change_order(self, new_healpix_order, sort=False): :Parameters: - new_healpix_order: `str` - The new HEALPix order. One of ``'nested'``, + new_index_scheme: `str` + The new HEALPix indexing scheme. One of ``'nested'``, ``'ring'``, or ``'nuniq'``. sort: `bool`, optional @@ -1965,7 +2027,7 @@ def healpix_change_order(self, new_healpix_order, sort=False): : height(1) = [1.5] m Auxiliary coords: healpix_index(healpix_index(48)) = [0, ..., 47] 1 Coord references: grid_mapping_name:healpix - >>> f.coordinate_reference('grid_mapping_name:healpix').coordinate_conversion.get_parameter('healpix_order') + >>> f.coordinate_reference('grid_mapping_name:healpix').coordinate_conversion.get_parameter('index_scheme') 'nested' >>> print(f.coordinate('healpix_index').array) [ 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 @@ -1999,10 +2061,10 @@ def healpix_change_order(self, new_healpix_order, sort=False): f = self.copy() - if new_healpix_order not in ("nested", "ring", "nuniq"): + if new_index_scheme not in ("nested", "ring", "nuniq"): raise ValueError( - "Can't change HEALPix order: new_healpix_order must be " - f"'nested', 'ring', or 'nuniq'. Got {new_healpix_order!r}" + "Can't change HEALPix order: new_index_scheme must be " + f"'nested', 'ring', or 'nuniq'. Got {new_index_scheme!r}" ) # Parse the HEALPix coordinate reference @@ -2022,16 +2084,16 @@ def healpix_change_order(self, new_healpix_order, sort=False): "has not been set" ) - healpix_order = parameters.get("healpix_order") - if healpix_order is None: + index_scheme = parameters.get("index_scheme") + if index_scheme is None: raise ValueError( - "Can't change HEALPix order: healpix_order has not been set" + "Can't change HEALPix order: index_scheme has not been set" ) - if healpix_order not in ("nested", "ring"): + if index_scheme not in ("nested", "ring"): raise ValueError( "Can't change HEALPix order: Can only change from " - f"healpix_order 'nested' or 'ring'. Got {healpix_order!r}" + f"index_scheme 'nested' or 'ring'. Got {index_scheme!r}" ) # Get the original healpix_index coordinates and the key of @@ -2050,29 +2112,29 @@ def healpix_change_order(self, new_healpix_order, sort=False): axis = f.get_data_axes(hp_key)[0] - if healpix_order != new_healpix_order: + if index_scheme != new_index_scheme: # Change the HEALPix indices dx = healpix_index.to_dask_array() dx = dx.map_blocks( cf_HEALPix_change_order, meta=np.array((), dtype="int64"), - healpix_order=healpix_order, - new_healpix_order=new_healpix_order, + index_scheme=index_scheme, + new_index_scheme=new_index_scheme, refinement_level=refinement_level, ) healpix_index.set_data(dx, copy=False) # Update the Coordinate Reference cr.coordinate_conversion.set_parameter( - "healpix_order", new_healpix_order + "index_scheme", new_index_scheme ) - if new_healpix_order == "nuniq": + if new_index_scheme == "nuniq": cr.coordinate_conversion.del_parameter( "refinement_level", None ) if healpix_index.construct_type == "dimension_coordinate": - # Convert healpix_indices to auxiliary coordinates + # Convert healpix indices to auxiliary coordinates healpix_index = f._AuxiliaryCoordinate( source=healpix_index, copy=False ) @@ -2229,7 +2291,7 @@ def healpix_to_ugrid(self, inplace=False): elif cr is not None: cc = cr.coordinate_conversion cc.del_parameter("grid_mapping_name", None) - cc.del_parameter("healpix_order", None) + cc.del_parameter("index_scheme", None) cc.del_parameter("refinement_level", None) if cr.coordinate_conversion.parameters() or cr.datum.parameters(): # The Coordinate Reference contains generic coordinate @@ -2374,18 +2436,18 @@ def create_latlon_coordinates( # HEALPix 1-d coordinates # -------------------------------------------------------- parameters = cr.coordinate_conversion.parameters() - healpix_order = parameters.get("healpix_order") - if healpix_order not in ("nested", "ring", "nuniq"): + index_scheme = parameters.get("index_scheme") + if index_scheme not in ("nested", "ring", "nuniq"): if is_log_level_info(logger): logger.info( "Can't create 1-d latitude and longitude coordinates: " - f"Invalid HEALPix order: {healpix_order!r}" + f"Invalid HEALPix order: {index_scheme!r}" ) # pragma: no cover return f refinement_level = parameters.get("refinement_level") - if refinement_level is None and healpix_order in ( + if refinement_level is None and index_scheme in ( "nested", "ring", ): @@ -2444,7 +2506,7 @@ def create_latlon_coordinates( dy = dx.map_blocks( cf_HEALPix_coordinates, meta=meta, - healpix_order=healpix_order, + index_scheme=index_scheme, refinement_level=refinement_level, lat=True, ) @@ -2458,7 +2520,7 @@ def create_latlon_coordinates( dy = dx.map_blocks( cf_HEALPix_coordinates, meta=meta, - healpix_order=healpix_order, + index_scheme=index_scheme, refinement_level=refinement_level, lon=True, ) @@ -2476,7 +2538,7 @@ def create_latlon_coordinates( "i", new_axes={"j": 4}, meta=meta, - healpix_order=healpix_order, + index_scheme=index_scheme, refinement_level=refinement_level, lat=True, ) @@ -2491,7 +2553,7 @@ def create_latlon_coordinates( "i", new_axes={"j": 4}, meta=meta, - healpix_order=healpix_order, + index_scheme=index_scheme, refinement_level=refinement_level, lon=True, ) @@ -3173,7 +3235,7 @@ def radius(self, default=None): .. versionadded:: 3.0.2 - .. seealso:: `bin`, `cell_area`, `collapse`, `weights` + .. seealso:: `bin`, `cell_area`, `collapse` :Parameters: @@ -3683,12 +3745,13 @@ def set_coordinate_reference( for value in coordinate_reference.coordinates(): if value in coordinates: identity = coordinates[value].identity(strict=strict) - key = self.coordinate(identity, key=True, default=None) - if key is not None: - ckeys.append(key) + ckey = self.coordinate(identity, key=True, default=None) + if ckey is not None: + ckeys.append(ckey) ref.clear_coordinates() - ref.set_coordinates(ckeys) + if ckeys: + ref.set_coordinates(ckeys) coordinate_conversion = coordinate_reference.coordinate_conversion diff --git a/cf/regrid/regrid.py b/cf/regrid/regrid.py index ed89fa1df4..2b18b91b01 100644 --- a/cf/regrid/regrid.py +++ b/cf/regrid/regrid.py @@ -131,10 +131,11 @@ class Grid: # The integer position in *coords* of a vertical coordinate. If # `None` then there are no vertical coordinates. z_index: Any = None - # TODOHEALPIX - original_domain: Any = None - # Whether or not the original domain is a HEALPix grid - original_healpix: bool = False + # The original field/domain before any transformations are applied + # (such as creating lat/lon cooridnates, or converting to UGRID). + domain: Any = None + # Whether or not the field/original domain is a HEALPix grid + healpix: bool = False def regrid( @@ -342,7 +343,7 @@ def regrid( """ if not inplace: src = src.copy() - + spherical = coord_sys == "spherical" cartesian = not spherical @@ -507,22 +508,24 @@ def regrid( conform_coordinates(src_grid, dst_grid) if method == "conservative": - if src_grid.original_healpix or dst_grid.original_healpix: + if src_grid.healpix or dst_grid.healpix: raise ValueError( f"{method!r} regridding is not available for HEALPix grids" ) - + elif method in ("conservative_2nd", "patch"): - if method == "conservative_2nd" and (src_grid.original_healpix or dst_grid.original_healpix): + if method == "conservative_2nd" and ( + src_grid.healpix or dst_grid.healpix + ): raise ValueError( f"{method!r} regridding is not available for HEALPix grids" ) - + if not (src_grid.dimensionality >= 2 and dst_grid.dimensionality >= 2): raise ValueError( f"{method!r} regridding is not available for 1-d regridding" ) - + elif method in ("nearest_dtos", "nearest_stod"): if not has_coordinate_arrays(src_grid) and not has_coordinate_arrays( dst_grid @@ -669,14 +672,15 @@ def regrid( start_index=start_index, src_axes=src_axes, dst_axes=dst_axes, - dst=dst, + dst=dst_grid.domain, weights_file=weights_file if from_file else None, src_mesh_location=src_grid.mesh_location, src_featureType=src_grid.featureType, dst_featureType=dst_grid.featureType, src_z=src_grid.z, dst_z=dst_grid.z, - ln_z=ln_z, # TODOHEALPIX + ln_z=ln_z, + dst_healpix=dst_grid.healpix, ) else: if weights_file is not None: @@ -730,7 +734,9 @@ def regrid( # ---------------------------------------------------------------- # Update the regridded metadata # ---------------------------------------------------------------- - cr_map = update_non_coordinates(src, dst, src_grid, dst_grid, regrid_operator) + cr_map = update_non_coordinates( + src, dst, src_grid, dst_grid, regrid_operator + ) update_coordinates(src, dst, src_grid, dst_grid, cr_map) @@ -738,7 +744,7 @@ def regrid( # Insert regridded data into the new field # ---------------------------------------------------------------- update_data(src, regridded_data, src_grid, dst_grid) - + if coord_sys == "spherical" and dst_grid.is_grid: # Set the cyclicity of the longitude axis of the new field key, x = src.dimension_coordinate("X", default=(None, None), item=True) @@ -1148,11 +1154,11 @@ def spherical_grid( The grid definition. """ - original_domain = f.copy() - - # Whether or not the original domain is a HEALPix grid - original_healpix = False - + domain = f.copy() + + # Whether or not the original field/domain is a HEALPix grid + healpix = False + # Create any implied lat/lon coordinates in-place try: # Try to convert a HEALPix grid to a UGRID grid (which will @@ -1161,7 +1167,7 @@ def spherical_grid( except ValueError: f.create_latlon_coordinates(inplace=True) else: - original_healpix = True + healpix = True data_axes = f.constructs.data_axes() @@ -1465,8 +1471,8 @@ def spherical_grid( z=z, ln_z=ln_z, z_index=z_index, - original_domain=original_domain, - original_healpix=original_healpix, + domain=domain, + healpix=healpix, ) set_grid_type(grid) @@ -1512,8 +1518,8 @@ def Cartesian_grid(f, name=None, method=None, axes=None, z=None, ln_z=None): The grid definition. """ - original_domain = f.copy() - + domain = f.copy() + if not axes: if name == "source": raise ValueError( @@ -1697,7 +1703,7 @@ def Cartesian_grid(f, name=None, method=None, axes=None, z=None, ln_z=None): z=z, ln_z=ln_z, z_index=z_index, - original_domain=original_domain + domain=domain, ) set_grid_type(grid) @@ -2810,18 +2816,23 @@ def update_coordinates(src, dst, src_grid, dst_grid, cr_map): The definition of the destination grid. cr_map `dict` - TODOHEALPIX + + The mapping of destination coordinate reference identities + to source coordinate reference identities, as output by + `update_non_coordinates`. :Returns: `None` """ - if dst_grid.original_domain is not None: - dst = dst_grid.original_domain + dst = dst_grid.domain + + # An HEALPix grid is converted to UGRID for the regridding, but we + # want the original HEALPic metadata to be copied to the regridded + # source field, rather than the UGRID view of it. + dst_grid_is_mesh = dst_grid.is_mesh and not dst_grid.healpix - dst_grid_is_mesh = dst_grid.is_mesh and not dst_grid.original_healpix - src_axis_keys = src_grid.axis_keys dst_axis_keys = dst_grid.axis_keys @@ -2839,7 +2850,7 @@ def update_coordinates(src, dst, src_grid, dst_grid, cr_map): todict=True, ): src.del_construct(key) - + # Domain axes src_domain_axes = src.domain_axes(todict=True) dst_domain_axes = dst.domain_axes(todict=True) @@ -2923,11 +2934,12 @@ def update_non_coordinates(src, dst, src_grid, dst_grid, regrid_operator): :Returns: `dict` - TODOHEALPIX + The mapping of destination coordinate reference identities + to source coordinate reference identities. """ - if dst_grid.original_domain is not None: - dst = dst_grid.original_domain + # if dst_grid.domain is not None: + dst = dst_grid.domain src_axis_keys = src_grid.axis_keys dst_axis_keys = dst_grid.axis_keys @@ -3022,25 +3034,26 @@ def update_non_coordinates(src, dst, src_grid, dst_grid, regrid_operator): # ---------------------------------------------------------------- # Copy selected coordinate references from the destination grid # - # Define the mapping of destination corodinate references to + # Define the mapping of destination coordinate references to # source coordinate references # ---------------------------------------------------------------- cr_map = {} - + dst_data_axes = dst.constructs.data_axes() for dst_key, ref in dst.coordinate_references(todict=True).items(): axes = set() for c_key in ref.coordinates(): axes.update(dst_data_axes[c_key]) - + if axes and set(axes).issubset(dst_axis_keys): src_cr = ref.copy() src_cr.clear_coordinates() src_key = src.set_construct(src_cr) cr_map[dst_key] = src_key - + return cr_map + def update_data(src, regridded_data, src_grid, dst_grid): """Insert the regridded field data. diff --git a/cf/regrid/regridoperator.py b/cf/regrid/regridoperator.py index 7621addc7e..001afe10ae 100644 --- a/cf/regrid/regridoperator.py +++ b/cf/regrid/regridoperator.py @@ -50,6 +50,7 @@ def __init__( src_z=None, dst_z=None, ln_z=False, + dst_healpix=False, ): """**Initialisation** @@ -124,7 +125,8 @@ def __init__( Use keyword parameters instead. dst: `Field` or `Domain` - The definition of the destination grid. + The definition of the destination grid from which the + weights were calculated. dst_axes: `dict` or sequence or `None`, optional The destination grid axes to be regridded. @@ -191,6 +193,11 @@ def __init__( .. versionadded:: 3.16.2 + dst_healpix: `bool`, optional + Whether or not the destination grid is HEALPix grid. + + .. versionadded:: NEXTVERSION + """ super().__init__() @@ -224,6 +231,7 @@ def __init__( self._set_component("src_z", src_z, copy=False) self._set_component("dst_z", dst_z, copy=False) self._set_component("ln_z", bool(ln_z), copy=False) + self._set_component("dst_healpix", bool(dst_healpix), copy=False) def __repr__(self): """x.__repr__() <==> repr(x)""" @@ -322,6 +330,15 @@ def dst_mesh_location(self): """ return self._get_component("dst_mesh_location") + @property + def dst_healpix(self): + """Whether or not the destination grid is HEALPix grid. + + .. versionadded:: NEXTVERSION + + """ + return self._get_component("dst_healpix") + @property def dst_shape(self): """The shape of the destination grid. @@ -549,6 +566,7 @@ def copy(self): src_z=self.src_z, dst_z=self.dst_z, ln_z=self.ln_z, + dst_healpix=self.dst_healpix, ) @_display_or_return diff --git a/cf/test/test_Domain.py b/cf/test/test_Domain.py index c42d806574..5a4734590b 100644 --- a/cf/test/test_Domain.py +++ b/cf/test/test_Domain.py @@ -486,6 +486,33 @@ def test_Domain_cyclic_iscyclic(self): d2.cyclic("X", iscyclic=False) self.assertTrue(d2.iscyclic("X")) + def test_Domain_create_healpix(self): + """Test Domain.create_healpix.""" + d = cf.Domain.create_healpix(0) + self.assertEqual(len(d.constructs), 3) + self.assertEqual(len(d.domain_axes()), 1) + self.assertEqual(len(d.auxiliary_coordinates()), 1) + self.assertEqual(len(d.coordinate_references()), 1) + + self.assertTrue( + (d.auxiliary_coordinate().array == np.arange(12)).all() + ) + + d = cf.Domain.create_healpix(0, "nuniq") + self.assertTrue( + (d.auxiliary_coordinate().array == np.arange(4, 16)).all() + ) + self.assertIsNone( + d.coordinate_reference().datum.get_parameter("earth_radius", None) + ) + + for radius in (1000, cf.Data(1, "km")): + d = cf.Domain.create_healpix(0, "ring", radius=radius) + self.assertEqual( + d.coordinate_reference().datum.get_parameter("earth_radius"), + 1000, + ) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index 1fdeafaa97..025f53be7e 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -965,10 +965,11 @@ def test_Field_radius(self): with self.assertRaises(ValueError): f.radius() - for default in ("earth", cf.field._earth_radius): + earth_radius = cf.Data(6371229, "m") + for default in ("earth", earth_radius): r = f.radius(default=default) self.assertEqual(r.Units, cf.Units("m")) - self.assertEqual(r, cf.field._earth_radius) + self.assertEqual(r, earth_radius) a = cf.Data(1234, "m") for default in ( @@ -1160,6 +1161,7 @@ def test_Field_indices(self): x = f.dimension_coordinate("X") x[...] = np.arange(0, 360, 40) + x.set_property("long_name", "grid_longitude") x.set_bounds(x.create_bounds()) f.cyclic("X", iscyclic=True, period=360) @@ -1441,6 +1443,55 @@ def test_Field_indices(self): with self.assertRaises(ValueError): f.indices(grid_longitude=cf.wo(-180, 180)) + # Multiple conditions for one axis + axis = f.get_data_axes("grid_longitude")[0] + + indices = f.indices( + **{"grid_longitude": cf.wi(0, 360), axis: [1, 3, 5, 6]} + ) + g = f[indices] + x = g.dimension_coordinate("X").array + self.assertEqual(g.shape, (1, 10, 4)) + self.assertTrue((x == [40, 120, 200, 240]).all()) + + indices = f.indices( + **{"grid_longitude": cf.wi(0, 180), axis: [1, 3, 5, 6]} + ) + g = f[indices] + x = g.dimension_coordinate("X").array + self.assertEqual(g.shape, (1, 10, 2)) + self.assertTrue((x == [40, 120]).all()) + + indices = f.indices( + "envelope", **{"grid_longitude": cf.wi(0, 180), axis: [1, 3, 5, 6]} + ) + g = f[indices] + x = g.dimension_coordinate("X").array + self.assertEqual(g.shape, (1, 10, 3)) + self.assertTrue((x == [40, 80, 120]).all()) + + indices = f.indices( + "full", **{"grid_longitude": cf.wi(0, 180), axis: [1, 3, 5]} + ) + g = f[indices] + x = g.dimension_coordinate("X").array + self.assertEqual(g.shape, (1, 10, 9)) + self.assertTrue((x == [0, 40, 80, 120, 160, 200, 240, 280, 320]).all()) + + indices = f.indices( + **{ + "grid_longitude": cf.wi(50, 350), + axis: [1, 2, 3, 4, 5, 6, 7], + "long_name=grid_longitude": slice(1, None, 2), + } + ) + g = f[indices] + x = g.dimension_coordinate("X").array + print(x) + self.assertEqual(g.shape, (1, 10, 3)) + self.assertTrue((x == [120, 200, 280]).all()) + + cf.log_level(1) # 2-d lon = f.construct("longitude").array lon = np.transpose(lon) @@ -2926,15 +2977,15 @@ def test_Field_auxiliary_to_dimension_to_auxiliary(self): def test_Field_subspace_ugrid(self): f = cf.read(self.ugrid_global)[0] - with self.assertRaises(ValueError): - # Can't specify 2 conditions for 1 axis - g = f.subspace(X=cf.wi(40, 70), Y=cf.wi(-20, 30)) - g = f.subspace(X=cf.wi(40, 70)) g = g.subspace(Y=cf.wi(-20, 30)) self.assertTrue(g.aux("X").data.range() < 30) self.assertTrue(g.aux("Y").data.range() < 50) + g = f.subspace(X=cf.wi(40, 70), Y=cf.wi(-20, 30)) + self.assertTrue(g.aux("X").data.range() < 30) + self.assertTrue(g.aux("Y").data.range() < 50) + def test_Field_pad_missing(self): """Test Field.pad_missing.""" f = cf.example_field(0) @@ -3131,6 +3182,19 @@ def test_Field_create_latlon_coordinates(self): self.assertIsNone(f.create_latlon_coordinates(inplace=True)) self.assertTrue(f.equals(g)) + def test_Field_subspace_healpix(self): + """Test Field.subspace for HEALPix""" + f = self.f12 + g = f.subspace(X=cf.wi(40, 70), Y=cf.wi(-20, 30)) + self.assertTrue(np.allclose(g.aux("healpix_index"), [0, 22, 35])) + g.create_latlon_coordinates(inplace=True) + self.assertTrue(np.allclose(g.aux("X"), [45.0, 67.5, 45.0])) + self.assertTrue( + np.allclose( + g.aux("Y"), [19.47122063449069, 0.0, -19.47122063449069] + ) + ) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) diff --git a/cf/test/test_RegridOperator.py b/cf/test/test_RegridOperator.py index dae6be8ce8..94577f56c9 100644 --- a/cf/test/test_RegridOperator.py +++ b/cf/test/test_RegridOperator.py @@ -38,6 +38,7 @@ def test_RegridOperator_attributes(self): self.assertIsNone(self.r.src_z) self.assertIsNone(self.r.dst_z) self.assertFalse(self.r.ln_z) + self.assertFalse(self.r.dst_healpix) def test_RegridOperator_copy(self): self.assertIsInstance(self.r.copy(), self.r.__class__) diff --git a/cf/weights.py b/cf/weights.py index 1601536861..ee38566988 100644 --- a/cf/weights.py +++ b/cf/weights.py @@ -2000,19 +2000,19 @@ def healpix_area( ) parameters = cr.coordinate_conversion.parameters() - healpix_order = parameters.get("healpix_order") - if healpix_order not in ("nested", "ring", "nuniq"): + index_scheme = parameters.get("index_scheme") + if index_scheme not in ("nested", "ring", "nuniq"): if auto: return False raise ValueError( - "Can't create weights: Invalid HEALPix healpix_order for " + "Can't create weights: Invalid HEALPix index_scheme for " f"{f.constructs.domain_axis_identity(axis)!r} axis: " - f"{healpix_order!r}" + f"{index_scheme!r}" ) refinement_level = parameters.get("refinement_level") - if refinement_level is None and healpix_order != "nuniq": + if refinement_level is None and index_scheme != "nuniq": # No refinement_level if auto: return False @@ -2025,8 +2025,8 @@ def healpix_area( if measure and not methods and radius is not None: radius = f.radius(default=radius) - # Create weights for 'nuniq' ordering - if healpix_order == "nuniq": + # Create weights for 'nuniq' indexed cells + if index_scheme == "nuniq": if methods: weights[(axis,)] = "HEALPix Multi-Order Coverage" return True @@ -2069,7 +2069,7 @@ def healpix_area( return True # Still here? Then create weights for 'nested' or 'ring' - # ordering. + # indexed cells. if methods: if measure: weights[(axis,)] = "HEALPix equal area" From c672208109e7abbbd7bf669177b577c14715cee9 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 3 Jul 2025 19:22:34 +0100 Subject: [PATCH 15/59] dev --- cf/dask_utils.py | 148 ++++++++++++++-------- cf/domain.py | 32 ++--- cf/field.py | 224 +++++++++++++++++++++------------ cf/healpix_utils.py | 69 ++++++++++ cf/mixin/fielddomain.py | 180 ++++++++++++-------------- cf/test/test_Domain.py | 2 +- cf/test/test_Field.py | 77 ++++++++++-- cf/test/test_collapse.py | 26 +++- cf/test/test_weights.py | 16 +++ cf/weights.py | 23 ++-- docs/source/field_analysis.rst | 2 +- docs/source/installation.rst | 6 +- 12 files changed, 528 insertions(+), 277 deletions(-) create mode 100644 cf/healpix_utils.py diff --git a/cf/dask_utils.py b/cf/dask_utils.py index 34fc42fe09..70421c6afa 100644 --- a/cf/dask_utils.py +++ b/cf/dask_utils.py @@ -8,8 +8,8 @@ import numpy as np -def cf_HEALPix_bounds( - a, index_scheme, refinement_level=None, lat=False, lon=False +def cf_healpix_bounds( + a, indexing_scheme, refinement_level=None, lat=False, lon=False ): """Calculate HEALPix cell bounds. @@ -25,15 +25,15 @@ def cf_HEALPix_bounds( a: `numpy.ndarray` The array of HEALPix indices. - index_scheme: `str` + indexing_scheme: `str` The HEALPix indexing scheme. One of ``'nested'``, - ``'ring'``, or ``'nuniq'``. + ``'ring'``, or ``'nested_unique'``. refinement_level: `int` or `None`, optional For a ``'nested'`` or ``'ring'`` ordered grid, the refinement level of the grid within the HEALPix hierarchy, starting at 0 for the base tesselation with 12 cells. Set - to `None` for a ``'nuniq'`` ordered grid, for which the + to `None` for a ``'nested_unique'`` ordered grid, for which the refinement level is ignored. lat: `bool`, optional @@ -49,19 +49,26 @@ def cf_HEALPix_bounds( **Examples** - >>> cf_HEALPix_bounds([0, 1, 2, 3], 'nested', 1, lat=True) + >>> cf_healpix_bounds([0, 1, 2, 3], 'nested', 1, lat=True) array([[19.47122063, 41.8103149 , 19.47122063, 0. ], [41.8103149 , 66.44353569, 41.8103149 , 19.47122063], [41.8103149 , 66.44353569, 41.8103149 , 19.47122063], [66.44353569, 90. , 66.44353569, 41.8103149 ]]) - >>> cf_HEALPix_bounds([0, 1, 2, 3], 'nested', 1, lon=True) + >>> cf_healpix_bounds([0, 1, 2, 3], 'nested', 1, lon=True) array([[67.5, 45. , 22.5, 45. ], [90. , 90. , 45. , 67.5], [45. , 0. , 0. , 22.5], [90. , 45. , 0. , 45. ]]) """ - import healpix + try: + import healpix + except ImportError as e: + raise ImportError( + f"{e}. Must install healpix (e.g. from " + "https://pypi.org/project/healpix) to allow the calculation " + "of latitude/longitude coordinate bounds for a HEALPix grid" + ) # Keep an eye on https://github.com/ntessore/healpix/issues/66 if a.ndim != 1: @@ -75,7 +82,7 @@ def cf_HEALPix_bounds( elif lon: pos = 0 - if index_scheme == "nested": + if indexing_scheme == "nested": bounds_func = healpix._chp.nest2ang_uv else: bounds_func = healpix._chp.ring2ang_uv @@ -91,8 +98,8 @@ def cf_HEALPix_bounds( # Initialise the output bounds array b = np.empty((a.size, 4), dtype="float64") - if index_scheme == "nuniq": - # Create bounds for 'nuniq' cells + if indexing_scheme == "nested_unique": + # Create bounds for 'nested_unique' cells orders, a = healpix.uniq2pix(a, nest=False) orders, index, inverse = np.unique( orders, return_index=True, return_inverse=True @@ -133,8 +140,8 @@ def cf_HEALPix_bounds( return b -def cf_HEALPix_change_order( - a, index_scheme, new_index_scheme, refinement_level +def cf_healpix_indexing_scheme( + a, indexing_scheme, new_indexing_scheme, refinement_level ): """Change the ordering of HEALPix indices. @@ -148,13 +155,13 @@ def cf_HEALPix_change_order( a: `numpy.ndarray` The array of HEALPix indices. - index_scheme: `str` + indexing_scheme: `str` The original HEALPix indexing scheme. One of ``'nested'`` or ``'ring'``. - new_index_scheme: `str` + new_indexing_scheme: `str` The new HEALPix indexing scheme to change to. One of - ``'nested'``, ``'ring'``, or ``'nuniq'``. + ``'nested'``, ``'ring'``, or ``'nested_unique'``. refinement_level: `int` The refinement level of the original grid within the @@ -168,37 +175,51 @@ def cf_HEALPix_change_order( **Examples** - >>> cf_HEALPix_change_order([0, 1, 2, 3], 'nested', 'ring', 1) + >>> cf_healpix_indexing_scheme([0, 1, 2, 3], 'nested', 'ring', 1) array([13, 5, 4, 0]) - >>> cf_HEALPix_change_order([0, 1, 2, 3], 'nuniq', 'ring', 1) + >>> cf_healpix_indexing_scheme([0, 1, 2, 3], 'nested_unique', 'ring', 1) array([16, 17, 18, 19]) """ - import healpix + if new_indexing_scheme == indexing_scheme: + # Null operation + return a + + try: + import healpix + except ImportError as e: + raise ImportError( + f"{e}. Must install healpix (e.g. from " + "https://pypi.org/project/healpix) to allow the changing of " + "the HEALPix index scheme" + ) - if index_scheme == "nested": - if new_index_scheme == "ring": + if indexing_scheme == "nested": + if new_indexing_scheme == "ring": return healpix.nest2ring(healpix.order2nside(refinement_level), a) - if new_index_scheme == "nuniq": + if new_indexing_scheme == "nested_unique": return healpix._chp.nest2uniq(refinement_level, a) - elif index_scheme == "ring": - if new_index_scheme == "nested": + elif indexing_scheme == "ring": + if new_indexing_scheme == "nested": return healpix.ring2nest(healpix.order2nside(refinement_level), a) - if new_index_scheme == "nuniq": + if new_indexing_scheme == "nested_unique": return healpix._chp.ring2uniq(refinement_level, a) - else: + elif indexing_scheme == "nested_unique": raise ValueError( - "Can't change HEALPix order: Can only change from HEALPix " - f"order 'nested' or 'ring'. Got {index_scheme!r}" + "Can't change HEALPix scheme from 'nested_unique' to " + f"{new_indexing_scheme!r}" ) + else: + raise ValueError(f"Invalid HEALPix scheme: {indexing_scheme!r}") + -def cf_HEALPix_coordinates( - a, index_scheme, refinement_level=None, lat=False, lon=False +def cf_healpix_coordinates( + a, indexing_scheme, refinement_level=None, lat=False, lon=False ): """Calculate HEALPix cell coordinates. @@ -211,15 +232,15 @@ def cf_HEALPix_coordinates( a: `numpy.ndarray` The array of HEALPix indices. - index_scheme: `str` + indexing_scheme: `str` The HEALPix indexing scheme. One of ``'nested'``, - ``'ring'``, or ``'nuniq'``. + ``'ring'``, or ``'nested_unique'``. refinement_level: `int` or `None`, optional For a ``'nested'`` or ``'ring'`` ordered grid, the refinement level of the grid within the HEALPix hierarchy, starting at 0 for the base tesselation with 12 cells. - Ignored for a ``'nuniq'`` ordered grid. + Ignored for a ``'nested_unique'`` ordered grid. lat: `bool`, optional If True then return latitude coordinates. @@ -234,13 +255,20 @@ def cf_HEALPix_coordinates( **Examples** - >>> cf_HEALPix_coordinates([0, 1, 2, 3], 'nested', 1, lat=True) + >>> cf_healpix_coordinates([0, 1, 2, 3], 'nested', 1, lat=True) array([19.47122063, 41.8103149 , 41.8103149 , 66.44353569]) - >>> cf_HEALPix_coordinates([0, 1, 2, 3], 'nested', 1, lon=True) + >>> cf_healpix_coordinates([0, 1, 2, 3], 'nested', 1, lon=True) array([45. , 67.5, 22.5, 45. ]) """ - import healpix + try: + import healpix + except ImportError as e: + raise ImportError( + f"{e}. Must install healpix (e.g. from " + "https://pypi.org/project/healpix) to allow the calculation " + "of latitude/longitude coordinates for a HEALPix grid" + ) if a.ndim != 1: raise ValueError( @@ -253,8 +281,8 @@ def cf_HEALPix_coordinates( elif lon: pos = 0 - if index_scheme == "nuniq": - # Create coordinates for 'nuniq' cells + if indexing_scheme == "nested_unique": + # Create coordinates for 'nested_unique' cells c = np.empty(a.shape, dtype="float64") nest = False @@ -270,7 +298,7 @@ def cf_HEALPix_coordinates( )[pos] else: # Create coordinates for 'nested' or 'ring' cells - nest = (index_scheme == "nested",) + nest = (indexing_scheme == "nested",) nside = healpix.order2nside(refinement_level) c = healpix.pix2ang( nside=nside, @@ -282,23 +310,24 @@ def cf_HEALPix_coordinates( return c -def cf_HEALPix_nuniq_area_weights(a, measure=False, radius=None): - """Calculate HEALPix cell area weights for 'nuniq' indices. +def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): + """Calculate HEALPix cell area weights. - For mathematical details, see section 4 of: - - K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, - et al.. HEALPix: A Framework for High‐Resolution - Discretization and Fast Analysis of Data Distributed on the - Sphere. The Astrophysical Journal, 2005, 622 (2), pp.759-771. - https://dx.doi.org/10.1086/427976 + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et + al.. HEALPix: A Framework for High-Resolution Discretization and + Fast Analysis of Data Distributed on the Sphere. The Astrophysical + Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 .. versionadded:: NEXTVERSION :Parameters: a: `numpy.ndarray` - The array of HEALPix 'nuniq' indices. + The array of HEALPix 'nested_unique' indices. + + indexing_scheme: `str` + The HEALPix indexing scheme. Must be ``'nested_unique'``. measure: `bool`, optional If True then create weights that are actual cell areas, in @@ -315,15 +344,28 @@ def cf_HEALPix_nuniq_area_weights(a, measure=False, radius=None): **Examples** - >>> cf_HEALPix_nuniq_weights([76, 77, 78, 79, 20, 21]) + >>> cf_healpix_weights([76, 77, 78, 79, 20, 21], 'nested_unique') array([0.0625, 0.0625, 0.0625, 0.0625, 0.25 , 0.25 ]) - >>> cf_HEALPix_nuniq_weights([76, 77, 78, 79, 20, 21], - ... measure=True, radius=6371000) + >>> cf_healpix_weights([76, 77, 78, 79, 20, 21], 'nested_unique', + ... measure=True, radius=6371000) array([2.65658579e+12, 2.65658579e+12, 2.65658579e+12, 2.65658579e+12, 1.06263432e+13, 1.06263432e+13]) """ - import healpix + try: + import healpix + except ImportError as e: + raise ImportError( + f"{e}. Must install healpix (e.g. from " + "https://pypi.org/project/healpix) to allow the calculation " + "of cell area weights for a HEALPix grid" + ) + + if indexing_scheme != "nested_unique": + raise ValueError( + "cf_healpix_weights: Can only calulate weights for the " + "'nested_unique' indexing scheme" + ) if measure: x = np.pi * (radius**2) / 3.0 diff --git a/cf/domain.py b/cf/domain.py index 46d8c1b36f..0e2653285a 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -265,7 +265,7 @@ def create_regular(cls, x_args, y_args, bounds=True): @classmethod def create_healpix( - cls, refinement_level, index_scheme="nested", radius=None + cls, refinement_level, indexing_scheme="nested", radius=None ): """Create a new global HEALPix domain. @@ -281,9 +281,9 @@ def create_healpix( 12 cells. The number of cells in the global HEALPix grid is :math:`(12 \times 4^refinement_level)`. - index_scheme: `str` + indexing_scheme: `str` The HEALPix indexing scheme. One of ``'nested'`` (the - default), ``'ring'``, or ``'nuniq'``. + default), ``'ring'``, or ``'nested_unique'``. radius: optional Specify the radius of the latitude-longitude plane @@ -322,11 +322,11 @@ def create_healpix( Coordinate reference: grid_mapping_name:healpix Coordinate conversion:grid_mapping_name = healpix - Coordinate conversion:index_scheme = nested + Coordinate conversion:indexing_scheme = nested Coordinate conversion:refinement_level = 4 Auxiliary Coordinate: healpix_index - >>> d = cf.Domain.create_healpix(8, "nuniq", radius=6371000) + >>> d = cf.Domain.create_healpix(8, "nested_unique", radius=6371000) >>> d.dump() -------- Domain: @@ -340,7 +340,7 @@ def create_healpix( Coordinate reference: grid_mapping_name:healpix Coordinate conversion:grid_mapping_name = healpix - Coordinate conversion:index_scheme = nuniq + Coordinate conversion:indexing_scheme = nested_unique Datum:earth_radius = 6371000.0 Auxiliary Coordinate: healpix_index @@ -353,13 +353,13 @@ def create_healpix( f"Got: {refinement_level!r}" ) - nuniq = index_scheme == "nuniq" - if nuniq: - index_scheme = "nested" - elif index_scheme not in ("nested", "ring"): + nested_unique = indexing_scheme == "nested_unique" + if nested_unique: + indexing_scheme = "nested" + elif indexing_scheme not in ("nested", "ring"): raise ValueError( - "'index_scheme' must be 'nested', 'ring', or 'nuniq'. " - f"Got: {index_scheme!r}" + "'indexing_scheme' must be 'nested', 'ring', or 'nested_unique'. " + f"Got: {indexing_scheme!r}" ) domain = Domain() @@ -389,16 +389,16 @@ def create_healpix( cr.coordinate_conversion.set_parameters( { "grid_mapping_name": "healpix", - "index_scheme": index_scheme, + "indexing_scheme": indexing_scheme, "refinement_level": refinement_level, } ) domain.set_construct(cr) - if nuniq: - # Change from 'nested' to 'nuniq' indexing scheme - domain = domain.healpix_change_order("nuniq") + if nested_unique: + # Change from 'nested' to 'nested_unique' indexing scheme + domain = domain.healpix_indexing_scheme("nested_unique") return domain diff --git a/cf/field.py b/cf/field.py index 0da76aae34..57e02fcc2a 100644 --- a/cf/field.py +++ b/cf/field.py @@ -3482,6 +3482,18 @@ def weights( ): # Found linear weights from dimension coordinates pass + elif Weights.healpix_area( + self, + da_key, + comp, + weights_axes, + measure=measure, + radius=radius, + methods=methods, + auto=True, + ): + # Found area weights from HEALPix cells + pass weights_axes = [] for key in comp: @@ -4912,23 +4924,23 @@ def bin( return out def healpix_decrease_refinement_level( - self, level, reduction, sort=True, check_healpix_index=True + self, level, reduction, conform=True, check_healpix_index=True ): """Decrease the refinement level of a HEALPix grid. .. versionadded:: NEXTVERSION - .. seealso:: `healpix_change_order` + .. seealso:: `healpix_indexing_scheme` :Parameters: level: `int` or `None` Specify the new refinement level. If a non-negative - integer then this explicitly defines the new - refinement level. If a negative integer then the new - refinement level is defined by the current refinement - level plus *level*. If `None` then the refinement - level is not changed. + integer then this will be the new refinement level. If + a negative integer then the new refinement level is + defined by the current refinement level plus + *level*. If `None` then the refinement level is not + changed. *Example:* If the current refinement level is 10 then a new @@ -4936,55 +4948,58 @@ def healpix_decrease_refinement_level( either ``8`` or ``-2``. reduction: function - The function used to calculate the values in the new - coarser cells from the original data defined on the - original finer cells. + The function used to calculate, from the data on the + original finer cells, the values in the new coarser + cells. *Example:* For an intensive field quantity (that does not depend on the size of the cells, such as - "precipitation_flux" with units of kg m-2 s-1), - ``np.mean`` may be appropriate. For an extensive - field quantity (that depends on the size of the - cells, such as "precipitation_amount" with units of - kg m-2), ``np.sum`` may be appropriate. - - sort: `bool`, optional - As a requirement of the coarsening algorithm, the - HEALPix indices are always automatically converted to - 'nested' ordering, and if *sort* is True (the default) - then the HEALPix axis will also be sorted so that - these nested HEALPix indices are monotonically - increasing. Only set to False, which will speed up the - operation, if it is known that the HEALPix indices - expressed in their nested ordering are already - monotonically increasing. + "sea_ice_amount" with units of kg m-2), ``np.mean`` + may be appropriate. For an extensive field quantity + (that depends on the size of the cells, such as + "sea_ice_mass" with units of kg), ``np.sum`` may be + appropriate. + + conform: `bool`, optional + If True (the default) the HEALPix grid is converted to + a form suitable for having its refinement level + changed, i.e. the indexing scheme is changed to + 'nested' and the HEALPix axis is sorted so that the + nested HEALPix indices are monotonically + increasing. If False then either an exeption is raised + if the HEALPix indexing scheme is not already + 'nested', or else the HEALPix axis is not sorted. + + .. note:: Setting to False will speed up the operation + when the HEALPix indexing scheme is already + nested and the HEALPix axis is already + sorted montonically. check_healpix_index: `bool`, optional - As a requirement of the coarsening algorithm, the - HEALPix indices are always automatically converted to - 'nested' ordering, and if *sort* is True then the - HEALPix axis will also be sorted so that these nested - HEALPix indices are monotonically increasing. If - *check_healpix_indices* is True (the default) then it - will be checked that 1) the nested HEALPix indices are - strictly monotonically increasing, and 2) a cell at - the new coarser refinement level contains the maximum - possible number of cells at the original finer - refinement level. + + If True (the default) then it will be checked (after + the HEALPix grid has been conformed, if *conform* is + True) that 1. the nested HEALPix indices are strictly + monotonically increasing, and 2. every cell at the new + coarser refinement level contains the maximum possible + number of cells at the original finer refinement + level. If either condition is not met then an + exception is raised. If False then these checks are + not carried out. .. warning:: Only set to False, which will speed up - the operation, if it is known these - conditions are met. If set to False and - any of the conditions is not met then - either an exception will be raised or, - much worse, the operation will complete - and return incorrect results. + the operation, if it is known in advance + that these conditions are met. If set to + False and any of the conditions is not + met then either an exception will be + raised or, much worse, the operation will + complete and return incorrect values. :Returns: `Field` - A new field with a coarsened HEALPix grid. + A new Field with a coarsened HEALPix grid. **Examples** @@ -4994,7 +5009,7 @@ def healpix_decrease_refinement_level( Set the refinement level to 0: - >>> g = f.healpix_decrease_refinement_level(0, np.mean)g + >>> g = f.healpix_decrease_refinement_level(0, np.mean) >>> g @@ -5008,32 +5023,8 @@ def healpix_decrease_refinement_level( np.float64(289.15) """ - # Try to change the order to nested, as that's the only order - # from which we can change the refinement level. - try: - f = self.healpix_change_order("nested", sort=sort) - except ValueError as error: - raise ValueError( - f"Can't decrease HEALPix refinement level: {error}" - ) - - # Get the healpix_index coordinates and the key of the HEALPix - # domain axis - hp_key, healpix_index = f.auxiliary_coordinate( - "healpix_index", - filter_by_naxes=(1,), - item=True, - default=(None, None), - ) - if healpix_index is None: - raise ValueError( - "Can't decrease HEALPix refinement level: There are no " - "healpix_index coordinates" - ) - - axis = f.get_data_axes(hp_key)[0] + f = self.copy() - # Parse the HEALPix coordinate reference cr = f.coordinate_reference("grid_mapping_name:healpix", default=None) if cr is None: raise ValueError( @@ -5041,12 +5032,42 @@ def healpix_decrease_refinement_level( "grid mapping has not been set" ) - parameters = cr.coordinate_conversion.parameters() - refinement_level = parameters.get("refinement_level") + indexing_scheme = cr.coordinate_conversion.get_parameter( + "indexing_scheme", None + ) + if indexing_scheme is None: + raise ValueError( + "Can't decrease HEALPix refinement level: indexing_scheme " + "has not been set in the HEALPix grid mapping coordinate " + "reference" + ) + + refinement_level = cr.coordinate_conversion.get_parameter( + "refinement_level" + ) if refinement_level is None: raise ValueError( "Can't decrease HEALPix refinement level: refinement_level " - "has not been set" + "has not been set in the HEALPix grid mapping coordinate " + "reference" + ) + + # Make sure that we have a nested indexing scheme + if conform: + try: + f = f.healpix_indexing_scheme("nested", sort=True) + except ValueError as error: + raise ValueError( + f"Can't decrease HEALPix refinement level: {error}" + ) + + cr = f.coordinate_reference("grid_mapping_name:healpix") + elif indexing_scheme != "nested": + raise ValueError( + "Can't decrease HEALPix refinement level: indexing_scheme in " + "the HEALPix grid mapping coordinate reference is " + f"{indexing_scheme!r}, and not 'nested'. " + "Consider setting conform=True" ) # Parse 'level' @@ -5074,17 +5095,34 @@ def healpix_decrease_refinement_level( # No change in refinement level return f - # The number of cells at the original refinement level which are + # Get the number of cells at the original refinement level which are # contained in one cell at the coarser refinement level ncells = 4**-level + # Get the healpix_index coordinates + hp_key, healpix_index = f.auxiliary_coordinate( + "healpix_index", + filter_by_naxes=(1,), + item=True, + default=(None, None), + ) + if healpix_index is None: + raise ValueError( + "Can't decrease HEALPix refinement level: There are no " + "healpix_index coordinates" + ) + + # Get the HEALPix axis + axis = f.get_data_axes(hp_key)[0] + iaxis = f.get_data_axes().index(axis) + if check_healpix_index: d = healpix_index.data - if not sort and not (d.diff() > 0).all(): + if not conform and not (d.diff() > 0).all(): raise ValueError( "Can't decrease HEALPix refinement level: Nested " "healpix_index coordinates are not strictly " - "monotonically increasing. Consider setting sort=True." + "monotonically increasing. Consider setting conform=True" ) if (d[::ncells] % ncells).any() or ( @@ -5101,9 +5139,8 @@ def healpix_decrease_refinement_level( # Coarsen (using the 'reduction' function) the field # data. Note that using the 'coarsen' technique only works for # 'nested' HEALPix ordering. - hp_iaxis = f.get_data_axes().index(axis) f.data.coarsen( - reduction, axes={hp_iaxis: ncells}, trim_excess=False, inplace=True + reduction, axes={iaxis: ncells}, trim_excess=False, inplace=True ) # Coarsen the domain ancillary constructs that span the @@ -5143,7 +5180,7 @@ def healpix_decrease_refinement_level( # Re-size the HEALPix axis domain_axis = f.domain_axis(axis) - domain_axis.set_size(f.shape[hp_iaxis]) + domain_axis.set_size(f.shape[iaxis]) # Set the healpix_index coordinates for the new refinement # level @@ -6734,8 +6771,6 @@ def collapse( else: f = self.copy() - f.create_latlon_coordinates(one_d=True, two_d=False, inplace=True) - # Whether or not to create null bounds for null # collapses. I.e. if the collapse axis has size 1 and no # bounds, whether or not to create upper and lower bounds to @@ -6803,6 +6838,7 @@ def collapse( input_axes = all_axes all_axes = [] + f_latlon = None for axes in input_axes: if axes is None: domain_axes = self.domain_axes( @@ -6811,6 +6847,10 @@ def collapse( all_axes.append(list(domain_axes)) continue + if f_latlon is None: + # Temporarily create any implied lat/lon coordinates + f_latlon = f.create_latlon_coordinates() + axes2 = [] for axis in axes: msg = ( @@ -6828,7 +6868,7 @@ def collapse( msg = "Can't find the collapse axis identified by {!r}" for x in iterate_over: - a = f.domain_axis(x, key=True, default=None) + a = f_latlon.domain_axis(x, key=True, default=None) if a is None: raise ValueError(msg.format(x)) @@ -6836,6 +6876,8 @@ def collapse( all_axes.append(axes2) + del f_latlon + if debug: logger.debug( " all_methods, all_axes, all_within, all_over = " @@ -7188,6 +7230,16 @@ def collapse( if set(ref_axes).intersection(flat(all_axes)): f.del_coordinate_reference(ref_key) + # -------------------------------------------------------- + # Remove a HEALPix coordinate reference + # -------------------------------------------------------- + axis = f.healpix_axis(None) + domain_axis = collapse_axes.get(axis) + if domain_axis is not None and domain_axis.get_size() > 1: + from .healpix_utils import del_healpix_coordinate_reference + + del_healpix_coordinate_reference(f, axis=axis) + # --------------------------------------------------------- # Update dimension coordinates, auxiliary coordinates, # cell measures and domain ancillaries @@ -13297,6 +13349,14 @@ def regrids( any other latitude-longitude grid, including other UGRID meshes and DSG feature types. + **HEALPix grids** + + Data on HEALPix grids may be regridded to any other + latitude-longitude grid, including other HEALPix grids, UGRID + meshes and DSG feature types. This is done internally by + converting HEALPix grids to UGRID meshes and carrying out a + UGRID regridding. + **DSG feature types** Data on any latitude-longitude grid (including tripolar and diff --git a/cf/healpix_utils.py b/cf/healpix_utils.py new file mode 100644 index 0000000000..61ebea7b3e --- /dev/null +++ b/cf/healpix_utils.py @@ -0,0 +1,69 @@ +"""TODOHEALPIX""" + + +def del_healpix_coordinate_reference(f, axis=None): + """TODOHEALPIX + + :Parameters: + + f: `Field` or `Domain` + + axis: optional + + :Returns: + + `CoordinateReference` or `None` + + """ + cr_key, cr = f.coordinate_reference( + "grid_mapping_name:healpix", item=True, default=(None, None) + ) + latlon = f.coordinate_reference( + "grid_mapping_name:latitude_longitude", default=None + ) + + if cr is None: + out = None + else: + out = cr.copy() + + if latlon is not None: + # There is already a latitude_longitude coordinate reference, + # so delete a healpix coordinate reference. + if cr is not None: + f.del_construct(cr_key) + + elif cr is not None: + cc = cr.coordinate_conversion + cc.del_parameter("grid_mapping_name", None) + cc.del_parameter("indexing_scheme", None) + cc.del_parameter("refinement_level", None) + if cc.parameters() or cr.datum.parameters(): + # The healpix coordinate reference contains generic + # coordinate conversion or datum parameters, so rename it + # as 'latitude_longitude'. + cr.coordinate_conversion.set_parameter( + "grid_mapping_name", "latitude_longitude" + ) + + # Remove a healpix_index coordinate from the coordinate + # reference + if axis is None: + axis = f.healpix_axis(None) + + if axis is not None: + key = f.coordinate( + "healpix_index", + filter_by_axis=(axis,), + axis_mode="exact", + key=True, + default=None, + ) + if key is not None: + cr.del_coordinate(key) + else: + # The healpix coordinate reference contains no generic + # parameters, so delete it. + f.del_construct(cr_key) + + return out diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 454a20f7a7..f415761535 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -1945,7 +1945,7 @@ def healpix_axis(self, default=ValueError()): .. versionadded:: NEXTVERSION - .. seealso:: `healpix_change_order`, `healpix_to_ugrid` + .. seealso:: `healpix_indexing_scheme`, `healpix_to_ugrid` :Parameters: @@ -1990,8 +1990,8 @@ def healpix_axis(self, default=ValueError()): return self.get_data_axes(key)[0] - def healpix_change_order(self, new_index_scheme, sort=False): - """Change the ordering of HEALPix indices. + def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): + """Change the indexing scheme of HEALPix indices. .. versionadded:: NEXTVERSION @@ -1999,21 +1999,20 @@ def healpix_change_order(self, new_index_scheme, sort=False): :Parameters: - new_index_scheme: `str` + new_indexing_scheme: `str` The new HEALPix indexing scheme. One of ``'nested'``, - ``'ring'``, or ``'nuniq'``. + ``'ring'``, or ``'nested_unique'``. sort: `bool`, optional - If True then sort the HEALPix axis of the output - {{class}} so that the HEALPix indices are - monotonically increasing. If False (the default) then - don't do this. + If True then sort the HEALPix axis of the output so + that its HEALPix indices are monotonically + increasing. If False (the default) then don't do this. :Returns: `{{class}}` The {{class}} with the HEALPix indices redefined for - the new ordering. + the new scheme. **Examples** @@ -2027,114 +2026,121 @@ def healpix_change_order(self, new_index_scheme, sort=False): : height(1) = [1.5] m Auxiliary coords: healpix_index(healpix_index(48)) = [0, ..., 47] 1 Coord references: grid_mapping_name:healpix - >>> f.coordinate_reference('grid_mapping_name:healpix').coordinate_conversion.get_parameter('index_scheme') + >>> f.coordinate_reference('grid_mapping_name:healpix').coordinate_conversion.get_parameter('indexing_scheme') 'nested' >>> print(f.coordinate('healpix_index').array) [ 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47] - >>> g = f.healpix_change_order('nuniq') + >>> g = f.healpix_indexing_scheme('nested_unique') >>> print(g.coordinate('healpix_index').array) [16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63] - >>> g = f.healpix_change_order('ring') + >>> g = f.healpix_indexing_scheme('ring') >>> print(g.coordinate('healpix_index').array) [13 5 4 0 15 7 6 1 17 9 8 2 19 11 10 3 28 20 27 12 30 22 21 14 32 24 23 16 34 26 25 18 44 37 36 29 45 39 38 31 46 41 40 33 47 43 42 35] - >>> g = f.healpix_change_order('ring', sort=True) + >>> g = f.healpix_indexing_scheme('ring', sort=True) >>> print(g.coordinate('healpix_index').array) [ 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47] - >>> h = g.healpix_change_order('nested') + >>> h = g.healpix_indexing_scheme('nested') >>> print(h.coordinate('healpix_index').array) [ 3 7 11 15 2 1 6 5 10 9 14 13 19 0 23 4 27 8 31 12 17 22 21 26 25 30 29 18 16 35 20 39 24 43 28 47 34 33 38 37 42 41 46 45 32 36 40 44] - >>> h = g.healpix_change_order('nested', sort=True) + >>> h = g.healpix_indexing_scheme('nested', sort=True) >>> print(h.coordinate('healpix_index').array) [ 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47] """ - from ..dask_utils import cf_HEALPix_change_order + from ..dask_utils import cf_healpix_indexing_scheme f = self.copy() - if new_index_scheme not in ("nested", "ring", "nuniq"): + if new_indexing_scheme not in ("nested", "ring", "nested_unique"): raise ValueError( - "Can't change HEALPix order: new_index_scheme must be " - f"'nested', 'ring', or 'nuniq'. Got {new_index_scheme!r}" + "Can't change HEALPix index scheme: new_indexing_scheme " + "keyword must be 'nested', 'ring', or 'nested_unique'. " + f"Got {new_indexing_scheme!r}" + ) + + # Get the original healpix_index coordinates and the key of + # the HEALPix domain axis + hp_key, healpix_index = f.coordinate( + "healpix_index", + filter_by_naxes=(1,), + item=True, + default=(None, None), + ) + if healpix_index is None: + raise ValueError( + "Can't change HEALPix index scheme: There are no " + "healpix_index coordinates" ) # Parse the HEALPix coordinate reference cr = f.coordinate_reference("grid_mapping_name:healpix", default=None) if cr is None: raise ValueError( - "Can't change HEALPix order: There is no HEALPix grid " + "Can't change HEALPix index scheme: There is no HEALPix grid " "mapping coordinate reference" ) parameters = cr.coordinate_conversion.parameters() - refinement_level = parameters.get("refinement_level") - if refinement_level is None: - raise ValueError( - "Can't change HEALPix order: refinement_level " - "has not been set" - ) + indexing_scheme = parameters.get("indexing_scheme") - index_scheme = parameters.get("index_scheme") - if index_scheme is None: + if indexing_scheme is None: raise ValueError( - "Can't change HEALPix order: index_scheme has not been set" + "Can't change HEALPix indexing scheme: indexing_scheme has " + "not been set in the HEALPix grid mapping coordinate reference" ) - if index_scheme not in ("nested", "ring"): + if indexing_scheme not in ("nested", "ring", "nested_unique"): raise ValueError( - "Can't change HEALPix order: Can only change from " - f"index_scheme 'nested' or 'ring'. Got {index_scheme!r}" + f"Can't change HEALPix indexing scheme: Invalid " + f"indexing_scheme: {indexing_scheme!r}" ) - # Get the original healpix_index coordinates and the key of - # the HEALPix domain axis - hp_key, healpix_index = f.coordinate( - "healpix_index", - filter_by_naxes=(1,), - item=True, - default=(None, None), - ) - if healpix_index is None: + if indexing_scheme == "nested_unique": + if new_indexing_scheme != "nested_unique": + raise ValueError( + "Can't change HEALPix indexing scheme from " + f"'nested_unique' to {new_indexing_scheme!r}" + ) + elif refinement_level is None: raise ValueError( - "Can't change HEALPix order: There are no " - "healpix_index coordinates" + "Can't change HEALPix indexing scheme from " + f"{indexing_scheme!r} when refinement_level has not been set " + "in the HEALPix grid mapping coordinate reference" ) axis = f.get_data_axes(hp_key)[0] - if index_scheme != new_index_scheme: + if indexing_scheme != new_indexing_scheme: # Change the HEALPix indices dx = healpix_index.to_dask_array() dx = dx.map_blocks( - cf_HEALPix_change_order, + cf_healpix_indexing_scheme, meta=np.array((), dtype="int64"), - index_scheme=index_scheme, - new_index_scheme=new_index_scheme, + indexing_scheme=indexing_scheme, + new_indexing_scheme=new_indexing_scheme, refinement_level=refinement_level, ) healpix_index.set_data(dx, copy=False) # Update the Coordinate Reference cr.coordinate_conversion.set_parameter( - "index_scheme", new_index_scheme + "indexing_scheme", new_indexing_scheme ) - if new_index_scheme == "nuniq": - cr.coordinate_conversion.del_parameter( - "refinement_level", None - ) + if new_indexing_scheme == "nested_unique": + cr.coordinate_conversion.del_parameter("refinement_level") + # Ensure that healpix indices are auxiliary coordinates if healpix_index.construct_type == "dimension_coordinate": - # Convert healpix indices to auxiliary coordinates healpix_index = f._AuxiliaryCoordinate( source=healpix_index, copy=False ) @@ -2145,7 +2151,8 @@ def healpix_change_order(self, new_index_scheme, sort=False): if sort: # Sort the HEALPix axis so that the HEALPix indices are # monotonically increasing. Test for the common case of - # already-ordered global nested indices. + # already-ordered global nested indices (which is fast + # compared to do doing any sorting). d = healpix_index.to_dask_array() if not (d == da.arange(d.size)).all(): index = healpix_index.data.compute() @@ -2278,33 +2285,9 @@ def healpix_to_ugrid(self, inplace=False): default=None, ) - # Update the Coordinate Reference - cr_key, cr = f.coordinate_reference( - "grid_mapping_name:healpix", item=True, default=(None, None) - ) - latlon = f.coordinate_reference( - "grid_mapping_name:latitude_longitude", default=None - ) - if latlon is not None: - latlon.set_coordinates((y_key, x_key)) - f.del_construct(cr_key) - elif cr is not None: - cc = cr.coordinate_conversion - cc.del_parameter("grid_mapping_name", None) - cc.del_parameter("index_scheme", None) - cc.del_parameter("refinement_level", None) - if cr.coordinate_conversion.parameters() or cr.datum.parameters(): - # The Coordinate Reference contains generic coordinate - # conversion or datum parameters, so rename it as - # 'latitude_longitude'. - cr.coordinate_conversion.set_parameter( - "grid_mapping_name", "latitude_longitude" - ) - cr.set_coordinates((y_key, x_key)) - else: - # The Coordinate Reference contains no generic - # parameters, so delete it. - f.del_construct(cr_key) + from ..healpix_utils import del_healpix_coordinate_reference + + del_healpix_coordinate_reference(f, axis=axis) return f @@ -2382,7 +2365,7 @@ def create_latlon_coordinates( """ f = _inplace_enabled_define_and_cleanup(self) - # Get all Coordinate References + # Get all Coordinate References in a dictionary identities = { cr.identity(""): cr for cr in f.coordinate_references(todict=True).values() @@ -2411,7 +2394,8 @@ def create_latlon_coordinates( return f - # Remove a 'latitude_longitude' grid mapping, if there is one. + # Remove a 'latitude_longitude' grid mapping from the + # dictionary latlon_cr = identities.pop( "grid_mapping_name:latitude_longitude", None ) @@ -2436,18 +2420,18 @@ def create_latlon_coordinates( # HEALPix 1-d coordinates # -------------------------------------------------------- parameters = cr.coordinate_conversion.parameters() - index_scheme = parameters.get("index_scheme") - if index_scheme not in ("nested", "ring", "nuniq"): + indexing_scheme = parameters.get("indexing_scheme") + if indexing_scheme not in ("nested", "ring", "nested_unique"): if is_log_level_info(logger): logger.info( "Can't create 1-d latitude and longitude coordinates: " - f"Invalid HEALPix order: {index_scheme!r}" + f"Invalid HEALPix index scheme: {indexing_scheme!r}" ) # pragma: no cover return f refinement_level = parameters.get("refinement_level") - if refinement_level is None and index_scheme in ( + if refinement_level is None and indexing_scheme in ( "nested", "ring", ): @@ -2496,7 +2480,7 @@ def create_latlon_coordinates( # Define functions to create latitudes and longitudes from # HEALPix indices - from ..dask_utils import cf_HEALPix_bounds, cf_HEALPix_coordinates + from ..dask_utils import cf_healpix_bounds, cf_healpix_coordinates # Create new latitude and longitude coordinates with bounds dx = healpix_index.to_dask_array() @@ -2504,9 +2488,9 @@ def create_latlon_coordinates( # Latitude coordinates dy = dx.map_blocks( - cf_HEALPix_coordinates, + cf_healpix_coordinates, meta=meta, - index_scheme=index_scheme, + indexing_scheme=indexing_scheme, refinement_level=refinement_level, lat=True, ) @@ -2518,9 +2502,9 @@ def create_latlon_coordinates( # Longitude coordinates dy = dx.map_blocks( - cf_HEALPix_coordinates, + cf_healpix_coordinates, meta=meta, - index_scheme=index_scheme, + indexing_scheme=indexing_scheme, refinement_level=refinement_level, lon=True, ) @@ -2532,13 +2516,13 @@ def create_latlon_coordinates( # Latitude bounds dy = da.blockwise( - cf_HEALPix_bounds, + cf_healpix_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta, - index_scheme=index_scheme, + indexing_scheme=indexing_scheme, refinement_level=refinement_level, lat=True, ) @@ -2547,13 +2531,13 @@ def create_latlon_coordinates( # Longitude bounds dy = da.blockwise( - cf_HEALPix_bounds, + cf_healpix_bounds, "ij", dx, "i", new_axes={"j": 4}, meta=meta, - index_scheme=index_scheme, + indexing_scheme=indexing_scheme, refinement_level=refinement_level, lon=True, ) @@ -2573,7 +2557,7 @@ def create_latlon_coordinates( pass # ------------------------------------------------------------ - # Update the approriate Coordinate Reference + # Update Coordinate References # ------------------------------------------------------------ if new_coords: if latlon_cr is not None: diff --git a/cf/test/test_Domain.py b/cf/test/test_Domain.py index 5a4734590b..8767854332 100644 --- a/cf/test/test_Domain.py +++ b/cf/test/test_Domain.py @@ -498,7 +498,7 @@ def test_Domain_create_healpix(self): (d.auxiliary_coordinate().array == np.arange(12)).all() ) - d = cf.Domain.create_healpix(0, "nuniq") + d = cf.Domain.create_healpix(0, "nested_unique") self.assertTrue( (d.auxiliary_coordinate().array == np.arange(4, 16)).all() ) diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index 025f53be7e..d3b23d7fb6 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -3101,49 +3101,55 @@ def test_Field_healpix_axis(self): with self.assertRaises(ValueError): self.f0.healpix_axis() - def test_Field_healpix_change_order(self): - """Test Field.healpix_change_order.""" + def test_Field_healpix_indexing_scheme(self): + """Test Field.healpix_indexing_scheme.""" # HEALPix field f = self.f12 - g = f.healpix_change_order("ring") + # Null change + g = f.healpix_indexing_scheme("nested") + self.assertTrue(g.equals(f)) + + g = f.healpix_indexing_scheme("ring") self.assertTrue( (g.coordinate("healpix_index")[:4].array == [13, 5, 4, 0]).all() ) - h = g.healpix_change_order("nested") + h = g.healpix_indexing_scheme("nested") self.assertTrue( (h.coordinate("healpix_index")[:4].array == [0, 1, 2, 3]).all() ) - h = g.healpix_change_order("nuniq") + h = g.healpix_indexing_scheme("nested_unique") self.assertTrue( (h.coordinate("healpix_index")[:4].array == [16, 17, 18, 19]).all() ) - g = f.healpix_change_order("ring", sort=True) + g = f.healpix_indexing_scheme("ring", sort=True) self.assertTrue( (g.coordinate("healpix_index")[:4].array == [0, 1, 2, 3]).all() ) - h = g.healpix_change_order("nested", sort=False) + h = g.healpix_indexing_scheme("nested", sort=False) self.assertTrue( (h.coordinate("healpix_index")[:4].array == [3, 7, 11, 15]).all() ) - h = g.healpix_change_order("nested", sort=True) + h = g.healpix_indexing_scheme("nested", sort=True) self.assertTrue( (h.coordinate("healpix_index")[:4].array == [0, 1, 2, 3]).all() ) - g = f.healpix_change_order("nuniq") + g = f.healpix_indexing_scheme("nested_unique") self.assertTrue( (g.coordinate("healpix_index")[:4].array == [16, 17, 18, 19]).all() ) + h = g.healpix_indexing_scheme("nested_unique") + self.assertTrue(h.equals(g)) - # Can't change from 'nuniq' + # Can't change from 'nested_unique' to 'nested' with self.assertRaises(ValueError): - g.healpix_change_order("nested") + g.healpix_indexing_scheme("nested") # Non-HEALPix field with self.assertRaises(ValueError): - self.f0.healpix_change_order("ring") + self.f0.healpix_indexing_scheme("ring") def test_Field_healpix_to_ugrid(self): """Test Field.healpix_to_ugrid.""" @@ -3163,6 +3169,10 @@ def test_Field_healpix_to_ugrid(self): self.assertIsNone(f.healpix_to_ugrid(inplace=True)) self.assertEqual(len(f.domain_topologies()), 1) + self.assertEqual(len(u.coordinate_references()), 1) + cr = u.coordinate_reference() + self.assertEqual(cr.identity(), "grid_mapping_name:latitude_longitude") + # Non-HEALPix field with self.assertRaises(ValueError): self.f0.healpix_to_ugrid() @@ -3195,6 +3205,49 @@ def test_Field_subspace_healpix(self): ) ) + def test_Field_healpix_decrease_refinement_level(self): + """Test Field.healpix_decrease_refinement_level""" + f = self.f12 + g = f.healpix_decrease_refinement_level(0, np.mean) + self.assertTrue(g.shape, (2, 12)) + self.assertTrue((g.coord("healpix_index") == np.arange(12)).all()) + + g = f.healpix_decrease_refinement_level(-1, np.mean) + self.assertTrue(g.shape, (2, 12)) + self.assertTrue((g.coord("healpix_index") == np.arange(12)).all()) + + f = f.healpix_indexing_scheme("ring") + g = f.healpix_decrease_refinement_level(0, np.mean) + self.assertTrue(g.shape, (2, 12)) + self.assertTrue((g.coord("healpix_index") == np.arange(12)).all()) + with self.assertRaises(ValueError): + f.healpix_decrease_refinement_level(0, np.mean, conform=False) + + f = f.healpix_indexing_scheme("ring", sort=True) + f = f.healpix_indexing_scheme("nested") + + g = f.healpix_decrease_refinement_level(0, np.mean, conform=True) + self.assertTrue(g.shape, (2, 12)) + self.assertTrue((g.coord("healpix_index") == np.arange(12)).all()) + + with self.assertRaises(ValueError): + f.healpix_decrease_refinement_level(0, np.mean, conform=False) + + # Bad results when check_healpix_index=False + h = f.healpix_decrease_refinement_level( + 0, np.mean, conform=False, check_healpix_index=False + ) + self.assertFalse((h.coord("healpix_index") == np.arange(12)).all()) + + # Can't change refinment level for a 'nested_unique' field + f13 = cf.example_field(13) + with self.assertRaises(ValueError): + f13.healpix_decrease_refinement_level(0, np.mean) + + # Non-HEALPix field + with self.assertRaises(ValueError): + self.f0.healpix_decrease_refinement_level(0, np.mean) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) diff --git a/cf/test/test_collapse.py b/cf/test/test_collapse.py index c7c866c8b6..f835b22d3e 100644 --- a/cf/test/test_collapse.py +++ b/cf/test/test_collapse.py @@ -3,7 +3,7 @@ import os import unittest -import numpy +import numpy as np faulthandler.enable() # to debug seg faults and timeouts @@ -350,7 +350,7 @@ def test_Field_collapse(self): self.assertEqual(g.shape, (12, 4, 5)) for m in range(1, 13): - a = numpy.empty((5, 4, 5)) + a = np.empty((5, 4, 5)) for i, year in enumerate( f.subspace(T=cf.month(m)).coord("T").year.unique() ): @@ -360,7 +360,7 @@ def test_Field_collapse(self): a[i] = x.array a = a.min(axis=0) - self.assertTrue(numpy.allclose(a, g.array[m % 12])) + self.assertTrue(np.allclose(a, g.array[m % 12])) g = f.collapse("T: mean", group=360) @@ -791,6 +791,26 @@ def test_Field_collapse_non_positive_weights(self): # compute time g.array + def test_Field_collapse_HEALPix(self): + """Test HEALpix collapses.""" + f0 = cf.example_field(12) + f1 = cf.example_field(13) + + g0 = f0.collapse("area: mean", weights=True) + g1 = f1.collapse("area: mean", weights=True) + self.assertTrue(g1.equals(g0)) + self.assertTrue( + g0.coordinate_reference("grid_mapping_name:latitude_longitude") + ) + + g0 = f0[:, 0].collapse("area: mean", weights=True) + g1 = f1[:, :4].collapse("area: mean", weights=True) + self.assertTrue(np.allclose(g0, g1)) + self.assertTrue(g0.coordinate_reference("grid_mapping_name:healpix")) + self.assertTrue( + g1.coordinate_reference("grid_mapping_name:latitude_longitude") + ) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) diff --git a/cf/test/test_weights.py b/cf/test/test_weights.py index bd202672a3..b2861d4a3e 100644 --- a/cf/test/test_weights.py +++ b/cf/test/test_weights.py @@ -336,6 +336,22 @@ def test_weights_exceptions(self): ): f.weights("area") + def test_weights_healpix(self): + """Test HEALPix weights.""" + # HEALPix grid with Multi-Order Coverage (a combination of + # refinement level 1 and 2 cells) + f = cf.example_field(13) + + w = f.weights(components=True)[(1,)].array + self.assertTrue(np.allclose(w[:16], 1 / (4**2))) + self.assertTrue(np.allclose(w[16:], 1 / (4**1))) + + w = f.weights(measure=True, components=True)[(1,)].array + radius = f.radius() + x = 4 * np.pi * (radius**2) / 12 + self.assertTrue(np.allclose(w[:16], x / (4**2))) + self.assertTrue(np.allclose(w[16:], x / (4**1))) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) diff --git a/cf/weights.py b/cf/weights.py index ee38566988..21459ca99d 100644 --- a/cf/weights.py +++ b/cf/weights.py @@ -2000,19 +2000,19 @@ def healpix_area( ) parameters = cr.coordinate_conversion.parameters() - index_scheme = parameters.get("index_scheme") - if index_scheme not in ("nested", "ring", "nuniq"): + indexing_scheme = parameters.get("indexing_scheme") + if indexing_scheme not in ("nested", "ring", "nested_unique"): if auto: return False raise ValueError( - "Can't create weights: Invalid HEALPix index_scheme for " + "Can't create weights: Invalid HEALPix indexing_scheme for " f"{f.constructs.domain_axis_identity(axis)!r} axis: " - f"{index_scheme!r}" + f"{indexing_scheme!r}" ) refinement_level = parameters.get("refinement_level") - if refinement_level is None and index_scheme != "nuniq": + if refinement_level is None and indexing_scheme != "nested_unique": # No refinement_level if auto: return False @@ -2025,8 +2025,8 @@ def healpix_area( if measure and not methods and radius is not None: radius = f.radius(default=radius) - # Create weights for 'nuniq' indexed cells - if index_scheme == "nuniq": + # Create weights for 'nested_unique' indexed cells + if indexing_scheme == "nested_unique": if methods: weights[(axis,)] = "HEALPix Multi-Order Coverage" return True @@ -2047,17 +2047,20 @@ def healpix_area( if measure: units = radius.Units**2 + r = radius.array else: units = "1" + r = None - from .dask_utils import cf_HEALPix_nuniq_area_weights + from .dask_utils import cf_healpix_weights dx = healpix_index.to_dask_array() dx = dx.map_blocks( - cf_HEALPix_nuniq_area_weights, + cf_healpix_weights, meta=np.array((), dtype="float64"), + indexing_scheme="nested_unique", measure=measure, - radius=radius.array, + radius=r, ) area = f._Data(dx, units=units, copy=False) diff --git a/docs/source/field_analysis.rst b/docs/source/field_analysis.rst index 638ee2d073..72e5fb558a 100644 --- a/docs/source/field_analysis.rst +++ b/docs/source/field_analysis.rst @@ -1395,7 +1395,7 @@ on the surface of the sphere, rather than in :ref:`Euclidean space Spherical regridding can occur between source and destination grids that comprise any pairing of `Latitude-longitude`_, `Rotated latitude-longitude`_, `Plane projection`_, `Tripolar`_, `UGRID mesh`_, -and `DSG feature type`_ coordinate systems. +`HEALPix_`, and `DSG feature type`_ coordinate systems. The most convenient usage is when the destination domain exists in another field construct. In this case, all you need to specify is the diff --git a/docs/source/installation.rst b/docs/source/installation.rst index cb973621c7..fa79ef4be9 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -271,7 +271,11 @@ environments for which these features are not required. .. rubric:: HEALPix manipulations * `healpix `_, version 2024.2 or - newer. + newer. This package is not required to read and write HEALPix + datasets, but may be needed for particular manipulations with + HEALPix grids, such as creating latitude and longitude coordinates, + regridding, some changes to the refinement level, and some + collapses. ---- From e4c5b2ba37f9506ea0737e364229eba7c987e48b Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 4 Jul 2025 15:08:26 +0100 Subject: [PATCH 16/59] dev --- cf/dask_utils.py | 38 +++++++++++----- cf/healpix_utils.py | 98 +++++++++++++++++++++------------------- cf/mixin/fielddomain.py | 96 +++++++++++++++++++++++++-------------- cf/regrid/regrid.py | 4 +- cf/test/test_Field.py | 21 +++++++-- cf/test/test_collapse.py | 6 ++- 6 files changed, 161 insertions(+), 102 deletions(-) diff --git a/cf/dask_utils.py b/cf/dask_utils.py index 70421c6afa..97cf8e1c6c 100644 --- a/cf/dask_utils.py +++ b/cf/dask_utils.py @@ -156,17 +156,20 @@ def cf_healpix_indexing_scheme( The array of HEALPix indices. indexing_scheme: `str` - The original HEALPix indexing scheme. One of ``'nested'`` - or ``'ring'``. + The original HEALPix indexing scheme. One of ``'nested'``, + ``'ring'``, or ``'nested_unique'``. new_indexing_scheme: `str` The new HEALPix indexing scheme to change to. One of ``'nested'``, ``'ring'``, or ``'nested_unique'``. - refinement_level: `int` + refinement_level: `int` or `None` The refinement level of the original grid within the HEALPix hierarchy, starting at 0 for the base tesselation - with 12 cells. + with 12 cells. Must be an `int` *indexing_scheme* for + ``'nested'`` or, ``'ring'``, but ignored for + *indexing_scheme* ``'nested_unique'`` (in which case + *refinement_level* may be `None`). :Returns: @@ -177,8 +180,10 @@ def cf_healpix_indexing_scheme( >>> cf_healpix_indexing_scheme([0, 1, 2, 3], 'nested', 'ring', 1) array([13, 5, 4, 0]) - >>> cf_healpix_indexing_scheme([0, 1, 2, 3], 'nested_unique', 'ring', 1) + >>> cf_healpix_indexing_scheme([0, 1, 2, 3], 'nested', 'nested_unique', 1) array([16, 17, 18, 19]) + >>> cf_healpix_indexing_scheme([16, 17, 18, 19], 'nested_unique', 'nest', None) + array([0, 1, 2, 3]) """ if new_indexing_scheme == indexing_scheme: @@ -209,13 +214,22 @@ def cf_healpix_indexing_scheme( return healpix._chp.ring2uniq(refinement_level, a) elif indexing_scheme == "nested_unique": - raise ValueError( - "Can't change HEALPix scheme from 'nested_unique' to " - f"{new_indexing_scheme!r}" - ) - - else: - raise ValueError(f"Invalid HEALPix scheme: {indexing_scheme!r}") + if new_indexing_scheme in ("nested", "ring"): + nest = new_indexing_scheme == "nested" + order, a = healpix.uniq2pix(a, nest=nest) + + refinement_levels = np.unique(order) + if refinement_levels.size > 1: + raise ValueError( + "Can't change HEALPix indexing scheme from " + f"'nested_unique' to {new_indexing_scheme!r} when the " + "HEALPix indices span multiple refinement levels (at " + f"least levels {refinement_levels.tolist()})" + ) + + return a + + raise ValueError("Failed to change the HEALPix indexing scheme") def cf_healpix_coordinates( diff --git a/cf/healpix_utils.py b/cf/healpix_utils.py index 61ebea7b3e..0879bb3d9d 100644 --- a/cf/healpix_utils.py +++ b/cf/healpix_utils.py @@ -1,18 +1,32 @@ -"""TODOHEALPIX""" +"""General functions useful for HEALPix functionality.""" def del_healpix_coordinate_reference(f, axis=None): - """TODOHEALPIX + """Remove a healpix grid mapping coordinate reference construct. + + A new latitude_longitude grid mapping coordinate reference will be + created in-place, if required, to store any generic coordinate + conversion or datum parameters found in the healpix grid mapping + coordinate reference. + + .. versionadded:: NEXTVERSION :Parameters: f: `Field` or `Domain` + The Field or Domain from which to delete the healpix grid + mapping coordinate reference. - axis: optional + axis: `str`, optional + The identifier of the HEALPix domain axis. If not set then + it will be inferred from the healpix_index corodinates, if + required. :Returns: `CoordinateReference` or `None` + The removed healpix grid mapping coordinate reference + construct, or `None` if there wasn't one. """ cr_key, cr = f.coordinate_reference( @@ -22,48 +36,38 @@ def del_healpix_coordinate_reference(f, axis=None): "grid_mapping_name:latitude_longitude", default=None ) - if cr is None: - out = None - else: - out = cr.copy() - - if latlon is not None: - # There is already a latitude_longitude coordinate reference, - # so delete a healpix coordinate reference. - if cr is not None: - f.del_construct(cr_key) - - elif cr is not None: - cc = cr.coordinate_conversion - cc.del_parameter("grid_mapping_name", None) - cc.del_parameter("indexing_scheme", None) - cc.del_parameter("refinement_level", None) - if cc.parameters() or cr.datum.parameters(): - # The healpix coordinate reference contains generic - # coordinate conversion or datum parameters, so rename it - # as 'latitude_longitude'. - cr.coordinate_conversion.set_parameter( - "grid_mapping_name", "latitude_longitude" - ) - - # Remove a healpix_index coordinate from the coordinate - # reference - if axis is None: - axis = f.healpix_axis(None) - - if axis is not None: - key = f.coordinate( - "healpix_index", - filter_by_axis=(axis,), - axis_mode="exact", - key=True, - default=None, + if cr is not None: + f.del_construct(cr_key) + + if latlon is None: + latlon = cr.copy() + cc = latlon.coordinate_conversion + cc.del_parameter("grid_mapping_name", None) + cc.del_parameter("indexing_scheme", None) + cc.del_parameter("refinement_level", None) + if cc.parameters() or latlon.datum.parameters(): + # The healpix coordinate reference contains generic + # coordinate conversion or datum parameters. + latlon.coordinate_conversion.set_parameter( + "grid_mapping_name", "latitude_longitude" ) - if key is not None: - cr.del_coordinate(key) - else: - # The healpix coordinate reference contains no generic - # parameters, so delete it. - f.del_construct(cr_key) - - return out + + # Remove a healpix_index coordinate from the coordinate + # reference + if axis is None: + axis = f.healpix_axis(None) + + if axis is not None: + key = f.coordinate( + "healpix_index", + filter_by_axis=(axis,), + axis_mode="exact", + key=True, + default=None, + ) + if key is not None: + latlon.del_coordinate(key) + + f.set_construct(latlon) + + return cr diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index f415761535..460ebd301c 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -1999,14 +1999,16 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): :Parameters: - new_indexing_scheme: `str` + new_indexing_scheme: `str` or `None` The new HEALPix indexing scheme. One of ``'nested'``, - ``'ring'``, or ``'nested_unique'``. + ``'ring'``, ``'nested_unique'``, or `None`. If `None` + then the indexing scheme is unchanged. sort: `bool`, optional If True then sort the HEALPix axis of the output so - that its HEALPix indices are monotonically - increasing. If False (the default) then don't do this. + that its HEALPix indices are monotonically increasing, + including when the indexing scheme is unchanged. If + False (the default) then don't do this. :Returns: @@ -2060,15 +2062,15 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): f = self.copy() - if new_indexing_scheme not in ("nested", "ring", "nested_unique"): + valid_indexing_schemes = ("nested", "ring", "nested_unique") + if new_indexing_scheme not in valid_indexing_schemes + (None,): raise ValueError( "Can't change HEALPix index scheme: new_indexing_scheme " - "keyword must be 'nested', 'ring', or 'nested_unique'. " + f"keyword must be None or one of {valid_indexing_schemes!r}. " f"Got {new_indexing_scheme!r}" ) - # Get the original healpix_index coordinates and the key of - # the HEALPix domain axis + # Get the original healpix_index coordinates hp_key, healpix_index = f.coordinate( "healpix_index", filter_by_naxes=(1,), @@ -2090,37 +2092,62 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): ) parameters = cr.coordinate_conversion.parameters() - refinement_level = parameters.get("refinement_level") indexing_scheme = parameters.get("indexing_scheme") - if indexing_scheme is None: raise ValueError( "Can't change HEALPix indexing scheme: indexing_scheme has " "not been set in the HEALPix grid mapping coordinate reference" ) - if indexing_scheme not in ("nested", "ring", "nested_unique"): + if indexing_scheme not in valid_indexing_schemes: raise ValueError( - f"Can't change HEALPix indexing scheme: Invalid " - f"indexing_scheme: {indexing_scheme!r}" + "Can't change HEALPix indexing scheme: indexing_scheme in " + "the HEALPix grid mapping coordinate reference must be one " + f"of {valid_indexing_schemes!r}. Got {indexing_scheme!r}" ) - if indexing_scheme == "nested_unique": - if new_indexing_scheme != "nested_unique": + if ( + new_indexing_scheme is not None + and new_indexing_scheme != indexing_scheme + ): + refinement_level = parameters.get("refinement_level") + if ( + indexing_scheme in ("nested", "ring") + and refinement_level is None + ): raise ValueError( "Can't change HEALPix indexing scheme from " - f"'nested_unique' to {new_indexing_scheme!r}" + f"{indexing_scheme!r} to {new_indexing_scheme!r} when " + "refinement_level has not been set in the HEALPix grid " + "mapping coordinate reference" ) - elif refinement_level is None: - raise ValueError( - "Can't change HEALPix indexing scheme from " - f"{indexing_scheme!r} when refinement_level has not been set " - "in the HEALPix grid mapping coordinate reference" - ) - axis = f.get_data_axes(hp_key)[0] + # Update the Coordinate Reference + cr.coordinate_conversion.set_parameter( + "indexing_scheme", new_indexing_scheme + ) + if new_indexing_scheme == "nested_unique": + cr.coordinate_conversion.del_parameter("refinement_level") + elif indexing_scheme == "nested_unique": + # Set the refinement level for the new indexing + # scheme. This is the largest integer, N, for which + # 2**(2(N+1)) <= healpix_index[0] (see "Efficient data + # structures for masks on 2D grids". M. Reinecke and + # E. Hivon. A&A, 580 (2015) A132. DOI: + # https://doi.org/10.1051/0004-6361/201526549) + # + # It doesn't matter if there are in fact multiple + # refinement levels, as this will get trapped as an + # exception when 'cf_healpix_indexing_scheme' is + # executed. + from math import log2 + + cr.coordinate_conversion.set_parameter( + "refinement_level", + int(log2(int(healpix_index.data.first_element())) // 2) + - 1, + ) - if indexing_scheme != new_indexing_scheme: # Change the HEALPix indices dx = healpix_index.to_dask_array() dx = dx.map_blocks( @@ -2132,12 +2159,8 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): ) healpix_index.set_data(dx, copy=False) - # Update the Coordinate Reference - cr.coordinate_conversion.set_parameter( - "indexing_scheme", new_indexing_scheme - ) - if new_indexing_scheme == "nested_unique": - cr.coordinate_conversion.del_parameter("refinement_level") + # Get the identifier fo the HEALPix domain axis + axis = f.get_data_axes(hp_key)[0] # Ensure that healpix indices are auxiliary coordinates if healpix_index.construct_type == "dimension_coordinate": @@ -2151,11 +2174,14 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): if sort: # Sort the HEALPix axis so that the HEALPix indices are # monotonically increasing. Test for the common case of - # already-ordered global nested indices (which is fast - # compared to do doing any sorting). - d = healpix_index.to_dask_array() - if not (d == da.arange(d.size)).all(): - index = healpix_index.data.compute() + # already-ordered global nested or ring indices (which is + # a fast test compared to do doing any actual sorting). + hp = healpix_index + if not ( + indexing_scheme in ("nested", "ring") + and (hp == da.arange(hp.size, chunks=hp.data.chunks)).all() + ): + index = hp.data.compute() f = f.subspace(**{axis: np.argsort(index)}) return f diff --git a/cf/regrid/regrid.py b/cf/regrid/regrid.py index 2b18b91b01..508c2c506f 100644 --- a/cf/regrid/regrid.py +++ b/cf/regrid/regrid.py @@ -2828,8 +2828,8 @@ def update_coordinates(src, dst, src_grid, dst_grid, cr_map): """ dst = dst_grid.domain - # An HEALPix grid is converted to UGRID for the regridding, but we - # want the original HEALPic metadata to be copied to the regridded + # A HEALPix grid is converted to UGRID for the regridding, but we + # want the original HEALPix metadata to be copied to the regridded # source field, rather than the UGRID view of it. dst_grid_is_mesh = dst_grid.is_mesh and not dst_grid.healpix diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index d3b23d7fb6..c80e7796d8 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -79,6 +79,7 @@ class FieldTest(unittest.TestCase): f0 = cf.example_field(0) f1 = cf.example_field(1) f12 = cf.example_field(12) + f13 = cf.example_field(13) def test_Field_creation_commands(self): for f in cf.example_fields(): @@ -1487,7 +1488,6 @@ def test_Field_indices(self): ) g = f[indices] x = g.dimension_coordinate("X").array - print(x) self.assertEqual(g.shape, (1, 10, 3)) self.assertTrue((x == [120, 200, 280]).all()) @@ -3106,6 +3106,10 @@ def test_Field_healpix_indexing_scheme(self): # HEALPix field f = self.f12 + # Null change + g = f.healpix_indexing_scheme(None) + self.assertTrue(g.equals(f)) + # Null change g = f.healpix_indexing_scheme("nested") self.assertTrue(g.equals(f)) @@ -3143,9 +3147,17 @@ def test_Field_healpix_indexing_scheme(self): h = g.healpix_indexing_scheme("nested_unique") self.assertTrue(h.equals(g)) - # Can't change from 'nested_unique' to 'nested' + # Can change from 'nested_unique' to 'nested' with a single + # refinement level + g = f.healpix_indexing_scheme("nested_unique") + h = g.healpix_indexing_scheme("nested") + self.assertTrue(h.equals(f, ignore_data_type=True)) + + # Can't change from 'nested_unique' to 'nested' with multiple + # refinement level (error comes at comute time) + g = self.f13.healpix_indexing_scheme("nested") with self.assertRaises(ValueError): - g.healpix_indexing_scheme("nested") + g.auxiliary_coordinate("healpix_index").array # Non-HEALPix field with self.assertRaises(ValueError): @@ -3240,9 +3252,8 @@ def test_Field_healpix_decrease_refinement_level(self): self.assertFalse((h.coord("healpix_index") == np.arange(12)).all()) # Can't change refinment level for a 'nested_unique' field - f13 = cf.example_field(13) with self.assertRaises(ValueError): - f13.healpix_decrease_refinement_level(0, np.mean) + self.f13.healpix_decrease_refinement_level(0, np.mean) # Non-HEALPix field with self.assertRaises(ValueError): diff --git a/cf/test/test_collapse.py b/cf/test/test_collapse.py index f835b22d3e..22db0bceb8 100644 --- a/cf/test/test_collapse.py +++ b/cf/test/test_collapse.py @@ -792,10 +792,14 @@ def test_Field_collapse_non_positive_weights(self): g.array def test_Field_collapse_HEALPix(self): - """Test HEALpix collapses.""" + """Test HEALPix collapses.""" f0 = cf.example_field(12) f1 = cf.example_field(13) + g0 = f0.collapse("area: mean", weights=False) + g1 = f1.collapse("area: mean", weights=False) + self.assertFalse(np.allclose(g0, g1)) + g0 = f0.collapse("area: mean", weights=True) g1 = f1.collapse("area: mean", weights=True) self.assertTrue(g1.equals(g0)) From 77265e4a8229dfc890e927e862dd385750e8f9af Mon Sep 17 00:00:00 2001 From: David Hassell Date: Sat, 5 Jul 2025 00:45:15 +0100 Subject: [PATCH 17/59] dev --- cf/data/data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cf/data/data.py b/cf/data/data.py index 3f370cf7ff..26c466bb16 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -6286,8 +6286,8 @@ def reshape(self, *shape, merge_chunks=True, limit=None, inplace=False): """Change the shape of the data without changing its values. It assumes that the array is stored in row-major order, and - only allows for reshapings that collapse or merge dimensions - like ``(1, 2, 3, 4) -> (1, 6, 4)`` or ``(64,) -> (4, 4, 4)``. + only allows for reshapings that collapse or merge dimensions, + e.g. ``(1, 2, 3, 4) -> (1, 6, 4)`` or ``(64,) -> (4, 4, 4)``. :Parameters: From 10cb5d5d367df417350c9908e0acb6de758151f3 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Sun, 6 Jul 2025 13:00:35 +0100 Subject: [PATCH 18/59] dev --- Changelog.rst | 7 +- cf/dask_utils.py | 193 ++++++++++++++++++++-------------------- cf/data/data.py | 3 +- cf/domain.py | 10 ++- cf/field.py | 8 +- cf/healpix_utils.py | 28 ++---- cf/mixin/fielddomain.py | 58 +----------- cf/test/test_Data.py | 2 +- cf/test/test_Field.py | 46 ++++++---- cf/weights.py | 2 +- 10 files changed, 155 insertions(+), 202 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 8e276d9d24..f619fc521a 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -5,10 +5,9 @@ Version NEXTVERSION * New method: `cf.Field.create_latlon_coordinates` (https://github.com/NCAS-CMS/cf-python/issues/???) -* New HEALPix methods: `cf.Field.healpix_axis`, - `cf.Field.healpix_change_order`, - `cf.Field.healpix_decrease_refinement_level`, - `cf.Field.healpix_to_ugrid`, `cf.Domain.create_healpix` +* New HEALPix methods: `cf.Field.healpix_decrease_refinement_level`, + `cf.Field.healpix_indexing_scheme`, `cf.Field.healpix_to_ugrid`, + `cf.Domain.create_healpix` (https://github.com/NCAS-CMS/cf-python/issues/???) * New method: `cf.Data.coarsen` (https://github.com/NCAS-CMS/cf-python/issues/???) diff --git a/cf/dask_utils.py b/cf/dask_utils.py index 97cf8e1c6c..dc4f39f92a 100644 --- a/cf/dask_utils.py +++ b/cf/dask_utils.py @@ -30,11 +30,11 @@ def cf_healpix_bounds( ``'ring'``, or ``'nested_unique'``. refinement_level: `int` or `None`, optional - For a ``'nested'`` or ``'ring'`` ordered grid, the + For a ``'nested'`` or ``'ring'`` indexed grid, the refinement level of the grid within the HEALPix hierarchy, starting at 0 for the base tesselation with 12 cells. Set - to `None` for a ``'nested_unique'`` ordered grid, for which the - refinement level is ignored. + to `None` for a ``'nested_unique'`` indexed grid, for + which the refinement level is ignored. lat: `bool`, optional If True then return latitude bounds. @@ -100,7 +100,8 @@ def cf_healpix_bounds( if indexing_scheme == "nested_unique": # Create bounds for 'nested_unique' cells - orders, a = healpix.uniq2pix(a, nest=False) + nest = False + orders, a = healpix.uniq2pix(a, nest=nest) orders, index, inverse = np.unique( orders, return_index=True, return_inverse=True ) @@ -140,6 +141,98 @@ def cf_healpix_bounds( return b +def cf_healpix_coordinates( + a, indexing_scheme, refinement_level=None, lat=False, lon=False +): + """Calculate HEALPix cell coordinates. + + THe coordinates are for the cell centres. + + .. versionadded:: NEXTVERSION + + :Parameters: + + a: `numpy.ndarray` + The array of HEALPix indices. + + indexing_scheme: `str` + The HEALPix indexing scheme. One of ``'nested'``, + ``'ring'``, or ``'nested_unique'``. + + refinement_level: `int` or `None`, optional + For a ``'nested'`` or ``'ring'`` indexed grid, the + refinement level of the grid within the HEALPix hierarchy, + starting at 0 for the base tesselation with 12 cells. + Ignored for a ``'nested_unique'`` indexed grid. + + lat: `bool`, optional + If True then return latitude coordinates. + + lon: `bool`, optional + If True then return longitude coordinates. + + :Returns: + + `numpy.ndarray` + A 1-d array containing the HEALPix cell coordinates. + + **Examples** + + >>> cf_healpix_coordinates([0, 1, 2, 3], 'nested', 1, lat=True) + array([19.47122063, 41.8103149 , 41.8103149 , 66.44353569]) + >>> cf_healpix_coordinates([0, 1, 2, 3], 'nested', 1, lon=True) + array([45. , 67.5, 22.5, 45. ]) + + """ + try: + import healpix + except ImportError as e: + raise ImportError( + f"{e}. Must install healpix (e.g. from " + "https://pypi.org/project/healpix) to allow the calculation " + "of latitude/longitude coordinates for a HEALPix grid" + ) + + if a.ndim != 1: + raise ValueError( + "Can't calculate HEALPix cell coordinates when the " + f"healpix_index array has one dimension. Got shape {a.shape}" + ) + + if lat: + pos = 1 + elif lon: + pos = 0 + + if indexing_scheme == "nested_unique": + # Create coordinates for 'nested_unique' cells + c = np.empty(a.shape, dtype="float64") + + nest = False + orders, a = healpix.uniq2pix(a, nest=nest) + orders, index, inverse = np.unique( + orders, return_index=True, return_inverse=True + ) + for order, i in zip(orders, index): + level = np.where(inverse == inverse[i])[0] + nside = healpix.order2nside(order) + c[level] = healpix.pix2ang( + nside=nside, ipix=a[level], nest=nest, lonlat=True + )[pos] + else: + # Create coordinates for 'nested' or 'ring' cells + nest = (indexing_scheme == "nested",) + nside = healpix.order2nside(refinement_level) + c = healpix.pix2ang( + nside=nside, + ipix=a, + nest=nest, + lonlat=True, + )[pos] + + return c + + def cf_healpix_indexing_scheme( a, indexing_scheme, new_indexing_scheme, refinement_level ): @@ -232,98 +325,6 @@ def cf_healpix_indexing_scheme( raise ValueError("Failed to change the HEALPix indexing scheme") -def cf_healpix_coordinates( - a, indexing_scheme, refinement_level=None, lat=False, lon=False -): - """Calculate HEALPix cell coordinates. - - THe coordinates are for the cell centres. - - .. versionadded:: NEXTVERSION - - :Parameters: - - a: `numpy.ndarray` - The array of HEALPix indices. - - indexing_scheme: `str` - The HEALPix indexing scheme. One of ``'nested'``, - ``'ring'``, or ``'nested_unique'``. - - refinement_level: `int` or `None`, optional - For a ``'nested'`` or ``'ring'`` ordered grid, the - refinement level of the grid within the HEALPix hierarchy, - starting at 0 for the base tesselation with 12 cells. - Ignored for a ``'nested_unique'`` ordered grid. - - lat: `bool`, optional - If True then return latitude coordinates. - - lon: `bool`, optional - If True then return longitude coordinates. - - :Returns: - - `numpy.ndarray` - A 1-d array containing the HEALPix cell coordinates. - - **Examples** - - >>> cf_healpix_coordinates([0, 1, 2, 3], 'nested', 1, lat=True) - array([19.47122063, 41.8103149 , 41.8103149 , 66.44353569]) - >>> cf_healpix_coordinates([0, 1, 2, 3], 'nested', 1, lon=True) - array([45. , 67.5, 22.5, 45. ]) - - """ - try: - import healpix - except ImportError as e: - raise ImportError( - f"{e}. Must install healpix (e.g. from " - "https://pypi.org/project/healpix) to allow the calculation " - "of latitude/longitude coordinates for a HEALPix grid" - ) - - if a.ndim != 1: - raise ValueError( - "Can't calculate HEALPix cell coordinates when the " - f"healpix_index array has one dimension. Got shape {a.shape}" - ) - - if lat: - pos = 1 - elif lon: - pos = 0 - - if indexing_scheme == "nested_unique": - # Create coordinates for 'nested_unique' cells - c = np.empty(a.shape, dtype="float64") - - nest = False - orders, a = healpix.uniq2pix(a, nest=nest) - orders, index, inverse = np.unique( - orders, return_index=True, return_inverse=True - ) - for order, i in zip(orders, index): - level = np.where(inverse == inverse[i])[0] - nside = healpix.order2nside(order) - c[level] = healpix.pix2ang( - nside=nside, ipix=a[level], nest=nest, lonlat=True - )[pos] - else: - # Create coordinates for 'nested' or 'ring' cells - nest = (indexing_scheme == "nested",) - nside = healpix.order2nside(refinement_level) - c = healpix.pix2ang( - nside=nside, - ipix=a, - nest=nest, - lonlat=True, - )[pos] - - return c - - def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): """Calculate HEALPix cell area weights. diff --git a/cf/data/data.py b/cf/data/data.py index 26c466bb16..a3a3ebe061 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1413,7 +1413,7 @@ def coarsen( [10] [16] [22]] - >>> e = d.coarsen(np.max, {-1: 5}) + >>> e = d.coarsen(np.max, {-1: 5}, trim_excess=False) ValueError: Coarsening factors {1: 5} do not align with array shape (4, 6). """ @@ -1425,6 +1425,7 @@ def coarsen( if k < -ndim or k > ndim: raise ValueError("axis {k} is out of bounds for {ndim}-d data") + # Make sure all axes are non-negative axes = {(k + ndim if k < 0 else k): v for k, v in axes.items()} dx = d.to_dask_array() diff --git a/cf/domain.py b/cf/domain.py index 0e2653285a..d44b44057a 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -271,7 +271,8 @@ def create_healpix( .. versionadded:: NEXTVERSION - .. seealso:: `cf.Domain.create_regular` + .. seealso:: `cf.Domain.create_regular`, + `cf.Domain.create_latlon_coordinates` :Parameters: @@ -343,6 +344,13 @@ def create_healpix( Coordinate conversion:indexing_scheme = nested_unique Datum:earth_radius = 6371000.0 Auxiliary Coordinate: healpix_index + >>> d = cf.Domain.create_healpix(4) + >>> d.create_latlon_coordinates(inplace=True) + >>> print(d) + Auxiliary coords: healpix_index(ncdim%cell(3072)) = [0, ..., 3071] 1 + : latitude(ncdim%cell(3072)) = [2.388015463268772, ..., -2.388015463268786] degrees_north + : longitude(ncdim%cell(3072)) = [45.0, ..., 315.0] degrees_east + Coord references: grid_mapping_name:healpix """ import dask.array as da diff --git a/cf/field.py b/cf/field.py index 57e02fcc2a..d469c048ad 100644 --- a/cf/field.py +++ b/cf/field.py @@ -7233,12 +7233,14 @@ def collapse( # -------------------------------------------------------- # Remove a HEALPix coordinate reference # -------------------------------------------------------- - axis = f.healpix_axis(None) - domain_axis = collapse_axes.get(axis) + healpix_axis = f.domain_axis( + "healpix_index", key=True, default=None + ) + domain_axis = collapse_axes.get(healpix_axis) if domain_axis is not None and domain_axis.get_size() > 1: from .healpix_utils import del_healpix_coordinate_reference - del_healpix_coordinate_reference(f, axis=axis) + del_healpix_coordinate_reference(f) # --------------------------------------------------------- # Update dimension coordinates, auxiliary coordinates, diff --git a/cf/healpix_utils.py b/cf/healpix_utils.py index 0879bb3d9d..6c135dfcf4 100644 --- a/cf/healpix_utils.py +++ b/cf/healpix_utils.py @@ -1,7 +1,7 @@ """General functions useful for HEALPix functionality.""" -def del_healpix_coordinate_reference(f, axis=None): +def del_healpix_coordinate_reference(f): """Remove a healpix grid mapping coordinate reference construct. A new latitude_longitude grid mapping coordinate reference will be @@ -17,11 +17,6 @@ def del_healpix_coordinate_reference(f, axis=None): The Field or Domain from which to delete the healpix grid mapping coordinate reference. - axis: `str`, optional - The identifier of the HEALPix domain axis. If not set then - it will be inferred from the healpix_index corodinates, if - required. - :Returns: `CoordinateReference` or `None` @@ -47,26 +42,17 @@ def del_healpix_coordinate_reference(f, axis=None): cc.del_parameter("refinement_level", None) if cc.parameters() or latlon.datum.parameters(): # The healpix coordinate reference contains generic - # coordinate conversion or datum parameters. + # coordinate conversion or datum parameters latlon.coordinate_conversion.set_parameter( "grid_mapping_name", "latitude_longitude" ) - # Remove a healpix_index coordinate from the coordinate + # Remove healpix_index coordinates from the coordinate # reference - if axis is None: - axis = f.healpix_axis(None) - - if axis is not None: - key = f.coordinate( - "healpix_index", - filter_by_axis=(axis,), - axis_mode="exact", - key=True, - default=None, - ) - if key is not None: - latlon.del_coordinate(key) + for key in f.coordinates( + "healpix_index", filter_by_naxes=(1,), todict=True + ): + latlon.del_coordinate(key, None) f.set_construct(latlon) diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 460ebd301c..3c5b3a4b01 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -1940,63 +1940,11 @@ def coordinate_reference_domain_axes(self, identity=None): return set(axes) - def healpix_axis(self, default=ValueError()): - """Return the HEALPix axis identifier. - - .. versionadded:: NEXTVERSION - - .. seealso:: `healpix_indexing_scheme`, `healpix_to_ugrid` - - :Parameters: - - default: optional - Return the value of the *default* parameter if an - HEALPix axis can not be found. - - {{default Exception}} - - :Returns: - - `str` - The identifier of the HEALPix domain axis construct. - - **Examples** - - >>> f = cf.example_field(12) - >>> print(f) - Field: air_temperature (ncvar%tas) - ---------------------------------- - Data : air_temperature(time(2), healpix_index(48)) K - Cell methods : time(2): mean area: mean - Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian - : height(1) = [1.5] m - Auxiliary coords: healpix_index(healpix_index(48)) = [0, ..., 47] 1 - Coord references: grid_mapping_name:healpix - >>> axis = f.healpix_axis() - >>> axis - 'domainaxis1' - >>> f.constructs.domain_axis_identity(axis) - 'healpix_index' - - """ - key = self.coordinate( - "healpix_index", filter_by_naxes=(1,), key=True, default=None - ) - if key is None: - if default is None: - return - - return self._default(default, "There is no HEALPix axis") - - return self.get_data_axes(key)[0] - def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): """Change the indexing scheme of HEALPix indices. .. versionadded:: NEXTVERSION - .. seealso:: `healpix_axis` - :Parameters: new_indexing_scheme: `str` or `None` @@ -2192,7 +2140,7 @@ def healpix_to_ugrid(self, inplace=False): .. versionadded:: NEXTVERSION - .. seealso:: `create_latlon_coordinates`, `healpix_axis` + .. seealso:: `create_latlon_coordinates` :Parameters: @@ -2229,7 +2177,7 @@ def healpix_to_ugrid(self, inplace=False): Topologies : cell:face(ncdim%cell(48), 4) = [[965, ..., 3074]] """ - axis = self.healpix_axis(None) + axis = self.domain_axis("healpix_index", key=True, default=None) if axis is None: raise ValueError( "Can't convert HEALPix to UGRID: There is no HEALPix domain " @@ -2313,7 +2261,7 @@ def healpix_to_ugrid(self, inplace=False): from ..healpix_utils import del_healpix_coordinate_reference - del_healpix_coordinate_reference(f, axis=axis) + del_healpix_coordinate_reference(f) return f diff --git a/cf/test/test_Data.py b/cf/test/test_Data.py index 52ea908c1f..d0eb7a5de7 100644 --- a/cf/test/test_Data.py +++ b/cf/test/test_Data.py @@ -4686,7 +4686,7 @@ def test_Data_collapse_axes_hdf_chunks(self): def test_Data_coarsen(self): """Test Data.coarsen.""" - d = cf.Data(np.arange(24).reshape((4, 6))) + d = cf.Data(np.arange(24).reshape((4, 6)), chunks=(3, 3)) a = d.array self.assertIsNone(d.coarsen(np.min, axes={}, inplace=True)) diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index c80e7796d8..c055723893 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -3087,20 +3087,6 @@ def test_Field_filled(self): self.assertEqual(values[0], -999) self.assertEqual(counts[0], 5) - def test_Field_healpix_axis(self): - """Test Field.healpix_axis.""" - # HEALPix field - f = self.f12 - - key = f.auxiliary_coordinate("healpix_index", key=True) - axis = f.get_data_axes(key)[0] - self.assertEqual(f.healpix_axis(), axis) - - # Non-HEALPix field - self.assertIsNone(self.f0.healpix_axis(None)) - with self.assertRaises(ValueError): - self.f0.healpix_axis() - def test_Field_healpix_indexing_scheme(self): """Test Field.healpix_indexing_scheme.""" # HEALPix field @@ -3204,21 +3190,43 @@ def test_Field_create_latlon_coordinates(self): self.assertIsNone(f.create_latlon_coordinates(inplace=True)) self.assertTrue(f.equals(g)) + g = self.f12.healpix_indexing_scheme("nested_unique") + g.create_latlon_coordinates(inplace=True) + for c in ("latitude", "longitude"): + self.assertTrue( + g.auxiliary_coordinate(c).equals(f.auxiliary_coordinate(c)) + ) + + # Check a Multi-Order Coverage grid + m = self.f13 + m = m.create_latlon_coordinates() + + l1 = f + l2 = cf.Domain.create_healpix(2) + l2.create_latlon_coordinates(inplace=True) + + for c in ("latitude", "longitude"): + mc = m.auxiliary_coordinate(c) + self.assertTrue(mc[:16].equals(l2.auxiliary_coordinate(c)[:16])) + self.assertTrue(mc[16:].equals(l1.auxiliary_coordinate(c)[4:])) + def test_Field_subspace_healpix(self): - """Test Field.subspace for HEALPix""" + """Test Field.subspace for HEALPix grids""" f = self.f12 g = f.subspace(X=cf.wi(40, 70), Y=cf.wi(-20, 30)) - self.assertTrue(np.allclose(g.aux("healpix_index"), [0, 22, 35])) + self.assertTrue( + np.allclose(g.coordinate("healpix_index"), [0, 22, 35]) + ) g.create_latlon_coordinates(inplace=True) - self.assertTrue(np.allclose(g.aux("X"), [45.0, 67.5, 45.0])) + self.assertTrue(np.allclose(g.coordinate("X"), [45.0, 67.5, 45.0])) self.assertTrue( np.allclose( - g.aux("Y"), [19.47122063449069, 0.0, -19.47122063449069] + g.coordinate("Y"), [19.47122063449069, 0.0, -19.47122063449069] ) ) def test_Field_healpix_decrease_refinement_level(self): - """Test Field.healpix_decrease_refinement_level""" + """Test Field.healpix_decrease_refinement_level.""" f = self.f12 g = f.healpix_decrease_refinement_level(0, np.mean) self.assertTrue(g.shape, (2, 12)) diff --git a/cf/weights.py b/cf/weights.py index 21459ca99d..19a6735f07 100644 --- a/cf/weights.py +++ b/cf/weights.py @@ -1957,7 +1957,7 @@ def healpix_area( the weights are returned. """ - axis = f.healpix_axis(None) + axis = f.domain_axis("healpix_index", key=True, default=None) if axis is None: if auto: return False From c3bf502f375b3395105276a4deef6707d78e1d32 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 7 Jul 2025 08:51:22 +0100 Subject: [PATCH 19/59] typo --- cf/mixin/propertiesdata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cf/mixin/propertiesdata.py b/cf/mixin/propertiesdata.py index 158bd2dc17..012777f6a6 100644 --- a/cf/mixin/propertiesdata.py +++ b/cf/mixin/propertiesdata.py @@ -530,6 +530,8 @@ def __pos__(self): def __query_isclose__(self, value, rtol, atol): """Query interface method for an "is close" condition. + .. versionadded:: 3.15.2 + :Parameters: value: @@ -541,8 +543,6 @@ def __query_isclose__(self, value, rtol, atol): atol: number The tolerance on absolute numerical differences. - .. versionadded:: 3.15.2 - """ data = self.get_data(None, _fill_value=None) if data is None: From d594a8e8e79993146ab0981f4a61e457ddc2959c Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 7 Jul 2025 20:42:59 +0100 Subject: [PATCH 20/59] dev --- cf/dask_utils.py | 115 +++++++++++++-------- cf/data/data.py | 19 ++-- cf/docstring/docstring.py | 22 ++++ cf/domain.py | 17 ++-- cf/field.py | 166 +++++++------------------------ cf/healpix_utils.py | 2 +- cf/mixin/fielddomain.py | 68 +++++++++---- cf/mixin/propertiesdatabounds.py | 4 +- cf/query.py | 54 ++++++++++ cf/test/test_Field.py | 85 ++++++++++++---- 10 files changed, 321 insertions(+), 231 deletions(-) diff --git a/cf/dask_utils.py b/cf/dask_utils.py index dc4f39f92a..561b99c649 100644 --- a/cf/dask_utils.py +++ b/cf/dask_utils.py @@ -9,7 +9,12 @@ def cf_healpix_bounds( - a, indexing_scheme, refinement_level=None, lat=False, lon=False + a, + indexing_scheme, + refinement_level=None, + lat=False, + lon=False, + pole_longitude=None, ): """Calculate HEALPix cell bounds. @@ -42,6 +47,13 @@ def cf_healpix_bounds( lon: `bool`, optional If True then return longitude bounds. + pole_longitude: `None` or number + The longitude of coordinate bounds that lie exactly on the + north or south pole. If `None` (the default) then the + longitudes of such a point will be identical to its + opposite vertex. If set to a number, then the longitudes + of such points will all be that value. + :Returns: `numpy.ndarray` @@ -55,19 +67,25 @@ def cf_healpix_bounds( [41.8103149 , 66.44353569, 41.8103149 , 19.47122063], [66.44353569, 90. , 66.44353569, 41.8103149 ]]) >>> cf_healpix_bounds([0, 1, 2, 3], 'nested', 1, lon=True) - array([[67.5, 45. , 22.5, 45. ], - [90. , 90. , 45. , 67.5], - [45. , 0. , 0. , 22.5], - [90. , 45. , 0. , 45. ]]) + array([[45. , 22.5, 45. , 67.5], + [90. , 45. , 67.5, 90. ], + [ 0. , 0. , 22.5, 45. ], + [45. , 0. , 45. , 90. ]]) + >>> cf_healpix_bounds([0, 1, 2, 3], 'nested', 1, lon=True, + ... pole_longitude=3.14159) + array([[45. , 22.5 , 45. , 67.5 ], + [90. , 45. , 67.5 , 90. ], + [ 0. , 0. , 22.5 , 45. ], + [ 3.14159, 0. , 45. , 90. ]]) """ try: import healpix except ImportError as e: raise ImportError( - f"{e}. Must install healpix (e.g. from " - "https://pypi.org/project/healpix) to allow the calculation " - "of latitude/longitude coordinate bounds for a HEALPix grid" + f"{e}. Must install healpix (https://pypi.org/project/healpix) " + "to allow the calculation of latitude/longitude coordinate " + "bounds for a HEALPix grid" ) # Keep an eye on https://github.com/ntessore/healpix/issues/66 @@ -82,26 +100,26 @@ def cf_healpix_bounds( elif lon: pos = 0 - if indexing_scheme == "nested": - bounds_func = healpix._chp.nest2ang_uv - else: + if indexing_scheme == "ring": bounds_func = healpix._chp.ring2ang_uv + else: + bounds_func = healpix._chp.nest2ang_uv # Define the cell vertices, in an anticlockwise direction, as seen - # from above, starting with the eastern-most vertex. - right = (1, 0) - top = (1, 1) - left = (0, 1) - bottom = (0, 0) - vertices = (right, top, left, bottom) + # from above, starting with the northern-most vertex. + east = (1, 0) + north = (1, 1) + west = (0, 1) + south = (0, 0) + vertices = (north, west, south, east) # Initialise the output bounds array b = np.empty((a.size, 4), dtype="float64") if indexing_scheme == "nested_unique": # Create bounds for 'nested_unique' cells - nest = False - orders, a = healpix.uniq2pix(a, nest=nest) + # nest = False + orders, a = healpix.uniq2pix(a, nest=True) orders, index, inverse = np.unique( orders, return_index=True, return_inverse=True ) @@ -124,19 +142,29 @@ def cf_healpix_bounds( if where_ge_360[0].size: b[where_ge_360] -= 360.0 - # Bounds on the north (south) pole come out with a longitude - # of NaN, so replace these with a sensible value, i.e. the - # longitude of the southern (northern) vertex. - # + # A vertex on the north (south) pole comes out with a + # longitude of NaN, so replace these with a sensible value, + # i.e. the longitude of the southern-most (northern-most) + # vertex. + # North pole - i = np.argwhere(np.isnan(b[:, 1])).flatten() + longitude = pole_longitude + north = 0 + south = 2 + i = np.argwhere(np.isnan(b[:, north])).flatten() if i.size: - b[i, 1] = b[i, 3] + if pole_longitude is None: + longitude = b[i, south] + + b[i, north] = longitude # South pole - i = np.argwhere(np.isnan(b[:, 3])).flatten() + i = np.argwhere(np.isnan(b[:, south])).flatten() if i.size: - b[i, 3] = b[i, 1] + if pole_longitude is None: + longitude = b[i, north] + + b[i, south] = longitude return b @@ -188,9 +216,9 @@ def cf_healpix_coordinates( import healpix except ImportError as e: raise ImportError( - f"{e}. Must install healpix (e.g. from " - "https://pypi.org/project/healpix) to allow the calculation " - "of latitude/longitude coordinates for a HEALPix grid" + f"{e}. Must install healpix (https://pypi.org/project/healpix) " + "to allow the calculation of latitude/longitude coordinates " + "for a HEALPix grid" ) if a.ndim != 1: @@ -208,8 +236,7 @@ def cf_healpix_coordinates( # Create coordinates for 'nested_unique' cells c = np.empty(a.shape, dtype="float64") - nest = False - orders, a = healpix.uniq2pix(a, nest=nest) + orders, a = healpix.uniq2pix(a, nest=True) orders, index, inverse = np.unique( orders, return_index=True, return_inverse=True ) @@ -217,7 +244,7 @@ def cf_healpix_coordinates( level = np.where(inverse == inverse[i])[0] nside = healpix.order2nside(order) c[level] = healpix.pix2ang( - nside=nside, ipix=a[level], nest=nest, lonlat=True + nside=nside, ipix=a[level], nest=True, lonlat=True )[pos] else: # Create coordinates for 'nested' or 'ring' cells @@ -287,24 +314,25 @@ def cf_healpix_indexing_scheme( import healpix except ImportError as e: raise ImportError( - f"{e}. Must install healpix (e.g. from " - "https://pypi.org/project/healpix) to allow the changing of " - "the HEALPix index scheme" + f"{e}. Must install healpix (https://pypi.org/project/healpix) " + "to allow the changing of the HEALPix index scheme" ) if indexing_scheme == "nested": if new_indexing_scheme == "ring": - return healpix.nest2ring(healpix.order2nside(refinement_level), a) + nside = healpix.order2nside(refinement_level) + return healpix.nest2ring(nside, a) if new_indexing_scheme == "nested_unique": - return healpix._chp.nest2uniq(refinement_level, a) + return healpix.pix2uniq(refinement_level, a, nest=True) elif indexing_scheme == "ring": if new_indexing_scheme == "nested": - return healpix.ring2nest(healpix.order2nside(refinement_level), a) + nside = healpix.order2nside(refinement_level) + return healpix.ring2nest(nside, a) if new_indexing_scheme == "nested_unique": - return healpix._chp.ring2uniq(refinement_level, a) + return healpix.pix2uniq(refinement_level, a, nest=False) elif indexing_scheme == "nested_unique": if new_indexing_scheme in ("nested", "ring"): @@ -371,9 +399,8 @@ def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): import healpix except ImportError as e: raise ImportError( - f"{e}. Must install healpix (e.g. from " - "https://pypi.org/project/healpix) to allow the calculation " - "of cell area weights for a HEALPix grid" + f"{e}. Must install healpix (https://pypi.org/project/healpix) " + "to allow the calculation of cell area weights for a HEALPix grid" ) if indexing_scheme != "nested_unique": @@ -387,7 +414,7 @@ def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): else: x = 1.0 - orders = healpix.uniq2pix(a)[0] + orders = healpix.uniq2pix(a, nest=True)[0] orders, index, inverse = np.unique( orders, return_index=True, return_inverse=True ) diff --git a/cf/data/data.py b/cf/data/data.py index a3a3ebe061..ce2bf2c066 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1365,26 +1365,25 @@ def coarsen( axes: `dict` Define how to coarsening neighbourhood for each axis. A dictionary key is an integer axis position, - with correponding value giving the non-negative - integer size of the coarsening neighbourhood for that - axis. Unspecified axes are not coarsened, which is - equivalent to providing a coarsening neighbourhood of - ``1``. + with correponding value giving the integer size of the + coarsening neighbourhood for that axis. Unspecified + axes are not coarsened, which is equivalent to + providing a coarsening neighbourhood of ``1``. *Example:* Coarsen the axis in position 1 by combining every 4 elements: ``{1: 4}`` *Example:* - Coarsen the axis in position 0 by combining every 3 + Coarsen the first axis by combining every 3 elements, and the last axis by combining every 4 elements: ``{0: 3, -1: 4}`` trim_excess: `bool`, optional - If True then do not return a partially-full - neighbourhood at the end of a coarsened axis. If False - (the default) then an exception is raised if there are - any partially-filled neighbourhoods. + If True then omit a partially-full neighbourhood at + the end of a coarsened axis. If False (the default) + then an exception is raised if there are any + partially-filled neighbourhoods. {{inplace: `bool`, optional}} diff --git a/cf/docstring/docstring.py b/cf/docstring/docstring.py index 9006299d75..b3198d54ac 100644 --- a/cf/docstring/docstring.py +++ b/cf/docstring/docstring.py @@ -564,6 +564,28 @@ If True then do not perform the regridding, rather return the `esmpy.Regrid` instance that defines the regridding operation.""", + # HEALPix indexing schemes + "{{HEALPix indexing schemes}}": """The "nested" scheme indexes the pixels inside a single + coarser refinement level cell with consecutive + indices. The "ring" scheme indexes the pixels moving + down from the north to the south pole along each + isolatitude ring. When the HEALPix axis is ordered + with monotonically increasing indices, each type of + indexing scheme is optmised for different types of + operation. For instance, the "ring" scheme is + optimised for Fourier transforms with spherical + harmonics; and the "nested" scheme is optimised for + geographical nearest-neighbour operations such as + decreasing the refinement level. + + For a Multi-Order Coverage (MOC), where pixels with + different refinment levels are stored in the same + array, the indexing scheme has a unique index for each + cell at each refinement level. The "nested_unique" + scheme for an MOC indexes pixels within each + refinement level with the nested scheme, but + geographically close pixels at different refinement + levels do not have close indices.""", # ---------------------------------------------------------------- # Method description substitutions (4 levels of indentation) # ---------------------------------------------------------------- diff --git a/cf/domain.py b/cf/domain.py index d44b44057a..7bf4133174 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -269,6 +269,9 @@ def create_healpix( ): """Create a new global HEALPix domain. + The HEALPix axis of the new Domain is ordered so that the + HEALPix indices are monotonically increasing. + .. versionadded:: NEXTVERSION .. seealso:: `cf.Domain.create_regular`, @@ -286,6 +289,8 @@ def create_healpix( The HEALPix indexing scheme. One of ``'nested'`` (the default), ``'ring'``, or ``'nested_unique'``. + {{HEALPix indexing schemes}} + radius: optional Specify the radius of the latitude-longitude plane defined in spherical polar coordinates. May be set to @@ -332,22 +337,22 @@ def create_healpix( -------- Domain: -------- - Domain Axis: healpix_index(786432) + Domain Axis: healpix_index(3072) Auxiliary coordinate: healpix_index standard_name = 'healpix_index' units = '1' - Data(healpix_index(786432)) = [262144, ..., 1048575] 1 + Data(healpix_index(3072)) = [1024, ..., 4095] 1 Coordinate reference: grid_mapping_name:healpix Coordinate conversion:grid_mapping_name = healpix Coordinate conversion:indexing_scheme = nested_unique Datum:earth_radius = 6371000.0 Auxiliary Coordinate: healpix_index - >>> d = cf.Domain.create_healpix(4) + >>> d.create_latlon_coordinates(inplace=True) >>> print(d) - Auxiliary coords: healpix_index(ncdim%cell(3072)) = [0, ..., 3071] 1 + Auxiliary coords: healpix_index(ncdim%cell(3072)) = [1024, ..., 4095] 1 : latitude(ncdim%cell(3072)) = [2.388015463268772, ..., -2.388015463268786] degrees_north : longitude(ncdim%cell(3072)) = [45.0, ..., 315.0] degrees_east Coord references: grid_mapping_name:healpix @@ -366,8 +371,8 @@ def create_healpix( indexing_scheme = "nested" elif indexing_scheme not in ("nested", "ring"): raise ValueError( - "'indexing_scheme' must be 'nested', 'ring', or 'nested_unique'. " - f"Got: {indexing_scheme!r}" + "'indexing_scheme' must be 'nested', 'ring', or '" + f"nested_unique'. Got: {indexing_scheme!r}" ) domain = Domain() diff --git a/cf/field.py b/cf/field.py index d469c048ad..60c6322f41 100644 --- a/cf/field.py +++ b/cf/field.py @@ -2629,112 +2629,6 @@ def get_domain(self): return domain - # def radius(self, default=None): - # """Return the radius of a latitude-longitude plane defined in - # spherical polar coordinates. - # - # The radius is taken from the datums of any coordinate - # reference constructs, but if and only if this is not possible - # then a default value may be used instead. - # - # .. versionadded:: 3.0.2 - # - # .. seealso:: `bin`, `cell_area`, `collapse`, `weights` - # - # :Parameters: - # - # default: optional - # The radius is taken from the datums of any coordinate - # reference constructs, but if and only if this is not - # possible then the value set by the *default* parameter - # is used. May be set to any numeric scalar object, - # including `numpy` and `Data` objects. The units of the - # radius are assumed to be metres, unless specified by a - # `Data` object. If the special value ``'earth'`` is - # given then the default radius taken as 6371229 - # metres. If *default* is `None` an exception will be - # raised if no unique datum can be found in the - # coordinate reference constructs. - # - # *Parameter example:* - # Five equivalent ways to set a default radius of - # 6371200 metres: ``6371200``, - # ``numpy.array(6371200)``, ``cf.Data(6371200)``, - # ``cf.Data(6371200, 'm')``, ``cf.Data(6371.2, - # 'km')``. - # - # :Returns: - # - # `Data` - # The radius of the sphere, in units of metres. - # - # **Examples** - # - # >>> f.radius() - # - # - # >>> g.radius() - # ValueError: No radius found in coordinate reference constructs and no default provided - # >>> g.radius('earth') - # - # >>> g.radius(1234) - # - # - # """ - # radii = [] - # for cr in self.coordinate_references(todict=True).values(): - # r = cr.datum.get_parameter("earth_radius", None) - # if r is not None: - # r = Data.asdata(r) - # if not r.Units: - # r.override_units("m", inplace=True) - # - # if r.size != 1: - # radii.append(r) - # continue - # - # got = False - # for _ in radii: - # if r == _: - # got = True - # break - # - # if not got: - # radii.append(r) - # - # if len(radii) > 1: - # raise ValueError( - # "Multiple radii found from coordinate reference " - # f"constructs: {radii!r}" - # ) - # - # if not radii: - # if default is None: - # raise ValueError( - # "No radius found from coordinate reference constructs " - # "and no default provided" - # ) - # - # if isinstance(default, str): - # if default != "earth": - # raise ValueError( - # "The default radius must be numeric, 'earth', " - # "or None" - # ) - # - # return _earth_radius.copy() - # - # r = Data.asdata(default).squeeze() - # else: - # r = Data.asdata(radii[0]).squeeze() - # - # if r.size != 1: - # raise ValueError(f"Multiple radii: {r!r}") - # - # r.Units = Units("m") - # r.dtype = float - # return r - def laplacian_xy( self, x_wrap=None, one_sided_at_boundary=False, radius=None ): @@ -4948,25 +4842,25 @@ def healpix_decrease_refinement_level( either ``8`` or ``-2``. reduction: function - The function used to calculate, from the data on the - original finer cells, the values in the new coarser + The function used to calculate the values in the new + coarser cells, from the data on the original finer cells. *Example:* For an intensive field quantity (that does not depend on the size of the cells, such as - "sea_ice_amount" with units of kg m-2), ``np.mean`` - may be appropriate. For an extensive field quantity - (that depends on the size of the cells, such as - "sea_ice_mass" with units of kg), ``np.sum`` may be - appropriate. + "sea_ice_amount" with units of kg m-2), `np.mean` + might be appropriate. For an extensive field + quantity (that depends on the size of the cells, + such as "sea_ice_mass" with units of kg), `np.sum` + might be appropriate. conform: `bool`, optional - If True (the default) the HEALPix grid is converted to - a form suitable for having its refinement level - changed, i.e. the indexing scheme is changed to - 'nested' and the HEALPix axis is sorted so that the - nested HEALPix indices are monotonically + If True (the default) the HEALPix grid is + automatically converted to a form suitable for having + its refinement level changed, i.e. the indexing scheme + is changed to 'nested' and the HEALPix axis is sorted + so that the nested HEALPix indices are monotonically increasing. If False then either an exeption is raised if the HEALPix indexing scheme is not already 'nested', or else the HEALPix axis is not sorted. @@ -4977,11 +4871,10 @@ def healpix_decrease_refinement_level( sorted montonically. check_healpix_index: `bool`, optional - If True (the default) then it will be checked (after the HEALPix grid has been conformed, if *conform* is - True) that 1. the nested HEALPix indices are strictly - monotonically increasing, and 2. every cell at the new + True) that a) the nested HEALPix indices are strictly + monotonically increasing, and b) every cell at the new coarser refinement level contains the maximum possible number of cells at the original finer refinement level. If either condition is not met then an @@ -5064,8 +4957,8 @@ def healpix_decrease_refinement_level( cr = f.coordinate_reference("grid_mapping_name:healpix") elif indexing_scheme != "nested": raise ValueError( - "Can't decrease HEALPix refinement level: indexing_scheme in " - "the HEALPix grid mapping coordinate reference is " + "Can't decrease HEALPix refinement level: indexing_scheme " + "in the HEALPix grid mapping coordinate reference is " f"{indexing_scheme!r}, and not 'nested'. " "Consider setting conform=True" ) @@ -5095,8 +4988,9 @@ def healpix_decrease_refinement_level( # No change in refinement level return f - # Get the number of cells at the original refinement level which are - # contained in one cell at the coarser refinement level + # Get the number of cells at the original refinement level + # which are contained in one cell at the coarser refinement + # level ncells = 4**-level # Get the healpix_index coordinates @@ -5136,6 +5030,17 @@ def healpix_decrease_refinement_level( f"({refinement_level})" ) + # Whether or not to Create lat/lon coordinates for the + # coarsened grid + create_coarsened_latlon = bool( + f.coordinates( + "latitude", + "longitude", + filter_by_axis=(axis,), + axis_mode="exact", + ) + ) + # Coarsen (using the 'reduction' function) the field # data. Note that using the 'coarsen' technique only works for # 'nested' HEALPix ordering. @@ -5198,6 +5103,11 @@ def healpix_decrease_refinement_level( ) cr.set_coordinate(new_key) + # Create lat/lon coordinates for the coarsened grid if they + # exist in the original grid + if create_coarsened_latlon: + f.create_latlon_coordinates(inplace=True) + return f def histogram(self, digitized): @@ -7202,7 +7112,7 @@ def collapse( # # This is because missing domain ancillaries in a # coordinate reference are assumed to have the value zero, - # which is most likely inapproriate. + # which is most likely inappropriate. # -------------------------------------------------------- if remove_vertical_crs: for ref_key, ref in f.coordinate_references( @@ -7238,9 +7148,9 @@ def collapse( ) domain_axis = collapse_axes.get(healpix_axis) if domain_axis is not None and domain_axis.get_size() > 1: - from .healpix_utils import del_healpix_coordinate_reference + from .healpix_utils import _del_healpix_coordinate_reference - del_healpix_coordinate_reference(f) + _del_healpix_coordinate_reference(f) # --------------------------------------------------------- # Update dimension coordinates, auxiliary coordinates, diff --git a/cf/healpix_utils.py b/cf/healpix_utils.py index 6c135dfcf4..429d3541ff 100644 --- a/cf/healpix_utils.py +++ b/cf/healpix_utils.py @@ -1,7 +1,7 @@ """General functions useful for HEALPix functionality.""" -def del_healpix_coordinate_reference(f): +def _del_healpix_coordinate_reference(f): """Remove a healpix grid mapping coordinate reference construct. A new latitude_longitude grid mapping coordinate reference will be diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 3c5b3a4b01..a9a5ccecd9 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -372,12 +372,12 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): zip(*axes_key_construct_value_id) ) - n_items = len(constructs) + n_constructs = len(constructs) n_axes = len(canonical_axes) - if n_axes > 1 and n_items > n_axes: + if n_axes > 1 and n_constructs > n_axes: raise ValueError( - f"Error: Can't specify {n_items} conditions for " + f"Error: Can't specify {n_constructs} conditions for " f"{n_axes} axes: {points}. Consider applying the " "conditions separately." ) @@ -403,6 +403,10 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): ind0 = None index0 = None + # Loop round each condition for this axis. When there + # are multiple conditions, each iteration produces a + # 1-d Boolean array, and the axis selection is the + # logical AND of these arrays. for item, value, identity in zip( constructs, points, identities ): @@ -433,7 +437,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): # Placeholder to be overwritten later index = None - if n_items > 1 and index is not None: + if n_constructs > 1 and index is not None: # Convert 'index' to a boolean array i = np.zeros((size,), bool) i[index] = True @@ -505,7 +509,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): # np.arange(size)[index] else: # Index is a cyclic slice - if n_items > 1: + if n_constructs > 1: raise ValueError( "Error: Can't specify multiple " "conditions for a single axis when " @@ -525,7 +529,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): # Placeholder to be overwritten later index = None - if n_items > 1 and index is not None: + if n_constructs > 1 and index is not None: # Convert 'index' to a boolean array i = np.zeros((size,), bool) i[index] = True @@ -558,7 +562,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): # Placeholder to be overwritten later index = None - if n_items > 1 and ind is not None: + if n_constructs > 1 and ind is not None: # Convert 'ind' to a boolean array (note # that 'index' is already a boolean # array) @@ -577,7 +581,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): f" index = {index}\n ind = {ind}" ) # pragma: no cover - if n_items > 1: + if n_constructs > 1: # Update the 'ind0' and 'index0' boolean # arrays with the latest 'ind' and 'index' if ind is not None: @@ -595,7 +599,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): index0 &= index # Finalise 'ind' and 'index' - if n_items > 1: + if n_constructs > 1: if ind0 is not None: ind = ind0 @@ -622,7 +626,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): # ---------------------------------------------------- if debug: logger.debug( - f" {n_items} N-d constructs: {constructs!r}\n" + f" {n_constructs} N-d constructs: {constructs!r}\n" f" {len(points)} points : {points!r}\n" ) # pragma: no cover @@ -694,7 +698,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): # outside of the cell. This could happen if the cells # are not rectilinear (e.g. for curvilinear latitudes # and longitudes arrays). - if n_items == constructs[0].ndim == len(bounds) == 2: + if n_constructs == constructs[0].ndim == len(bounds) == 2: point2 = [] for v, construct in zip(points, transposed_constructs): if isinstance(v, Query) and v.iscontains(): @@ -1952,6 +1956,8 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): ``'ring'``, ``'nested_unique'``, or `None`. If `None` then the indexing scheme is unchanged. + {{HEALPix indexing schemes}} + sort: `bool`, optional If True then sort the HEALPix axis of the output so that its HEALPix indices are monotonically increasing, @@ -2187,8 +2193,12 @@ def healpix_to_ugrid(self, inplace=False): f = _inplace_enabled_define_and_cleanup(self) # If lat/lon coordinates do not exist, then derive them from - # the HEALPix indices. - f.create_latlon_coordinates(one_d=True, two_d=False, inplace=True) + # the HEALPix indices. It's important to set pole_longitude to + # something arbitrarily other than None so that the polar + # vertex comes out as a single node in the domain topology. + f.create_latlon_coordinates( + one_d=True, two_d=False, pole_longitude=0, inplace=True + ) x_key, x = f.auxiliary_coordinate( "Y", @@ -2259,16 +2269,21 @@ def healpix_to_ugrid(self, inplace=False): default=None, ) - from ..healpix_utils import del_healpix_coordinate_reference + from ..healpix_utils import _del_healpix_coordinate_reference - del_healpix_coordinate_reference(f) + _del_healpix_coordinate_reference(f) return f @_inplace_enabled(default=False) @_manage_log_level_via_verbosity def create_latlon_coordinates( - self, one_d=True, two_d=True, inplace=False, verbose=None + self, + one_d=True, + two_d=True, + pole_longitude=None, + inplace=False, + verbose=None, ): """Create latitude and longitude coordinates. @@ -2290,14 +2305,22 @@ def create_latlon_coordinates( :Parameters: one_d: `bool`, optional` - If True (the default) then create 1-d latitude and - longitude coordinates, if possible. Otherwise do not - attempt this. + If True (the default) then attempt to create 1-d + latitude and longitude coordinates, if + possible. Otherwise do not attempt this. two_d: `bool`, optional` - If True (the default) then create 2-d latitude and - longitude coordinates, if possible. Otherwise do not - attempt this. + If True (the default) then attempt to create 2-d + latitude and longitude coordinates, if + possible. Otherwise do not attempt this. + + pole_longitude: `None` or number + The longitude of coordinates, or coordinate bounds, + that lie exactly on the north or south pole. If `None` + (the default) then the longitudes of such points will + vary according to the alogrithm being used to create + them. If set to a number, then the longitudes of such + points will all be that value. {{inplace: `bool`, optional}} @@ -2514,6 +2537,7 @@ def create_latlon_coordinates( indexing_scheme=indexing_scheme, refinement_level=refinement_level, lon=True, + pole_longitude=pole_longitude, ) bounds = f._Bounds(data=dy) lon.set_bounds(bounds) diff --git a/cf/mixin/propertiesdatabounds.py b/cf/mixin/propertiesdatabounds.py index 21e8b9f803..43a9299455 100644 --- a/cf/mixin/propertiesdatabounds.py +++ b/cf/mixin/propertiesdatabounds.py @@ -960,8 +960,8 @@ def lower_bounds(self): return data.copy() raise AttributeError( - "Can't get lower bounds when there are no bounds nor coordinate " - "data" + f"Can't get lower bounds from {self!r} when there are no bounds " + "nor coordinate data" ) @property diff --git a/cf/query.py b/cf/query.py index 268ca7b6d0..fe62768660 100644 --- a/cf/query.py +++ b/cf/query.py @@ -2707,6 +2707,60 @@ def seasons(n=4, start=12): return out +def heapix_contains(f, lat, lon): + """TODOHEALPIX""" + from collections.abc import Iterable + + try: + import healpix + except ImportError as e: + raise ImportError( + f"{e}. Must install healpix (e.g. from " + "https://pypi.org/project/healpix) to allow the calculation " + "TODOHEALPIX of latitude/longitude coordinate bounds for a HEALPix grid" + ) + + if not isinstance(lat, Iterable): + lat = (lat,) + + if not isinstance(lon, Iterable): + lon = (lon,) + + cr = f.coordinate_reference("grid_mapping_name:healpix", default=None) + cc = cr.coordinate_conversion.parameters() + indexing_scheme = cc.get("indexing_scheme") + order = cc.get("refinement_level") + + if indexing_scheme in ("nested", "ring"): + nest = indexing_scheme == "nested" + nside = healpix.order2nside(order) + print(nside, lon, lat, nest) + index = healpix.ang2pix(nside, lon, lat, nest=nest, lonlat=True) + + elif indexing_scheme == "nested_unique": + healpix_index = f.coordinate( + "healpix_index", filter_by_naxes=(1,), default=None + ) + if healpix_index is None: + raise ValueError("TODOHEALPIX hjscdjhdscjdsj") + + index = [] + orders = healpix.uniq2pix(healpix_index.array, nest=True)[0] + orders = np.unique(orders) + for order in orders: + nside = healpix.order2nside(order) + nested_pix = healpix.ang2pix( + nside, lon, lat, nest=True, lonlat=True + ) + i = healpix.pix2uniq(order, nested_pix, nest=True) + index.extend(i.tolist()) + + else: + raise ValueError("TODOHEALPIX 1ooooooooooooooooo1") + + return set(index) + + # -------------------------------------------------------------------- # Deprecated functions # -------------------------------------------------------------------- diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index c055723893..7797d8faea 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -1644,6 +1644,7 @@ def test_Field_indices(self): f.set_construct(lat_2d_coord, axes=axes, copy=False) for mode in ("compress", "full", "envelope"): + print(repr(lon_2d_coord.bounds)) indices = f.indices(mode, aux_x=cf.contains(160.1)) g = f[indices] if mode == "full": @@ -3102,33 +3103,33 @@ def test_Field_healpix_indexing_scheme(self): g = f.healpix_indexing_scheme("ring") self.assertTrue( - (g.coordinate("healpix_index")[:4].array == [13, 5, 4, 0]).all() + np.allclose(g.coordinate("healpix_index")[:4], [13, 5, 4, 0]) ) h = g.healpix_indexing_scheme("nested") self.assertTrue( - (h.coordinate("healpix_index")[:4].array == [0, 1, 2, 3]).all() + np.allclose(h.coordinate("healpix_index")[:4], [0, 1, 2, 3]) ) h = g.healpix_indexing_scheme("nested_unique") self.assertTrue( - (h.coordinate("healpix_index")[:4].array == [16, 17, 18, 19]).all() + np.allclose(h.coordinate("healpix_index")[:4], [16, 17, 18, 19]) ) g = f.healpix_indexing_scheme("ring", sort=True) self.assertTrue( - (g.coordinate("healpix_index")[:4].array == [0, 1, 2, 3]).all() + np.allclose(g.coordinate("healpix_index")[:4], [0, 1, 2, 3]) ) h = g.healpix_indexing_scheme("nested", sort=False) self.assertTrue( - (h.coordinate("healpix_index")[:4].array == [3, 7, 11, 15]).all() + np.allclose(h.coordinate("healpix_index")[:4], [3, 7, 11, 15]) ) h = g.healpix_indexing_scheme("nested", sort=True) self.assertTrue( - (h.coordinate("healpix_index")[:4].array == [0, 1, 2, 3]).all() + np.allclose(h.coordinate("healpix_index")[:4], [0, 1, 2, 3]) ) g = f.healpix_indexing_scheme("nested_unique") self.assertTrue( - (g.coordinate("healpix_index")[:4].array == [16, 17, 18, 19]).all() + np.allclose(g.coordinate("healpix_index")[:4], [16, 17, 18, 19]) ) h = g.healpix_indexing_scheme("nested_unique") self.assertTrue(h.equals(g)) @@ -3157,13 +3158,27 @@ def test_Field_healpix_to_ugrid(self): u = f.healpix_to_ugrid() self.assertEqual(len(u.domain_topologies()), 1) self.assertEqual(len(u.auxiliary_coordinates()), 2) + + topology = u.domain_topology().normalise().array + self.assertEqual(np.unique(topology).size, 53) self.assertTrue( - ( - u.domain_topology()[:4].normalise().array - == [[6, 4, 2, 3], [7, 8, 4, 6], [4, 1, 0, 2], [8, 5, 1, 4]] - ).all() + np.allclose( + topology[:4], + [ + [14, 11, 13, 16], + [21, 14, 16, 20], + [8, 7, 11, 14], + [9, 8, 14, 21], + ], + ) ) + # North pole + self.assertTrue(np.allclose(topology[3:16:4, 0], 9)) + + # South pole + self.assertTrue(np.allclose(topology[32:48:4, 2], 3)) + self.assertIsNone(f.healpix_to_ugrid(inplace=True)) self.assertEqual(len(f.domain_topologies()), 1) @@ -3177,7 +3192,9 @@ def test_Field_healpix_to_ugrid(self): def test_Field_create_latlon_coordinates(self): """Test Field.create_latlon_coordinates.""" + # ------------------------------------------------------------ # HEALPix field + # ------------------------------------------------------------ f = self.f12.copy() self.assertEqual(len(f.auxiliary_coordinates()), 1) self.assertEqual(len(f.auxiliary_coordinates("healpix_index")), 1) @@ -3197,11 +3214,31 @@ def test_Field_create_latlon_coordinates(self): g.auxiliary_coordinate(c).equals(f.auxiliary_coordinate(c)) ) + # pole_longitude + f = self.f12 + g = f.create_latlon_coordinates() + longitude = g.auxiliary_coordinate("X").bounds.array + # North pole + self.assertTrue( + np.allclose(longitude[3:16:4, 0], longitude[3:16:4, 2]) + ) + # South pole + self.assertTrue( + np.allclose(longitude[32:48:4, 2], longitude[32:48:4, 0]) + ) + + g = f.create_latlon_coordinates(pole_longitude=3.14) + longitude = g.auxiliary_coordinate("X").bounds.array + # North pole + self.assertTrue(np.allclose(longitude[3:16:4, 0], 3.14)) + # South pole + self.assertTrue(np.allclose(longitude[32:48:4, 2], 3.14)) + # Check a Multi-Order Coverage grid m = self.f13 m = m.create_latlon_coordinates() - l1 = f + l1 = self.f12.create_latlon_coordinates() l2 = cf.Domain.create_healpix(2) l2.create_latlon_coordinates(inplace=True) @@ -3210,9 +3247,14 @@ def test_Field_create_latlon_coordinates(self): self.assertTrue(mc[:16].equals(l2.auxiliary_coordinate(c)[:16])) self.assertTrue(mc[16:].equals(l1.auxiliary_coordinate(c)[4:])) - def test_Field_subspace_healpix(self): + def test_Field_healpix_subspace(self): """Test Field.subspace for HEALPix grids""" f = self.f12 + + index = [47, 3, 2, 0] + g = f.subspace(healpix_index=index) + self.assertTrue(np.allclose(g.coordinate("healpix_index"), index)) + g = f.subspace(X=cf.wi(40, 70), Y=cf.wi(-20, 30)) self.assertTrue( np.allclose(g.coordinate("healpix_index"), [0, 22, 35]) @@ -3230,16 +3272,16 @@ def test_Field_healpix_decrease_refinement_level(self): f = self.f12 g = f.healpix_decrease_refinement_level(0, np.mean) self.assertTrue(g.shape, (2, 12)) - self.assertTrue((g.coord("healpix_index") == np.arange(12)).all()) + self.assertTrue(np.allclose(g.coord("healpix_index"), np.arange(12))) g = f.healpix_decrease_refinement_level(-1, np.mean) self.assertTrue(g.shape, (2, 12)) - self.assertTrue((g.coord("healpix_index") == np.arange(12)).all()) + self.assertTrue(np.allclose(g.coord("healpix_index"), np.arange(12))) f = f.healpix_indexing_scheme("ring") g = f.healpix_decrease_refinement_level(0, np.mean) self.assertTrue(g.shape, (2, 12)) - self.assertTrue((g.coord("healpix_index") == np.arange(12)).all()) + self.assertTrue(np.allclose(g.coord("healpix_index"), np.arange(12))) with self.assertRaises(ValueError): f.healpix_decrease_refinement_level(0, np.mean, conform=False) @@ -3248,7 +3290,14 @@ def test_Field_healpix_decrease_refinement_level(self): g = f.healpix_decrease_refinement_level(0, np.mean, conform=True) self.assertTrue(g.shape, (2, 12)) - self.assertTrue((g.coord("healpix_index") == np.arange(12)).all()) + self.assertTrue(np.allclose(g.coord("healpix_index"), np.arange(12))) + + # Check that lat/lon coords get created when they're present + # in the original field + f = f.create_latlon_coordinates() + g = f.healpix_decrease_refinement_level(0, np.mean) + self.assertEqual(g.auxiliary_coordinate("latitude"), (12,)) + self.assertEqual(g.auxiliary_coordinate("longitude"), (12,)) with self.assertRaises(ValueError): f.healpix_decrease_refinement_level(0, np.mean, conform=False) @@ -3257,7 +3306,7 @@ def test_Field_healpix_decrease_refinement_level(self): h = f.healpix_decrease_refinement_level( 0, np.mean, conform=False, check_healpix_index=False ) - self.assertFalse((h.coord("healpix_index") == np.arange(12)).all()) + self.assertFalse(np.allclose(h.coord("healpix_index"), np.arange(12))) # Can't change refinment level for a 'nested_unique' field with self.assertRaises(ValueError): From 7130e54fde006c7d4bfd1bbb9882064cc8a1f76d Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 8 Jul 2025 15:49:29 +0100 Subject: [PATCH 21/59] dev --- Changelog.rst | 2 +- cf/__init__.py | 3 + cf/dask_utils.py | 63 ++++++------ cf/field.py | 4 +- cf/healpix_utils.py | 116 +++++++++++++++++++++- cf/mixin/fielddomain.py | 186 ++++++++++++++++++++++------------- cf/test/test_Field.py | 1 - docs/source/installation.rst | 2 +- 8 files changed, 265 insertions(+), 112 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index f619fc521a..34afdb0fbf 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,7 +11,7 @@ Version NEXTVERSION (https://github.com/NCAS-CMS/cf-python/issues/???) * New method: `cf.Data.coarsen` (https://github.com/NCAS-CMS/cf-python/issues/???) -* New optional dependency: ``healpix>=2024.2`` +* New optional dependency: ``healpix>=2025.1`` * Changed dependency: ``cfdm>=1.13.0.0, <1.13.1.0`` ---- diff --git a/cf/__init__.py b/cf/__init__.py index 581f044ae9..bcc1b74f77 100644 --- a/cf/__init__.py +++ b/cf/__init__.py @@ -365,3 +365,6 @@ def detail(self, message, *args, **kwargs): logging.Logger.detail = detail + + +from .healpix_utils import * diff --git a/cf/dask_utils.py b/cf/dask_utils.py index 561b99c649..d79aa8b556 100644 --- a/cf/dask_utils.py +++ b/cf/dask_utils.py @@ -35,11 +35,11 @@ def cf_healpix_bounds( ``'ring'``, or ``'nested_unique'``. refinement_level: `int` or `None`, optional - For a ``'nested'`` or ``'ring'`` indexed grid, the - refinement level of the grid within the HEALPix hierarchy, - starting at 0 for the base tesselation with 12 cells. Set - to `None` for a ``'nested_unique'`` indexed grid, for - which the refinement level is ignored. + The refinement level of the grid within the HEALPix + hierarchy, starting at 0 for the base tesselation with 12 + cells. Must be an `int` for *indexing_scheme* ``'nested'`` + or ``'ring'``, but is ignored for ``'nested_unique'`` (in + which case *refinement_level* may be `None`). lat: `bool`, optional If True then return latitude bounds. @@ -118,17 +118,13 @@ def cf_healpix_bounds( if indexing_scheme == "nested_unique": # Create bounds for 'nested_unique' cells - # nest = False orders, a = healpix.uniq2pix(a, nest=True) - orders, index, inverse = np.unique( - orders, return_index=True, return_inverse=True - ) - for order, i in zip(orders, index): - level = np.where(inverse == inverse[i])[0] + for order in np.unique(orders): nside = healpix.order2nside(order) + indices = np.where(orders == order)[0] for j, (u, v) in enumerate(vertices): - thetaphi = bounds_func(nside, a[level], u, v) - b[level, j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] + thetaphi = bounds_func(nside, a[indices], u, v) + b[indices, j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] else: # Create bounds for 'nested' or 'ring' cells nside = healpix.order2nside(refinement_level) @@ -188,10 +184,11 @@ def cf_healpix_coordinates( ``'ring'``, or ``'nested_unique'``. refinement_level: `int` or `None`, optional - For a ``'nested'`` or ``'ring'`` indexed grid, the - refinement level of the grid within the HEALPix hierarchy, - starting at 0 for the base tesselation with 12 cells. - Ignored for a ``'nested_unique'`` indexed grid. + The refinement level of the grid within the HEALPix + hierarchy, starting at 0 for the base tesselation with 12 + cells. Must be an `int` for *indexing_scheme* ``'nested'`` + or ``'ring'``, but is ignored for ``'nested_unique'`` (in + which case *refinement_level* may be `None`). lat: `bool`, optional If True then return latitude coordinates. @@ -237,14 +234,11 @@ def cf_healpix_coordinates( c = np.empty(a.shape, dtype="float64") orders, a = healpix.uniq2pix(a, nest=True) - orders, index, inverse = np.unique( - orders, return_index=True, return_inverse=True - ) - for order, i in zip(orders, index): - level = np.where(inverse == inverse[i])[0] + for order in np.unique(orders): nside = healpix.order2nside(order) - c[level] = healpix.pix2ang( - nside=nside, ipix=a[level], nest=True, lonlat=True + indices = np.where(orders == order)[0] + c[indices] = healpix.pix2ang( + nside=nside, ipix=a[indices], nest=True, lonlat=True )[pos] else: # Create coordinates for 'nested' or 'ring' cells @@ -261,7 +255,7 @@ def cf_healpix_coordinates( def cf_healpix_indexing_scheme( - a, indexing_scheme, new_indexing_scheme, refinement_level + a, indexing_scheme, new_indexing_scheme, refinement_level=None ): """Change the ordering of HEALPix indices. @@ -283,13 +277,12 @@ def cf_healpix_indexing_scheme( The new HEALPix indexing scheme to change to. One of ``'nested'``, ``'ring'``, or ``'nested_unique'``. - refinement_level: `int` or `None` - The refinement level of the original grid within the - HEALPix hierarchy, starting at 0 for the base tesselation - with 12 cells. Must be an `int` *indexing_scheme* for - ``'nested'`` or, ``'ring'``, but ignored for - *indexing_scheme* ``'nested_unique'`` (in which case - *refinement_level* may be `None`). + refinement_level: `int` or `None`, optional + The refinement level of the grid within the HEALPix + hierarchy, starting at 0 for the base tesselation with 12 + cells. Must be an `int` for *indexing_scheme* ``'nested'`` + or ``'ring'``, but is ignored for ``'nested_unique'`` (in + which case *refinement_level* may be `None`). :Returns: @@ -377,8 +370,8 @@ def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): units of the square of the radius units. radius: number, optional - The radius of the sphere, in units of length. Must be set - if *measure * is True, otherwise ignored. + The radius of the sphere. Must be set if *measure * is + True, otherwise ignored. :Returns: @@ -408,7 +401,7 @@ def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): "cf_healpix_weights: Can only calulate weights for the " "'nested_unique' indexing scheme" ) - + if measure: x = np.pi * (radius**2) / 3.0 else: diff --git a/cf/field.py b/cf/field.py index 60c6322f41..0542eb5900 100644 --- a/cf/field.py +++ b/cf/field.py @@ -7148,9 +7148,9 @@ def collapse( ) domain_axis = collapse_axes.get(healpix_axis) if domain_axis is not None and domain_axis.get_size() > 1: - from .healpix_utils import _del_healpix_coordinate_reference + from .healpix_utils import del_healpix_coordinate_reference - _del_healpix_coordinate_reference(f) + del_healpix_coordinate_reference(f) # --------------------------------------------------------- # Update dimension coordinates, auxiliary coordinates, diff --git a/cf/healpix_utils.py b/cf/healpix_utils.py index 429d3541ff..0921cb3e59 100644 --- a/cf/healpix_utils.py +++ b/cf/healpix_utils.py @@ -1,7 +1,121 @@ """General functions useful for HEALPix functionality.""" +import numpy as np +import dask.array as da -def _del_healpix_coordinate_reference(f): +def _healpix_info(f): + """TODOHEALPIX + + + + >>> _healpix_info(f) + {} + + >>> _healpix_info(f) + {'coordinate_reference_key': 'coordinatereference0', + 'grid_mapping_name:healpix': , + 'indexing_scheme': 'nested', + 'refinement_level': 1, + 'axis_key': 'domainaxis1', + 'coordinate_key': 'auxiliarycoordinate0', + 'healpix_index': } + +""" + info = {} + + # Parse the HEALPix coordinate reference + cr_key, cr = f.coordinate_reference("grid_mapping_name:healpix", + item=True, default=(None, None)) + if cr is not None: + info['coordinate_reference_key'] = cr_key + info['grid_mapping_name:healpix'] = cr + parameters = cr.coordinate_conversion.parameters() + for p in ('indexing_scheme','refinement_level'): + value = parameters.get(p) + if value is not None: + info[p] = value + + hp_key, healpix_index = f.coordinate( + "healpix_index", + filter_by_naxes=(1,), + item=True, + default=(None, None), + ) + if healpix_index is not None: + info['axis_key'] = f.get_data_axes(hp_key)[0] + info['coordinate_key'] = hp_key + info['healpix_index'] = healpix_index + + return info + +def contains_latlon(lat, lon, f=None): + + def latlon(lat, lon, f): + if _healpix_info(f): + return _healpix_indices(lat, lon, f): + + if _ugrid_info(f): + return _ugrid_indices(lat, lon, f): + + if f is None: + return partial(latlon, lat=lat, lon=lon) + + return latlon(lat, lon, f) + + +def _healpix_indices(lat, lon, f): + """TODOHEALPIX""" + + try: + import healpix + except ImportError as e: + raise ImportError( + f"{e}. Must install healpix (https://pypi.org/project/healpix) " + "to TODOHEALPIX allow the calculation of latitude/longitude coordinate " + "bounds for a HEALPix grid" + ) + + hp = _healpix_info(f) + if not hp: + raise ValueError("TODOHEALPIX") + + healpix_index = hp.get('healpix_index') + if healpix_index is None: + raise ValueError("TODOHEALPIX") + + indexing_scheme = hp.get("indexing_scheme") + if indexing_scheme is None: + raise ValueError("TODOHEALPIX") + + if indexing_scheme == 'nested_unique': + index = [] + healpix_index = healpix_index.array + orders = healpix.uniq2pix(healpix_index, nest=True)[0] + orders = np.unique(orders) + for order in orders: + nside = healpix.order2nside(order) + pix = healpix.ang2pix(nside, lon, lat, nest=True, lonlat=True) + pix = np.unique(pix) + pix = healpix._chp.nest2uniq(order, pix, pix) + index.append(da.where(da.isin(healpix_index, pix))[0]) + + index = da.unique(da.concatenate(index, axis=0)) + else: + refinement_level = hp.get("refinement_level") + if refinement_level is None: + raise ValueError("TODOHEALPIX") + + nest = indexing_scheme == 'nested' + nside = healpix.order2nside(refinement_level) + pix = healpix.ang2pix(nside, lon, lat, nest=nest, lonlat=True) + pix = np.unique(pix) + index = da.where(da.isin(healpix_index, pix))[0] + + return index.compute() + + + +def del_healpix_coordinate_reference(f): """Remove a healpix grid mapping coordinate reference construct. A new latitude_longitude grid mapping coordinate reference will be diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index a9a5ccecd9..c268bce3fe 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -296,9 +296,13 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): f"non-negative integer. Got {halo!r}" ) - # Create any implied latitude and longitude coordinates - # (e.g. as implied by a non-latitude_longitude grid mapping - # coordinate reference). Do not do this in-place. + # Create any latitude and longitude coordinates (e.g. as + # implied by a non-latitude_longitude grid mapping coordinate + # reference). This is so that latitude and longitude kwarg + # selections can work, even if the actual latitude and + # longitude coordinates are not part of the metadata. + # + # Do not do this in-place. self = self.create_latlon_coordinates() domain_axes = self.domain_axes(todict=True) @@ -403,10 +407,14 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): ind0 = None index0 = None - # Loop round each condition for this axis. When there - # are multiple conditions, each iteration produces a - # 1-d Boolean array, and the axis selection is the - # logical AND of these arrays. + # Loop round the conditions for this axis. + # + # When there are multiple conditions, each iteration + # produces a 1-d Boolean array, and the axis selection + # is the logical AND of these arrays. + # + # When there is a single condition, the axis selection + # could be a slice or list of integers. for item, value, identity in zip( constructs, points, identities ): @@ -438,7 +446,8 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): index = None if n_constructs > 1 and index is not None: - # Convert 'index' to a boolean array + # Multiple conditions: Convert 'index' to + # a boolean array i = np.zeros((size,), bool) i[index] = True index = i @@ -530,7 +539,8 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): index = None if n_constructs > 1 and index is not None: - # Convert 'index' to a boolean array + # Multiple conditions: Convert 'index' to + # a boolean array i = np.zeros((size,), bool) i[index] = True index = i @@ -563,9 +573,9 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): index = None if n_constructs > 1 and ind is not None: - # Convert 'ind' to a boolean array (note - # that 'index' is already a boolean - # array) + # Multiple conditions: Convert 'ind' to a + # boolean array (note that 'index' is + # already a boolean array) i = np.zeros((size,), bool) i[ind] = True ind = i @@ -582,8 +592,9 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): ) # pragma: no cover if n_constructs > 1: - # Update the 'ind0' and 'index0' boolean - # arrays with the latest 'ind' and 'index' + # Multiple conditions: Update the 'ind0' and + # 'index0' boolean arrays with the latest + # 'ind' and 'index' if ind is not None: # Note that 'index' must be None when # 'ind' is not None, so no need to update @@ -600,6 +611,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): # Finalise 'ind' and 'index' if n_constructs > 1: + # Multiple conditions if ind0 is not None: ind = ind0 @@ -2013,9 +2025,12 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): """ from ..dask_utils import cf_healpix_indexing_scheme - + from ..healpix_utils import _healpix_info + f = self.copy() + hp = _healpix_info(f) + valid_indexing_schemes = ("nested", "ring", "nested_unique") if new_indexing_scheme not in valid_indexing_schemes + (None,): raise ValueError( @@ -2024,29 +2039,41 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): f"Got {new_indexing_scheme!r}" ) - # Get the original healpix_index coordinates - hp_key, healpix_index = f.coordinate( - "healpix_index", - filter_by_naxes=(1,), - item=True, - default=(None, None), - ) + # Get the healpix_index coordinates + healpix_index = hp.get('healpix_index') if healpix_index is None: raise ValueError( "Can't change HEALPix index scheme: There are no " "healpix_index coordinates" ) - # Parse the HEALPix coordinate reference - cr = f.coordinate_reference("grid_mapping_name:healpix", default=None) - if cr is None: - raise ValueError( - "Can't change HEALPix index scheme: There is no HEALPix grid " - "mapping coordinate reference" - ) - - parameters = cr.coordinate_conversion.parameters() - indexing_scheme = parameters.get("indexing_scheme") + # Get the healpix_index axis + axis = hp['axis_key'] + + # Get the healpix_index coordinates +# hp_key, healpix_index = f.coordinate( +# "healpix_index", +# filter_by_naxes=(1,), +# item=True, +# default=(None, None), +# ) + # if healpix_index is None: + # raise ValueError( +# "Can't change HEALPix index scheme: There are no " +# "healpix_index coordinates" +# ) + +# # Parse the HEALPix coordinate reference +# cr = f.coordinate_reference("grid_mapping_name:healpix", default=None) +# if cr is None: +# raise ValueError( +# "Can't change HEALPix index scheme: There is no HEALPix grid " +# "mapping coordinate reference" +# ) + + indexing_scheme = hp.get("indexing_scheme") +# parameters = cr.coordinate_conversion.parameters() +# indexing_scheme = parameters.get("indexing_scheme") if indexing_scheme is None: raise ValueError( "Can't change HEALPix indexing scheme: indexing_scheme has " @@ -2064,7 +2091,8 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): new_indexing_scheme is not None and new_indexing_scheme != indexing_scheme ): - refinement_level = parameters.get("refinement_level") + # refinement_level = parameters.get("refinement_level") + refinement_level = hp.get("refinement_level") if ( indexing_scheme in ("nested", "ring") and refinement_level is None @@ -2077,6 +2105,7 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): ) # Update the Coordinate Reference + cr = hp["grid_mapping_name:healpix"] cr.coordinate_conversion.set_parameter( "indexing_scheme", new_indexing_scheme ) @@ -2085,15 +2114,17 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): elif indexing_scheme == "nested_unique": # Set the refinement level for the new indexing # scheme. This is the largest integer, N, for which - # 2**(2(N+1)) <= healpix_index[0] (see "Efficient data - # structures for masks on 2D grids". M. Reinecke and - # E. Hivon. A&A, 580 (2015) A132. DOI: - # https://doi.org/10.1051/0004-6361/201526549) + # 2**(2(N+1)) <= healpix_index[0]. Therefore N = + # int(log2(healpix_index[0]) // 2 - 1) # # It doesn't matter if there are in fact multiple - # refinement levels, as this will get trapped as an - # exception when 'cf_healpix_indexing_scheme' is - # executed. + # refinement levels in the grid, as this will get + # trapped as an exception when + # `cf_healpix_indexing_scheme` is executed. + # + # M. Reinecke and E. Hivon: Efficient data structures + # for masks on 2D grids. A&A, 580 (2015) A132. + # https://doi.org/10.1051/0004-6361/201526549 from math import log2 cr.coordinate_conversion.set_parameter( @@ -2113,15 +2144,15 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): ) healpix_index.set_data(dx, copy=False) - # Get the identifier fo the HEALPix domain axis - axis = f.get_data_axes(hp_key)[0] +# # Get the identifier for the HEALPix domain axis +# axis = f.get_data_axes(hp_key)[0] # Ensure that healpix indices are auxiliary coordinates if healpix_index.construct_type == "dimension_coordinate": healpix_index = f._AuxiliaryCoordinate( source=healpix_index, copy=False ) - f.del_construct(hp_key) + f.del_construct(hp['coordinate_key']) hp_key = f.set_construct(healpix_index, axes=axis, copy=False) cr.set_coordinate(hp_key) @@ -2183,7 +2214,12 @@ def healpix_to_ugrid(self, inplace=False): Topologies : cell:face(ncdim%cell(48), 4) = [[965, ..., 3074]] """ - axis = self.domain_axis("healpix_index", key=True, default=None) + from ..healpix_utils import del_healpix_coordinate_reference, _healpix_info + + hp = _healpix_info(self) + + axis = hp.get('axis_key') +# axis = self.domain_axis("healpix_index", key=True, default=None) if axis is None: raise ValueError( "Can't convert HEALPix to UGRID: There is no HEALPix domain " @@ -2194,8 +2230,9 @@ def healpix_to_ugrid(self, inplace=False): # If lat/lon coordinates do not exist, then derive them from # the HEALPix indices. It's important to set pole_longitude to - # something arbitrarily other than None so that the polar - # vertex comes out as a single node in the domain topology. + # something other than None (it doesn't matter what) so that + # the polar vertices come out as a single node in the domain + # topology. f.create_latlon_coordinates( one_d=True, two_d=False, pole_longitude=0, inplace=True ) @@ -2216,14 +2253,14 @@ def healpix_to_ugrid(self, inplace=False): ) if x is None: raise ValueError( - "Can't convert HEALPix to UGRID: Not able to find nor " - "create longitude coordinates" + "Can't convert HEALPix to UGRID: Not able to find (or " + "create) longitude coordinates" ) if y is None: raise ValueError( - "Can't convert HEALPix to UGRID: Not able to find nor " - "create latitude coordinates" + "Can't convert HEALPix to UGRID: Not able to find (or " + "create) latitude coordinates" ) bounds_y = y.get_bounds(None) @@ -2258,20 +2295,19 @@ def healpix_to_ugrid(self, inplace=False): f.set_construct(domain_topology, axes=axis, copy=False) # Remove the HEALPix index coordinates - f.del_construct( - "healpix_index", - filter_by_type=( - "auxiliary_coordinate", - "dimension_coordinate", - ), - filter_by_axis=(axis,), - axis_mode="exact", - default=None, - ) - - from ..healpix_utils import _del_healpix_coordinate_reference - - _del_healpix_coordinate_reference(f) + f.del_construct(hp['coordinate_key']) +# "healpix_index", +# filter_by_type=( +# "auxiliary_coordinate", +# "dimension_coordinate", +# ), +# filter_by_axis=(axis,), +# axis_mode="exact", +# default=None, +# ) + + # Remove the HEALPix coordinate reference + del_healpix_coordinate_reference(f) return f @@ -2416,8 +2452,13 @@ def create_latlon_coordinates( # -------------------------------------------------------- # HEALPix 1-d coordinates # -------------------------------------------------------- - parameters = cr.coordinate_conversion.parameters() - indexing_scheme = parameters.get("indexing_scheme") + from ..healpix_utils import _healpix_info + + hp = _healpix_info(f) + +# parameters = cr.coordinate_conversion.parameters() +# indexing_scheme = parameters.get("indexing_scheme") + indexing_scheme = hp.get('indexing_scheme') if indexing_scheme not in ("nested", "ring", "nested_unique"): if is_log_level_info(logger): logger.info( @@ -2427,7 +2468,8 @@ def create_latlon_coordinates( return f - refinement_level = parameters.get("refinement_level") +# refinement_level = parameters.get("refinement_level") + refinement_level = hp.get("refinement_level") if refinement_level is None and indexing_scheme in ( "nested", "ring", @@ -2447,7 +2489,8 @@ def create_latlon_coordinates( item=True, default=(None, None), ) - if healpix_index is None: +# if healpix_index is None: + if hp.get('healpix_index') is None: if is_log_level_info(logger): logger.info( "Can't create 1-d latitude and longitude coordinates: " @@ -2457,7 +2500,8 @@ def create_latlon_coordinates( return f # Get the HEALPix axis - axis = f.get_data_axes(key)[0] + axis = hp['axis_key'] +# axis = f.get_data_axes(key)[0] if f.coordinates( "X", @@ -2552,10 +2596,10 @@ def create_latlon_coordinates( # -------------------------------------------------------- # Plane projection or rotated pole # -------------------------------------------------------- - pass + pass # Add some code here! # ------------------------------------------------------------ - # Update Coordinate References + # Update coordinate references # ------------------------------------------------------------ if new_coords: if latlon_cr is not None: diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index 7797d8faea..b68cf40459 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -1644,7 +1644,6 @@ def test_Field_indices(self): f.set_construct(lat_2d_coord, axes=axes, copy=False) for mode in ("compress", "full", "envelope"): - print(repr(lon_2d_coord.bounds)) indices = f.indices(mode, aux_x=cf.contains(160.1)) g = f[indices] if mode == "full": diff --git a/docs/source/installation.rst b/docs/source/installation.rst index fa79ef4be9..9f62d6f3bc 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -270,7 +270,7 @@ environments for which these features are not required. .. rubric:: HEALPix manipulations -* `healpix `_, version 2024.2 or +* `healpix `_, version 2025.1 or newer. This package is not required to read and write HEALPix datasets, but may be needed for particular manipulations with HEALPix grids, such as creating latitude and longitude coordinates, From 7b522117ee522de412924505c37f36c3c82872fb Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 8 Jul 2025 16:06:09 +0100 Subject: [PATCH 22/59] dev --- cf/healpix_utils.py | 12 +++++++----- cf/mixin/fielddomain.py | 6 +++++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cf/healpix_utils.py b/cf/healpix_utils.py index 0921cb3e59..9575aa4366 100644 --- a/cf/healpix_utils.py +++ b/cf/healpix_utils.py @@ -52,10 +52,12 @@ def contains_latlon(lat, lon, f=None): def latlon(lat, lon, f): if _healpix_info(f): - return _healpix_indices(lat, lon, f): + return _healpix_contains_latlon(lat, lon, f) if _ugrid_info(f): - return _ugrid_indices(lat, lon, f): + return _ugrid_contains_latlon(lat, lon, f) + + raise ValueError("Can only use with discretet axes") if f is None: return partial(latlon, lat=lat, lon=lon) @@ -63,7 +65,7 @@ def latlon(lat, lon, f): return latlon(lat, lon, f) -def _healpix_indices(lat, lon, f): +def _healpix_contains_latlon(lat, lon, f): """TODOHEALPIX""" try: @@ -71,8 +73,8 @@ def _healpix_indices(lat, lon, f): except ImportError as e: raise ImportError( f"{e}. Must install healpix (https://pypi.org/project/healpix) " - "to TODOHEALPIX allow the calculation of latitude/longitude coordinate " - "bounds for a HEALPix grid" + "to allow the calculation of which cells contain " + "latitude/longitude locations for a HEALPix grid" ) hp = _healpix_info(f) diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index c268bce3fe..e85168d264 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -546,10 +546,14 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): index = i elif item is not None: + print (9999999, value, callable(value)) # 1-d CASE 3: All other 1-d cases if debug: logger.debug(" 1-d CASE 3:") # pragma: no cover - + + if callable(value): + value = value(self) # case 1? TODOHEALPIX + index = item == value # Performance: Convert the 1-d 'index' to a From edaa78a4908b17fe4defe21233d84c2d551d78ab Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 8 Jul 2025 16:09:18 +0100 Subject: [PATCH 23/59] dev --- cf/mixin/fielddomain.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index e85168d264..bb5e3421ee 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -428,6 +428,9 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): ind = None + if callable(value): + value = value(self) # TODOHEALPIX + if isinstance(value, (list, slice, tuple, np.ndarray)): # 1-d CASE 1: Value is already an index, # e.g. [0], [7,4,2], slice(0,4,2), @@ -551,9 +554,6 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): if debug: logger.debug(" 1-d CASE 3:") # pragma: no cover - if callable(value): - value = value(self) # case 1? TODOHEALPIX - index = item == value # Performance: Convert the 1-d 'index' to a From 65b7a5350fa1d9585434976d583e3a115e12a8c7 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 9 Jul 2025 00:34:13 +0100 Subject: [PATCH 24/59] dev --- cf/functions.py | 21 ++++++++++++++++ cf/healpix_utils.py | 53 +++++++++++++++++++---------------------- cf/mixin/fielddomain.py | 8 +++---- 3 files changed, 49 insertions(+), 33 deletions(-) diff --git a/cf/functions.py b/cf/functions.py index 82799009ec..c4e867fcaa 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -3318,6 +3318,27 @@ def unique_constructs(constructs, ignore_properties=None, copy=True): ) +def contains_latlon(lat, lon, f=None): + """TODOHEALPIX""" + def latlon(f, lat, lon): + if f.coordinate("healpix_index", filter_by_naxes=(1,) default=None): + # HEALPix + from .healpix_utils import _healpix_contains_latlon + + return _healpix_contains_latlon(f, lat, lon) + + if f.domain_topologies(todict=True): + # UGRID + return _ugrid_contains_latlon(f, lat, lon) + + raise ValueError("Can only use with discretet axes") + + if f is None: + return partial(latlon, lat=lat, lon=lon) + + return latlon(f, lat, lon) + + def _DEPRECATION_ERROR(message="", version="3.0.0", removed_at="4.0.0"): if removed_at: removed_at = f" and will be removed at version {removed_at}" diff --git a/cf/healpix_utils.py b/cf/healpix_utils.py index 9575aa4366..3e9cc68ab9 100644 --- a/cf/healpix_utils.py +++ b/cf/healpix_utils.py @@ -1,16 +1,28 @@ """General functions useful for HEALPix functionality.""" +from functools import partial import numpy as np import dask.array as da def _healpix_info(f): - """TODOHEALPIX + """Get information about the HEALPix axis, if there is one. + .. versionadded:: NEXTVERSION + + :Parameters: + f: `Field` or `Domain` + The field or domain. + + :Returns: - >>> _healpix_info(f) - {} + `dict` + The information about the HEALPix axis. The dictionary + will be empty if there is no HEALPix axis. + + **Examples** + >>> f = cf.example_field(12) >>> _healpix_info(f) {'coordinate_reference_key': 'coordinatereference0', 'grid_mapping_name:healpix': , @@ -20,20 +32,19 @@ def _healpix_info(f): 'coordinate_key': 'auxiliarycoordinate0', 'healpix_index': } -""" + """ info = {} - # Parse the HEALPix coordinate reference cr_key, cr = f.coordinate_reference("grid_mapping_name:healpix", item=True, default=(None, None)) if cr is not None: info['coordinate_reference_key'] = cr_key info['grid_mapping_name:healpix'] = cr parameters = cr.coordinate_conversion.parameters() - for p in ('indexing_scheme','refinement_level'): - value = parameters.get(p) + for param in ('indexing_scheme','refinement_level'): + value = parameters.get(param) if value is not None: - info[p] = value + info[param] = value hp_key, healpix_index = f.coordinate( "healpix_index", @@ -48,26 +59,14 @@ def _healpix_info(f): return info -def contains_latlon(lat, lon, f=None): - - def latlon(lat, lon, f): - if _healpix_info(f): - return _healpix_contains_latlon(lat, lon, f) - - if _ugrid_info(f): - return _ugrid_contains_latlon(lat, lon, f) - - raise ValueError("Can only use with discretet axes") - if f is None: - return partial(latlon, lat=lat, lon=lon) - - return latlon(lat, lon, f) - - -def _healpix_contains_latlon(lat, lon, f): +def _healpix_contains_latlon(f, lat, lon): """TODOHEALPIX""" + hp = _healpix_info(f) + if not hp: + raise ValueError("TODOHEALPIX") + try: import healpix except ImportError as e: @@ -77,10 +76,6 @@ def _healpix_contains_latlon(lat, lon, f): "latitude/longitude locations for a HEALPix grid" ) - hp = _healpix_info(f) - if not hp: - raise ValueError("TODOHEALPIX") - healpix_index = hp.get('healpix_index') if healpix_index is None: raise ValueError("TODOHEALPIX") diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index bb5e3421ee..402ae6c0b6 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -332,6 +332,10 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): f"defined by {identity!r}" ) + if callable(value): + print ('value=', value) + value = value(self) + if axes in parsed: # The axes are the same as an existing key parsed[axes].append((axes, key, construct, value, identity)) @@ -428,9 +432,6 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): ind = None - if callable(value): - value = value(self) # TODOHEALPIX - if isinstance(value, (list, slice, tuple, np.ndarray)): # 1-d CASE 1: Value is already an index, # e.g. [0], [7,4,2], slice(0,4,2), @@ -549,7 +550,6 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): index = i elif item is not None: - print (9999999, value, callable(value)) # 1-d CASE 3: All other 1-d cases if debug: logger.debug(" 1-d CASE 3:") # pragma: no cover From e7a6c0fad866cfbc35b252084c1c5f1c99e10e9f Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 9 Jul 2025 23:25:36 +0100 Subject: [PATCH 25/59] dev --- cf/__init__.py | 3 - cf/dask_utils.py | 8 +- cf/dimensioncoordinate.py | 37 ++--- cf/domain.py | 7 +- cf/field.py | 7 +- cf/functions.py | 115 ++++++++++++++-- cf/{healpix_utils.py => healpix.py} | 202 ++++++++++++++++++---------- cf/mixin/fielddomain.py | 178 +++++++++++------------- cf/query.py | 58 +------- cf/test/test_DimensionCoordinate.py | 6 + cf/test/test_Domain.py | 8 ++ cf/test/test_Field.py | 9 ++ cf/test/test_weights.py | 14 ++ docs/source/function.rst | 1 + 14 files changed, 381 insertions(+), 272 deletions(-) rename cf/{healpix_utils.py => healpix.py} (54%) diff --git a/cf/__init__.py b/cf/__init__.py index bcc1b74f77..581f044ae9 100644 --- a/cf/__init__.py +++ b/cf/__init__.py @@ -365,6 +365,3 @@ def detail(self, message, *args, **kwargs): logging.Logger.detail = detail - - -from .healpix_utils import * diff --git a/cf/dask_utils.py b/cf/dask_utils.py index d79aa8b556..f706f78f6b 100644 --- a/cf/dask_utils.py +++ b/cf/dask_utils.py @@ -36,7 +36,7 @@ def cf_healpix_bounds( refinement_level: `int` or `None`, optional The refinement level of the grid within the HEALPix - hierarchy, starting at 0 for the base tesselation with 12 + hierarchy, starting at 0 for the base tessellation with 12 cells. Must be an `int` for *indexing_scheme* ``'nested'`` or ``'ring'``, but is ignored for ``'nested_unique'`` (in which case *refinement_level* may be `None`). @@ -185,7 +185,7 @@ def cf_healpix_coordinates( refinement_level: `int` or `None`, optional The refinement level of the grid within the HEALPix - hierarchy, starting at 0 for the base tesselation with 12 + hierarchy, starting at 0 for the base tessellation with 12 cells. Must be an `int` for *indexing_scheme* ``'nested'`` or ``'ring'``, but is ignored for ``'nested_unique'`` (in which case *refinement_level* may be `None`). @@ -279,7 +279,7 @@ def cf_healpix_indexing_scheme( refinement_level: `int` or `None`, optional The refinement level of the grid within the HEALPix - hierarchy, starting at 0 for the base tesselation with 12 + hierarchy, starting at 0 for the base tessellation with 12 cells. Must be an `int` for *indexing_scheme* ``'nested'`` or ``'ring'``, but is ignored for ``'nested_unique'`` (in which case *refinement_level* may be `None`). @@ -401,7 +401,7 @@ def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): "cf_healpix_weights: Can only calulate weights for the " "'nested_unique' indexing scheme" ) - + if measure: x = np.pi * (radius**2) / 3.0 else: diff --git a/cf/dimensioncoordinate.py b/cf/dimensioncoordinate.py index 644709bbda..6b0752a00a 100644 --- a/cf/dimensioncoordinate.py +++ b/cf/dimensioncoordinate.py @@ -440,8 +440,7 @@ def direction(self): @classmethod def create_regular(cls, args, units=None, standard_name=None, bounds=True): - """ - Create a new `DimensionCoordinate` with the given range and cellsize. + """Create a new `DimensionCoordinate` with the given range and cellsize. .. versionadded:: 3.15.0 @@ -455,9 +454,9 @@ def create_regular(cls, args, units=None, standard_name=None, bounds=True): bounds: `bool`, optional If True (the default) then the given range represents - the bounds, and the coordinate points will be the midpoints of - the bounds. If False, the range represents the coordinate points - directly. + the bounds, and the coordinate points will be the + midpoints of the bounds. If False, the range + represents the coordinate points directly. units: str or `Units`, optional The units of the new `DimensionCoordinate` object. @@ -492,31 +491,34 @@ def create_regular(cls, args, units=None, standard_name=None, bounds=True): f"Expected a sequence of three numbers, got {args}." ) - range = (args[0], args[1]) + r = (args[0], args[1]) cellsize = args[2] - range_diff = range[1] - range[0] + range_diff = r[1] - r[0] if cellsize > 0 and range_diff <= 0: raise ValueError( - f"Range ({range[0], range[1]}) must be increasing for a " + f"Range {r} must be increasing for a " f"positive cellsize ({cellsize})" ) elif cellsize < 0 and range_diff >= 0: raise ValueError( - f"Range ({range[0], range[1]}) must be decreasing for a " + f"Range {r} must be decreasing for a " f"negative cellsize ({cellsize})" ) + elif cellsize > abs(range_diff): + raise ValueError( + f"cellsize ({cellsize}) can not exceed the range {r}" + ) if standard_name is not None and not isinstance(standard_name, str): raise ValueError("standard_name must be either None or a string.") + start = r[0] + end = r[1] if bounds: cellsize2 = cellsize / 2 - start = range[0] + cellsize2 - end = range[1] - cellsize2 - else: - start = range[0] - end = range[1] + start += cellsize2 + end -= cellsize2 points = np.arange(start, end + cellsize, cellsize) @@ -527,8 +529,11 @@ def create_regular(cls, args, units=None, standard_name=None, bounds=True): ) if bounds: - b = coordinate.create_bounds() - coordinate.set_bounds(b, copy=False) + if points.size > 1: + coordinate.create_bounds(inplace=True) + else: + b = Bounds(data=Data([r])) + coordinate.set_bounds(b, copy=False) return coordinate diff --git a/cf/domain.py b/cf/domain.py index 7bf4133174..d8e44a714a 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -281,9 +281,10 @@ def create_healpix( refinement_level: `int` The refinement level of the grid within the HEALPix - hierarchy, starting at 0 for the base tesselation with - 12 cells. The number of cells in the global HEALPix - grid is :math:`(12 \times 4^refinement_level)`. + hierarchy, starting at 0 for the base tessellation + with 12 cells. The number of cells in the global + HEALPix grid is :math:`(12 \times + 4^refinement_level)`. indexing_scheme: `str` The HEALPix indexing scheme. One of ``'nested'`` (the diff --git a/cf/field.py b/cf/field.py index 0542eb5900..118b61e6e6 100644 --- a/cf/field.py +++ b/cf/field.py @@ -7148,7 +7148,7 @@ def collapse( ) domain_axis = collapse_axes.get(healpix_axis) if domain_axis is not None and domain_axis.get_size() > 1: - from .healpix_utils import del_healpix_coordinate_reference + from .healpix import del_healpix_coordinate_reference del_healpix_coordinate_reference(f) @@ -13267,7 +13267,8 @@ def regrids( latitude-longitude grid, including other HEALPix grids, UGRID meshes and DSG feature types. This is done internally by converting HEALPix grids to UGRID meshes and carrying out a - UGRID regridding. + UGRID regridding. See also + `healpix_decrease_refinement_level`. **DSG feature types** @@ -13291,7 +13292,7 @@ def regrids( .. versionadded:: 1.0.4 - .. seealso:: `regridc` + .. seealso:: `regridc`, `healpix_decrease_refinement_level` :Parameters: diff --git a/cf/functions.py b/cf/functions.py index c4e867fcaa..76388ac037 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -3319,24 +3319,115 @@ def unique_constructs(constructs, ignore_properties=None, copy=True): def contains_latlon(lat, lon, f=None): - """TODOHEALPIX""" - def latlon(f, lat, lon): - if f.coordinate("healpix_index", filter_by_naxes=(1,) default=None): + """Return indices of cells containing latitude-longitude locations. + + The cells must be defined by a discrete axis that has, or could + hav, 1-d latitude and longitude coordinates. At present, only + HEALPix axes are supported. + + If a single latitude is given then it is paired with each + longitude, and if a single longitude is given then it is paired + with each latitude. If multiple latitudes and multiple longitudes + are provided then they are paired element-wise. + + A cell index appears at most onxce in the output, even if that cell + contains more than one of the given latitude-longitude locations. + + .. versionadded:: NEXTVERSION + + .. seealso:: `cf.contains` + + :Parameters: + + lat: (sequence of) number + The latitude(s), in degrees north, for which to find the + cell indices. Must be in the range [-90, 90]. + + lon: (sequence of) number + The longitude(s), in degrees east, for which to find the + cell indices. + + f: `Field` or `Domain` or `None`, optional + The Field or Domain containing the UGRID or HEALPix + grid. + + If `None` (the default) then a callable function is + returned, which when called with an argument of *f* + returns the indices, i.e ``contains_latlon(lat, lon, f)`` + is equivalent to ``contains_latlon(lat, lon)(f)``. This + new function may be used as a condition in the `subspace` + and `indices` methods of *f*, since such conditions may be + functions that take the calling Field or Domain construct + as an argument. For instance, + ``f.subspace(X=contains_latlon(0, 45)`` is equivalent to + ``f.subspace(X=contains_latlon(0, 45, f)``. + + `numpy.ndarray` or function + Indices for the discrete axis that contain the + latitude-longitude locations, or if *f* is `None`, a + function that will return those indices. + + **Examples** + + >>> f = cf.example_field(12) + >>> print(f) + Field: air_temperature (ncvar%tas) + ---------------------------------- + Data : air_temperature(time(2), healpix_index(48)) K + Cell methods : time(2): mean area: mean + Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian + : height(1) = [1.5] m + Auxiliary coords: healpix_index(healpix_index(48)) = [0, ..., 47] 1 + Coord references: grid_mapping_name:healpix + >>> cf.contains_latlon(20, 90, f) + array([23]) + >>> cf.contains_latlon(-70, 90, f) + array([36]) + >>> cf.contains_latlon(20, 90, f) + array([23]) + >>> cf.contains_latlon([-70, 20], 90, f) + array([23, 36]) + >>> cf.contains_latlon([-70, 20], [90, 280], f) + array([31, 36]) + >>> cf.contains_latlon([-70, 20], [90, 280])(f) + array([31, 36]) + >>> cf.contains_latlon(20, [280, 280.001], f) + array([31]) + + >>> func = cf.contains_latlon(20, [280, 280.001]) + >>> func(f) + array([31]) + + """ + + def _contains_latlon(lat, lon, f): + if f.coordinate("healpix_index", filter_by_naxes=(1,), default=None): # HEALPix - from .healpix_utils import _healpix_contains_latlon - - return _healpix_contains_latlon(f, lat, lon) - + from .healpix import _healpix_contains_latlon + + return _healpix_contains_latlon(lat, lon, f) + if f.domain_topologies(todict=True): - # UGRID - return _ugrid_contains_latlon(f, lat, lon) + # UGRID - not coded up, yet. + pass + # from .ugrid import _ugrid_contains_latlon + # return _ugrid_contains_latlon(lat, lon, f) - raise ValueError("Can only use with discretet axes") + raise ValueError( + "'contains_latlon' can only calculate indices for a " + "HEALPix axis" + ) + + if np.abs(lat).max() > 90: + raise ValueError( + "Can't find cell locations: All latitudes must be in " + f"the range [-90, 90]. Got: {lat}" + ) if f is None: - return partial(latlon, lat=lat, lon=lon) + return partial(_contains_latlon, lat, lon) - return latlon(f, lat, lon) + return _contains_latlon(lat, lon, f) def _DEPRECATION_ERROR(message="", version="3.0.0", removed_at="4.0.0"): diff --git a/cf/healpix_utils.py b/cf/healpix.py similarity index 54% rename from cf/healpix_utils.py rename to cf/healpix.py index 3e9cc68ab9..4c8a7e8b5f 100644 --- a/cf/healpix_utils.py +++ b/cf/healpix.py @@ -1,117 +1,115 @@ """General functions useful for HEALPix functionality.""" -from functools import partial -import numpy as np import dask.array as da +import numpy as np -def _healpix_info(f): - """Get information about the HEALPix axis, if there is one. - .. versionadded:: NEXTVERSION - - :Parameters: +def _healpix_contains_latlon(lat, lon, f): + """Return indices of cells containing latitude-longitude locations. - f: `Field` or `Domain` - The field or domain. - - :Returns: + The cells must be defined by a HEALPix grid. - `dict` - The information about the HEALPix axis. The dictionary - will be empty if there is no HEALPix axis. + If a single latitude is given then it is paired with each + longitude, and if a single longitude is given then it is paired + with each latitude. If multiple latitudes and multiple longitudes + are provided then they are paired element-wise. - **Examples** + A cell index appears at most onxce in the output, even if that cell + contains more than one of the given latitude-longitude locations. - >>> f = cf.example_field(12) - >>> _healpix_info(f) - {'coordinate_reference_key': 'coordinatereference0', - 'grid_mapping_name:healpix': , - 'indexing_scheme': 'nested', - 'refinement_level': 1, - 'axis_key': 'domainaxis1', - 'coordinate_key': 'auxiliarycoordinate0', - 'healpix_index': } + .. versionadded:: NEXTVERSION - """ - info = {} - - cr_key, cr = f.coordinate_reference("grid_mapping_name:healpix", - item=True, default=(None, None)) - if cr is not None: - info['coordinate_reference_key'] = cr_key - info['grid_mapping_name:healpix'] = cr - parameters = cr.coordinate_conversion.parameters() - for param in ('indexing_scheme','refinement_level'): - value = parameters.get(param) - if value is not None: - info[param] = value - - hp_key, healpix_index = f.coordinate( - "healpix_index", - filter_by_naxes=(1,), - item=True, - default=(None, None), - ) - if healpix_index is not None: - info['axis_key'] = f.get_data_axes(hp_key)[0] - info['coordinate_key'] = hp_key - info['healpix_index'] = healpix_index - - return info + .. seealso:: `cf.contains_latlon` + :Parameters: + + lat: (sequence of) number + The latitude(s), in degrees north, for which to find the + cell indices. Must be in the range [-90, 90], although + this is not checked. -def _healpix_contains_latlon(f, lat, lon): - """TODOHEALPIX""" + lon: (sequence of) number + The longitude(s), in degrees east, for which to find the + cell indices. - hp = _healpix_info(f) - if not hp: - raise ValueError("TODOHEALPIX") + f: `Field` or `Domain` + The Field or Domain containing the HEALPix grid. + :Returns: + + `numpy.ndarray` + Indices for the HEALPix axis that contain the + latitude-longitude locations. + + """ try: import healpix except ImportError as e: raise ImportError( f"{e}. Must install healpix (https://pypi.org/project/healpix) " - "to allow the calculation of which cells contain " - "latitude/longitude locations for a HEALPix grid" + "to allow the calculation of which HEALPix cells contain " + "latitude-longitude locations" ) - - healpix_index = hp.get('healpix_index') + + hp = healpix_info(f) + + healpix_index = hp.get("healpix_index") if healpix_index is None: - raise ValueError("TODOHEALPIX") + raise ValueError( + "Can't find HEALPix cell locations: There are no healpix_index " + "coordinates" + ) indexing_scheme = hp.get("indexing_scheme") if indexing_scheme is None: - raise ValueError("TODOHEALPIX") - - if indexing_scheme == 'nested_unique': + raise ValueError( + "Can't find HEALPix cell locations: indexing_scheme has " + "not been set in the HEALPix grid mapping coordinate reference" + ) + + if indexing_scheme == "nested_unique": + # nested_unique indexing scheme index = [] healpix_index = healpix_index.array orders = healpix.uniq2pix(healpix_index, nest=True)[0] orders = np.unique(orders) for order in orders: - nside = healpix.order2nside(order) + # For this refinement level, find the HEALPix nested + # indices of the cells that contain the lat-lon points. + nside = healpix.order2nside(order) pix = healpix.ang2pix(nside, lon, lat, nest=True, lonlat=True) pix = np.unique(pix) + # Convert back to HEALPix nested_unique indices pix = healpix._chp.nest2uniq(order, pix, pix) + # Find where these HEALPix nested_unique indices are + # located in the original healpix_index coordinates index.append(da.where(da.isin(healpix_index, pix))[0]) index = da.unique(da.concatenate(index, axis=0)) else: + # nested or ring indexing scheme refinement_level = hp.get("refinement_level") if refinement_level is None: - raise ValueError("TODOHEALPIX") - - nest = indexing_scheme == 'nested' + raise ValueError( + "Can't find HEALPix cell locations: refinement_level has " + "not been set in the HEALPix grid mapping coordinate " + "reference" + ) + + # Find the HEALPix indices of the cells that contain the + # lat-lon points nside = healpix.order2nside(refinement_level) + nest = indexing_scheme == "nested" pix = healpix.ang2pix(nside, lon, lat, nest=nest, lonlat=True) pix = np.unique(pix) + # Find where these HEALPix indices are located in the + # healpix_index coordinates index = da.where(da.isin(healpix_index, pix))[0] + # Return the cell locations as a numpy array of element indices return index.compute() - def del_healpix_coordinate_reference(f): """Remove a healpix grid mapping coordinate reference construct. @@ -138,13 +136,12 @@ def del_healpix_coordinate_reference(f): cr_key, cr = f.coordinate_reference( "grid_mapping_name:healpix", item=True, default=(None, None) ) - latlon = f.coordinate_reference( - "grid_mapping_name:latitude_longitude", default=None - ) - if cr is not None: f.del_construct(cr_key) + latlon = f.coordinate_reference( + "grid_mapping_name:latitude_longitude", default=None + ) if latlon is None: latlon = cr.copy() cc = latlon.coordinate_conversion @@ -168,3 +165,64 @@ def del_healpix_coordinate_reference(f): f.set_construct(latlon) return cr + + +def healpix_info(f): + """Get information about the HEALPix axis, if there is one. + + .. versionadded:: NEXTVERSION + + :Parameters: + + f: `Field` or `Domain` + The field or domain. + + :Returns: + + `dict` + The information about the HEALPix axis. The dictionary + will be empty if there is no HEALPix axis. + + **Examples** + + >>> f = cf.example_field(12) + >>> healpix_info(f) + {'coordinate_reference_key': 'coordinatereference0', + 'grid_mapping_name:healpix': , + 'indexing_scheme': 'nested', + 'refinement_level': 1, + 'axis_key': 'domainaxis1', + 'coordinate_key': 'auxiliarycoordinate0', + 'healpix_index': } + + >>> f = cf.example_field(0) + >>> healpix_info(f) + {} + + """ + info = {} + + cr_key, cr = f.coordinate_reference( + "grid_mapping_name:healpix", item=True, default=(None, None) + ) + if cr is not None: + info["coordinate_reference_key"] = cr_key + info["grid_mapping_name:healpix"] = cr + parameters = cr.coordinate_conversion.parameters() + for param in ("indexing_scheme", "refinement_level"): + value = parameters.get(param) + if value is not None: + info[param] = value + + hp_key, healpix_index = f.coordinate( + "healpix_index", + filter_by_naxes=(1,), + item=True, + default=(None, None), + ) + if healpix_index is not None: + info["axis_key"] = f.get_data_axes(hp_key)[0] + info["coordinate_key"] = hp_key + info["healpix_index"] = healpix_index + + return info diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 402ae6c0b6..fc3f1e13fe 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -300,7 +300,8 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): # implied by a non-latitude_longitude grid mapping coordinate # reference). This is so that latitude and longitude kwarg # selections can work, even if the actual latitude and - # longitude coordinates are not part of the metadata. + # longitude coordinates are not currently part of the + # metadata. # # Do not do this in-place. self = self.create_latlon_coordinates() @@ -333,9 +334,8 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): ) if callable(value): - print ('value=', value) value = value(self) - + if axes in parsed: # The axes are the same as an existing key parsed[axes].append((axes, key, construct, value, identity)) @@ -412,7 +412,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): index0 = None # Loop round the conditions for this axis. - # + # # When there are multiple conditions, each iteration # produces a 1-d Boolean array, and the axis selection # is the logical AND of these arrays. @@ -553,7 +553,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): # 1-d CASE 3: All other 1-d cases if debug: logger.debug(" 1-d CASE 3:") # pragma: no cover - + index = item == value # Performance: Convert the 1-d 'index' to a @@ -2029,12 +2029,12 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): """ from ..dask_utils import cf_healpix_indexing_scheme - from ..healpix_utils import _healpix_info - + from ..healpix import healpix_info + f = self.copy() - hp = _healpix_info(f) - + hp = healpix_info(f) + valid_indexing_schemes = ("nested", "ring", "nested_unique") if new_indexing_scheme not in valid_indexing_schemes + (None,): raise ValueError( @@ -2044,7 +2044,7 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): ) # Get the healpix_index coordinates - healpix_index = hp.get('healpix_index') + healpix_index = hp.get("healpix_index") if healpix_index is None: raise ValueError( "Can't change HEALPix index scheme: There are no " @@ -2052,32 +2052,9 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): ) # Get the healpix_index axis - axis = hp['axis_key'] - - # Get the healpix_index coordinates -# hp_key, healpix_index = f.coordinate( -# "healpix_index", -# filter_by_naxes=(1,), -# item=True, -# default=(None, None), -# ) - # if healpix_index is None: - # raise ValueError( -# "Can't change HEALPix index scheme: There are no " -# "healpix_index coordinates" -# ) - -# # Parse the HEALPix coordinate reference -# cr = f.coordinate_reference("grid_mapping_name:healpix", default=None) -# if cr is None: -# raise ValueError( -# "Can't change HEALPix index scheme: There is no HEALPix grid " -# "mapping coordinate reference" -# ) + axis = hp["axis_key"] indexing_scheme = hp.get("indexing_scheme") -# parameters = cr.coordinate_conversion.parameters() -# indexing_scheme = parameters.get("indexing_scheme") if indexing_scheme is None: raise ValueError( "Can't change HEALPix indexing scheme: indexing_scheme has " @@ -2095,7 +2072,6 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): new_indexing_scheme is not None and new_indexing_scheme != indexing_scheme ): - # refinement_level = parameters.get("refinement_level") refinement_level = hp.get("refinement_level") if ( indexing_scheme in ("nested", "ring") @@ -2148,15 +2124,12 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): ) healpix_index.set_data(dx, copy=False) -# # Get the identifier for the HEALPix domain axis -# axis = f.get_data_axes(hp_key)[0] - # Ensure that healpix indices are auxiliary coordinates if healpix_index.construct_type == "dimension_coordinate": healpix_index = f._AuxiliaryCoordinate( source=healpix_index, copy=False ) - f.del_construct(hp['coordinate_key']) + f.del_construct(hp["coordinate_key"]) hp_key = f.set_construct(healpix_index, axes=axis, copy=False) cr.set_coordinate(hp_key) @@ -2218,12 +2191,11 @@ def healpix_to_ugrid(self, inplace=False): Topologies : cell:face(ncdim%cell(48), 4) = [[965, ..., 3074]] """ - from ..healpix_utils import del_healpix_coordinate_reference, _healpix_info + from ..healpix import del_healpix_coordinate_reference, healpix_info - hp = _healpix_info(self) + hp = healpix_info(self) - axis = hp.get('axis_key') -# axis = self.domain_axis("healpix_index", key=True, default=None) + axis = hp.get("axis_key") if axis is None: raise ValueError( "Can't convert HEALPix to UGRID: There is no HEALPix domain " @@ -2299,16 +2271,7 @@ def healpix_to_ugrid(self, inplace=False): f.set_construct(domain_topology, axes=axis, copy=False) # Remove the HEALPix index coordinates - f.del_construct(hp['coordinate_key']) -# "healpix_index", -# filter_by_type=( -# "auxiliary_coordinate", -# "dimension_coordinate", -# ), -# filter_by_axis=(axis,), -# axis_mode="exact", -# default=None, -# ) + f.del_construct(hp["coordinate_key"]) # Remove the HEALPix coordinate reference del_healpix_coordinate_reference(f) @@ -2322,6 +2285,7 @@ def create_latlon_coordinates( one_d=True, two_d=True, pole_longitude=None, + overwrite=False, inplace=False, verbose=None, ): @@ -2335,8 +2299,9 @@ def create_latlon_coordinates( construct. When it is not possible to create latitude and longitude - coordinates, the reason why will be reported if *verbose* is - at ``'INFO'`` or higher. + coordinates, the reason why will be reported if the log level + at ``2``/``'INFO'`` or higher (as set by `cf.log_level` or the + *verbose* parameter). .. versionadded:: NEXTVERSION @@ -2362,6 +2327,13 @@ def create_latlon_coordinates( them. If set to a number, then the longitudes of such points will all be that value. + overwrite: `bool`, optional + If True then remove any existing latitude and + longitude coordinates, prior to attempting to create + new ones. If False (the default) then if any latitude + and longitude coordinates already exist, new ones will + not be created. + {{inplace: `bool`, optional}} {{verbose: `int` or `str` or `None`, optional}} @@ -2402,6 +2374,29 @@ def create_latlon_coordinates( """ f = _inplace_enabled_define_and_cleanup(self) + # See if there are any existing latitude/longutude coordinates + latlon_coordinates = { + key: c + for key, c in f.coordinates(todict=True).items() + if c.Units.islatitude or c.Units.islongitude + } + if latlon_coordinates: + if overwrite: + # Remove existing latitude/longutude coordinates + # before carrying on + for key in latlon_coordinates: + f.del_construct(key) + else: + if is_log_level_info(logger): + logger.info( + "Can't create latitude and longitude coordinates: " + "overwrite=False and latitude and/or longitude " + "coordinates already exist: " + f"{', '.join(map(repr, latlon_coordinates.values()))}" + ) # pragma: no cover + + return f + # Get all Coordinate References in a dictionary identities = { cr.identity(""): cr @@ -2422,14 +2417,6 @@ def create_latlon_coordinates( for identity, cr in identities.items() if identity.startswith("grid_mapping_name:") } - if len(identities) > 2: - if is_log_level_info(logger): - logger.info( - "Can't create latitude and longitude coordinates: There " - "are more than two grid mapping coordinate references" - ) # pragma: no cover - - return f # Remove a 'latitude_longitude' grid mapping from the # dictionary @@ -2447,6 +2434,16 @@ def create_latlon_coordinates( return f + if len(identities) > 1: + if is_log_level_info(logger): + logger.info( + "Can't create latitude and longitude coordinates: There " + "is more than one non-latitude_longitude grid mapping " + "coordinate reference" + ) # pragma: no cover + + return f + # Still here? Then get the non-latitude_longitude grid mapping # and calulate the lat/lon coordinates. identity, cr = identities.popitem() @@ -2456,13 +2453,11 @@ def create_latlon_coordinates( # -------------------------------------------------------- # HEALPix 1-d coordinates # -------------------------------------------------------- - from ..healpix_utils import _healpix_info + from ..healpix import healpix_info + + hp = healpix_info(f) - hp = _healpix_info(f) - -# parameters = cr.coordinate_conversion.parameters() -# indexing_scheme = parameters.get("indexing_scheme") - indexing_scheme = hp.get('indexing_scheme') + indexing_scheme = hp.get("indexing_scheme") if indexing_scheme not in ("nested", "ring", "nested_unique"): if is_log_level_info(logger): logger.info( @@ -2472,7 +2467,6 @@ def create_latlon_coordinates( return f -# refinement_level = parameters.get("refinement_level") refinement_level = hp.get("refinement_level") if refinement_level is None and indexing_scheme in ( "nested", @@ -2486,15 +2480,8 @@ def create_latlon_coordinates( return f - # The the HEALPix indices - key, healpix_index = f.coordinate( - "healpix_index", - filter_by_naxes=(1,), - item=True, - default=(None, None), - ) -# if healpix_index is None: - if hp.get('healpix_index') is None: + healpix_index = hp.get("healpix_index") + if healpix_index is None: if is_log_level_info(logger): logger.info( "Can't create 1-d latitude and longitude coordinates: " @@ -2503,25 +2490,7 @@ def create_latlon_coordinates( return f - # Get the HEALPix axis - axis = hp['axis_key'] -# axis = f.get_data_axes(key)[0] - - if f.coordinates( - "X", - "Y", - filter_by_axis=(axis,), - axis_mode="exact", - todict=True, - ): - if is_log_level_info(logger): - logger.info( - "Can't create 1-d latitude and longitude coordinates: " - "1-d X or Y coordinates already exist for axis " - f"{f.constructs.domain_axis_identity(axis)!r}" - ) # pragma: no cover - - return f + axis = hp["axis_key"] # Define functions to create latitudes and longitudes from # HEALPix indices @@ -3151,11 +3120,14 @@ def is_discrete_axis(self, *identity, **filter_kwargs): return True # HEALPix - if self.coordinate( - "healpix_index", - filter_by_axis=(axis,), - axis_mode="exact", - default=None, + if ( + self.coordinate( + "healpix_index", + filter_by_axis=(axis,), + axis_mode="exact", + default=None, + ) + is not None ): return True diff --git a/cf/query.py b/cf/query.py index fe62768660..6ac3908e25 100644 --- a/cf/query.py +++ b/cf/query.py @@ -1921,9 +1921,9 @@ def contains(value, units=None): .. versionadded:: 3.0.0 - .. seealso:: `cf.Query.iscontains`, `cf.cellsize`, `cf.cellge`, + .. seealso:: `cf.contains_latlon`, `cf.cellsize`, `cf.cellge`, `cf.cellgt`, `cf.cellne`, `cf.cellle`, `cf.celllt`, - `cf.cellwi`, `cf.cellwo` + `cf.cellwi`, `cf.cellwo`, `cf.Query.iscontains` :Parameters: @@ -2707,60 +2707,6 @@ def seasons(n=4, start=12): return out -def heapix_contains(f, lat, lon): - """TODOHEALPIX""" - from collections.abc import Iterable - - try: - import healpix - except ImportError as e: - raise ImportError( - f"{e}. Must install healpix (e.g. from " - "https://pypi.org/project/healpix) to allow the calculation " - "TODOHEALPIX of latitude/longitude coordinate bounds for a HEALPix grid" - ) - - if not isinstance(lat, Iterable): - lat = (lat,) - - if not isinstance(lon, Iterable): - lon = (lon,) - - cr = f.coordinate_reference("grid_mapping_name:healpix", default=None) - cc = cr.coordinate_conversion.parameters() - indexing_scheme = cc.get("indexing_scheme") - order = cc.get("refinement_level") - - if indexing_scheme in ("nested", "ring"): - nest = indexing_scheme == "nested" - nside = healpix.order2nside(order) - print(nside, lon, lat, nest) - index = healpix.ang2pix(nside, lon, lat, nest=nest, lonlat=True) - - elif indexing_scheme == "nested_unique": - healpix_index = f.coordinate( - "healpix_index", filter_by_naxes=(1,), default=None - ) - if healpix_index is None: - raise ValueError("TODOHEALPIX hjscdjhdscjdsj") - - index = [] - orders = healpix.uniq2pix(healpix_index.array, nest=True)[0] - orders = np.unique(orders) - for order in orders: - nside = healpix.order2nside(order) - nested_pix = healpix.ang2pix( - nside, lon, lat, nest=True, lonlat=True - ) - i = healpix.pix2uniq(order, nested_pix, nest=True) - index.extend(i.tolist()) - - else: - raise ValueError("TODOHEALPIX 1ooooooooooooooooo1") - - return set(index) - - # -------------------------------------------------------------------- # Deprecated functions # -------------------------------------------------------------------- diff --git a/cf/test/test_DimensionCoordinate.py b/cf/test/test_DimensionCoordinate.py index 3fbcb22a92..4aab5fb55d 100644 --- a/cf/test/test_DimensionCoordinate.py +++ b/cf/test/test_DimensionCoordinate.py @@ -712,6 +712,12 @@ def test_DimensionCoordinate_create_regular(self): ) self.assertEqual(longitude_decreasing_no_bounds.units, "degrees_east") + # Size 1 with bounds + d = cf.DimensionCoordinate.create_regular((-180, 180, 360)) + self.assertEqual(d.size, 1) + self.assertTrue(np.allclose(d, 0)) + self.assertTrue(np.allclose(d.bounds, [-180, 180])) + def test_DimensionCoordinate_cell_characteristics(self): """Test the `cell_characteristic` DimensionCoordinate methods.""" d = self.dim.copy() diff --git a/cf/test/test_Domain.py b/cf/test/test_Domain.py index 8767854332..9a85a9400f 100644 --- a/cf/test/test_Domain.py +++ b/cf/test/test_Domain.py @@ -391,6 +391,14 @@ def test_Domain_create_regular(self): np.allclose(latitude_specific.array - y_points_specific, 0) ) + # Size 1 X axis + d = cf.Domain.create_regular((-180, 180, 360), (-90, 90, 18)) + self.assertEqual(d.size, 10) + self.assertTrue(d.cyclic()) + x = d.coordinate("X") + self.assertTrue(np.allclose(x, 0)) + self.assertTrue(np.allclose(x.bounds, [-180, 180])) + def test_Domain_del_construct(self): """Test the `del_construct` Domain method.""" # Test a domain without cyclic axes. These are equivalent tests to diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index b68cf40459..cc1845fd40 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -3266,6 +3266,15 @@ def test_Field_healpix_subspace(self): ) ) + g = f.subspace(healpix_index=cf.contains_latlon(20, 46)) + self.assertEqual(g.coordinate("healpix_index").array, 0) + + g = f.subspace(healpix_index=cf.contains_latlon(20, 1)) + self.assertEqual(g.coordinate("healpix_index").array, 19) + + g = f.subspace(healpix_index=cf.contains_latlon(20, [1, 46])) + self.assertTrue(np.allclose(g.coordinate("healpix_index"), [0, 19])) + def test_Field_healpix_decrease_refinement_level(self): """Test Field.healpix_decrease_refinement_level.""" f = self.f12 diff --git a/cf/test/test_weights.py b/cf/test/test_weights.py index b2861d4a3e..d258489305 100644 --- a/cf/test/test_weights.py +++ b/cf/test/test_weights.py @@ -217,6 +217,20 @@ def test_weights_polygon_area_ugrid(self): self.assertTrue((w.array == correct_weights).all()) self.assertEqual(w.Units, cf.Units("m2")) + # Check that the global sum of cell areas is equal between a + # global HEALPix grid (with a single refinment level) and its + # UGRID version. This test assumes that the HEALPix areas are + # correct. + h = cf.example_field(12) + u = h.healpix_to_ugrid() + h_weights = h.weights(measure=True, components=True)[(1,)] + u_weights = u.weights(measure=True, components=True,great_circle=True)[(1,)] + global_area = float(4 * np.pi * (h.radius()**2)) + h_area = h_weights.sum().array + u_area = u_weights.sum().array + self.assertTrue(np.allclose(h_area, global_area)) + self.assertTrue(np.allclose(h_area, u_area)) + def test_weights_line_length_geometry(self): # Spherical line geometry gls = gps.copy() diff --git a/docs/source/function.rst b/docs/source/function.rst index 8892f325c1..0cdff526c6 100644 --- a/docs/source/function.rst +++ b/docs/source/function.rst @@ -101,6 +101,7 @@ Condition constructors :template: function.rst cf.contains + cf.contains_latlon cf.cellsize cf.cellgt cf.cellge From 177e5d52fdc775a867f997f02864a93244a401a5 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 10 Jul 2025 15:17:03 +0100 Subject: [PATCH 26/59] dev --- Changelog.rst | 2 + cf/dask_utils.py | 28 +++--- cf/data/data.py | 3 +- cf/field.py | 15 +++- cf/functions.py | 56 ++++++------ cf/healpix.py | 26 +++--- cf/query.py | 6 +- cf/regrid/regrid.py | 14 +-- cf/test/test_Field.py | 6 +- cf/test/test_functions.py | 26 ++++++ cf/test/test_regrid.py | 13 +-- cf/test/test_regrid_featureType.py | 13 +-- cf/test/test_regrid_healpix.py | 135 +++++++++++++++++++++++++++++ cf/test/test_regrid_mesh.py | 13 +-- cf/test/test_weights.py | 28 +++--- 15 files changed, 251 insertions(+), 133 deletions(-) create mode 100644 cf/test/test_regrid_healpix.py diff --git a/Changelog.rst b/Changelog.rst index 34afdb0fbf..3090dc3801 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,8 @@ Version NEXTVERSION (https://github.com/NCAS-CMS/cf-python/issues/???) * New method: `cf.Data.coarsen` (https://github.com/NCAS-CMS/cf-python/issues/???) +* New function: `cf.locate` + (https://github.com/NCAS-CMS/cf-python/issues/???) * New optional dependency: ``healpix>=2025.1`` * Changed dependency: ``cfdm>=1.13.0.0, <1.13.1.0`` diff --git a/cf/dask_utils.py b/cf/dask_utils.py index f706f78f6b..7871eade90 100644 --- a/cf/dask_utils.py +++ b/cf/dask_utils.py @@ -139,28 +139,22 @@ def cf_healpix_bounds( b[where_ge_360] -= 360.0 # A vertex on the north (south) pole comes out with a - # longitude of NaN, so replace these with a sensible value, - # i.e. the longitude of the southern-most (northern-most) - # vertex. - - # North pole - longitude = pole_longitude + # longitude of NaN, so replace these with a sensible value: + # Either the constant 'pole_longitude', or else the longitude + # of the southern-most (northern-most) vertex. north = 0 south = 2 - i = np.argwhere(np.isnan(b[:, north])).flatten() - if i.size: - if pole_longitude is None: - longitude = b[i, south] - - b[i, north] = longitude + for pole, vertex in ((north, south), (south, north)): + indices = np.argwhere(np.isnan(b[:, pole])).flatten() + if not indices.size: + continue - # South pole - i = np.argwhere(np.isnan(b[:, south])).flatten() - if i.size: if pole_longitude is None: - longitude = b[i, north] + longitude = b[indices, vertex] + else: + longitude = pole_longitude - b[i, south] = longitude + b[indices, pole] = longitude return b diff --git a/cf/data/data.py b/cf/data/data.py index ce2bf2c066..0c99bbc1dd 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1418,13 +1418,12 @@ def coarsen( """ d = _inplace_enabled_define_and_cleanup(self) - # Parse axes + # Parse axes, making sure that all axes are non-negative. ndim = self.ndim for k in axes: if k < -ndim or k > ndim: raise ValueError("axis {k} is out of bounds for {ndim}-d data") - # Make sure all axes are non-negative axes = {(k + ndim if k < 0 else k): v for k, v in axes.items()} dx = d.to_dask_array() diff --git a/cf/field.py b/cf/field.py index 118b61e6e6..6337d1da49 100644 --- a/cf/field.py +++ b/cf/field.py @@ -5030,14 +5030,16 @@ def healpix_decrease_refinement_level( f"({refinement_level})" ) - # Whether or not to Create lat/lon coordinates for the - # coarsened grid + # Whether or not to create lat/lon coordinates for the + # coarsened grid. Only do so if the original grid has lat/lon + # coordinates. create_coarsened_latlon = bool( f.coordinates( "latitude", "longitude", filter_by_axis=(axis,), axis_mode="exact", + todict=True, ) ) @@ -5103,9 +5105,8 @@ def healpix_decrease_refinement_level( ) cr.set_coordinate(new_key) - # Create lat/lon coordinates for the coarsened grid if they - # exist in the original grid if create_coarsened_latlon: + # Create lat/lon coordinates for the coarsened grid f.create_latlon_coordinates(inplace=True) return f @@ -13014,6 +13015,12 @@ def subspace(self): acting along orthogonal dimensions, some missing data may still need to be inserted into the field construct's data. + * If a condition is a given as a callable function, then it is + replaced with the output of it being called with the field + as its ony argument. For instance, + ``f.subspace(X=cf.locate(30, 180))`` is equivalent to + ``f.subspace(X=cf.locate(30, 180)(f))``. + **Subspacing by index** Subspacing by indexing, signified by the use of square brackets, diff --git a/cf/functions.py b/cf/functions.py index 76388ac037..dbbac1dccc 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -3318,11 +3318,11 @@ def unique_constructs(constructs, ignore_properties=None, copy=True): ) -def contains_latlon(lat, lon, f=None): +def locate(lat, lon, f=None): """Return indices of cells containing latitude-longitude locations. The cells must be defined by a discrete axis that has, or could - hav, 1-d latitude and longitude coordinates. At present, only + have, 1-d latitude and longitude coordinates. At present, only HEALPix axes are supported. If a single latitude is given then it is paired with each @@ -3353,14 +3353,14 @@ def contains_latlon(lat, lon, f=None): If `None` (the default) then a callable function is returned, which when called with an argument of *f* - returns the indices, i.e ``contains_latlon(lat, lon, f)`` - is equivalent to ``contains_latlon(lat, lon)(f)``. This + returns the indices, i.e ``contains(lat, lon, f)`` + is equivalent to ``contains(lat, lon)(f)``. This new function may be used as a condition in the `subspace` and `indices` methods of *f*, since such conditions may be functions that take the calling Field or Domain construct as an argument. For instance, - ``f.subspace(X=contains_latlon(0, 45)`` is equivalent to - ``f.subspace(X=contains_latlon(0, 45, f)``. + ``f.subspace(X=locate(0, 45)`` is equivalent to + ``f.subspace(X=locate(0, 45, f)``. `numpy.ndarray` or function Indices for the discrete axis that contain the @@ -3379,55 +3379,51 @@ def contains_latlon(lat, lon, f=None): : height(1) = [1.5] m Auxiliary coords: healpix_index(healpix_index(48)) = [0, ..., 47] 1 Coord references: grid_mapping_name:healpix - >>> cf.contains_latlon(20, 90, f) + >>> cf.locate(20, 90, f) array([23]) - >>> cf.contains_latlon(-70, 90, f) + >>> cf.locate(-70, 90, f) array([36]) - >>> cf.contains_latlon(20, 90, f) - array([23]) - >>> cf.contains_latlon([-70, 20], 90, f) + >>> cf.locate([-70, 20], 90, f) array([23, 36]) - >>> cf.contains_latlon([-70, 20], [90, 280], f) + >>> cf.locate([-70, 20], [90, 280], f) array([31, 36]) - >>> cf.contains_latlon([-70, 20], [90, 280])(f) + >>> cf.locate([-70, 20], [90, 280])(f) array([31, 36]) - >>> cf.contains_latlon(20, [280, 280.001], f) + >>> cf.locate(20, [280, 280.001], f) array([31]) - - >>> func = cf.contains_latlon(20, [280, 280.001]) + >>> func = cf.locate(20, [280, 280.001]) >>> func(f) array([31]) """ - def _contains_latlon(lat, lon, f): + def _locate(lat, lon, f): if f.coordinate("healpix_index", filter_by_naxes=(1,), default=None): # HEALPix - from .healpix import _healpix_contains_latlon + from .healpix import _healpix_locate - return _healpix_contains_latlon(lat, lon, f) + return _healpix_locate(lat, lon, f) + + raise ValueError( + "'locate' can only calculate indices for a HEALPix axis" + ) if f.domain_topologies(todict=True): # UGRID - not coded up, yet. pass - # from .ugrid import _ugrid_contains_latlon - # return _ugrid_contains_latlon(lat, lon, f) - - raise ValueError( - "'contains_latlon' can only calculate indices for a " - "HEALPix axis" - ) + # from .ugrid import _ugrid_locate + # return _ugrid_locate(lat, lon, f) if np.abs(lat).max() > 90: raise ValueError( - "Can't find cell locations: All latitudes must be in " - f"the range [-90, 90]. Got: {lat}" + "Can't find cell locations: Latitudes must be in the range " + f"[-90, 90]. Got: {lat}" ) if f is None: - return partial(_contains_latlon, lat, lon) + return partial(_locate, lat, lon) - return _contains_latlon(lat, lon, f) + return _locate(lat, lon, f) def _DEPRECATION_ERROR(message="", version="3.0.0", removed_at="4.0.0"): diff --git a/cf/healpix.py b/cf/healpix.py index 4c8a7e8b5f..cc1dc93858 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -4,7 +4,7 @@ import numpy as np -def _healpix_contains_latlon(lat, lon, f): +def _healpix_locate(lat, lon, f): """Return indices of cells containing latitude-longitude locations. The cells must be defined by a HEALPix grid. @@ -19,7 +19,7 @@ def _healpix_contains_latlon(lat, lon, f): .. versionadded:: NEXTVERSION - .. seealso:: `cf.contains_latlon` + .. seealso:: `cf.locate` :Parameters: @@ -47,8 +47,7 @@ def _healpix_contains_latlon(lat, lon, f): except ImportError as e: raise ImportError( f"{e}. Must install healpix (https://pypi.org/project/healpix) " - "to allow the calculation of which HEALPix cells contain " - "latitude-longitude locations" + "to allow the location of HEALPix cells" ) hp = healpix_info(f) @@ -56,15 +55,15 @@ def _healpix_contains_latlon(lat, lon, f): healpix_index = hp.get("healpix_index") if healpix_index is None: raise ValueError( - "Can't find HEALPix cell locations: There are no healpix_index " + "Can't locate HEALPix cells: There are no healpix_index " "coordinates" ) indexing_scheme = hp.get("indexing_scheme") if indexing_scheme is None: raise ValueError( - "Can't find HEALPix cell locations: indexing_scheme has " - "not been set in the HEALPix grid mapping coordinate reference" + "Can't locate HEALPix cells: indexing_scheme has not been set " + "in the HEALPix grid mapping coordinate reference" ) if indexing_scheme == "nested_unique": @@ -78,11 +77,12 @@ def _healpix_contains_latlon(lat, lon, f): # indices of the cells that contain the lat-lon points. nside = healpix.order2nside(order) pix = healpix.ang2pix(nside, lon, lat, nest=True, lonlat=True) + # Remove duplicates pix = np.unique(pix) # Convert back to HEALPix nested_unique indices pix = healpix._chp.nest2uniq(order, pix, pix) - # Find where these HEALPix nested_unique indices are - # located in the original healpix_index coordinates + # Find where these HEALPix indices are located in the + # healpix_index coordinates index.append(da.where(da.isin(healpix_index, pix))[0]) index = da.unique(da.concatenate(index, axis=0)) @@ -91,16 +91,16 @@ def _healpix_contains_latlon(lat, lon, f): refinement_level = hp.get("refinement_level") if refinement_level is None: raise ValueError( - "Can't find HEALPix cell locations: refinement_level has " - "not been set in the HEALPix grid mapping coordinate " - "reference" + "Can't locate HEALPix cells: refinement_level has not been " + "set in the HEALPix grid mapping coordinate reference" ) # Find the HEALPix indices of the cells that contain the # lat-lon points - nside = healpix.order2nside(refinement_level) nest = indexing_scheme == "nested" + nside = healpix.order2nside(refinement_level) pix = healpix.ang2pix(nside, lon, lat, nest=nest, lonlat=True) + # Remove duplicates pix = np.unique(pix) # Find where these HEALPix indices are located in the # healpix_index coordinates diff --git a/cf/query.py b/cf/query.py index 6ac3908e25..304398e78a 100644 --- a/cf/query.py +++ b/cf/query.py @@ -1921,9 +1921,9 @@ def contains(value, units=None): .. versionadded:: 3.0.0 - .. seealso:: `cf.contains_latlon`, `cf.cellsize`, `cf.cellge`, - `cf.cellgt`, `cf.cellne`, `cf.cellle`, `cf.celllt`, - `cf.cellwi`, `cf.cellwo`, `cf.Query.iscontains` + .. seealso:: `cf.locate`, `cf.cellsize`, `cf.cellge`, `cf.cellgt`, + `cf.cellne`, `cf.cellle`, `cf.celllt`, `cf.cellwi`, + `cf.cellwo`, `cf.Query.iscontains` :Parameters: diff --git a/cf/regrid/regrid.py b/cf/regrid/regrid.py index 508c2c506f..29d0751467 100644 --- a/cf/regrid/regrid.py +++ b/cf/regrid/regrid.py @@ -14,21 +14,11 @@ from ..units import Units from .regridoperator import RegridOperator -# ESMF renamed its Python module to `esmpy` at ESMF version 8.4.0. Allow -# either for now for backwards compatibility. -esmpy_imported = False +esmpy_imported = True try: import esmpy - - esmpy_imported = True except ImportError: - try: - # Take the new name to use in preference to the old one. - import ESMF as esmpy - - esmpy_imported = True - except ImportError: - pass + esmpy_imported = False logger = logging.getLogger(__name__) diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index cc1845fd40..703a3dbe2c 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -3266,13 +3266,13 @@ def test_Field_healpix_subspace(self): ) ) - g = f.subspace(healpix_index=cf.contains_latlon(20, 46)) + g = f.subspace(healpix_index=cf.locate(20, 46)) self.assertEqual(g.coordinate("healpix_index").array, 0) - g = f.subspace(healpix_index=cf.contains_latlon(20, 1)) + g = f.subspace(healpix_index=cf.locate(20, 1)) self.assertEqual(g.coordinate("healpix_index").array, 19) - g = f.subspace(healpix_index=cf.contains_latlon(20, [1, 46])) + g = f.subspace(healpix_index=cf.locate(20, [1, 46])) self.assertTrue(np.allclose(g.coordinate("healpix_index"), [0, 19])) def test_Field_healpix_decrease_refinement_level(self): diff --git a/cf/test/test_functions.py b/cf/test/test_functions.py index 1a26a1afdc..522c89c012 100644 --- a/cf/test/test_functions.py +++ b/cf/test/test_functions.py @@ -430,6 +430,32 @@ def test_normalize_slice(self): with self.assertRaises(IndexError): cf.normalize_slice(index, 8, cyclic=True) + def test_locate(self): + """Test cf.locate""" + # HEALPix + f = cf.example_field(12) + self.assertEqual(cf.locate(20, 90, f), 23) + self.assertEqual(cf.locate(-70, 90, f), 36) + self.assertEqual(cf.locate(20, [280, 280.001], f), 31) + + self.assertTrue(np.allclose(cf.locate([-70, 20], 90, f), [23, 36])) + self.assertTrue( + np.allclose(cf.locate([-70, 20], [90, 280], f), [31, 36]) + ) + self.assertTrue( + np.allclose(cf.locate([-70, 20], [90, 280])(f), [31, 36]) + ) + + # Bad latitudes + for lat in (-91, 91): + with self.assertRaises(ValueError): + cf.locate(lat, 30, f) + + # Invalid grid types (regulat lat/lon, geometry, UGRID) + for f in cf.example_fields(0, 6, 8): + with self.assertRaises(ValueError): + cf.locate(60, 30, f) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) diff --git a/cf/test/test_regrid.py b/cf/test/test_regrid.py index 2001b4cce6..ef3e58f162 100644 --- a/cf/test/test_regrid.py +++ b/cf/test/test_regrid.py @@ -31,21 +31,12 @@ def _remove_tmpfiles(): atexit.register(_remove_tmpfiles) -# ESMF renamed its Python module to `esmpy` at ESMF version 8.4.0. Allow -# either for now for backwards compatibility. -esmpy_imported = False +esmpy_imported = True try: import esmpy - - esmpy_imported = True except ImportError: - try: - # Take the new name to use in preference to the old one. - import ESMF as esmpy + esmpy_imported = False - esmpy_imported = True - except ImportError: - pass all_methods = ( "linear", diff --git a/cf/test/test_regrid_featureType.py b/cf/test/test_regrid_featureType.py index 198b44dd97..7581e32f80 100644 --- a/cf/test/test_regrid_featureType.py +++ b/cf/test/test_regrid_featureType.py @@ -9,21 +9,12 @@ import cf -# ESMF renamed its Python module to `esmpy` at ESMF version 8.4.0. Allow -# either for now for backwards compatibility. -esmpy_imported = False +esmpy_imported = True try: import esmpy - - esmpy_imported = True except ImportError: - try: - # Take the new name to use in preference to the old one. - import ESMF as esmpy + esmpy_imported = False - esmpy_imported = True - except ImportError: - pass methods = ( "linear", diff --git a/cf/test/test_regrid_healpix.py b/cf/test/test_regrid_healpix.py new file mode 100644 index 0000000000..7df2020ffc --- /dev/null +++ b/cf/test/test_regrid_healpix.py @@ -0,0 +1,135 @@ +import datetime +import faulthandler +import os +import unittest + +faulthandler.enable() # to debug seg faults and timeouts + +import numpy as np + +import cf + +esmpy_imported = True +try: + import esmpy # noqa: F401 +except ImportError: + esmpy_imported = False + + +all_methods = ( + "linear", + "conservative", + "conservative_2nd", + "nearest_dtos", + "nearest_stod", + "patch", +) + + +# Set numerical comparison tolerances +atol = 0 +rtol = 0 + + +class RegridMeshTest(unittest.TestCase): + # Get the test source and destination fields + src_mesh_file = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "ugrid_global_1.nc" + ) + dst_mesh_file = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "ugrid_global_2.nc" + ) + src_mesh = cf.read(src_mesh_file)[0] + dst_mesh = cf.read(dst_mesh_file)[0] + + def setUp(self): + """Preparations called immediately before each test method.""" + # Disable log messages to silence expected warnings + cf.log_level("DISABLE") + # Note: to enable all messages for given methods, lines or calls (those + # without a 'verbose' option to do the same) e.g. to debug them, wrap + # them (for methods, start-to-end internally) as follows: + # cfdm.log_level('DEBUG') + # < ... test code ... > + # cfdm.log_level('DISABLE') + + @unittest.skipUnless(esmpy_imported, "Requires esmpy/ESMF package.") + def test_Field_regrid_to_healpix(self): + # Check that UGRID -> healpix is the same as UGRID -> UGRUD + self.assertFalse(cf.regrid_logging()) + + dst = cf.Domain.create_healpix(3) # 768 cells + dst_ugrid = dst.healpix_to_ugrid() + src = self.src_mesh.copy() + + for src_masked in (False, True): + if src_masked: + src = src.copy() + src[100:200] = cf.masked + + # Loop over whether or not to use the destination grid + # masked points + for method in all_methods: + if method in ("conservative", "conservative_2nd"): + continue + + x = src.regrids(dst, method=method) + y = src.regrids(dst_ugrid, method=method) + a = x.array + b = y.array + self.assertTrue(np.allclose(b, a, atol=atol, rtol=rtol)) + + if np.ma.isMA(a): + self.assertTrue((b.mask == a.mask).all()) + + # Check that the result is a HEALPix grid + self.assertTrue(cf.healpix.healpix_info(x)) + + # Methods that don't currently work + for method in ("conservative", "conservative_2nd"): + with self.assertRaises(ValueError): + src.regrids(dst, method=method) + + @unittest.skipUnless(esmpy_imported, "Requires esmpy/ESMF package.") + def test_Field_regrid_from_healpix(self): + # Check that healpix -> UGRID is the same as UGRID -> UGRUD + self.assertFalse(cf.regrid_logging()) + + src = cf.Field(source=cf.Domain.create_healpix(3)) # 768 cells + src.set_data(np.arange(768)) + + src_ugrid = src.healpix_to_ugrid() + + dst = self.dst_mesh.copy() + + for src_masked in (False, True): + if src_masked: + src[100:200] = cf.masked + src_ugrid[100:200] = cf.masked + + # Loop over whether or not to use the destination grid + # masked points + for method in all_methods: + if method in ("conservative", "conservative_2nd"): + continue + + x = src.regrids(dst, method=method) + y = src_ugrid.regrids(dst, method=method) + a = x.array + b = y.array + self.assertTrue(np.allclose(b, a, atol=atol, rtol=rtol)) + + if np.ma.isMA(a): + self.assertTrue((b.mask == a.mask).all()) + + # Methods that don't currently work + for method in ("conservative", "conservative_2nd"): + with self.assertRaises(ValueError): + src.regrids(dst, method=method) + + +if __name__ == "__main__": + print("Run date:", datetime.datetime.now()) + cf.environment() + print("") + unittest.main(verbosity=2) diff --git a/cf/test/test_regrid_mesh.py b/cf/test/test_regrid_mesh.py index 3095640135..68c5c0e7da 100644 --- a/cf/test/test_regrid_mesh.py +++ b/cf/test/test_regrid_mesh.py @@ -9,21 +9,12 @@ import cf -# ESMF renamed its Python module to `esmpy` at ESMF version 8.4.0. Allow -# either for now for backwards compatibility. -esmpy_imported = False +esmpy_imported = True try: import esmpy - - esmpy_imported = True except ImportError: - try: - # Take the new name to use in preference to the old one. - import ESMF as esmpy + esmpy_imported = False - esmpy_imported = True - except ImportError: - pass all_methods = ( "linear", diff --git a/cf/test/test_weights.py b/cf/test/test_weights.py index d258489305..5043a6b796 100644 --- a/cf/test/test_weights.py +++ b/cf/test/test_weights.py @@ -217,20 +217,15 @@ def test_weights_polygon_area_ugrid(self): self.assertTrue((w.array == correct_weights).all()) self.assertEqual(w.Units, cf.Units("m2")) - # Check that the global sum of cell areas is equal between a - # global HEALPix grid (with a single refinment level) and its - # UGRID version. This test assumes that the HEALPix areas are - # correct. - h = cf.example_field(12) - u = h.healpix_to_ugrid() - h_weights = h.weights(measure=True, components=True)[(1,)] - u_weights = u.weights(measure=True, components=True,great_circle=True)[(1,)] - global_area = float(4 * np.pi * (h.radius()**2)) - h_area = h_weights.sum().array - u_area = u_weights.sum().array - self.assertTrue(np.allclose(h_area, global_area)) - self.assertTrue(np.allclose(h_area, u_area)) - + # For a UGRID derived from a global HEALPix grid, check that + # the global sum of cell areas is correct + u = cf.example_field(12).healpix_to_ugrid() + u_weights = u.weights( + measure=True, components=True, great_circle=True + )[(1,)] + global_area = 4 * np.pi * (u.radius() ** 2) + self.assertTrue(np.allclose(u_weights.sum(), global_area)) + def test_weights_line_length_geometry(self): # Spherical line geometry gls = gps.copy() @@ -361,10 +356,11 @@ def test_weights_healpix(self): self.assertTrue(np.allclose(w[16:], 1 / (4**1))) w = f.weights(measure=True, components=True)[(1,)].array - radius = f.radius() - x = 4 * np.pi * (radius**2) / 12 + x = 4 * np.pi * (f.radius() ** 2) / 12 self.assertTrue(np.allclose(w[:16], x / (4**2))) self.assertTrue(np.allclose(w[16:], x / (4**1))) + # Total global area + self.assertTrue(np.allclose(w.sum(), x * 12)) if __name__ == "__main__": From cf756e2f695d8b6b80c6edf96666300d3f58c830 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 11 Jul 2025 14:59:49 +0100 Subject: [PATCH 27/59] dev --- cf/dask_utils.py | 23 ++++++++++++++++------- cf/field.py | 21 ++++++++++++--------- cf/mixin/fielddomain.py | 18 ++++++++++++------ cf/regrid/regrid.py | 15 +-------------- cf/test/test_regrid_healpix.py | 16 ---------------- 5 files changed, 41 insertions(+), 52 deletions(-) diff --git a/cf/dask_utils.py b/cf/dask_utils.py index 7871eade90..d7312a520d 100644 --- a/cf/dask_utils.py +++ b/cf/dask_utils.py @@ -6,6 +6,7 @@ """ import numpy as np +from cfdm.data.dask_utils import cfdm_to_memory def cf_healpix_bounds( @@ -88,6 +89,8 @@ def cf_healpix_bounds( "bounds for a HEALPix grid" ) + a = cfdm_to_memory(a) + # Keep an eye on https://github.com/ntessore/healpix/issues/66 if a.ndim != 1: raise ValueError( @@ -138,19 +141,19 @@ def cf_healpix_bounds( if where_ge_360[0].size: b[where_ge_360] -= 360.0 - # A vertex on the north (south) pole comes out with a - # longitude of NaN, so replace these with a sensible value: + # Vertices on the north or south pole come out with a + # longitude of NaN, so replace these with sensible values: # Either the constant 'pole_longitude', or else the longitude - # of the southern-most (northern-most) vertex. + # of the cell vertex that is opposite the vertex on the pole. north = 0 south = 2 - for pole, vertex in ((north, south), (south, north)): + for pole, replacement in ((north, south), (south, north)): indices = np.argwhere(np.isnan(b[:, pole])).flatten() if not indices.size: continue if pole_longitude is None: - longitude = b[indices, vertex] + longitude = b[indices, replacement] else: longitude = pole_longitude @@ -212,9 +215,11 @@ def cf_healpix_coordinates( "for a HEALPix grid" ) + a = cfdm_to_memory(a) + if a.ndim != 1: raise ValueError( - "Can't calculate HEALPix cell coordinates when the " + "Can only calculate HEALPix cell coordinates when the " f"healpix_index array has one dimension. Got shape {a.shape}" ) @@ -236,7 +241,7 @@ def cf_healpix_coordinates( )[pos] else: # Create coordinates for 'nested' or 'ring' cells - nest = (indexing_scheme == "nested",) + nest = indexing_scheme == "nested" nside = healpix.order2nside(refinement_level) c = healpix.pix2ang( nside=nside, @@ -305,6 +310,8 @@ def cf_healpix_indexing_scheme( "to allow the changing of the HEALPix index scheme" ) + a = cfdm_to_memory(a) + if indexing_scheme == "nested": if new_indexing_scheme == "ring": nside = healpix.order2nside(refinement_level) @@ -396,6 +403,8 @@ def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): "'nested_unique' indexing scheme" ) + a = cfdm_to_memory(a) + if measure: x = np.pi * (radius**2) / 3.0 else: diff --git a/cf/field.py b/cf/field.py index 6337d1da49..b236c638c2 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4945,8 +4945,9 @@ def healpix_decrease_refinement_level( "reference" ) - # Make sure that we have a nested indexing scheme if conform: + # Make sure that we have 'nested' indexing scheme with + # ordered HEALPix indices try: f = f.healpix_indexing_scheme("nested", sort=True) except ValueError as error: @@ -6819,12 +6820,8 @@ def collapse( if debug: logger.debug( - f" axes = {axes}" - ) # pragma: no cover - logger.debug( - f" method = {method}" - ) # pragma: no cover - logger.debug( + f" axes = {axes}\n" + f" method = {method}\n" f" collapse_axes_all_sizes = {collapse_axes_all_sizes}" ) # pragma: no cover @@ -13266,7 +13263,9 @@ def regrids( Data defined on UGRID face or node cells may be regridded to any other latitude-longitude grid, including other UGRID - meshes and DSG feature types. + meshes and DSG feature types. Note that for conservative + regridding, a cell edge is assumed to be the great circle arc + that connects its two vertices. **HEALPix grids** @@ -13274,7 +13273,11 @@ def regrids( latitude-longitude grid, including other HEALPix grids, UGRID meshes and DSG feature types. This is done internally by converting HEALPix grids to UGRID meshes and carrying out a - UGRID regridding. See also + UGRID regridding. This means that for conservative regridding, + a cell edge is assumed to be the great circle arc that + connects its two vertices, which is certainly not true for + HEALPix cells. However, the errors are likely to be small, + particularly at higher resolutions. See also `healpix_decrease_refinement_level`. **DSG feature types** diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index fc3f1e13fe..90967751ad 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -2114,7 +2114,9 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): ) # Change the HEALPix indices - dx = healpix_index.to_dask_array() + dx = healpix_index.data.to_dask_array( + _force_mask_hardness=False, _force_to_memory=False + ) dx = dx.map_blocks( cf_healpix_indexing_scheme, meta=np.array((), dtype="int64"), @@ -2254,8 +2256,8 @@ def healpix_to_ugrid(self, inplace=False): ) # Create a unique integer identifer for each node location - bounds_y = bounds_y.to_dask_array() - bounds_x = bounds_x.to_dask_array() + bounds_y = bounds_y.data.to_dask_array(_force_mask_hardness=False) + bounds_x = bounds_x.data.to_dask_array(_force_mask_hardness=False) _, y_indices = np.unique(bounds_y, return_inverse=True) _, x_indices = np.unique(bounds_x, return_inverse=True) @@ -2474,8 +2476,10 @@ def create_latlon_coordinates( ): if is_log_level_info(logger): logger.info( - "Can't create 1-d latitude and longitude coordinates: " - "Missing HEALPix refinemnt level" + "Can't create 1-d latitude and longitude coordinates " + "from {indexing_scheme!r} HEALPix indices: " + "refinement_indexing_scheme has not been set in the " + "HEALPix grid mapping coordinate reference" ) # pragma: no cover return f @@ -2497,7 +2501,9 @@ def create_latlon_coordinates( from ..dask_utils import cf_healpix_bounds, cf_healpix_coordinates # Create new latitude and longitude coordinates with bounds - dx = healpix_index.to_dask_array() + dx = healpix_index.data.to_dask_array( + _force_mask_hardness=False, _force_to_memory=False + ) meta = np.array((), dtype="float64") # Latitude coordinates diff --git a/cf/regrid/regrid.py b/cf/regrid/regrid.py index 29d0751467..11587dd561 100644 --- a/cf/regrid/regrid.py +++ b/cf/regrid/regrid.py @@ -497,20 +497,7 @@ def regrid( conform_coordinates(src_grid, dst_grid) - if method == "conservative": - if src_grid.healpix or dst_grid.healpix: - raise ValueError( - f"{method!r} regridding is not available for HEALPix grids" - ) - - elif method in ("conservative_2nd", "patch"): - if method == "conservative_2nd" and ( - src_grid.healpix or dst_grid.healpix - ): - raise ValueError( - f"{method!r} regridding is not available for HEALPix grids" - ) - + if method in ("conservative_2nd", "patch"): if not (src_grid.dimensionality >= 2 and dst_grid.dimensionality >= 2): raise ValueError( f"{method!r} regridding is not available for 1-d regridding" diff --git a/cf/test/test_regrid_healpix.py b/cf/test/test_regrid_healpix.py index 7df2020ffc..5f655ec1b6 100644 --- a/cf/test/test_regrid_healpix.py +++ b/cf/test/test_regrid_healpix.py @@ -70,9 +70,6 @@ def test_Field_regrid_to_healpix(self): # Loop over whether or not to use the destination grid # masked points for method in all_methods: - if method in ("conservative", "conservative_2nd"): - continue - x = src.regrids(dst, method=method) y = src.regrids(dst_ugrid, method=method) a = x.array @@ -85,11 +82,6 @@ def test_Field_regrid_to_healpix(self): # Check that the result is a HEALPix grid self.assertTrue(cf.healpix.healpix_info(x)) - # Methods that don't currently work - for method in ("conservative", "conservative_2nd"): - with self.assertRaises(ValueError): - src.regrids(dst, method=method) - @unittest.skipUnless(esmpy_imported, "Requires esmpy/ESMF package.") def test_Field_regrid_from_healpix(self): # Check that healpix -> UGRID is the same as UGRID -> UGRUD @@ -110,9 +102,6 @@ def test_Field_regrid_from_healpix(self): # Loop over whether or not to use the destination grid # masked points for method in all_methods: - if method in ("conservative", "conservative_2nd"): - continue - x = src.regrids(dst, method=method) y = src_ugrid.regrids(dst, method=method) a = x.array @@ -122,11 +111,6 @@ def test_Field_regrid_from_healpix(self): if np.ma.isMA(a): self.assertTrue((b.mask == a.mask).all()) - # Methods that don't currently work - for method in ("conservative", "conservative_2nd"): - with self.assertRaises(ValueError): - src.regrids(dst, method=method) - if __name__ == "__main__": print("Run date:", datetime.datetime.now()) From 84492f929dd9c51b4a14defc4e1ff42fd5da88b6 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 18 Jul 2025 12:14:44 +0100 Subject: [PATCH 28/59] dev --- cf/dask_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cf/dask_utils.py b/cf/dask_utils.py index d7312a520d..02372958e6 100644 --- a/cf/dask_utils.py +++ b/cf/dask_utils.py @@ -406,6 +406,9 @@ def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): a = cfdm_to_memory(a) if measure: + # Surface area of sphere is 4*pi*(r**2) + # Number of HEALPix cells at refinement level N is 12*(4**N) + # => Area of one cell is pi*(r**2)/(3*(4**N)) x = np.pi * (radius**2) / 3.0 else: x = 1.0 @@ -419,7 +422,6 @@ def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): w = np.empty(a.shape, dtype="float64") for order, i in zip(orders, index): - nside = healpix.order2nside(order) - w = np.where(inverse == inverse[i], x / (nside**2), w) + w = np.where(inverse == inverse[i], x / (4**order), w) return w From f3b7695e2d0d7543770710ad0d9bc3b5f0358233 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 21 Jul 2025 18:51:08 +0100 Subject: [PATCH 29/59] dev --- cf/docstring/docstring.py | 2 +- cf/field.py | 107 ++++++++++++--------- cf/healpix.py | 163 ++++++++++++++++++++++++++++++- cf/mixin/fielddomain.py | 195 +++++++++++++++----------------------- cf/test/test_Field.py | 2 +- 5 files changed, 298 insertions(+), 171 deletions(-) diff --git a/cf/docstring/docstring.py b/cf/docstring/docstring.py index b3198d54ac..9bcbe1d44a 100644 --- a/cf/docstring/docstring.py +++ b/cf/docstring/docstring.py @@ -579,7 +579,7 @@ decreasing the refinement level. For a Multi-Order Coverage (MOC), where pixels with - different refinment levels are stored in the same + different refinement levels are stored in the same array, the indexing scheme has a unique index for each cell at each refinement level. The "nested_unique" scheme for an MOC indexes pixels within each diff --git a/cf/field.py b/cf/field.py index b236c638c2..088bc9e60f 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4822,6 +4822,11 @@ def healpix_decrease_refinement_level( ): """Decrease the refinement level of a HEALPix grid. + Decreasing the refinement level coarsens the horizontal grid + to a lower-level HEALPix grid by combining, using the + *reduction* function, all cells that lie inside each larger + cell at the new refinement level. + .. versionadded:: NEXTVERSION .. seealso:: `healpix_indexing_scheme` @@ -4838,7 +4843,7 @@ def healpix_decrease_refinement_level( *Example:* If the current refinement level is 10 then a new - coarser refinment level of 8 can be specified by + coarser refinement level of 8 can be specified by either ``8`` or ``-2``. reduction: function @@ -4850,20 +4855,23 @@ def healpix_decrease_refinement_level( For an intensive field quantity (that does not depend on the size of the cells, such as "sea_ice_amount" with units of kg m-2), `np.mean` - might be appropriate. For an extensive field - quantity (that depends on the size of the cells, - such as "sea_ice_mass" with units of kg), `np.sum` might be appropriate. + *Example:* + For an extensive field quantity (that depends on the + size of the cells, such as "sea_ice_mass" with units + of kg), `np.sum` might be appropriate. + conform: `bool`, optional If True (the default) the HEALPix grid is automatically converted to a form suitable for having its refinement level changed, i.e. the indexing scheme is changed to 'nested' and the HEALPix axis is sorted - so that the nested HEALPix indices are monotonically - increasing. If False then either an exeption is raised - if the HEALPix indexing scheme is not already - 'nested', or else the HEALPix axis is not sorted. + so that the nested HEALPix indices are strictly + monotonically increasing. If False then either an + exeption is raised if the HEALPix indexing scheme is + not already 'nested', or else the HEALPix axis is not + sorted. .. note:: Setting to False will speed up the operation when the HEALPix indexing scheme is already @@ -4871,23 +4879,31 @@ def healpix_decrease_refinement_level( sorted montonically. check_healpix_index: `bool`, optional - If True (the default) then it will be checked (after - the HEALPix grid has been conformed, if *conform* is - True) that a) the nested HEALPix indices are strictly - monotonically increasing, and b) every cell at the new - coarser refinement level contains the maximum possible - number of cells at the original finer refinement - level. If either condition is not met then an - exception is raised. If False then these checks are - not carried out. + If True (the default) then the following conditions + will be checked before the creation of the new Field + (but after the HEALPix grid has been conformed, if + *conform* is True): + + 1. The nested HEALPix indices are strictly + monotonically increasing + + 2. Every cell at the new coarser refinement level + contains the maximum possible number of cells at + the original finer refinement level. + + If True and any of these conditions is not met, then an + exception is raised. + + If False then these checks are not carried out. .. warning:: Only set to False, which will speed up the operation, if it is known in advance - that these conditions are met. If set to - False and any of the conditions is not - met then either an exception will be + that these conditions are already met. If + set to False and any of the conditions is + not met then either an exception will be raised or, much worse, the operation will - complete and return incorrect values. + complete and return incorrect data + values. :Returns: @@ -4899,35 +4915,39 @@ def healpix_decrease_refinement_level( >>> f = cf.example_field(12) >>> f + >>> f.healpix_info()['refinement_level'] + 1 Set the refinement level to 0: >>> g = f.healpix_decrease_refinement_level(0, np.mean) >>> g + >>> g.healpix_info()['refinement_level'] + 0 - Decrease the refinement level by 1: + Decrease the refinement level by 1, showing that every four + cells in the orginal field correspond to one cell at the lower + level: - >>> f.healpix_decrease_refinement_level(-1, np.mean) + >>> g = f.healpix_decrease_refinement_level(-1, np.mean) - >>> f.array[0, :4].mean() + >>> g.healpix_info()['refinement_level'] + 0 + >>> np.mean(g.array[0, 0]) np.float64(289.15) - >>> g.array[0, 0] + >>> f.healpix_info()['refinement_level'] + 1 + >>> np.mean(f.array[0, :4]) np.float64(289.15) """ f = self.copy() - cr = f.coordinate_reference("grid_mapping_name:healpix", default=None) - if cr is None: - raise ValueError( - "Can't decrease HEALPix refinement level: HEALPix " - "grid mapping has not been set" - ) + # Get HEALPix info + hp = f.healpix_info() - indexing_scheme = cr.coordinate_conversion.get_parameter( - "indexing_scheme", None - ) + indexing_scheme = hp.get("indexing_scheme") if indexing_scheme is None: raise ValueError( "Can't decrease HEALPix refinement level: indexing_scheme " @@ -4935,9 +4955,7 @@ def healpix_decrease_refinement_level( "reference" ) - refinement_level = cr.coordinate_conversion.get_parameter( - "refinement_level" - ) + refinement_level = hp.get("refinement_level") if refinement_level is None: raise ValueError( "Can't decrease HEALPix refinement level: refinement_level " @@ -4955,7 +4973,8 @@ def healpix_decrease_refinement_level( f"Can't decrease HEALPix refinement level: {error}" ) - cr = f.coordinate_reference("grid_mapping_name:healpix") + # Re-get HEALPix info + hp = f.healpix_info() elif indexing_scheme != "nested": raise ValueError( "Can't decrease HEALPix refinement level: indexing_scheme " @@ -4995,12 +5014,7 @@ def healpix_decrease_refinement_level( ncells = 4**-level # Get the healpix_index coordinates - hp_key, healpix_index = f.auxiliary_coordinate( - "healpix_index", - filter_by_naxes=(1,), - item=True, - default=(None, None), - ) + healpix_index = hp.get("healpix_index") if healpix_index is None: raise ValueError( "Can't decrease HEALPix refinement level: There are no " @@ -5008,7 +5022,7 @@ def healpix_decrease_refinement_level( ) # Get the HEALPix axis - axis = f.get_data_axes(hp_key)[0] + axis = hp["domain_axis_key"] iaxis = f.get_data_axes().index(axis) if check_healpix_index: @@ -5097,10 +5111,13 @@ def healpix_decrease_refinement_level( # Convert indices to auxiliary coordinates new_index = f._AuxiliaryCoordinate(source=new_index, copy=False) hp_key = None + else: + hp_key = hp["coordinate_key"] new_key = f.set_construct(new_index, axes=axis, key=hp_key, copy=False) # Set the new refinement level + cr = hp.get("grid_mapping_name:healpix") cr.coordinate_conversion.set_parameter( "refinement_level", new_refinement_level ) diff --git a/cf/healpix.py b/cf/healpix.py index cc1dc93858..a9a612b4a7 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -4,6 +4,159 @@ import numpy as np +def _healpix_indexing_scheme(healpix_index, hp, new_indexing_scheme): + """Change the indexing scheme of HEALPix indices. + + .. versionadded:: NEXTVERSION + + .. seealso:: `cf.Field.healpix_indexing_scheme` + + :Parameters: + + healpix_index: `Coordinate` + The healpix_index coordinates, which will be updated + in-place. + + hp: `dict` + The HEALPix info dictionary. + + new_indexing_scheme: `str` + The new indexing scheme. + + :Returns: + + `None` + + """ + from .dask_utils import cf_healpix_indexing_scheme + + indexing_scheme = hp["indexing_scheme"] + refinement_level = hp.get("refinement_level") + + # Change the HEALPix indices + dx = healpix_index.data.to_dask_array( + _force_mask_hardness=False, _force_to_memory=False + ) + dx = dx.map_blocks( + cf_healpix_indexing_scheme, + meta=np.array((), dtype="int64"), + indexing_scheme=indexing_scheme, + new_indexing_scheme=new_indexing_scheme, + refinement_level=refinement_level, + ) + healpix_index.set_data(dx, copy=False) + + +def _healpix_create_latlon_coordinates(f, hp, pole_longitude): + """Create latitude and longitude coordinates for a HEALPix grid. + + .. versionadded:: NEXTVERSION + + .. seealso:: `cf.Field.create_latlon_coordinates` + + :Parameters: + + f: `Field` or `Domain` + The Field or Domain containing the HEALPix grid, which + will be updated in-place. + + hp: `dict` + The HEALPix info dictionary. + + pole_longitude: `None` or number + The longitude of coordinates, or coordinate bounds, that + lie exactly on the north or south pole. If `None` then the + longitudes of such points will vary according to the + alogrithm being used to create them. If set to a number, + then the longitudes of such points will all be given that + value. + + :Returns: + + `str`, `str` + The keys of the new latitude and longitude coordinate + constructs. + + """ + from .dask_utils import cf_healpix_bounds, cf_healpix_coordinates + + healpix_index = hp["healpix_index"] + indexing_scheme = hp["indexing_scheme"] + refinement_level = hp.get("refinement_level") + + # Create new latitude and longitude coordinates with bounds + dx = healpix_index.data.to_dask_array( + _force_mask_hardness=False, _force_to_memory=False + ) + meta = np.array((), dtype="float64") + + # Latitude coordinates + dy = dx.map_blocks( + cf_healpix_coordinates, + meta=meta, + indexing_scheme=indexing_scheme, + refinement_level=refinement_level, + lat=True, + ) + lat = f._AuxiliaryCoordinate( + data=f._Data(dy, "degrees_north", copy=False), + properties={"standard_name": "latitude"}, + copy=False, + ) + + # Longitude coordinates + dy = dx.map_blocks( + cf_healpix_coordinates, + meta=meta, + indexing_scheme=indexing_scheme, + refinement_level=refinement_level, + lon=True, + ) + lon = f._AuxiliaryCoordinate( + data=f._Data(dy, "degrees_east", copy=False), + properties={"standard_name": "longitude"}, + copy=False, + ) + + # Latitude bounds + dy = da.blockwise( + cf_healpix_bounds, + "ij", + dx, + "i", + new_axes={"j": 4}, + meta=meta, + indexing_scheme=indexing_scheme, + refinement_level=refinement_level, + lat=True, + ) + bounds = f._Bounds(data=dy) + lat.set_bounds(bounds) + + # Longitude bounds + dy = da.blockwise( + cf_healpix_bounds, + "ij", + dx, + "i", + new_axes={"j": 4}, + meta=meta, + indexing_scheme=indexing_scheme, + refinement_level=refinement_level, + lon=True, + pole_longitude=pole_longitude, + ) + bounds = f._Bounds(data=dy) + lon.set_bounds(bounds) + + # Set the new latitude and longitude coordinates + axis = hp["domain_axis_key"] + lat_key = f.set_construct(lat, axes=axis, copy=False) + lon_key = f.set_construct(lon, axes=axis, copy=False) + + return lat_key, lon_key + + def _healpix_locate(lat, lon, f): """Return indices of cells containing latitude-longitude locations. @@ -77,7 +230,7 @@ def _healpix_locate(lat, lon, f): # indices of the cells that contain the lat-lon points. nside = healpix.order2nside(order) pix = healpix.ang2pix(nside, lon, lat, nest=True, lonlat=True) - # Remove duplicates + # Remove duplicate indices pix = np.unique(pix) # Convert back to HEALPix nested_unique indices pix = healpix._chp.nest2uniq(order, pix, pix) @@ -100,7 +253,7 @@ def _healpix_locate(lat, lon, f): nest = indexing_scheme == "nested" nside = healpix.order2nside(refinement_level) pix = healpix.ang2pix(nside, lon, lat, nest=nest, lonlat=True) - # Remove duplicates + # Remove duplicate indices pix = np.unique(pix) # Find where these HEALPix indices are located in the # healpix_index coordinates @@ -168,7 +321,7 @@ def del_healpix_coordinate_reference(f): def healpix_info(f): - """Get information about the HEALPix axis, if there is one. + """Get information about the HEALPix grid, if there is one. .. versionadded:: NEXTVERSION @@ -191,7 +344,7 @@ def healpix_info(f): 'grid_mapping_name:healpix': , 'indexing_scheme': 'nested', 'refinement_level': 1, - 'axis_key': 'domainaxis1', + 'domain_axis_key': 'domainaxis1', 'coordinate_key': 'auxiliarycoordinate0', 'healpix_index': } @@ -221,7 +374,7 @@ def healpix_info(f): default=(None, None), ) if healpix_index is not None: - info["axis_key"] = f.get_data_axes(hp_key)[0] + info["domain_axis_key"] = f.get_data_axes(hp_key)[0] info["coordinate_key"] = hp_key info["healpix_index"] = healpix_index diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 90967751ad..afe59eb712 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -1998,18 +1998,22 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): : height(1) = [1.5] m Auxiliary coords: healpix_index(healpix_index(48)) = [0, ..., 47] 1 Coord references: grid_mapping_name:healpix - >>> f.coordinate_reference('grid_mapping_name:healpix').coordinate_conversion.get_parameter('indexing_scheme') + >>> f.healpix_info()['indexing_scheme'] 'nested' >>> print(f.coordinate('healpix_index').array) [ 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47] >>> g = f.healpix_indexing_scheme('nested_unique') + >>> g.healpix_info()['indexing_scheme'] + 'nested_unique' >>> print(g.coordinate('healpix_index').array) [16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63] >>> g = f.healpix_indexing_scheme('ring') + >>> g.healpix_info()['indexing_scheme'] + 'ring' >>> print(g.coordinate('healpix_index').array) [13 5 4 0 15 7 6 1 17 9 8 2 19 11 10 3 28 20 27 12 30 22 21 14 32 24 23 16 34 26 25 18 44 37 36 29 45 39 38 31 46 41 40 33 47 43 42 35] @@ -2028,44 +2032,43 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47] """ - from ..dask_utils import cf_healpix_indexing_scheme - from ..healpix import healpix_info - f = self.copy() - hp = healpix_info(f) + hp = f.healpix_info() valid_indexing_schemes = ("nested", "ring", "nested_unique") if new_indexing_scheme not in valid_indexing_schemes + (None,): raise ValueError( - "Can't change HEALPix index scheme: new_indexing_scheme " - f"keyword must be None or one of {valid_indexing_schemes!r}. " - f"Got {new_indexing_scheme!r}" + f"Can't change HEALPix index scheme of {f!r}: " + "new_indexing_scheme keyword must be None or one of " + f"{valid_indexing_schemes!r}. Got {new_indexing_scheme!r}" ) # Get the healpix_index coordinates healpix_index = hp.get("healpix_index") if healpix_index is None: raise ValueError( - "Can't change HEALPix index scheme: There are no " + f"Can't change HEALPix index scheme of {f!r}: There are no " "healpix_index coordinates" ) # Get the healpix_index axis - axis = hp["axis_key"] + axis = hp["domain_axis_key"] indexing_scheme = hp.get("indexing_scheme") if indexing_scheme is None: raise ValueError( - "Can't change HEALPix indexing scheme: indexing_scheme has " - "not been set in the HEALPix grid mapping coordinate reference" + f"Can't change HEALPix indexing scheme of {f!r}: " + "indexing_scheme has not been set in the HEALPix grid " + "mapping coordinate reference" ) if indexing_scheme not in valid_indexing_schemes: raise ValueError( - "Can't change HEALPix indexing scheme: indexing_scheme in " - "the HEALPix grid mapping coordinate reference must be one " - f"of {valid_indexing_schemes!r}. Got {indexing_scheme!r}" + f"Can't change HEALPix indexing scheme of {f!r}: " + "indexing_scheme in the HEALPix grid mapping coordinate " + f"reference must be one of {valid_indexing_schemes!r}. " + f"Got {indexing_scheme!r}" ) if ( @@ -2078,7 +2081,7 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): and refinement_level is None ): raise ValueError( - "Can't change HEALPix indexing scheme from " + f"Can't change HEALPix indexing scheme of {f!r} from " f"{indexing_scheme!r} to {new_indexing_scheme!r} when " "refinement_level has not been set in the HEALPix grid " "mapping coordinate reference" @@ -2114,17 +2117,9 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): ) # Change the HEALPix indices - dx = healpix_index.data.to_dask_array( - _force_mask_hardness=False, _force_to_memory=False - ) - dx = dx.map_blocks( - cf_healpix_indexing_scheme, - meta=np.array((), dtype="int64"), - indexing_scheme=indexing_scheme, - new_indexing_scheme=new_indexing_scheme, - refinement_level=refinement_level, - ) - healpix_index.set_data(dx, copy=False) + from ..healpix import _healpix_indexing_scheme + + _healpix_indexing_scheme(healpix_index, hp, new_indexing_scheme) # Ensure that healpix indices are auxiliary coordinates if healpix_index.construct_type == "dimension_coordinate": @@ -2150,6 +2145,38 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): return f + def healpix_info(self): + """Get information about the HEALPix grid, if there is one. + + .. versionadded:: NEXTVERSION + + :Returns: + + `dict` + The information about the HEALPix axis. The dictionary + will be empty if there is no HEALPix axis. + + **Examples** + + >>> f = cf.example_field(12) + >>> f.healpix_info() + {'coordinate_reference_key': 'coordinatereference0', + 'grid_mapping_name:healpix': , + 'indexing_scheme': 'nested', + 'refinement_level': 1, + 'domain_axis_key': 'domainaxis1', + 'coordinate_key': 'auxiliarycoordinate0', + 'healpix_index': } + + >>> f = cf.example_field(0) + >>> healpix_info(f) + {} + + """ + from ..healpix import healpix_info + + return healpix_info(self) + @_inplace_enabled(default=False) def healpix_to_ugrid(self, inplace=False): """Convert a HEALPix domain to a UGRID domain. @@ -2193,11 +2220,11 @@ def healpix_to_ugrid(self, inplace=False): Topologies : cell:face(ncdim%cell(48), 4) = [[965, ..., 3074]] """ - from ..healpix import del_healpix_coordinate_reference, healpix_info + from ..healpix import del_healpix_coordinate_reference - hp = healpix_info(self) + hp = self.healpix_info() - axis = hp.get("axis_key") + axis = hp.get("domain_axis_key") if axis is None: raise ValueError( "Can't convert HEALPix to UGRID: There is no HEALPix domain " @@ -2327,7 +2354,7 @@ def create_latlon_coordinates( (the default) then the longitudes of such points will vary according to the alogrithm being used to create them. If set to a number, then the longitudes of such - points will all be that value. + points will all be given that value. overwrite: `bool`, optional If True then remove any existing latitude and @@ -2450,36 +2477,35 @@ def create_latlon_coordinates( # and calulate the lat/lon coordinates. identity, cr = identities.popitem() + # Initialize the flag that tells us if any new coordinates + # were created new_coords = False + if one_d and identity == "grid_mapping_name:healpix": # -------------------------------------------------------- - # HEALPix 1-d coordinates + # HEALPix: 1-d lat/lon coordinates # -------------------------------------------------------- - from ..healpix import healpix_info - - hp = healpix_info(f) + hp = f.healpix_info() indexing_scheme = hp.get("indexing_scheme") if indexing_scheme not in ("nested", "ring", "nested_unique"): if is_log_level_info(logger): logger.info( - "Can't create 1-d latitude and longitude coordinates: " - f"Invalid HEALPix index scheme: {indexing_scheme!r}" + "Can't create 1-d latitude and longitude coordinates " + f"for {f!r}: Invalid HEALPix index scheme: " + f"{indexing_scheme!r}" ) # pragma: no cover return f refinement_level = hp.get("refinement_level") - if refinement_level is None and indexing_scheme in ( - "nested", - "ring", - ): + if refinement_level is None and indexing_scheme != "nested_unique": if is_log_level_info(logger): logger.info( "Can't create 1-d latitude and longitude coordinates " - "from {indexing_scheme!r} HEALPix indices: " - "refinement_indexing_scheme has not been set in the " - "HEALPix grid mapping coordinate reference" + f"for {f!r} from {indexing_scheme!r} HEALPix indices: " + "refinement_level has not been set in the HEALPix " + "grid mapping coordinate reference" ) # pragma: no cover return f @@ -2488,87 +2514,18 @@ def create_latlon_coordinates( if healpix_index is None: if is_log_level_info(logger): logger.info( - "Can't create 1-d latitude and longitude coordinates: " - "Missing healpix_index coordinates" + "Can't create 1-d latitude and longitude coordinates " + f"for {f!r}: Missing healpix_index coordinates" ) # pragma: no cover return f - axis = hp["axis_key"] + # Create the new lat/lon coordinates + from ..healpix import _healpix_create_latlon_coordinates - # Define functions to create latitudes and longitudes from - # HEALPix indices - from ..dask_utils import cf_healpix_bounds, cf_healpix_coordinates - - # Create new latitude and longitude coordinates with bounds - dx = healpix_index.data.to_dask_array( - _force_mask_hardness=False, _force_to_memory=False - ) - meta = np.array((), dtype="float64") - - # Latitude coordinates - dy = dx.map_blocks( - cf_healpix_coordinates, - meta=meta, - indexing_scheme=indexing_scheme, - refinement_level=refinement_level, - lat=True, + lat_key, lon_key = _healpix_create_latlon_coordinates( + f, hp, pole_longitude ) - lat = f._AuxiliaryCoordinate( - data=f._Data(dy, "degrees_north", copy=False), - properties={"standard_name": "latitude"}, - copy=False, - ) - - # Longitude coordinates - dy = dx.map_blocks( - cf_healpix_coordinates, - meta=meta, - indexing_scheme=indexing_scheme, - refinement_level=refinement_level, - lon=True, - ) - lon = f._AuxiliaryCoordinate( - data=f._Data(dy, "degrees_east", copy=False), - properties={"standard_name": "longitude"}, - copy=False, - ) - - # Latitude bounds - dy = da.blockwise( - cf_healpix_bounds, - "ij", - dx, - "i", - new_axes={"j": 4}, - meta=meta, - indexing_scheme=indexing_scheme, - refinement_level=refinement_level, - lat=True, - ) - bounds = f._Bounds(data=dy) - lat.set_bounds(bounds) - - # Longitude bounds - dy = da.blockwise( - cf_healpix_bounds, - "ij", - dx, - "i", - new_axes={"j": 4}, - meta=meta, - indexing_scheme=indexing_scheme, - refinement_level=refinement_level, - lon=True, - pole_longitude=pole_longitude, - ) - bounds = f._Bounds(data=dy) - lon.set_bounds(bounds) - - # Set the new latitude and longitude coordinates - lat_key = f.set_construct(lat, axes=axis, copy=False) - lon_key = f.set_construct(lon, axes=axis, copy=False) - new_coords = True elif two_d: diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index 05b036248b..b79d216f42 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -3334,7 +3334,7 @@ def test_Field_healpix_decrease_refinement_level(self): ) self.assertFalse(np.allclose(h.coord("healpix_index"), np.arange(12))) - # Can't change refinment level for a 'nested_unique' field + # Can't change refinement level for a 'nested_unique' field with self.assertRaises(ValueError): self.f13.healpix_decrease_refinement_level(0, np.mean) From 84d86f14796da4c3ec5f75e1ecdd0c635399dbb0 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 22 Jul 2025 15:17:48 +0100 Subject: [PATCH 30/59] dev --- cf/dask_utils.py | 427 ----------------------------------- cf/data/dask_utils.py | 484 ++++++++++++++++++++++++++++++++++++++++ cf/field.py | 34 ++- cf/functions.py | 45 ++-- cf/healpix.py | 189 ++++++++++------ cf/mixin/fielddomain.py | 105 +++++---- cf/weights.py | 4 +- 7 files changed, 726 insertions(+), 562 deletions(-) delete mode 100644 cf/dask_utils.py diff --git a/cf/dask_utils.py b/cf/dask_utils.py deleted file mode 100644 index 02372958e6..0000000000 --- a/cf/dask_utils.py +++ /dev/null @@ -1,427 +0,0 @@ -"""Functions intended to be passed to be dask. - -These will typically be functions that operate on dask chunks. For -instance, as would be passed to `dask.array.map_blocks`. - -""" - -import numpy as np -from cfdm.data.dask_utils import cfdm_to_memory - - -def cf_healpix_bounds( - a, - indexing_scheme, - refinement_level=None, - lat=False, - lon=False, - pole_longitude=None, -): - """Calculate HEALPix cell bounds. - - Latitude or longitude locations of the cell vertices are derived - from HEALPix indices.x Each cell has four bounds, which are - returned in an anticlockwise direction, as seen from above, - starting with the eastern-most vertex. - - .. versionadded:: NEXTVERSION - - :Parameters: - - a: `numpy.ndarray` - The array of HEALPix indices. - - indexing_scheme: `str` - The HEALPix indexing scheme. One of ``'nested'``, - ``'ring'``, or ``'nested_unique'``. - - refinement_level: `int` or `None`, optional - The refinement level of the grid within the HEALPix - hierarchy, starting at 0 for the base tessellation with 12 - cells. Must be an `int` for *indexing_scheme* ``'nested'`` - or ``'ring'``, but is ignored for ``'nested_unique'`` (in - which case *refinement_level* may be `None`). - - lat: `bool`, optional - If True then return latitude bounds. - - lon: `bool`, optional - If True then return longitude bounds. - - pole_longitude: `None` or number - The longitude of coordinate bounds that lie exactly on the - north or south pole. If `None` (the default) then the - longitudes of such a point will be identical to its - opposite vertex. If set to a number, then the longitudes - of such points will all be that value. - - :Returns: - - `numpy.ndarray` - A 2-d array containing the HEALPix cell bounds. - - **Examples** - - >>> cf_healpix_bounds([0, 1, 2, 3], 'nested', 1, lat=True) - array([[19.47122063, 41.8103149 , 19.47122063, 0. ], - [41.8103149 , 66.44353569, 41.8103149 , 19.47122063], - [41.8103149 , 66.44353569, 41.8103149 , 19.47122063], - [66.44353569, 90. , 66.44353569, 41.8103149 ]]) - >>> cf_healpix_bounds([0, 1, 2, 3], 'nested', 1, lon=True) - array([[45. , 22.5, 45. , 67.5], - [90. , 45. , 67.5, 90. ], - [ 0. , 0. , 22.5, 45. ], - [45. , 0. , 45. , 90. ]]) - >>> cf_healpix_bounds([0, 1, 2, 3], 'nested', 1, lon=True, - ... pole_longitude=3.14159) - array([[45. , 22.5 , 45. , 67.5 ], - [90. , 45. , 67.5 , 90. ], - [ 0. , 0. , 22.5 , 45. ], - [ 3.14159, 0. , 45. , 90. ]]) - - """ - try: - import healpix - except ImportError as e: - raise ImportError( - f"{e}. Must install healpix (https://pypi.org/project/healpix) " - "to allow the calculation of latitude/longitude coordinate " - "bounds for a HEALPix grid" - ) - - a = cfdm_to_memory(a) - - # Keep an eye on https://github.com/ntessore/healpix/issues/66 - if a.ndim != 1: - raise ValueError( - "Can only calculate HEALPix cell bounds when the " - f"healpix_index array has one dimension. Got shape {a.shape}" - ) - - if lat: - pos = 1 - elif lon: - pos = 0 - - if indexing_scheme == "ring": - bounds_func = healpix._chp.ring2ang_uv - else: - bounds_func = healpix._chp.nest2ang_uv - - # Define the cell vertices, in an anticlockwise direction, as seen - # from above, starting with the northern-most vertex. - east = (1, 0) - north = (1, 1) - west = (0, 1) - south = (0, 0) - vertices = (north, west, south, east) - - # Initialise the output bounds array - b = np.empty((a.size, 4), dtype="float64") - - if indexing_scheme == "nested_unique": - # Create bounds for 'nested_unique' cells - orders, a = healpix.uniq2pix(a, nest=True) - for order in np.unique(orders): - nside = healpix.order2nside(order) - indices = np.where(orders == order)[0] - for j, (u, v) in enumerate(vertices): - thetaphi = bounds_func(nside, a[indices], u, v) - b[indices, j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] - else: - # Create bounds for 'nested' or 'ring' cells - nside = healpix.order2nside(refinement_level) - for j, (u, v) in enumerate(vertices): - thetaphi = bounds_func(nside, a, u, v) - b[..., j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] - - if not pos: - # Ensure that longitude bounds are less than 360 - where_ge_360 = np.where(b >= 360) - if where_ge_360[0].size: - b[where_ge_360] -= 360.0 - - # Vertices on the north or south pole come out with a - # longitude of NaN, so replace these with sensible values: - # Either the constant 'pole_longitude', or else the longitude - # of the cell vertex that is opposite the vertex on the pole. - north = 0 - south = 2 - for pole, replacement in ((north, south), (south, north)): - indices = np.argwhere(np.isnan(b[:, pole])).flatten() - if not indices.size: - continue - - if pole_longitude is None: - longitude = b[indices, replacement] - else: - longitude = pole_longitude - - b[indices, pole] = longitude - - return b - - -def cf_healpix_coordinates( - a, indexing_scheme, refinement_level=None, lat=False, lon=False -): - """Calculate HEALPix cell coordinates. - - THe coordinates are for the cell centres. - - .. versionadded:: NEXTVERSION - - :Parameters: - - a: `numpy.ndarray` - The array of HEALPix indices. - - indexing_scheme: `str` - The HEALPix indexing scheme. One of ``'nested'``, - ``'ring'``, or ``'nested_unique'``. - - refinement_level: `int` or `None`, optional - The refinement level of the grid within the HEALPix - hierarchy, starting at 0 for the base tessellation with 12 - cells. Must be an `int` for *indexing_scheme* ``'nested'`` - or ``'ring'``, but is ignored for ``'nested_unique'`` (in - which case *refinement_level* may be `None`). - - lat: `bool`, optional - If True then return latitude coordinates. - - lon: `bool`, optional - If True then return longitude coordinates. - - :Returns: - - `numpy.ndarray` - A 1-d array containing the HEALPix cell coordinates. - - **Examples** - - >>> cf_healpix_coordinates([0, 1, 2, 3], 'nested', 1, lat=True) - array([19.47122063, 41.8103149 , 41.8103149 , 66.44353569]) - >>> cf_healpix_coordinates([0, 1, 2, 3], 'nested', 1, lon=True) - array([45. , 67.5, 22.5, 45. ]) - - """ - try: - import healpix - except ImportError as e: - raise ImportError( - f"{e}. Must install healpix (https://pypi.org/project/healpix) " - "to allow the calculation of latitude/longitude coordinates " - "for a HEALPix grid" - ) - - a = cfdm_to_memory(a) - - if a.ndim != 1: - raise ValueError( - "Can only calculate HEALPix cell coordinates when the " - f"healpix_index array has one dimension. Got shape {a.shape}" - ) - - if lat: - pos = 1 - elif lon: - pos = 0 - - if indexing_scheme == "nested_unique": - # Create coordinates for 'nested_unique' cells - c = np.empty(a.shape, dtype="float64") - - orders, a = healpix.uniq2pix(a, nest=True) - for order in np.unique(orders): - nside = healpix.order2nside(order) - indices = np.where(orders == order)[0] - c[indices] = healpix.pix2ang( - nside=nside, ipix=a[indices], nest=True, lonlat=True - )[pos] - else: - # Create coordinates for 'nested' or 'ring' cells - nest = indexing_scheme == "nested" - nside = healpix.order2nside(refinement_level) - c = healpix.pix2ang( - nside=nside, - ipix=a, - nest=nest, - lonlat=True, - )[pos] - - return c - - -def cf_healpix_indexing_scheme( - a, indexing_scheme, new_indexing_scheme, refinement_level=None -): - """Change the ordering of HEALPix indices. - - Does not change the position of each cell in the array, but - redefines their indices according to the new ordering scheme. - - .. versionadded:: NEXTVERSION - - :Parameters: - - a: `numpy.ndarray` - The array of HEALPix indices. - - indexing_scheme: `str` - The original HEALPix indexing scheme. One of ``'nested'``, - ``'ring'``, or ``'nested_unique'``. - - new_indexing_scheme: `str` - The new HEALPix indexing scheme to change to. One of - ``'nested'``, ``'ring'``, or ``'nested_unique'``. - - refinement_level: `int` or `None`, optional - The refinement level of the grid within the HEALPix - hierarchy, starting at 0 for the base tessellation with 12 - cells. Must be an `int` for *indexing_scheme* ``'nested'`` - or ``'ring'``, but is ignored for ``'nested_unique'`` (in - which case *refinement_level* may be `None`). - - :Returns: - - `numpy.ndarray` - An array containing the new HEALPix indices. - - **Examples** - - >>> cf_healpix_indexing_scheme([0, 1, 2, 3], 'nested', 'ring', 1) - array([13, 5, 4, 0]) - >>> cf_healpix_indexing_scheme([0, 1, 2, 3], 'nested', 'nested_unique', 1) - array([16, 17, 18, 19]) - >>> cf_healpix_indexing_scheme([16, 17, 18, 19], 'nested_unique', 'nest', None) - array([0, 1, 2, 3]) - - """ - if new_indexing_scheme == indexing_scheme: - # Null operation - return a - - try: - import healpix - except ImportError as e: - raise ImportError( - f"{e}. Must install healpix (https://pypi.org/project/healpix) " - "to allow the changing of the HEALPix index scheme" - ) - - a = cfdm_to_memory(a) - - if indexing_scheme == "nested": - if new_indexing_scheme == "ring": - nside = healpix.order2nside(refinement_level) - return healpix.nest2ring(nside, a) - - if new_indexing_scheme == "nested_unique": - return healpix.pix2uniq(refinement_level, a, nest=True) - - elif indexing_scheme == "ring": - if new_indexing_scheme == "nested": - nside = healpix.order2nside(refinement_level) - return healpix.ring2nest(nside, a) - - if new_indexing_scheme == "nested_unique": - return healpix.pix2uniq(refinement_level, a, nest=False) - - elif indexing_scheme == "nested_unique": - if new_indexing_scheme in ("nested", "ring"): - nest = new_indexing_scheme == "nested" - order, a = healpix.uniq2pix(a, nest=nest) - - refinement_levels = np.unique(order) - if refinement_levels.size > 1: - raise ValueError( - "Can't change HEALPix indexing scheme from " - f"'nested_unique' to {new_indexing_scheme!r} when the " - "HEALPix indices span multiple refinement levels (at " - f"least levels {refinement_levels.tolist()})" - ) - - return a - - raise ValueError("Failed to change the HEALPix indexing scheme") - - -def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): - """Calculate HEALPix cell area weights. - - K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et - al.. HEALPix: A Framework for High-Resolution Discretization and - Fast Analysis of Data Distributed on the Sphere. The Astrophysical - Journal, 2005, 622 (2), pp.759-771. - https://dx.doi.org/10.1086/427976 - - .. versionadded:: NEXTVERSION - - :Parameters: - - a: `numpy.ndarray` - The array of HEALPix 'nested_unique' indices. - - indexing_scheme: `str` - The HEALPix indexing scheme. Must be ``'nested_unique'``. - - measure: `bool`, optional - If True then create weights that are actual cell areas, in - units of the square of the radius units. - - radius: number, optional - The radius of the sphere. Must be set if *measure * is - True, otherwise ignored. - - :Returns: - - `numpy.ndarray` - An array containing the HEALPix cell weights. - - **Examples** - - >>> cf_healpix_weights([76, 77, 78, 79, 20, 21], 'nested_unique') - array([0.0625, 0.0625, 0.0625, 0.0625, 0.25 , 0.25 ]) - >>> cf_healpix_weights([76, 77, 78, 79, 20, 21], 'nested_unique', - ... measure=True, radius=6371000) - array([2.65658579e+12, 2.65658579e+12, 2.65658579e+12, 2.65658579e+12, - 1.06263432e+13, 1.06263432e+13]) - - """ - try: - import healpix - except ImportError as e: - raise ImportError( - f"{e}. Must install healpix (https://pypi.org/project/healpix) " - "to allow the calculation of cell area weights for a HEALPix grid" - ) - - if indexing_scheme != "nested_unique": - raise ValueError( - "cf_healpix_weights: Can only calulate weights for the " - "'nested_unique' indexing scheme" - ) - - a = cfdm_to_memory(a) - - if measure: - # Surface area of sphere is 4*pi*(r**2) - # Number of HEALPix cells at refinement level N is 12*(4**N) - # => Area of one cell is pi*(r**2)/(3*(4**N)) - x = np.pi * (radius**2) / 3.0 - else: - x = 1.0 - - orders = healpix.uniq2pix(a, nest=True)[0] - orders, index, inverse = np.unique( - orders, return_index=True, return_inverse=True - ) - - # Initialise the output weights array - w = np.empty(a.shape, dtype="float64") - - for order, i in zip(orders, index): - w = np.where(inverse == inverse[i], x / (4**order), w) - - return w diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index 4c6923541d..3412e1f5ba 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -463,3 +463,487 @@ def cf_filled(a, fill_value=None): """ a = cfdm_to_memory(a) return np.ma.filled(a, fill_value=fill_value) + + +def cf_healpix_bounds( + a, + indexing_scheme, + refinement_level=None, + lat=False, + lon=False, + pole_longitude=None, +): + """Calculate HEALPix cell bounds. + + Latitude or longitude locations of the cell vertices are derived + from HEALPix indices. Each cell has four bounds which are returned + in an anticlockwise direction, as seen from above, starting with + the northern-most vertex. + + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et + al.. HEALPix: A Framework for High-Resolution Discretization and + Fast Analysis of Data Distributed on the Sphere. The Astrophysical + Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + + M. Reinecke and E. Hivon: Efficient data structures for masks on + 2D grids. A&A, 580 (2015) + A132. https://doi.org/10.1051/0004-6361/201526549 + + .. versionadded:: NEXTVERSION + + :Parameters: + + a: `numpy.ndarray` + The array of HEALPix indices. + + indexing_scheme: `str` + The HEALPix indexing scheme. One of ``'nested'``, + ``'ring'``, or ``'nested_unique'``. + + refinement_level: `int` or `None`, optional + The refinement level of the grid within the HEALPix + hierarchy, starting at 0 for the base tessellation with 12 + cells. Must be an `int` for *indexing_scheme* ``'nested'`` + or ``'ring'``, but is ignored for *indexing_scheme* + ``'nested_unique'`` (in which case *refinement_level* may + be `None`). + + lat: `bool`, optional + If True then return latitude bounds. + + lon: `bool`, optional + If True then return longitude bounds. + + pole_longitude: `None` or number + The longitude of coordinate bounds that lie exactly on the + north or south pole. If `None` (the default) then the + longitudes of such a point will be identical to its + opposite vertex. If set to a number, then the longitudes + of such points will all be that value. + + :Returns: + + `numpy.ndarray` + A 2-d array containing the HEALPix cell bounds. + + **Examples** + + >>> cf.data.dask_utils.cf_healpix_bounds( + ... np.array([0, 1, 2, 3]), 'nested', 1, lat=True + ) + array([[41.8103149 , 19.47122063, 0. , 19.47122063], + [66.44353569, 41.8103149 , 19.47122063, 41.8103149 ], + [66.44353569, 41.8103149 , 19.47122063, 41.8103149 ], + [90. , 66.44353569, 41.8103149 , 66.44353569]]) + >>> cf.data.dask_utils.cf_healpix_bounds( + ... np.array([0, 1, 2, 3]), 'nested', 1, lon=True + ) + array([[45. , 22.5, 45. , 67.5], + [90. , 45. , 67.5, 90. ], + [ 0. , 0. , 22.5, 45. ], + [45. , 0. , 45. , 90. ]]) + >>> cf.data.dask_utils.cf_healpix_bounds( + ... np.array([0, 1, 2, 3]), 'nested', 1, lon=True, + ... pole_longitude=3.14159 + ) + array([[45. , 22.5 , 45. , 67.5 ], + [90. , 45. , 67.5 , 90. ], + [ 0. , 0. , 22.5 , 45. ], + [ 3.14159, 0. , 45. , 90. ]]) + + """ + try: + import healpix + except ImportError as e: + raise ImportError( + f"{e}. Must install healpix (https://pypi.org/project/healpix) " + "to allow the calculation of latitude/longitude coordinate " + "bounds for a HEALPix grid" + ) + + a = cfdm_to_memory(a) + + # Keep an eye on https://github.com/ntessore/healpix/issues/66 + if a.ndim != 1: + raise ValueError( + "Can only calculate HEALPix cell bounds when the " + f"healpix_index array has one dimension. Got shape {a.shape}" + ) + + if lat: + pos = 1 + elif lon: + pos = 0 + + if indexing_scheme == "ring": + bounds_func = healpix._chp.ring2ang_uv + else: + bounds_func = healpix._chp.nest2ang_uv + + # Define the cell vertices, in an anticlockwise direction, as seen + # from above, starting with the northern-most vertex. + east = (1, 0) + north = (1, 1) + west = (0, 1) + south = (0, 0) + vertices = (north, west, south, east) + + # Initialise the output bounds array + b = np.empty((a.size, 4), dtype="float64") + + if indexing_scheme == "nested_unique": + # Create bounds for 'nested_unique' cells + orders, a = healpix.uniq2pix(a, nest=True) + for order in np.unique(orders): + nside = healpix.order2nside(order) + indices = np.where(orders == order)[0] + for j, (u, v) in enumerate(vertices): + thetaphi = bounds_func(nside, a[indices], u, v) + b[indices, j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] + else: + # Create bounds for 'nested' or 'ring' cells + nside = healpix.order2nside(refinement_level) + for j, (u, v) in enumerate(vertices): + thetaphi = bounds_func(nside, a, u, v) + b[..., j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] + + if not pos: + # Ensure that longitude bounds are less than 360 + where_ge_360 = np.where(b >= 360) + if where_ge_360[0].size: + b[where_ge_360] -= 360.0 + + # Vertices on the north or south pole come out with a + # longitude of NaN, so replace these with sensible values: + # Either the constant 'pole_longitude', or else the longitude + # of the cell vertex that is opposite the vertex on the pole. + north = 0 + south = 2 + for pole, replacement in ((north, south), (south, north)): + indices = np.argwhere(np.isnan(b[:, pole])).flatten() + if not indices.size: + continue + + if pole_longitude is None: + longitude = b[indices, replacement] + else: + longitude = pole_longitude + + b[indices, pole] = longitude + + return b + + +def cf_healpix_coordinates( + a, indexing_scheme, refinement_level=None, lat=False, lon=False +): + """Calculate HEALPix cell coordinates. + + THe coordinates are the cell centres. + + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et + al.. HEALPix: A Framework for High-Resolution Discretization and + Fast Analysis of Data Distributed on the Sphere. The Astrophysical + Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + + M. Reinecke and E. Hivon: Efficient data structures for masks on + 2D grids. A&A, 580 (2015) + A132. https://doi.org/10.1051/0004-6361/201526549 + + .. versionadded:: NEXTVERSION + + :Parameters: + + a: `numpy.ndarray` + The array of HEALPix indices. + + indexing_scheme: `str` + The HEALPix indexing scheme. One of ``'nested'``, + ``'ring'``, or ``'nested_unique'``. + + refinement_level: `int` or `None`, optional + The refinement level of the grid within the HEALPix + hierarchy, starting at 0 for the base tessellation with 12 + cells. Must be an `int` for *indexing_scheme* ``'nested'`` + or ``'ring'``, but is ignored for *indexing_scheme* + ``'nested_unique'`` (in which case *refinement_level* may + be `None`). + + lat: `bool`, optional + If True then return latitude coordinates. + + lon: `bool`, optional + If True then return longitude coordinates. + + :Returns: + + `numpy.ndarray` + A 1-d array containing the HEALPix cell coordinates. + + **Examples** + + >>> cf.data.dask_utils.cf_healpix_coordinates( + ... np.array([0, 1, 2, 3]), 'nested', 1, lat=True + ) + array([19.47122063, 41.8103149 , 41.8103149 , 66.44353569]) + >>> cf.data.dask_utils.cf_healpix_coordinates( + ... np.array([0, 1, 2, 3]), 'nested', 1, lon=True + ) + array([45. , 67.5, 22.5, 45. ]) + + """ + try: + import healpix + except ImportError as e: + raise ImportError( + f"{e}. Must install healpix (https://pypi.org/project/healpix) " + "to allow the calculation of latitude/longitude coordinates " + "for a HEALPix grid" + ) + + a = cfdm_to_memory(a) + + if a.ndim != 1: + raise ValueError( + "Can only calculate HEALPix cell coordinates when the " + f"healpix_index array has one dimension. Got shape {a.shape}" + ) + + if lat: + pos = 1 + elif lon: + pos = 0 + + if indexing_scheme == "nested_unique": + # Create coordinates for 'nested_unique' cells + c = np.empty(a.shape, dtype="float64") + + nest = True + orders, a = healpix.uniq2pix(a, nest=nest) + for order in np.unique(orders): + nside = healpix.order2nside(order) + indices = np.where(orders == order)[0] + c[indices] = healpix.pix2ang( + nside=nside, ipix=a[indices], nest=nest, lonlat=True + )[pos] + else: + # Create coordinates for 'nested' or 'ring' cells + nest = indexing_scheme == "nested" + nside = healpix.order2nside(refinement_level) + c = healpix.pix2ang( + nside=nside, + ipix=a, + nest=nest, + lonlat=True, + )[pos] + + return c + + +def cf_healpix_indexing_scheme( + a, indexing_scheme, new_indexing_scheme, refinement_level=None +): + """Change the ordering of HEALPix indices. + + Does not change the position of each cell in the array, but + redefines their indices according to the new ordering scheme. + + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et + al.. HEALPix: A Framework for High-Resolution Discretization and + Fast Analysis of Data Distributed on the Sphere. The Astrophysical + Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + + M. Reinecke and E. Hivon: Efficient data structures for masks on + 2D grids. A&A, 580 (2015) + A132. https://doi.org/10.1051/0004-6361/201526549 + + .. versionadded:: NEXTVERSION + + :Parameters: + + a: `numpy.ndarray` + The array of HEALPix indices. + + indexing_scheme: `str` + The original HEALPix indexing scheme. One of ``'nested'``, + ``'ring'``, or ``'nested_unique'``. + + new_indexing_scheme: `str` + The new HEALPix indexing scheme to change to. One of + ``'nested'``, ``'ring'``, or ``'nested_unique'``. + + refinement_level: `int` or `None`, optional + The refinement level of the grid within the HEALPix + hierarchy, starting at 0 for the base tessellation with 12 + cells. Must be an `int` for *indexing_scheme* ``'nested'`` + or ``'ring'``, but is ignored for *indexing_scheme* + ``'nested_unique'`` (in which case *refinement_level* may + be `None`). + + :Returns: + + `numpy.ndarray` + An array containing the new HEALPix indices. + + **Examples** + + >>> cf.data.dask_utils.cf_healpix_indexing_scheme( + ... [0, 1, 2, 3], 'nested', 'ring', 1 + ... ) + array([13, 5, 4, 0]) + >>> cf.data.dask_utils.cf_healpix_indexing_scheme( + ... [0, 1, 2, 3], 'nested', 'nested_unique', 1 + ) + array([16, 17, 18, 19]) + >>> cf.data.dask_utils.cf_healpix_indexing_scheme( + ... [16, 17, 18, 19], 'nested_unique', 'nest', None + ) + array([0, 1, 2, 3]) + + """ + if new_indexing_scheme == indexing_scheme: + # Null operation + return a + + try: + import healpix + except ImportError as e: + raise ImportError( + f"{e}. Must install healpix (https://pypi.org/project/healpix) " + "to allow the changing of the HEALPix index scheme" + ) + + a = cfdm_to_memory(a) + + if indexing_scheme == "nested": + if new_indexing_scheme == "ring": + nside = healpix.order2nside(refinement_level) + return healpix.nest2ring(nside, a) + + if new_indexing_scheme == "nested_unique": + return healpix.pix2uniq(refinement_level, a, nest=True) + + elif indexing_scheme == "ring": + if new_indexing_scheme == "nested": + nside = healpix.order2nside(refinement_level) + return healpix.ring2nest(nside, a) + + if new_indexing_scheme == "nested_unique": + return healpix.pix2uniq(refinement_level, a, nest=False) + + elif indexing_scheme == "nested_unique": + if new_indexing_scheme in ("nested", "ring"): + nest = new_indexing_scheme == "nested" + order, a = healpix.uniq2pix(a, nest=nest) + + refinement_levels = np.unique(order) + if refinement_levels.size > 1: + raise ValueError( + "Can't change HEALPix indexing scheme from " + f"'nested_unique' to {new_indexing_scheme!r} when the " + "HEALPix indices span multiple refinement levels (at " + f"least levels {refinement_levels.tolist()})" + ) + + return a + + raise ValueError("Failed to change the HEALPix indexing scheme") + + +def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): + """Calculate HEALPix cell area weights. + + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et + al.. HEALPix: A Framework for High-Resolution Discretization and + Fast Analysis of Data Distributed on the Sphere. The Astrophysical + Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + + M. Reinecke and E. Hivon: Efficient data structures for masks on + 2D grids. A&A, 580 (2015) + A132. https://doi.org/10.1051/0004-6361/201526549 + + .. versionadded:: NEXTVERSION + + :Parameters: + + a: `numpy.ndarray` + The array of HEALPix 'nested_unique' indices. + + indexing_scheme: `str` + The HEALPix indexing scheme. Must be ``'nested_unique'``. + + measure: `bool`, optional + If True then create weights that are actual cell areas, in + units of the square of the radius units. + + radius: number, optional + The radius of the sphere. Must be set if *measure * is + True, otherwise ignored. + + :Returns: + + `numpy.ndarray` + An array containing the HEALPix cell weights. + + **Examples** + + >>> cf.data.dask_utils.cf_healpix_weights( + ... [76, 77, 78, 79, 20, 21], 'nested_unique' + ) + array([0.0625, 0.0625, 0.0625, 0.0625, 0.25 , 0.25 ]) + >>> cf.data.dask_utils.cf_healpix_weights( + ... [76, 77, 78, 79, 20, 21], 'nested_unique', + ... measure=True, radius=6371000 + ) + array([2.65658579e+12, 2.65658579e+12, 2.65658579e+12, 2.65658579e+12, + 1.06263432e+13, 1.06263432e+13]) + + """ + try: + import healpix + except ImportError as e: + raise ImportError( + f"{e}. Must install healpix (https://pypi.org/project/healpix) " + "to allow the calculation of cell area weights for a HEALPix grid" + ) + + if indexing_scheme != "nested_unique": + raise ValueError( + "cf_healpix_weights: Can only calulate weights for the " + "'nested_unique' indexing scheme" + ) + + a = cfdm_to_memory(a) + + if a.ndim != 1: + raise ValueError( + "Can only calculate HEALPix cell area weights when the " + f"healpix_index array has one dimension. Got shape {a.shape}" + ) + + if measure: + # Surface area of sphere is 4*pi*(r**2) + # Number of HEALPix cells at refinement level N is 12*(4**N) + # => Area of one cell is pi*(r**2)/(3* (4**N)) + x = np.pi * (radius**2) / 3.0 + else: + x = 1.0 + + orders = healpix.uniq2pix(a, nest=True)[0] + orders, index, inverse = np.unique( + orders, return_index=True, return_inverse=True + ) + + # Initialise the output weights array + w = np.empty(a.shape, dtype="float64") + + # For each refinement level N, put the weights (= x/4**N) into 'w' + # at the correct locations + for order, i in zip(orders, index): + w = np.where(inverse == inverse[i], x / (4**order), w) + + return w diff --git a/cf/field.py b/cf/field.py index 088bc9e60f..ce34f0b080 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4824,8 +4824,23 @@ def healpix_decrease_refinement_level( Decreasing the refinement level coarsens the horizontal grid to a lower-level HEALPix grid by combining, using the - *reduction* function, all cells that lie inside each larger - cell at the new refinement level. + *reduction* function, all original cells that lie inside each + larger cell at the new refinement level. + + The operation requires that each larger cell at the new + refinement level either contains no original cells (in which + case that new cell is not included in the output), or is + completely covered by original cells. It is not allowed for a + larger cell to be only partially covered by original + cells. For instance, if the original refinement level is 10 + and the new refinement level is 8, then each output cell will + be the combination of 16 (:math:`=4^(10-8)`) original cells. + + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, + et al.. HEALPix: A Framework for High-Resolution + Discretization and Fast Analysis of Data Distributed on the + Sphere. The Astrophysical Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 .. versionadded:: NEXTVERSION @@ -4944,7 +4959,7 @@ def healpix_decrease_refinement_level( """ f = self.copy() - # Get HEALPix info + # Get the HEALPix info hp = f.healpix_info() indexing_scheme = hp.get("indexing_scheme") @@ -4973,7 +4988,7 @@ def healpix_decrease_refinement_level( f"Can't decrease HEALPix refinement level: {error}" ) - # Re-get HEALPix info + # Re-get the HEALPix info hp = f.healpix_info() elif indexing_scheme != "nested": raise ValueError( @@ -5027,7 +5042,7 @@ def healpix_decrease_refinement_level( if check_healpix_index: d = healpix_index.data - if not conform and not (d.diff() > 0).all(): + if not (d.diff() > 0).all(): raise ValueError( "Can't decrease HEALPix refinement level: Nested " "healpix_index coordinates are not strictly " @@ -5058,9 +5073,12 @@ def healpix_decrease_refinement_level( ) ) - # Coarsen (using the 'reduction' function) the field - # data. Note that using the 'coarsen' technique only works for - # 'nested' HEALPix ordering. + # Coarsen the field data using the 'reduction' function. + # + # Note: Using 'Data.coarsen' only works because a) we have + # 'nested' HEALPix ordering, and b) each coarser cell + # contains the maximum possible number of original + # cells. f.data.coarsen( reduction, axes={iaxis: ncells}, trim_excess=False, inplace=True ) diff --git a/cf/functions.py b/cf/functions.py index dbbac1dccc..9b3d7ccd69 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -3321,17 +3321,20 @@ def unique_constructs(constructs, ignore_properties=None, copy=True): def locate(lat, lon, f=None): """Return indices of cells containing latitude-longitude locations. - The cells must be defined by a discrete axis that has, or could - have, 1-d latitude and longitude coordinates. At present, only - HEALPix axes are supported. + The cells must be defined by a discrete axis that either has 1-d + latitude and longitude coordinates, or else those coordinates are + implied by the axis definition (as is the case for a HEALPix + axis). + + At present, only a HEALPix axis is supported. If a single latitude is given then it is paired with each longitude, and if a single longitude is given then it is paired with each latitude. If multiple latitudes and multiple longitudes are provided then they are paired element-wise. - A cell index appears at most onxce in the output, even if that cell - contains more than one of the given latitude-longitude locations. + If a cell contains more than one of the given latitude-longitude + locations then that cell's index appears only once in the output. .. versionadded:: NEXTVERSION @@ -3353,14 +3356,14 @@ def locate(lat, lon, f=None): If `None` (the default) then a callable function is returned, which when called with an argument of *f* - returns the indices, i.e ``contains(lat, lon, f)`` - is equivalent to ``contains(lat, lon)(f)``. This - new function may be used as a condition in the `subspace` - and `indices` methods of *f*, since such conditions may be + returns the indices, i.e ``cf.contains(lat, lon, f)`` is + equivalent to ``cf.contains(lat, lon)(f)``. This new + function may be used as a condition in the `subspace` and + `indices` methods of *f*, since such conditions may be functions that take the calling Field or Domain construct - as an argument. For instance, - ``f.subspace(X=locate(0, 45)`` is equivalent to - ``f.subspace(X=locate(0, 45, f)``. + as an argument. For instance, ``f.subspace(X=cf.locate(0, + 45)`` is equivalent to ``f.subspace(X=cf.locate(0, 45, + f)``. `numpy.ndarray` or function Indices for the discrete axis that contain the @@ -3405,7 +3408,8 @@ def _locate(lat, lon, f): return _healpix_locate(lat, lon, f) raise ValueError( - "'locate' can only calculate indices for a HEALPix axis" + f"Can't find cell locations for {f!r}: Can only find locations " + "for HEALPix cells" ) if f.domain_topologies(todict=True): @@ -3414,10 +3418,21 @@ def _locate(lat, lon, f): # from .ugrid import _ugrid_locate # return _ugrid_locate(lat, lon, f) + if f.domain_topologies(todict=True): + # Geometries - not coded up, yet. + pass + # from .geometry import _geometry_locate + # return _geometry_locate(lat, lon, f) + + raise ValueError( + f"Can't find cell locations for {f!r}: Can only find locations " + "for UGRID, HEALPix, and geometry cells" + ) + if np.abs(lat).max() > 90: raise ValueError( - "Can't find cell locations: Latitudes must be in the range " - f"[-90, 90]. Got: {lat}" + f"Can't find cell locations for {f!r}: Latitudes must be in " + f"the range [-90, 90]. Got: {lat}" ) if f is None: diff --git a/cf/healpix.py b/cf/healpix.py index a9a612b4a7..5993076967 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -1,55 +1,23 @@ -"""General functions useful for HEALPix functionality.""" +"""HEALPix functionality.""" + +import logging import dask.array as da import numpy as np +from cfdm import is_log_level_info +logger = logging.getLogger(__name__) -def _healpix_indexing_scheme(healpix_index, hp, new_indexing_scheme): - """Change the indexing scheme of HEALPix indices. - - .. versionadded:: NEXTVERSION - - .. seealso:: `cf.Field.healpix_indexing_scheme` - - :Parameters: - - healpix_index: `Coordinate` - The healpix_index coordinates, which will be updated - in-place. - - hp: `dict` - The HEALPix info dictionary. - - new_indexing_scheme: `str` - The new indexing scheme. - :Returns: - - `None` - - """ - from .dask_utils import cf_healpix_indexing_scheme - - indexing_scheme = hp["indexing_scheme"] - refinement_level = hp.get("refinement_level") - - # Change the HEALPix indices - dx = healpix_index.data.to_dask_array( - _force_mask_hardness=False, _force_to_memory=False - ) - dx = dx.map_blocks( - cf_healpix_indexing_scheme, - meta=np.array((), dtype="int64"), - indexing_scheme=indexing_scheme, - new_indexing_scheme=new_indexing_scheme, - refinement_level=refinement_level, - ) - healpix_index.set_data(dx, copy=False) - - -def _healpix_create_latlon_coordinates(f, hp, pole_longitude): +def _healpix_create_latlon_coordinates(f, pole_longitude): """Create latitude and longitude coordinates for a HEALPix grid. + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et + al.. HEALPix: A Framework for High-Resolution Discretization and + Fast Analysis of Data Distributed on the Sphere. The Astrophysical + Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + .. versionadded:: NEXTVERSION .. seealso:: `cf.Field.create_latlon_coordinates` @@ -60,9 +28,6 @@ def _healpix_create_latlon_coordinates(f, hp, pole_longitude): The Field or Domain containing the HEALPix grid, which will be updated in-place. - hp: `dict` - The HEALPix info dictionary. - pole_longitude: `None` or number The longitude of coordinates, or coordinate bounds, that lie exactly on the north or south pole. If `None` then the @@ -73,24 +38,54 @@ def _healpix_create_latlon_coordinates(f, hp, pole_longitude): :Returns: - `str`, `str` + (`str`, `str`) or (`None`, `None`) The keys of the new latitude and longitude coordinate - constructs. + constructs, or `None` if the coordinates could not be + created. """ - from .dask_utils import cf_healpix_bounds, cf_healpix_coordinates + from .data.dask_utils import cf_healpix_bounds, cf_healpix_coordinates + + hp = f.healpix_info() + + indexing_scheme = hp.get("indexing_scheme") + if indexing_scheme not in ("nested", "ring", "nested_unique"): + if is_log_level_info(logger): + logger.info( + "Can't create 1-d latitude and longitude coordinates for " + f"{f!r}: Invalid HEALPix index scheme: {indexing_scheme!r}" + ) # pragma: no cover + + return (None, None) - healpix_index = hp["healpix_index"] - indexing_scheme = hp["indexing_scheme"] refinement_level = hp.get("refinement_level") + if refinement_level is None and indexing_scheme != "nested_unique": + if is_log_level_info(logger): + logger.info( + "Can't create 1-d latitude and longitude coordinates for " + f"{f!r}: refinement_level has not been set in the HEALPix " + "grid mapping coordinate reference" + ) # pragma: no cover - # Create new latitude and longitude coordinates with bounds + return (None, None) + + healpix_index = hp.get("healpix_index") + if healpix_index is None: + if is_log_level_info(logger): + logger.info( + "Can't create 1-d latitude and longitude coordinates for " + f"{f!r}: Missing healpix_index coordinates" + ) # pragma: no cover + + return (None, None) + + # Get the Dask array of HEALPix indices dx = healpix_index.data.to_dask_array( _force_mask_hardness=False, _force_to_memory=False ) meta = np.array((), dtype="float64") - # Latitude coordinates + # Create latitude coordinates dy = dx.map_blocks( cf_healpix_coordinates, meta=meta, @@ -104,7 +99,7 @@ def _healpix_create_latlon_coordinates(f, hp, pole_longitude): copy=False, ) - # Longitude coordinates + # Create longitude coordinates dy = dx.map_blocks( cf_healpix_coordinates, meta=meta, @@ -118,7 +113,7 @@ def _healpix_create_latlon_coordinates(f, hp, pole_longitude): copy=False, ) - # Latitude bounds + # Create latitude bounds dy = da.blockwise( cf_healpix_bounds, "ij", @@ -133,7 +128,7 @@ def _healpix_create_latlon_coordinates(f, hp, pole_longitude): bounds = f._Bounds(data=dy) lat.set_bounds(bounds) - # Longitude bounds + # Create longitude bounds dy = da.blockwise( cf_healpix_bounds, "ij", @@ -157,18 +152,71 @@ def _healpix_create_latlon_coordinates(f, hp, pole_longitude): return lat_key, lon_key -def _healpix_locate(lat, lon, f): - """Return indices of cells containing latitude-longitude locations. +def _healpix_indexing_scheme(healpix_index, hp, new_indexing_scheme): + """Change the indexing scheme of HEALPix indices. + + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et + al.. HEALPix: A Framework for High-Resolution Discretization and + Fast Analysis of Data Distributed on the Sphere. The Astrophysical + Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + + .. versionadded:: NEXTVERSION + + .. seealso:: `cf.Field.healpix_indexing_scheme` + + :Parameters: + + healpix_index: `Coordinate` + The healpix_index coordinates, which will be updated + in-place. + + hp: `dict` + The HEALPix info dictionary. + + new_indexing_scheme: `str` + The new indexing scheme. - The cells must be defined by a HEALPix grid. + :Returns: + + `None` + + """ + from .data.dask_utils import cf_healpix_indexing_scheme + + indexing_scheme = hp["indexing_scheme"] + refinement_level = hp.get("refinement_level") + + # Change the HEALPix indices + dx = healpix_index.data.to_dask_array( + _force_mask_hardness=False, _force_to_memory=False + ) + dx = dx.map_blocks( + cf_healpix_indexing_scheme, + meta=np.array((), dtype="int64"), + indexing_scheme=indexing_scheme, + new_indexing_scheme=new_indexing_scheme, + refinement_level=refinement_level, + ) + healpix_index.set_data(dx, copy=False) + + +def _healpix_locate(lat, lon, f): + """Locate HEALPix cells containing latitude-longitude locations. If a single latitude is given then it is paired with each longitude, and if a single longitude is given then it is paired with each latitude. If multiple latitudes and multiple longitudes are provided then they are paired element-wise. - A cell index appears at most onxce in the output, even if that cell - contains more than one of the given latitude-longitude locations. + If a cell contains more than one of the given latitude-longitude + locations then that cell's index appears only once in the output. + + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et + al.. HEALPix: A Framework for High-Resolution Discretization and + Fast Analysis of Data Distributed on the Sphere. The Astrophysical + Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 .. versionadded:: NEXTVERSION @@ -192,7 +240,9 @@ def _healpix_locate(lat, lon, f): `numpy.ndarray` Indices for the HEALPix axis that contain the - latitude-longitude locations. + latitude-longitude locations. Note that these indices + identify locations along the HEALPix axis, and are not the + HEALPix indices defined by the indexing scheme. """ try: @@ -208,15 +258,15 @@ def _healpix_locate(lat, lon, f): healpix_index = hp.get("healpix_index") if healpix_index is None: raise ValueError( - "Can't locate HEALPix cells: There are no healpix_index " - "coordinates" + f"Can't locate HEALPix cells for {f!r}: There are no " + "healpix_index coordinates" ) indexing_scheme = hp.get("indexing_scheme") if indexing_scheme is None: raise ValueError( - "Can't locate HEALPix cells: indexing_scheme has not been set " - "in the HEALPix grid mapping coordinate reference" + f"Can't locate HEALPix cells for {f!r}: indexing_scheme has " + "not been set in the HEALPix grid mapping coordinate reference" ) if indexing_scheme == "nested_unique": @@ -244,8 +294,9 @@ def _healpix_locate(lat, lon, f): refinement_level = hp.get("refinement_level") if refinement_level is None: raise ValueError( - "Can't locate HEALPix cells: refinement_level has not been " - "set in the HEALPix grid mapping coordinate reference" + f"Can't locate HEALPix cells for {f!r}: refinement_level " + "has not been set in the HEALPix grid mapping coordinate " + "reference" ) # Find the HEALPix indices of the cells that contain the diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index afe59eb712..473d89e82b 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -1963,6 +1963,16 @@ def coordinate_reference_domain_axes(self, identity=None): def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): """Change the indexing scheme of HEALPix indices. + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, + et al.. HEALPix: A Framework for High-Resolution + Discretization and Fast Analysis of Data Distributed on the + Sphere. The Astrophysical Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + + M. Reinecke and E. Hivon: Efficient data structures for masks + on 2D grids. A&A, 580 (2015) + A132. https://doi.org/10.1051/0004-6361/201526549 + .. versionadded:: NEXTVERSION :Parameters: @@ -2135,12 +2145,12 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): # monotonically increasing. Test for the common case of # already-ordered global nested or ring indices (which is # a fast test compared to do doing any actual sorting). - hp = healpix_index + h = healpix_index if not ( indexing_scheme in ("nested", "ring") - and (hp == da.arange(hp.size, chunks=hp.data.chunks)).all() + and (h == da.arange(h.size, chunks=h.data.chunks)).all() ): - index = hp.data.compute() + index = h.data.compute() f = f.subspace(**{axis: np.argsort(index)}) return f @@ -2148,6 +2158,12 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): def healpix_info(self): """Get information about the HEALPix grid, if there is one. + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, + et al.. HEALPix: A Framework for High-Resolution + Discretization and Fast Analysis of Data Distributed on the + Sphere. The Astrophysical Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + .. versionadded:: NEXTVERSION :Returns: @@ -2181,6 +2197,12 @@ def healpix_info(self): def healpix_to_ugrid(self, inplace=False): """Convert a HEALPix domain to a UGRID domain. + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, + et al.. HEALPix: A Framework for High-Resolution + Discretization and Fast Analysis of Data Distributed on the + Sphere. The Astrophysical Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + .. versionadded:: NEXTVERSION .. seealso:: `create_latlon_coordinates` @@ -2236,8 +2258,8 @@ def healpix_to_ugrid(self, inplace=False): # If lat/lon coordinates do not exist, then derive them from # the HEALPix indices. It's important to set pole_longitude to # something other than None (it doesn't matter what) so that - # the polar vertices come out as a single node in the domain - # topology. + # the north and south polar vertices come out as a single node + # in the domain topology. f.create_latlon_coordinates( one_d=True, two_d=False, pole_longitude=0, inplace=True ) @@ -2485,48 +2507,49 @@ def create_latlon_coordinates( # -------------------------------------------------------- # HEALPix: 1-d lat/lon coordinates # -------------------------------------------------------- - hp = f.healpix_info() - - indexing_scheme = hp.get("indexing_scheme") - if indexing_scheme not in ("nested", "ring", "nested_unique"): - if is_log_level_info(logger): - logger.info( - "Can't create 1-d latitude and longitude coordinates " - f"for {f!r}: Invalid HEALPix index scheme: " - f"{indexing_scheme!r}" - ) # pragma: no cover - - return f - - refinement_level = hp.get("refinement_level") - if refinement_level is None and indexing_scheme != "nested_unique": - if is_log_level_info(logger): - logger.info( - "Can't create 1-d latitude and longitude coordinates " - f"for {f!r} from {indexing_scheme!r} HEALPix indices: " - "refinement_level has not been set in the HEALPix " - "grid mapping coordinate reference" - ) # pragma: no cover - - return f - - healpix_index = hp.get("healpix_index") - if healpix_index is None: - if is_log_level_info(logger): - logger.info( - "Can't create 1-d latitude and longitude coordinates " - f"for {f!r}: Missing healpix_index coordinates" - ) # pragma: no cover - - return f + # hp = f.healpix_info() + + # indexing_scheme = hp.get("indexing_scheme") + # if indexing_scheme not in ("nested", "ring", "nested_unique"): + # if is_log_level_info(logger): + # logger.info( + # "Can't create 1-d latitude and longitude coordinates " + # f"for {f!r}: Invalid HEALPix index scheme: " + # f"{indexing_scheme!r}" + # ) # pragma: no cover + # + # return f + # + # if ( + # "refinement_level" not in hp + # and indexing_scheme != "nested_unique" + # ): + # if is_log_level_info(logger): + # logger.info( + # "Can't create 1-d latitude and longitude coordinates " + # f"for {f!r} from {indexing_scheme!r} HEALPix indices: " + # "refinement_level has not been set in the HEALPix " + # "grid mapping coordinate reference" + # ) # pragma: no cover + # + # return f + # + # if "healpix_index" not in hp: + # if is_log_level_info(logger): + # logger.info( + # "Can't create 1-d latitude and longitude coordinates " + # f"for {f!r}: Missing healpix_index coordinates" + # ) # pragma: no cover + # + # return f # Create the new lat/lon coordinates from ..healpix import _healpix_create_latlon_coordinates lat_key, lon_key = _healpix_create_latlon_coordinates( - f, hp, pole_longitude + f, pole_longitude ) - new_coords = True + new_coords = lat_key is not None elif two_d: # -------------------------------------------------------- diff --git a/cf/weights.py b/cf/weights.py index 19a6735f07..01ba1d500d 100644 --- a/cf/weights.py +++ b/cf/weights.py @@ -2052,7 +2052,7 @@ def healpix_area( units = "1" r = None - from .dask_utils import cf_healpix_weights + from .data.dask_utils import cf_healpix_weights dx = healpix_index.to_dask_array() dx = dx.map_blocks( @@ -2080,7 +2080,7 @@ def healpix_area( return True if not measure: - # Non-measure Weights are all equal, so no need to create any. + # Weights are all equal, so no need to create any. return True r2 = radius**2 From 5b36f056dee01711f77ab8999d7f978d7f2fc619 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 23 Jul 2025 11:16:59 +0100 Subject: [PATCH 31/59] dev --- cf/data/dask_utils.py | 73 +++++++++++++++++++++------------------ cf/domain.py | 34 +++++++++++------- cf/field.py | 6 ++-- cf/functions.py | 34 ++++++++++++------ cf/healpix.py | 27 +++++++++++---- cf/mixin/fielddomain.py | 64 +++++++--------------------------- cf/test/test_Domain.py | 12 +++++++ cf/test/test_Field.py | 52 ++++++++++++++++++---------- cf/test/test_functions.py | 6 ++-- 9 files changed, 167 insertions(+), 141 deletions(-) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index 3412e1f5ba..53a4831ccf 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -818,39 +818,46 @@ def cf_healpix_indexing_scheme( a = cfdm_to_memory(a) - if indexing_scheme == "nested": - if new_indexing_scheme == "ring": - nside = healpix.order2nside(refinement_level) - return healpix.nest2ring(nside, a) - - if new_indexing_scheme == "nested_unique": - return healpix.pix2uniq(refinement_level, a, nest=True) - - elif indexing_scheme == "ring": - if new_indexing_scheme == "nested": - nside = healpix.order2nside(refinement_level) - return healpix.ring2nest(nside, a) - - if new_indexing_scheme == "nested_unique": - return healpix.pix2uniq(refinement_level, a, nest=False) - - elif indexing_scheme == "nested_unique": - if new_indexing_scheme in ("nested", "ring"): - nest = new_indexing_scheme == "nested" - order, a = healpix.uniq2pix(a, nest=nest) - - refinement_levels = np.unique(order) - if refinement_levels.size > 1: - raise ValueError( - "Can't change HEALPix indexing scheme from " - f"'nested_unique' to {new_indexing_scheme!r} when the " - "HEALPix indices span multiple refinement levels (at " - f"least levels {refinement_levels.tolist()})" - ) - - return a - - raise ValueError("Failed to change the HEALPix indexing scheme") + match indexing_scheme: + case "nested": + match new_indexing_scheme: + case "ring": + nside = healpix.order2nside(refinement_level) + return healpix.nest2ring(nside, a) + + case "nested_unique": + return healpix.pix2uniq(refinement_level, a, nest=True) + + case "ring": + match new_indexing_scheme: + case "nested": + nside = healpix.order2nside(refinement_level) + return healpix.ring2nest(nside, a) + + case "nested_unique": + return healpix.pix2uniq(refinement_level, a, nest=False) + + case "nested_unique": + match new_indexing_scheme: + case "nested" | "ring": + nest = new_indexing_scheme == "nested" + order, a = healpix.uniq2pix(a, nest=nest) + + refinement_levels = np.unique(order) + if refinement_levels.size > 1: + raise ValueError( + "Can't change HEALPix indexing scheme from " + f"'nested_unique' to {new_indexing_scheme!r} " + "when the HEALPix indices span multiple " + "refinement levels (at least levels " + f"{refinement_levels.tolist()})" + ) + + return a + + raise RuntimeError( + "cf_healpix_indexing_scheme: Failed during Dask computation" + ) def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): diff --git a/cf/domain.py b/cf/domain.py index d8e44a714a..530d918446 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -361,6 +361,14 @@ def create_healpix( """ import dask.array as da + from .healpix import HEALPix_indexing_schemes + + if indexing_scheme not in HEALPix_indexing_schemes: + raise ValueError( + "Can't create HEALPix Domain: 'indexing_scheme' must be one " + f"of {HEALPix_indexing_schemes!r}. Got {indexing_scheme!r}" + ) + if not isinstance(refinement_level, Integral) or refinement_level < 0: raise ValueError( "'refinement_level' must be a non-negative integer. " @@ -368,13 +376,6 @@ def create_healpix( ) nested_unique = indexing_scheme == "nested_unique" - if nested_unique: - indexing_scheme = "nested" - elif indexing_scheme not in ("nested", "ring"): - raise ValueError( - "'indexing_scheme' must be 'nested', 'ring', or '" - f"nested_unique'. Got: {indexing_scheme!r}" - ) domain = Domain() ncells = 12 * (4**refinement_level) @@ -388,7 +389,15 @@ def create_healpix( c = domain._AuxiliaryCoordinate() c.set_properties({"standard_name": "healpix_index"}) c.nc_set_variable("healpix_index") - c.set_data(Data(da.arange(ncells), units="1"), copy=False) + + # Create the healpix_index data + if nested_unique: + i0 = 4 ** (refinement_level + 1) + else: + i0 = 0 + + c.set_data(Data(da.arange(i0, i0 + ncells), units="1"), copy=False) + key = domain.set_construct(c, axes=axis, copy=False) # coordinate_reference: grid_mapping_name:healpix @@ -404,16 +413,15 @@ def create_healpix( { "grid_mapping_name": "healpix", "indexing_scheme": indexing_scheme, - "refinement_level": refinement_level, } ) + if not nested_unique: + cr.coordinate_conversion.set_parameter( + "refinement_level", refinement_level + ) domain.set_construct(cr) - if nested_unique: - # Change from 'nested' to 'nested_unique' indexing scheme - domain = domain.healpix_indexing_scheme("nested_unique") - return domain @_inplace_enabled(default=False) diff --git a/cf/field.py b/cf/field.py index ce34f0b080..e11f950b2e 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4966,7 +4966,7 @@ def healpix_decrease_refinement_level( if indexing_scheme is None: raise ValueError( "Can't decrease HEALPix refinement level: indexing_scheme " - "has not been set in the HEALPix grid mapping coordinate " + "has not been set in the healpix grid mapping coordinate " "reference" ) @@ -4974,7 +4974,7 @@ def healpix_decrease_refinement_level( if refinement_level is None: raise ValueError( "Can't decrease HEALPix refinement level: refinement_level " - "has not been set in the HEALPix grid mapping coordinate " + "has not been set in the healpix grid mapping coordinate " "reference" ) @@ -4993,7 +4993,7 @@ def healpix_decrease_refinement_level( elif indexing_scheme != "nested": raise ValueError( "Can't decrease HEALPix refinement level: indexing_scheme " - "in the HEALPix grid mapping coordinate reference is " + "in the healpix grid mapping coordinate reference is " f"{indexing_scheme!r}, and not 'nested'. " "Consider setting conform=True" ) diff --git a/cf/functions.py b/cf/functions.py index 9b3d7ccd69..b287dff2d5 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -3319,26 +3319,28 @@ def unique_constructs(constructs, ignore_properties=None, copy=True): def locate(lat, lon, f=None): - """Return indices of cells containing latitude-longitude locations. + """Locate cells containing latitude-longitude locations. The cells must be defined by a discrete axis that either has 1-d latitude and longitude coordinates, or else those coordinates are implied by the axis definition (as is the case for a HEALPix - axis). + axis). At present, only HEALPix grids are supported. - At present, only a HEALPix axis is supported. + Returns the discrete axis indices of the cells containing the + latitude-longitude locations. If a single latitude is given then it is paired with each longitude, and if a single longitude is given then it is paired with each latitude. If multiple latitudes and multiple longitudes are provided then they are paired element-wise. - If a cell contains more than one of the given latitude-longitude - locations then that cell's index appears only once in the output. + If a cell contains more than one of the latitude-longitude + locations, then that cell's index appears only once in the output. .. versionadded:: NEXTVERSION - .. seealso:: `cf.contains` + .. seealso:: `cf.contains`, `cf.Field.subspace`, + `cf.Field.indices` :Parameters: @@ -3401,7 +3403,10 @@ def locate(lat, lon, f=None): """ def _locate(lat, lon, f): - if f.coordinate("healpix_index", filter_by_naxes=(1,), default=None): + healpix = f.coordinate( + "healpix_index", filter_by_naxes=(1,), default=None + ) + if healpix: # HEALPix from .healpix import _healpix_locate @@ -3409,16 +3414,23 @@ def _locate(lat, lon, f): raise ValueError( f"Can't find cell locations for {f!r}: Can only find locations " - "for HEALPix cells" + "for HEALPix cells (at present)" ) - if f.domain_topologies(todict=True): + ugrid = f.domain_topologies(todict=True) + if ugrid: # UGRID - not coded up, yet. pass # from .ugrid import _ugrid_locate # return _ugrid_locate(lat, lon, f) - if f.domain_topologies(todict=True): + geometry = any( + aux.get_geometry(False) + for aux in f.auxiliary_coordinates( + filter_by_naxes=(1,), todict=True + ).values() + ) + if geometry: # Geometries - not coded up, yet. pass # from .geometry import _geometry_locate @@ -3426,7 +3438,7 @@ def _locate(lat, lon, f): raise ValueError( f"Can't find cell locations for {f!r}: Can only find locations " - "for UGRID, HEALPix, and geometry cells" + "for UGRID, HEALPix, or geometry cells" ) if np.abs(lat).max() > 90: diff --git a/cf/healpix.py b/cf/healpix.py index 5993076967..47b67eb9fc 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -8,6 +8,8 @@ logger = logging.getLogger(__name__) +HEALPix_indexing_schemes = ("nested", "ring", "nested_unique") + def _healpix_create_latlon_coordinates(f, pole_longitude): """Create latitude and longitude coordinates for a HEALPix grid. @@ -49,11 +51,13 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): hp = f.healpix_info() indexing_scheme = hp.get("indexing_scheme") - if indexing_scheme not in ("nested", "ring", "nested_unique"): + if indexing_scheme not in HEALPix_indexing_schemes: if is_log_level_info(logger): logger.info( "Can't create 1-d latitude and longitude coordinates for " - f"{f!r}: Invalid HEALPix index scheme: {indexing_scheme!r}" + f"{f!r}: indexing_scheme in the healpix grid mapping " + "coordinate reference must be one of " + f"{HEALPix_indexing_schemes!r}. Got {indexing_scheme!r}" ) # pragma: no cover return (None, None) @@ -63,7 +67,7 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): if is_log_level_info(logger): logger.info( "Can't create 1-d latitude and longitude coordinates for " - f"{f!r}: refinement_level has not been set in the HEALPix " + f"{f!r}: refinement_level has not been set in the healpix " "grid mapping coordinate reference" ) # pragma: no cover @@ -204,6 +208,9 @@ def _healpix_indexing_scheme(healpix_index, hp, new_indexing_scheme): def _healpix_locate(lat, lon, f): """Locate HEALPix cells containing latitude-longitude locations. + Returns the discrete axis indices of the cells containing the + latitude-longitude locations. + If a single latitude is given then it is paired with each longitude, and if a single longitude is given then it is paired with each latitude. If multiple latitudes and multiple longitudes @@ -253,7 +260,7 @@ def _healpix_locate(lat, lon, f): "to allow the location of HEALPix cells" ) - hp = healpix_info(f) + hp = f.healpix_info() healpix_index = hp.get("healpix_index") if healpix_index is None: @@ -266,7 +273,7 @@ def _healpix_locate(lat, lon, f): if indexing_scheme is None: raise ValueError( f"Can't locate HEALPix cells for {f!r}: indexing_scheme has " - "not been set in the HEALPix grid mapping coordinate reference" + "not been set in the healpix grid mapping coordinate reference" ) if indexing_scheme == "nested_unique": @@ -289,13 +296,13 @@ def _healpix_locate(lat, lon, f): index.append(da.where(da.isin(healpix_index, pix))[0]) index = da.unique(da.concatenate(index, axis=0)) - else: + elif indexing_scheme in ("nested", "ring"): # nested or ring indexing scheme refinement_level = hp.get("refinement_level") if refinement_level is None: raise ValueError( f"Can't locate HEALPix cells for {f!r}: refinement_level " - "has not been set in the HEALPix grid mapping coordinate " + "has not been set in the healpix grid mapping coordinate " "reference" ) @@ -309,6 +316,12 @@ def _healpix_locate(lat, lon, f): # Find where these HEALPix indices are located in the # healpix_index coordinates index = da.where(da.isin(healpix_index, pix))[0] + else: + raise ValueError( + f"Can't locate HEALPix cells for {f!r}: indexing_scheme in the " + "healpix grid mapping coordinate reference must be one of " + f"{HEALPix_indexing_schemes!r}. Got {indexing_scheme!r}" + ) # Return the cell locations as a numpy array of element indices return index.compute() diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 473d89e82b..643e859587 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -2042,16 +2042,17 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47] """ + from ..healpix import HEALPix_indexing_schemes + f = self.copy() hp = f.healpix_info() - valid_indexing_schemes = ("nested", "ring", "nested_unique") - if new_indexing_scheme not in valid_indexing_schemes + (None,): + if new_indexing_scheme not in HEALPix_indexing_schemes + (None,): raise ValueError( f"Can't change HEALPix index scheme of {f!r}: " "new_indexing_scheme keyword must be None or one of " - f"{valid_indexing_schemes!r}. Got {new_indexing_scheme!r}" + f"{HEALPix_indexing_schemes!r}. Got {new_indexing_scheme!r}" ) # Get the healpix_index coordinates @@ -2069,16 +2070,16 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): if indexing_scheme is None: raise ValueError( f"Can't change HEALPix indexing scheme of {f!r}: " - "indexing_scheme has not been set in the HEALPix grid " + "indexing_scheme has not been set in the healpix grid " "mapping coordinate reference" ) - if indexing_scheme not in valid_indexing_schemes: + if indexing_scheme not in HEALPix_indexing_schemes: raise ValueError( f"Can't change HEALPix indexing scheme of {f!r}: " - "indexing_scheme in the HEALPix grid mapping coordinate " - f"reference must be one of {valid_indexing_schemes!r}. " - f"Got {indexing_scheme!r}" + "indexing_scheme in the healpix grid mapping coordinate " + "reference must be one of " + f"{HEALPix_indexing_schemes!r}. Got {indexing_scheme!r}" ) if ( @@ -2093,7 +2094,7 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): raise ValueError( f"Can't change HEALPix indexing scheme of {f!r} from " f"{indexing_scheme!r} to {new_indexing_scheme!r} when " - "refinement_level has not been set in the HEALPix grid " + "refinement_level has not been set in the healpix grid " "mapping coordinate reference" ) @@ -2112,12 +2113,8 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): # # It doesn't matter if there are in fact multiple # refinement levels in the grid, as this will get - # trapped as an exception when - # `cf_healpix_indexing_scheme` is executed. - # - # M. Reinecke and E. Hivon: Efficient data structures - # for masks on 2D grids. A&A, 580 (2015) A132. - # https://doi.org/10.1051/0004-6361/201526549 + # trapped as an exception when the lazy calculations + # are computed. from math import log2 cr.coordinate_conversion.set_parameter( @@ -2507,43 +2504,6 @@ def create_latlon_coordinates( # -------------------------------------------------------- # HEALPix: 1-d lat/lon coordinates # -------------------------------------------------------- - # hp = f.healpix_info() - - # indexing_scheme = hp.get("indexing_scheme") - # if indexing_scheme not in ("nested", "ring", "nested_unique"): - # if is_log_level_info(logger): - # logger.info( - # "Can't create 1-d latitude and longitude coordinates " - # f"for {f!r}: Invalid HEALPix index scheme: " - # f"{indexing_scheme!r}" - # ) # pragma: no cover - # - # return f - # - # if ( - # "refinement_level" not in hp - # and indexing_scheme != "nested_unique" - # ): - # if is_log_level_info(logger): - # logger.info( - # "Can't create 1-d latitude and longitude coordinates " - # f"for {f!r} from {indexing_scheme!r} HEALPix indices: " - # "refinement_level has not been set in the HEALPix " - # "grid mapping coordinate reference" - # ) # pragma: no cover - # - # return f - # - # if "healpix_index" not in hp: - # if is_log_level_info(logger): - # logger.info( - # "Can't create 1-d latitude and longitude coordinates " - # f"for {f!r}: Missing healpix_index coordinates" - # ) # pragma: no cover - # - # return f - - # Create the new lat/lon coordinates from ..healpix import _healpix_create_latlon_coordinates lat_key, lon_key = _healpix_create_latlon_coordinates( diff --git a/cf/test/test_Domain.py b/cf/test/test_Domain.py index 9a85a9400f..f43662bedc 100644 --- a/cf/test/test_Domain.py +++ b/cf/test/test_Domain.py @@ -506,6 +506,13 @@ def test_Domain_create_healpix(self): (d.auxiliary_coordinate().array == np.arange(12)).all() ) + self.assertEqual( + d.coordinate_reference().coordinate_conversion.get_parameter( + "refinement_level" + ), + 0, + ) + d = cf.Domain.create_healpix(0, "nested_unique") self.assertTrue( (d.auxiliary_coordinate().array == np.arange(4, 16)).all() @@ -513,6 +520,11 @@ def test_Domain_create_healpix(self): self.assertIsNone( d.coordinate_reference().datum.get_parameter("earth_radius", None) ) + self.assertIsNone( + d.coordinate_reference().coordinate_conversion.get_parameter( + "refinement_level", None + ) + ) for radius in (1000, cf.Data(1, "km")): d = cf.Domain.create_healpix(0, "ring", radius=radius) diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index b79d216f42..4ac30336b5 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -3120,33 +3120,33 @@ def test_Field_healpix_indexing_scheme(self): g = f.healpix_indexing_scheme("ring") self.assertTrue( - np.allclose(g.coordinate("healpix_index")[:4], [13, 5, 4, 0]) + np.array_equal(g.coordinate("healpix_index")[:4], [13, 5, 4, 0]) ) h = g.healpix_indexing_scheme("nested") self.assertTrue( - np.allclose(h.coordinate("healpix_index")[:4], [0, 1, 2, 3]) + np.array_equal(h.coordinate("healpix_index")[:4], [0, 1, 2, 3]) ) h = g.healpix_indexing_scheme("nested_unique") self.assertTrue( - np.allclose(h.coordinate("healpix_index")[:4], [16, 17, 18, 19]) + np.array_equal(h.coordinate("healpix_index")[:4], [16, 17, 18, 19]) ) g = f.healpix_indexing_scheme("ring", sort=True) self.assertTrue( - np.allclose(g.coordinate("healpix_index")[:4], [0, 1, 2, 3]) + np.array_equal(g.coordinate("healpix_index")[:4], [0, 1, 2, 3]) ) h = g.healpix_indexing_scheme("nested", sort=False) self.assertTrue( - np.allclose(h.coordinate("healpix_index")[:4], [3, 7, 11, 15]) + np.array_equal(h.coordinate("healpix_index")[:4], [3, 7, 11, 15]) ) h = g.healpix_indexing_scheme("nested", sort=True) self.assertTrue( - np.allclose(h.coordinate("healpix_index")[:4], [0, 1, 2, 3]) + np.array_equal(h.coordinate("healpix_index")[:4], [0, 1, 2, 3]) ) g = f.healpix_indexing_scheme("nested_unique") self.assertTrue( - np.allclose(g.coordinate("healpix_index")[:4], [16, 17, 18, 19]) + np.array_equal(g.coordinate("healpix_index")[:4], [16, 17, 18, 19]) ) h = g.healpix_indexing_scheme("nested_unique") self.assertTrue(h.equals(g)) @@ -3179,7 +3179,7 @@ def test_Field_healpix_to_ugrid(self): topology = u.domain_topology().normalise().array self.assertEqual(np.unique(topology).size, 53) self.assertTrue( - np.allclose( + np.array_equal( topology[:4], [ [14, 11, 13, 16], @@ -3231,7 +3231,9 @@ def test_Field_create_latlon_coordinates(self): g.auxiliary_coordinate(c).equals(f.auxiliary_coordinate(c)) ) - # pole_longitude + # pole_longitude. Note that bounds index 0 is the + # northern-most vertex, and bounds index 2 is the + # southern-most vertex. f = self.f12 g = f.create_latlon_coordinates() longitude = g.auxiliary_coordinate("X").bounds.array @@ -3251,12 +3253,14 @@ def test_Field_create_latlon_coordinates(self): # South pole self.assertTrue(np.allclose(longitude[32:48:4, 2], 3.14)) - # Check a Multi-Order Coverage grid + # Multi-Order Coverage (MOC) grid with refinement levels 1 and + # 2 m = self.f13 m = m.create_latlon_coordinates() - l1 = self.f12.create_latlon_coordinates() + l1 = cf.Domain.create_healpix(1) l2 = cf.Domain.create_healpix(2) + l1.create_latlon_coordinates(inplace=True) l2.create_latlon_coordinates(inplace=True) for c in ("latitude", "longitude"): @@ -3270,11 +3274,11 @@ def test_Field_healpix_subspace(self): index = [47, 3, 2, 0] g = f.subspace(healpix_index=index) - self.assertTrue(np.allclose(g.coordinate("healpix_index"), index)) + self.assertTrue(np.array_equal(g.coordinate("healpix_index"), index)) g = f.subspace(X=cf.wi(40, 70), Y=cf.wi(-20, 30)) self.assertTrue( - np.allclose(g.coordinate("healpix_index"), [0, 22, 35]) + np.array_equal(g.coordinate("healpix_index"), [0, 22, 35]) ) g.create_latlon_coordinates(inplace=True) self.assertTrue(np.allclose(g.coordinate("X"), [45.0, 67.5, 45.0])) @@ -3291,23 +3295,29 @@ def test_Field_healpix_subspace(self): self.assertEqual(g.coordinate("healpix_index").array, 19) g = f.subspace(healpix_index=cf.locate(20, [1, 46])) - self.assertTrue(np.allclose(g.coordinate("healpix_index"), [0, 19])) + self.assertTrue(np.array_equal(g.coordinate("healpix_index"), [0, 19])) def test_Field_healpix_decrease_refinement_level(self): """Test Field.healpix_decrease_refinement_level.""" f = self.f12 g = f.healpix_decrease_refinement_level(0, np.mean) self.assertTrue(g.shape, (2, 12)) - self.assertTrue(np.allclose(g.coord("healpix_index"), np.arange(12))) + self.assertTrue( + np.array_equal(g.coord("healpix_index"), np.arange(12)) + ) g = f.healpix_decrease_refinement_level(-1, np.mean) self.assertTrue(g.shape, (2, 12)) - self.assertTrue(np.allclose(g.coord("healpix_index"), np.arange(12))) + self.assertTrue( + np.array_equal(g.coord("healpix_index"), np.arange(12)) + ) f = f.healpix_indexing_scheme("ring") g = f.healpix_decrease_refinement_level(0, np.mean) self.assertTrue(g.shape, (2, 12)) - self.assertTrue(np.allclose(g.coord("healpix_index"), np.arange(12))) + self.assertTrue( + np.array_equal(g.coord("healpix_index"), np.arange(12)) + ) with self.assertRaises(ValueError): f.healpix_decrease_refinement_level(0, np.mean, conform=False) @@ -3316,7 +3326,9 @@ def test_Field_healpix_decrease_refinement_level(self): g = f.healpix_decrease_refinement_level(0, np.mean, conform=True) self.assertTrue(g.shape, (2, 12)) - self.assertTrue(np.allclose(g.coord("healpix_index"), np.arange(12))) + self.assertTrue( + np.array_equal(g.coord("healpix_index"), np.arange(12)) + ) # Check that lat/lon coords get created when they're present # in the original field @@ -3332,7 +3344,9 @@ def test_Field_healpix_decrease_refinement_level(self): h = f.healpix_decrease_refinement_level( 0, np.mean, conform=False, check_healpix_index=False ) - self.assertFalse(np.allclose(h.coord("healpix_index"), np.arange(12))) + self.assertFalse( + np.array_equal(h.coord("healpix_index"), np.arange(12)) + ) # Can't change refinement level for a 'nested_unique' field with self.assertRaises(ValueError): diff --git a/cf/test/test_functions.py b/cf/test/test_functions.py index 522c89c012..ca25b4607d 100644 --- a/cf/test/test_functions.py +++ b/cf/test/test_functions.py @@ -438,12 +438,12 @@ def test_locate(self): self.assertEqual(cf.locate(-70, 90, f), 36) self.assertEqual(cf.locate(20, [280, 280.001], f), 31) - self.assertTrue(np.allclose(cf.locate([-70, 20], 90, f), [23, 36])) + self.assertTrue(np.array_equal(cf.locate([-70, 20], 90, f), [23, 36])) self.assertTrue( - np.allclose(cf.locate([-70, 20], [90, 280], f), [31, 36]) + np.array_equal(cf.locate([-70, 20], [90, 280], f), [31, 36]) ) self.assertTrue( - np.allclose(cf.locate([-70, 20], [90, 280])(f), [31, 36]) + np.array_equal(cf.locate([-70, 20], [90, 280])(f), [31, 36]) ) # Bad latitudes From 34a3a0f635e52b37883391d6f134b8e1de3a5a59 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 23 Jul 2025 21:28:33 +0100 Subject: [PATCH 32/59] dev --- cf/mixin/fielddomain.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 643e859587..d2a24c1e44 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -2252,11 +2252,11 @@ def healpix_to_ugrid(self, inplace=False): f = _inplace_enabled_define_and_cleanup(self) - # If lat/lon coordinates do not exist, then derive them from - # the HEALPix indices. It's important to set pole_longitude to - # something other than None (it doesn't matter what) so that - # the north and south polar vertices come out as a single node - # in the domain topology. + # If 1-d lat/lon coordinates do not exist, then derive them + # from the HEALPix indices. Setting the pole_longitude to + # something other than None - it doesn't matter what - ensures + # that the north (south) polar vertex comes out as a single + # node in the domain topology. f.create_latlon_coordinates( one_d=True, two_d=False, pole_longitude=0, inplace=True ) @@ -2301,7 +2301,8 @@ def healpix_to_ugrid(self, inplace=False): "bounds" ) - # Create a unique integer identifer for each node location + # Create the Domain Topology construct, by creating a unique + # integer identifer for each node location. bounds_y = bounds_y.data.to_dask_array(_force_mask_hardness=False) bounds_x = bounds_x.data.to_dask_array(_force_mask_hardness=False) @@ -2310,11 +2311,10 @@ def healpix_to_ugrid(self, inplace=False): nodes = y_indices * y_indices.size + x_indices - # Create the Domain Topology construct domain_topology = f._DomainTopology(data=f._Data(nodes)) domain_topology.set_cell("face") domain_topology.set_property( - "long_name", "UGRID topology derived from HEALPix" + "long_name", "UGRID domain topology derived from HEALPix" ) f.set_construct(domain_topology, axes=axis, copy=False) From cde98f03f1b7ac4fcc852e8acc827dfe0474d4ac Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 24 Jul 2025 10:02:57 +0100 Subject: [PATCH 33/59] dev --- cf/domain.py | 18 +++++++++++++----- cf/field.py | 4 ++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/cf/domain.py b/cf/domain.py index 530d918446..2a9d521a09 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -272,6 +272,12 @@ def create_healpix( The HEALPix axis of the new Domain is ordered so that the HEALPix indices are monotonically increasing. + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, + et al.. HEALPix: A Framework for High-Resolution + Discretization and Fast Analysis of Data Distributed on the + Sphere. The Astrophysical Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + .. versionadded:: NEXTVERSION .. seealso:: `cf.Domain.create_regular`, @@ -371,8 +377,8 @@ def create_healpix( if not isinstance(refinement_level, Integral) or refinement_level < 0: raise ValueError( - "'refinement_level' must be a non-negative integer. " - f"Got: {refinement_level!r}" + "Can't create HEALPix Domain: 'refinement_level' must be a " + f"non-negative integer. Got: {refinement_level!r}" ) nested_unique = indexing_scheme == "nested_unique" @@ -392,11 +398,13 @@ def create_healpix( # Create the healpix_index data if nested_unique: - i0 = 4 ** (refinement_level + 1) + index0 = 4 ** (refinement_level + 1) else: - i0 = 0 + index0 = 0 - c.set_data(Data(da.arange(i0, i0 + ncells), units="1"), copy=False) + c.set_data( + Data(da.arange(index0, index0 + ncells), units="1"), copy=False + ) key = domain.set_construct(c, axes=axis, copy=False) diff --git a/cf/field.py b/cf/field.py index e11f950b2e..6b3e6d38ce 100644 --- a/cf/field.py +++ b/cf/field.py @@ -2452,8 +2452,8 @@ def cell_area( Specify the radius used for calculating the areas of cells defined in spherical polar coordinates. The radius is that which would be returned by this call of - the field construct's `~cf.Field.radius` method: - ``f.radius(radius)``. See `cf.Field.radius` for + the field construct's `radius` method: + ``f.radius(default=radius)``. See `radius` for details. By default *radius* is ``'earth'`` which means that if From 0a2d5a49fc48fdc88f35a0468a1bc52baec3b4f8 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 29 Jul 2025 21:55:03 +0100 Subject: [PATCH 34/59] dev --- cf/data/dask_utils.py | 15 +++++---- cf/functions.py | 17 ++++------ cf/healpix.py | 45 ++++++++++++++++++++----- cf/mixin/fielddomain.py | 73 +++++++++++++++++++++++++++++++---------- cf/test/test_Field.py | 6 ++++ 5 files changed, 112 insertions(+), 44 deletions(-) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index 53a4831ccf..1acf30445d 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -517,10 +517,10 @@ def cf_healpix_bounds( pole_longitude: `None` or number The longitude of coordinate bounds that lie exactly on the - north or south pole. If `None` (the default) then the - longitudes of such a point will be identical to its - opposite vertex. If set to a number, then the longitudes - of such points will all be that value. + north (south) pole. If `None` then the longitude of such a + vertex will be the same as the south (north) vertex of the + same cell. If set to a number, then the longitudes of such + vertices will all be given that value. :Returns: @@ -581,12 +581,13 @@ def cf_healpix_bounds( else: bounds_func = healpix._chp.nest2ang_uv - # Define the cell vertices, in an anticlockwise direction, as seen - # from above, starting with the northern-most vertex. - east = (1, 0) + # Define the cell vertices in an anticlockwise direction, as seen + # from above, starting with the northern-most vertex. Each vertex + # is defined in the form needed by `bounds_func`. north = (1, 1) west = (0, 1) south = (0, 0) + east = (1, 0) vertices = (north, west, south, east) # Initialise the output bounds array diff --git a/cf/functions.py b/cf/functions.py index b287dff2d5..5beb18baf9 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -3321,13 +3321,14 @@ def unique_constructs(constructs, ignore_properties=None, copy=True): def locate(lat, lon, f=None): """Locate cells containing latitude-longitude locations. - The cells must be defined by a discrete axis that either has 1-d - latitude and longitude coordinates, or else those coordinates are - implied by the axis definition (as is the case for a HEALPix - axis). At present, only HEALPix grids are supported. + The cells must be defined by a discrete axis that has 1-d latitude + and longitude coordinate constructs, or for which it is possible + to create 1-d latitude and longitude coordinate constructs (as is + the case for a HEALPix axis). At present, only a HEALPix axis is + supported. - Returns the discrete axis indices of the cells containing the - latitude-longitude locations. + Returns the indices of the discrete axis for the cells containing + the latitude-longitude locations. If a single latitude is given then it is paired with each longitude, and if a single longitude is given then it is paired @@ -3421,8 +3422,6 @@ def _locate(lat, lon, f): if ugrid: # UGRID - not coded up, yet. pass - # from .ugrid import _ugrid_locate - # return _ugrid_locate(lat, lon, f) geometry = any( aux.get_geometry(False) @@ -3433,8 +3432,6 @@ def _locate(lat, lon, f): if geometry: # Geometries - not coded up, yet. pass - # from .geometry import _geometry_locate - # return _geometry_locate(lat, lon, f) raise ValueError( f"Can't find cell locations for {f!r}: Can only find locations " diff --git a/cf/healpix.py b/cf/healpix.py index 47b67eb9fc..80eb4b45b9 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -31,12 +31,11 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): will be updated in-place. pole_longitude: `None` or number - The longitude of coordinates, or coordinate bounds, that - lie exactly on the north or south pole. If `None` then the - longitudes of such points will vary according to the - alogrithm being used to create them. If set to a number, - then the longitudes of such points will all be given that - value. + The longitude of coordinate bounds that lie exactly on the + north (south) pole. If `None` then the longitude of such a + vertex will be the same as the south (north) vertex of the + same cell. If set to a number, then the longitudes of such + vertices will all be given that value. :Returns: @@ -165,6 +164,10 @@ def _healpix_indexing_scheme(healpix_index, hp, new_indexing_scheme): Journal, 2005, 622 (2), pp.759-771. https://dx.doi.org/10.1086/427976 + M. Reinecke and E. Hivon: Efficient data structures for masks on + 2D grids. A&A, 580 (2015) + A132. https://doi.org/10.1051/0004-6361/201526549 + .. versionadded:: NEXTVERSION .. seealso:: `cf.Field.healpix_indexing_scheme` @@ -384,7 +387,7 @@ def del_healpix_coordinate_reference(f): return cr -def healpix_info(f): + def healpix_info(f): """Get information about the HEALPix grid, if there is one. .. versionadded:: NEXTVERSION @@ -397,8 +400,32 @@ def healpix_info(f): :Returns: `dict` - The information about the HEALPix axis. The dictionary - will be empty if there is no HEALPix axis. + The information about the HEALPix axis, with some or all + of the following dictionary keys: + + * ``'coordinate_reference_key'``: The construct key of the + healpix coordinate + reference construct. + + * ``'grid_mapping_name:healpix'``: The healpix coordinate + reference construct. + + * ``'indexing_scheme'``: The HEALPix indexing scheme. + + * ``'refinement_level'``: The refinement level of the + HEALPix grid. + + * ``'domain_axis_key'``: The construct key of the HEALPix + domain axis construct. + + * ``'coordinate_key'``: The construct key of the + healpix_index coordinate + construct. + + * ``'healpix_index'``: The healpix_index coordinate + construct. + + The dictionary will be empty if there is no HEALPix axis. **Examples** diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index d2a24c1e44..6c04b918e8 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -1624,8 +1624,15 @@ def auxiliary_to_dimension( axis = f.get_data_axes(key) dim = f._DimensionCoordinate(source=aux) - f.set_construct(dim, axes=axis) + dim_key = f.set_construct(dim, axes=axis, copy=False) + + # Update Coordinates listed in Coordinate References + for cr in f.coordinate_references().values(): + if key in cr.coordinates(): + cr.set_coordinate(dim_key) + f.del_construct(key) + return f def del_coordinate_reference( @@ -2130,12 +2137,7 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): # Ensure that healpix indices are auxiliary coordinates if healpix_index.construct_type == "dimension_coordinate": - healpix_index = f._AuxiliaryCoordinate( - source=healpix_index, copy=False - ) - f.del_construct(hp["coordinate_key"]) - hp_key = f.set_construct(healpix_index, axes=axis, copy=False) - cr.set_coordinate(hp_key) + f.dimension_to_auxiliary(hp["coordinate_key"], inplace=True) if sort: # Sort the HEALPix axis so that the HEALPix indices are @@ -2166,8 +2168,37 @@ def healpix_info(self): :Returns: `dict` - The information about the HEALPix axis. The dictionary - will be empty if there is no HEALPix axis. + + The information about the HEALPix axis, with some or + all of the following dictionary keys: + + * ``'coordinate_reference_key'``: The construct key of + the healpix + coordinate reference + construct. + + * ``'grid_mapping_name:healpix'``: The healpix + coordinate + reference + construct. + + * ``'indexing_scheme'``: The HEALPix indexing scheme. + + * ``'refinement_level'``: The refinement level of the + HEALPix grid. + + * ``'domain_axis_key'``: The construct key of the + HEALPix domain axis + construct. + + * ``'coordinate_key'``: The construct key of the + healpix_index coordinate + construct. + + * ``'healpix_index'``: The healpix_index coordinate + construct. + + The dictionary will be empty if there is no HEALPix axis. **Examples** @@ -2445,7 +2476,7 @@ def create_latlon_coordinates( return f - # Get all Coordinate References in a dictionary + # Store all of the Coordinate References in a dictionary ... identities = { cr.identity(""): cr for cr in f.coordinate_references(todict=True).values() @@ -2459,7 +2490,7 @@ def create_latlon_coordinates( return f - # Keep only those that are grid mappings + # ... keeping only those that are grid mappings identities = { identity: cr for identity, cr in identities.items() @@ -2467,12 +2498,11 @@ def create_latlon_coordinates( } # Remove a 'latitude_longitude' grid mapping from the - # dictionary + # dictionary, saving it for later. latlon_cr = identities.pop( "grid_mapping_name:latitude_longitude", None ) if not identities: - # There is no non-latitude_longitude grid mapping if is_log_level_info(logger): logger.info( "Can't create latitude and longitude coordinates: There " @@ -2492,17 +2522,17 @@ def create_latlon_coordinates( return f - # Still here? Then get the non-latitude_longitude grid mapping - # and calulate the lat/lon coordinates. + # Still here? Then get the unique non-latitude_longitude grid + # mapping, and use it to calculate the lat/lon coordinates. identity, cr = identities.popitem() # Initialize the flag that tells us if any new coordinates - # were created + # have been created new_coords = False if one_d and identity == "grid_mapping_name:healpix": # -------------------------------------------------------- - # HEALPix: 1-d lat/lon coordinates + # 1-d lat/lon coordinates: HEALPix # -------------------------------------------------------- from ..healpix import _healpix_create_latlon_coordinates @@ -2728,8 +2758,15 @@ def dimension_to_auxiliary( axis = f.get_data_axes(key) aux = f._AuxiliaryCoordinate(source=dim) - f.set_construct(aux, axes=axis) + aux_key = f.set_construct(aux, axes=axis, copy=False) + + # Update Coordinates listed in Coordinate References + for cr in f.coordinate_references().values(): + if key in cr.coordinates(): + cr.set_coordinate(aux_key) + f.del_construct(key) + return f @_deprecated_kwarg_check("axes", version="3.0.0", removed_at="4.0.0") diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index 4ac30336b5..baf99afa84 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -2969,6 +2969,12 @@ def test_Field_auxiliary_to_dimension_to_auxiliary(self): self.assertIsNone(f.dimension_to_auxiliary("Y", inplace=True)) self.assertIsNone(g.auxiliary_to_dimension("Y", inplace=True)) + f = self.f12.copy() + g = f.auxiliary_to_dimension("healpix_index") + h = g.dimension_to_auxiliary("healpix_index") + self.assertFalse(f.equals(g)) + self.assertTrue(f.equals(h)) + f = cf.read("geometry_1.nc")[0] with self.assertRaises(ValueError): From 27d19dcf7293a3920fe7eb57bfc38a7c28f61c30 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 30 Jul 2025 09:08:54 +0100 Subject: [PATCH 35/59] dev --- cf/healpix.py | 8 ++--- cf/mixin/fielddomain.py | 77 +++++++++++++++++++++-------------------- 2 files changed, 44 insertions(+), 41 deletions(-) diff --git a/cf/healpix.py b/cf/healpix.py index 80eb4b45b9..be36401231 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -387,7 +387,7 @@ def del_healpix_coordinate_reference(f): return cr - def healpix_info(f): +def healpix_info(f): """Get information about the HEALPix grid, if there is one. .. versionadded:: NEXTVERSION @@ -406,18 +406,18 @@ def healpix_info(f): * ``'coordinate_reference_key'``: The construct key of the healpix coordinate reference construct. - + * ``'grid_mapping_name:healpix'``: The healpix coordinate reference construct. * ``'indexing_scheme'``: The HEALPix indexing scheme. - + * ``'refinement_level'``: The refinement level of the HEALPix grid. * ``'domain_axis_key'``: The construct key of the HEALPix domain axis construct. - + * ``'coordinate_key'``: The construct key of the healpix_index coordinate construct. diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 6c04b918e8..a0e4e8707d 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -2176,28 +2176,28 @@ def healpix_info(self): the healpix coordinate reference construct. - + * ``'grid_mapping_name:healpix'``: The healpix coordinate reference construct. - + * ``'indexing_scheme'``: The HEALPix indexing scheme. - + * ``'refinement_level'``: The refinement level of the HEALPix grid. - + * ``'domain_axis_key'``: The construct key of the HEALPix domain axis construct. - + * ``'coordinate_key'``: The construct key of the healpix_index coordinate construct. - + * ``'healpix_index'``: The healpix_index coordinate construct. - + The dictionary will be empty if there is no HEALPix axis. **Examples** @@ -2370,12 +2370,11 @@ def create_latlon_coordinates( ): """Create latitude and longitude coordinates. - Creates any 1-d or 2-d latitude and longitude coordinate - constructs that are implied by the {{class}}, but are not - atually present as part of the {{class}}'s metadata. Such - coordinates may be created if there is an appropriate - non-latitude_longitude grid mapping coordinate reference - construct. + Creates 1-d or 2-d latitude and longitude coordinate + constructs that are implied by the {{class}} coordinate + reference constructs. By default (or if *overwrite* is False), + new coordinates are only created if the {{class}} metadata + doesn't already include any latitude or longitude coordinates. When it is not possible to create latitude and longitude coordinates, the reason why will be reported if the log level @@ -2402,7 +2401,7 @@ def create_latlon_coordinates( The longitude of coordinates, or coordinate bounds, that lie exactly on the north or south pole. If `None` (the default) then the longitudes of such points will - vary according to the alogrithm being used to create + vary according to the algorithm being used to create them. If set to a number, then the longitudes of such points will all be given that value. @@ -2476,7 +2475,8 @@ def create_latlon_coordinates( return f - # Store all of the Coordinate References in a dictionary ... + # Store all of the grid mapping Coordinate References in a + # dictionary identities = { cr.identity(""): cr for cr in f.coordinate_references(todict=True).values() @@ -2490,15 +2490,14 @@ def create_latlon_coordinates( return f - # ... keeping only those that are grid mappings identities = { identity: cr for identity, cr in identities.items() if identity.startswith("grid_mapping_name:") } - # Remove a 'latitude_longitude' grid mapping from the - # dictionary, saving it for later. + # Remove a 'latitude_longitude' grid mapping (if there is one) + # from the dictionary, saving it for later. latlon_cr = identities.pop( "grid_mapping_name:latitude_longitude", None ) @@ -2522,35 +2521,42 @@ def create_latlon_coordinates( return f + # ------------------------------------------------------------ # Still here? Then get the unique non-latitude_longitude grid # mapping, and use it to calculate the lat/lon coordinates. + # ------------------------------------------------------------ identity, cr = identities.popitem() # Initialize the flag that tells us if any new coordinates # have been created new_coords = False - if one_d and identity == "grid_mapping_name:healpix": + if one_d: # -------------------------------------------------------- - # 1-d lat/lon coordinates: HEALPix + # 1-d lat/lon coordinates # -------------------------------------------------------- - from ..healpix import _healpix_create_latlon_coordinates + if identity == "grid_mapping_name:healpix": + # ---------------------------------------------------- + # HEALPix + # ---------------------------------------------------- + from ..healpix import _healpix_create_latlon_coordinates - lat_key, lon_key = _healpix_create_latlon_coordinates( - f, pole_longitude - ) - new_coords = lat_key is not None + lat_key, lon_key = _healpix_create_latlon_coordinates( + f, pole_longitude + ) + new_coords = lat_key is not None elif two_d: # -------------------------------------------------------- - # Plane projection or rotated pole + # 2-d lat/lon coordinates # -------------------------------------------------------- pass # Add some code here! - # ------------------------------------------------------------ - # Update coordinate references - # ------------------------------------------------------------ if new_coords: + # -------------------------------------------------------- + # Update the approrpriate coordinate reference with the + # new coordinate keys + # -------------------------------------------------------- if latlon_cr is not None: latlon_cr.set_coordinates((lat_key, lon_key)) else: @@ -3103,14 +3109,11 @@ def is_discrete_axis(self, *identity, **filter_kwargs): return True # HEALPix - if ( - self.coordinate( - "healpix_index", - filter_by_axis=(axis,), - axis_mode="exact", - default=None, - ) - is not None + if self.coordinates( + "healpix_index", + filter_by_axis=(axis,), + axis_mode="exact", + todict=True, ): return True From 66163faab17e9ae27b53bd013674bcfabf2806fa Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 30 Jul 2025 16:28:04 +0100 Subject: [PATCH 36/59] dev --- cf/data/dask_utils.py | 58 +++++++ cf/field.py | 341 ++++++++++++++++++++++++++++++++++++---- cf/healpix.py | 13 ++ cf/mixin/fielddomain.py | 4 +- 4 files changed, 388 insertions(+), 28 deletions(-) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index 1acf30445d..7af8ffb050 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -465,6 +465,64 @@ def cf_filled(a, fill_value=None): return np.ma.filled(a, fill_value=fill_value) +def cf_healpix_func(a, ncells, iaxis, conserve_integral): + """Calculate HEALPix cell bounds. + + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et + al.. HEALPix: A Framework for High-Resolution Discretization and + Fast Analysis of Data Distributed on the Sphere. The Astrophysical + Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + + """ + if conserve_integral: + a = a / ncells + + iaxis = iaxis + 1 + a = np.expand_dims(a, iaxis) + + shape = list(a.shape) + shape[iaxis] = ncells + + a = np.broadcast_to(a, shape, subok=True) + a = a.reshape( + shape[: iaxis - 1] + + [shape[iaxis - 1] * shape[iaxis]] + + shape[iaxis + 1 :] + ) + + return a + + +def cf_healpix_funcy(a, ncells): + """Calculate HEALPix cell bounds. + + For instance, when going from refinement level 1 to refinement + level 2, if *a* is ``(2, 23, 17)`` then it will transformed to + ``(8, 9, 10, 11, 92, 93, 94, 95, 68, 69, 70, 71)`` where + ``8=2*ncells, 9=2*ncells+1, ..., 71=17*ncells+3``, and where + ``ncells`` is the number of cells at refinement level 2 that lie + inside one cell at refinement level 1, i.e. ``ncells=4**(2-1)=4``. + + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et + al.. HEALPix: A Framework for High-Resolution Discretization and + Fast Analysis of Data Distributed on the Sphere. The Astrophysical + Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + + """ + # PERFORMANCE: This function can use a lot of memory when 'a' + # and/or 'ncells' are large. + + a = a * ncells + a = np.expand_dims(a, -1) + a = np.broadcast_to(a, (a.size, ncells), subok=True).copy() + a += np.arange(ncells) + a = a.flatten() + + return a + + def cf_healpix_bounds( a, indexing_scheme, diff --git a/cf/field.py b/cf/field.py index 6b3e6d38ce..17ba401040 100644 --- a/cf/field.py +++ b/cf/field.py @@ -1,6 +1,7 @@ import logging from dataclasses import dataclass from functools import reduce +from numbers import Integral from operator import mul as operator_mul import cfdm @@ -4844,7 +4845,9 @@ def healpix_decrease_refinement_level( .. versionadded:: NEXTVERSION - .. seealso:: `healpix_indexing_scheme` + .. seealso:: `healpix_increase_refinement_level`, + `healpix_info`, `healpix_indexing_scheme`, + `healpix_to_ugrid` :Parameters: @@ -5036,10 +5039,6 @@ def healpix_decrease_refinement_level( "healpix_index coordinates" ) - # Get the HEALPix axis - axis = hp["domain_axis_key"] - iaxis = f.get_data_axes().index(axis) - if check_healpix_index: d = healpix_index.data if not (d.diff() > 0).all(): @@ -5060,10 +5059,14 @@ def healpix_decrease_refinement_level( f"({refinement_level})" ) - # Whether or not to create lat/lon coordinates for the - # coarsened grid. Only do so if the original grid has lat/lon - # coordinates. - create_coarsened_latlon = bool( + # Get the HEALPix axis + axis = hp["domain_axis_key"] + iaxis = f.get_data_axes().index(axis) + + # Whether or not to create lat/lon coordinates for the new + # refinement level. Only do so if the original grid has + # lat/lon coordinates. + create_latlon = bool( f.coordinates( "latitude", "longitude", @@ -5083,6 +5086,10 @@ def healpix_decrease_refinement_level( reduction, axes={iaxis: ncells}, trim_excess=False, inplace=True ) + # Re-size the HEALPix axis + domain_axis = f.domain_axis(axis) + domain_axis.set_size(f.shape[iaxis]) + # Coarsen the domain ancillary constructs that span the # HEALPix axis. We're assuming that domain ancillary data are # intensive (i.e. do not depend on the size of the cell), so @@ -5118,35 +5125,315 @@ def healpix_decrease_refinement_level( ): f.del_construct(key) - # Re-size the HEALPix axis - domain_axis = f.domain_axis(axis) - domain_axis.set_size(f.shape[iaxis]) - - # Set the healpix_index coordinates for the new refinement + # Create the healpix_index coordinates for the new refinement # level - new_index = healpix_index[::ncells] // ncells - if new_index.construct_type == "dimension_coordinate": - # Convert indices to auxiliary coordinates - new_index = f._AuxiliaryCoordinate(source=new_index, copy=False) - hp_key = None - else: - hp_key = hp["coordinate_key"] + if healpix_index.construct_type == "dimension_coordinate": + # Ensure that healpix indices are auxiliary coordinates + healpix_index = f._AuxiliaryCoordinate( + source=healpix_index, copy=False + ) - new_key = f.set_construct(new_index, axes=axis, key=hp_key, copy=False) + healpix_index = healpix_index[::ncells] // ncells + hp_key = f.set_construct(healpix_index, axes=axis, copy=False) - # Set the new refinement level + # Update the healpix Coordinate Reference cr = hp.get("grid_mapping_name:healpix") cr.coordinate_conversion.set_parameter( "refinement_level", new_refinement_level ) - cr.set_coordinate(new_key) + cr.set_coordinate(hp_key) + + if create_latlon: + # Create lat/lon coordinates for the new refinement level + f.create_latlon_coordinates(two_d=False, inplace=True) + + return f + + # 00000 + + def healpix_increase_refinement_level(self, level, conserve): + """Decrease the refinement level of a HEALPix grid. + + Decreasing the refinement level coarsens the horizontal grid + to a lower-level HEALPix grid by combining, using the + *reduction* function, all original cells that lie inside each + larger cell at the new refinement level. + + The operation requires that each larger cell at the new + refinement level either contains no original cells (in which + case that new cell is not included in the output), or is + completely covered by original cells. It is not allowed for a + larger cell to be only partially covered by original + cells. For instance, if the original refinement level is 10 + and the new refinement level is 8, then each output cell will + be the combination of 16 (:math:`=4^(10-8)`) original cells. + + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, + et al.. HEALPix: A Framework for High-Resolution + Discretization and Fast Analysis of Data Distributed on the + Sphere. The Astrophysical Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + + .. versionadded:: NEXTVERSION + + .. seealso:: `healpix_decrease_refinement_level`, + `healpix_info`, `healpix_indexing_scheme`, + `healpix_to_ugrid` + + :Parameters: + + level: `int` or `None` + Specify the new refinement level as an integer greater + than or equal to the current refinement level, or if + `None` then the refinement level is not changed. + + reduction: function + The function used to calculate the values in the new + coarser cells, from the data on the original finer + cells. + + *Example:* + For an intensive field quantity (that does not + depend on the size of the cells, such as + "sea_ice_amount" with units of kg m-2), `np.mean` + might be appropriate. + + *Example:* + For an extensive field quantity (that depends on the + size of the cells, such as "sea_ice_mass" with units + of kg), `np.sum` might be appropriate. + + :Returns: + + `Field` + A new Field with a coarsened TODOHEALPIX HEALPix grid. + + **Examples** + + >>> f = cf.example_field(12) + >>> f + + >>> f.healpix_info()['refinement_level'] + 1 + + Set the refinement level to 0: + + >>> g = f.healpix_decrease_refinement_level(0, np.mean) + >>> g + + >>> g.healpix_info()['refinement_level'] + 0 + + Decrease the refinement level by 1, showing that every four + cells in the orginal field correspond to one cell at the lower + level: + + >>> g = f.healpix_decrease_refinement_level(-1, np.mean) + + >>> g.healpix_info()['refinement_level'] + 0 + >>> np.mean(g.array[0, 0]) + np.float64(289.15) + >>> f.healpix_info()['refinement_level'] + 1 + >>> np.mean(f.array[0, :4]) + np.float64(289.15) + + """ + from .data.dask_utils import cf_healpix_func, cf_healpix_funcy + from .healpix import healpix_max_refinement_level + + try: + f = self.healpix_indexing_scheme("nested", sort=False) + except ValueError as error: + raise ValueError( + f"Can't increase HEALPix refinement level: {error}" + ) + + # Get the HEALPix info + hp = f.healpix_info() + refinement_level = hp["refinement_level"] + + # Parse 'level' + if level is None: + # No change in refinement level + return f + + if ( + not isinstance(level, Integral) + or level < refinement_level + or level > healpix_max_refinement_level() + ): + raise ValueError( + "Can't increase refinement level: 'level' keyword must be " + "an integer greater than or equal to the current refinement " + f"level of {refinement_level}, and less than or equal to " + f"{healpix_max_refinement_level()}. Got {level!r}" + ) + + if level == refinement_level: + # No change in refinement level + return f + + # Get the number of cells at the new refinement level which + # are contained in one cell at the original refinement level + ncells = 4 ** (level - refinement_level) + + # Get the HEALPix axis + axis = hp["domain_axis_key"] + try: + iaxis = f.get_data_axes().index(axis) + except ValueError: + # Field data doesn't span the HEALPix axis, so insert it + # (note that it must be size 1, given that the Field data + # doen't span it). + f.insert_dimension(axis, -1, inplace=True) + iaxis = f.get_data_axes().index(axis) + + # Whether or not to create lat/lon coordinates for the new + # refinement level. Only do so if the original grid has + # lat/lon coordinates. + create_latlon = bool( + f.coordinates( + "latitude", + "longitude", + filter_by_axis=(axis,), + axis_mode="exact", + todict=True, + ) + ) + + conserve_integral = conserve == "integral" + + dx = f.data.to_dask_array(_force_mask_hardness=False) + + if conserve_integral: + dtype = np.dtype("float64") + else: + dtype = dx.dtype + + chunks = list(dx.chunks) + chunks[iaxis] = (np.array(chunks[iaxis]) * ncells).tolist() + + dx = dx.map_blocks( + cf_healpix_func, + chunks=tuple(chunks), + dtype=dtype, + meta=np.array((), dtype=dtype), + ncells=ncells, + iaxis=iaxis, + conserve_integral=conserve_integral, + ) + + # Re-size the HEALPix axis + domain_axis = f.domain_axis(axis) + domain_axis.set_size(dx.shape[iaxis]) + + f.set_data(dx, copy=False) + + # Increase the refinement level of domain ancillary constructs + # that span the HEALPix axis. We're assuming that domain + # ancillary data are intensive (i.e. do not depend on the size + # of the cell), so we use conserve_integral=False. + meta = np.array((), dtype=dtype) + for key, domain_ancillary in f.domain_ancillaries( + filter_by_axis=(axis,), axis_mode="and", todict=True + ).items(): + iaxis = f.get_data_axes(key).index(axis) + dx = domain_ancillary.data.to_dask_array( + _force_mask_hardness=False + ) + dtype = dx.dtype + chunks = list(dx.chunks) + chunks[iaxis] = (np.array(chunks[iaxis]) * ncells).tolist() + dx = dx.map_blocks( + cf_healpix_func, + chunks=tuple(chunks), + dtype=dtype, + meta=meta, + ncells=ncells, + iaxis=iaxis, + conserve_integral=False, + ) + domain_ancillary.set_data(dx, copy=False) + + # Increase the refinement level of cell measure constructs + # that span the HEALPix axis. Cell measure data are extensive + # (i.e. depend on the size of the cell), so we use + # conserve_integral=True. + dtype = np.dtype("float64") + meta = np.array((), dtype=dtype) + for key, cell_measure in f.cell_measures( + filter_by_axis=(axis,), axis_mode="and", todict=True + ).items(): + iaxis = f.get_data_axes(key).index(axis) + dx = cell_measure.data.to_dask_array(_force_mask_hardness=False) + chunks = list(dx.chunks) + chunks[iaxis] = (np.array(chunks[iaxis]) * ncells).tolist() + dx = dx.map_blocks( + cf_healpix_func, + chunks=tuple(chunks), + dtype=dtype, + meta=meta, + ncells=ncells, + iaxis=iaxis, + conserve_integral=True, + ) + cell_measure.set_data(dx, copy=False) + + # Remove all other metadata constructs that span the HEALPix + # axis (including the original healpix_index coordinate + # construct, and any lat/lon coordinate constructs that span + # the HEALPix axis) + for key in ( + f.constructs.filter_by_axis(axis, axis_mode="and") + .filter_by_type("cell_measure", "domain_ancillary") + .inverse_filter(1) + .todict() + ): + f.del_construct(key) + + # Create the healpix_index coordinates for the new refinement + # level + healpix_index = hp["healpix_index"] + if healpix_index.construct_type == "dimension_coordinate": + # Ensure that healpix indices are auxiliary coordinates + healpix_index = f._AuxiliaryCoordinate( + source=healpix_index, copy=False + ) + + dx = healpix_index.data.to_dask_array(_force_mask_hardness=False) + + dtype = cfdm.integer_dtype(12 * (4**level) - 1) + if dx.dtype != dtype: + dx = dx.astype(dtype, copy=False) + + chunks = [(np.array(dx.chunks[0]) * ncells).tolist()] + + dx = dx.map_blocks( + cf_healpix_funcy, + chunks=tuple(chunks), + dtype=dtype, + meta=np.array((), dtype=dtype), + ncells=ncells, + ) + + healpix_index.set_data(dx, copy=False) + hp_key = f.set_construct(healpix_index, axes=axis, copy=False) - if create_coarsened_latlon: - # Create lat/lon coordinates for the coarsened grid - f.create_latlon_coordinates(inplace=True) + # Update the healpix Coordinate Reference + cr = hp.get("grid_mapping_name:healpix") + cr.coordinate_conversion.set_parameter("refinement_level", level) + cr.set_coordinate(hp_key) + + if create_latlon: + # Create lat/lon coordinates for the new refinement level + f.create_latlon_coordinates(two_d=False, inplace=True) return f + # 999999 + def histogram(self, digitized): """Return a multi-dimensional histogram of the data. diff --git a/cf/healpix.py b/cf/healpix.py index be36401231..5b914d504d 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -470,3 +470,16 @@ def healpix_info(f): info["healpix_index"] = healpix_index return info + + +def healpix_max_refinement_level(): + """TODOHEALPIX""" + try: + import healpix + except ImportError as e: + raise ImportError( + f"{e}. Must install healpix (https://pypi.org/project/healpix) " + "to find the HEALPix maximum refinement level" + ) + + return healpix.nside2order(healpix._chp.NSIDE_MAX) diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index a0e4e8707d..aa1c32b184 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -1982,6 +1982,8 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): .. versionadded:: NEXTVERSION + .. seealso:: `healpix_info`, `healpix_to_ugrid` + :Parameters: new_indexing_scheme: `str` or `None` @@ -2233,7 +2235,7 @@ def healpix_to_ugrid(self, inplace=False): .. versionadded:: NEXTVERSION - .. seealso:: `create_latlon_coordinates` + .. seealso:: `healpix_info`, `create_latlon_coordinates` :Parameters: From 1d8be48a1c955cd897cd7e3b836410a2cc9c7105 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 30 Jul 2025 23:41:23 +0100 Subject: [PATCH 37/59] dev --- cf/data/dask_utils.py | 180 ++++++++++++++++++++++++++++-------------- cf/field.py | 75 ++++++++++-------- cf/healpix.py | 21 ++++- 3 files changed, 186 insertions(+), 90 deletions(-) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index 7af8ffb050..719af98f65 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -465,64 +465,6 @@ def cf_filled(a, fill_value=None): return np.ma.filled(a, fill_value=fill_value) -def cf_healpix_func(a, ncells, iaxis, conserve_integral): - """Calculate HEALPix cell bounds. - - K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et - al.. HEALPix: A Framework for High-Resolution Discretization and - Fast Analysis of Data Distributed on the Sphere. The Astrophysical - Journal, 2005, 622 (2), pp.759-771. - https://dx.doi.org/10.1086/427976 - - """ - if conserve_integral: - a = a / ncells - - iaxis = iaxis + 1 - a = np.expand_dims(a, iaxis) - - shape = list(a.shape) - shape[iaxis] = ncells - - a = np.broadcast_to(a, shape, subok=True) - a = a.reshape( - shape[: iaxis - 1] - + [shape[iaxis - 1] * shape[iaxis]] - + shape[iaxis + 1 :] - ) - - return a - - -def cf_healpix_funcy(a, ncells): - """Calculate HEALPix cell bounds. - - For instance, when going from refinement level 1 to refinement - level 2, if *a* is ``(2, 23, 17)`` then it will transformed to - ``(8, 9, 10, 11, 92, 93, 94, 95, 68, 69, 70, 71)`` where - ``8=2*ncells, 9=2*ncells+1, ..., 71=17*ncells+3``, and where - ``ncells`` is the number of cells at refinement level 2 that lie - inside one cell at refinement level 1, i.e. ``ncells=4**(2-1)=4``. - - K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et - al.. HEALPix: A Framework for High-Resolution Discretization and - Fast Analysis of Data Distributed on the Sphere. The Astrophysical - Journal, 2005, 622 (2), pp.759-771. - https://dx.doi.org/10.1086/427976 - - """ - # PERFORMANCE: This function can use a lot of memory when 'a' - # and/or 'ncells' are large. - - a = a * ncells - a = np.expand_dims(a, -1) - a = np.broadcast_to(a, (a.size, ncells), subok=True).copy() - a += np.arange(ncells) - a = a.flatten() - - return a - - def cf_healpix_bounds( a, indexing_scheme, @@ -550,6 +492,8 @@ def cf_healpix_bounds( .. versionadded:: NEXTVERSION + .. seealso:: `cf.Field.create_latlon_coordinates` + :Parameters: a: `numpy.ndarray` @@ -713,6 +657,8 @@ def cf_healpix_coordinates( .. versionadded:: NEXTVERSION + .. seealso:: `cf.Field.create_latlon_coordinates` + :Parameters: a: `numpy.ndarray` @@ -801,6 +747,120 @@ def cf_healpix_coordinates( return c +def cf_healpix_increase_refinement(a, ncells, iaxis, quantity): + """Increase the HEALPix refinement level. + + Array elements are broadcast to cells of a higher refinement + level. For an extensive quantity (that depends on the size of the + cells, such as "sea_ice_mass" with units of kg), the new values + are reduced so that they are consistent with the new smaller cell + areas. For an intensive quantity (that does not depend on the size + of the cells, such as "sea_ice_amount" with units of kg m-2), the + values are not changed. + + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et + al.. HEALPix: A Framework for High-Resolution Discretization and + Fast Analysis of Data Distributed on the Sphere. The Astrophysical + Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + + .. versionadded:: NEXTVERSION + + .. seealso:: `cf.Field.healpix_increase_refinement_level` + + :Parameters: + + a: `numpy.ndarray` + The array. + + ncells: `int` + The number of cells at the new refinement level which are + contained in one cell at the original refinement level + + iaxis: `int` + The position of the HEALPix axis in the array dimensions. + + quantity: `str` + Whether the array values represent intensive or extensive + quantities, specified with ``'intensive'`` and + ``'extensive'`` respectively. + + :Returns: + + `numpy.ndarray` + TODOHEALPIX + + """ + a = cfdm_to_memory(a) + + if quantity == "extensive": + a = a / ncells + + iaxis = iaxis + 1 + a = np.expand_dims(a, iaxis) + + shape = list(a.shape) + shape[iaxis] = ncells + + a = np.broadcast_to(a, shape, subok=True) + a = a.reshape( + shape[: iaxis - 1] + + [shape[iaxis - 1] * shape[iaxis]] + + shape[iaxis + 1 :] + ) + + return a + + +def cf_healpix_increase_refinement_indices(a, ncells): + """Calculate HEALPix cell bounds.TODOHEALPIX + + For instance, when going from refinement level 1 to refinement + level 2, if *a* is ``(2, 23, 17)`` then it will be transformed to + ``(8, 9, 10, 11, 92, 93, 94, 95, 68, 69, 70, 71)`` where + ``8=2*ncells, 9=2*ncells+1, ..., 71=17*ncells+3``, and where + ``ncells`` is the number of cells at refinement level 2 that lie + inside one cell at refinement level 1, i.e. ``ncells=4**(2-1)=4``. + + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et + al.. HEALPix: A Framework for High-Resolution Discretization and + Fast Analysis of Data Distributed on the Sphere. The Astrophysical + Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + + .. versionadded:: NEXTVERSION + + .. seealso:: `cf.Field.healpix_increase_refinement_level` + + :Parameters: + + a: `numpy.ndarray` + The array of HEALPix nested indices. + + ncells: `int` + The number of cells at the new refinement level which are + contained in one cell at the original refinement level + + :Returns: + + `numpy.ndarray` + TODOHEALPIX + + """ + # PERFORMANCE: This function can use a lot of memory when 'a' + # and/or 'ncells' are large. + + a = cfdm_to_memory(a) + + a = a * ncells + a = np.expand_dims(a, -1) + a = np.broadcast_to(a, (a.size, ncells), subok=True).copy() + a += np.arange(ncells) + a = a.flatten() + + return a + + def cf_healpix_indexing_scheme( a, indexing_scheme, new_indexing_scheme, refinement_level=None ): @@ -821,6 +881,8 @@ def cf_healpix_indexing_scheme( .. versionadded:: NEXTVERSION + .. seealso:: `cf.Field.healpix_indexing_scheme` + :Parameters: a: `numpy.ndarray` @@ -934,6 +996,8 @@ def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): .. versionadded:: NEXTVERSION + .. seealso:: `cf.weights.Weights.healpix_area` + :Parameters: a: `numpy.ndarray` diff --git a/cf/field.py b/cf/field.py index 17ba401040..824950997e 100644 --- a/cf/field.py +++ b/cf/field.py @@ -5151,22 +5151,16 @@ def healpix_decrease_refinement_level( # 00000 - def healpix_increase_refinement_level(self, level, conserve): + def healpix_increase_refinement_level(self, level, quantity): """Decrease the refinement level of a HEALPix grid. - Decreasing the refinement level coarsens the horizontal grid - to a lower-level HEALPix grid by combining, using the - *reduction* function, all original cells that lie inside each - larger cell at the new refinement level. - - The operation requires that each larger cell at the new - refinement level either contains no original cells (in which - case that new cell is not included in the output), or is - completely covered by original cells. It is not allowed for a - larger cell to be only partially covered by original - cells. For instance, if the original refinement level is 10 - and the new refinement level is 8, then each output cell will - be the combination of 16 (:math:`=4^(10-8)`) original cells. + Data are broadcast to cells of a higher refinement level. For + an extensive field quantity (that depends on the size of the + cells, such as "sea_ice_mass" with units of kg), the new + values are reduced so that they are consistent with the new + smaller cell areas. For an intensive field quantity (that does + not depend on the size of the cells, such as "sea_ice_amount" + with units of kg m-2), the values are not changed. K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution @@ -5240,7 +5234,10 @@ def healpix_increase_refinement_level(self, level, conserve): np.float64(289.15) """ - from .data.dask_utils import cf_healpix_func, cf_healpix_funcy + from .data.dask_utils import ( + cf_healpix_increase_refinement, + cf_healpix_increase_refinement_indices, + ) from .healpix import healpix_max_refinement_level try: @@ -5275,6 +5272,14 @@ def healpix_increase_refinement_level(self, level, conserve): # No change in refinement level return f + # Parse 'quantity' + valid_quantities = ("intensive", "extensive") + if quantity not in valid_quantities: + raise ValueError( + "Can't increase refinement level: 'quantity' keyword must " + f"be one of {valid_quantities}. Got {quantity!r}" + ) + # Get the number of cells at the new refinement level which # are contained in one cell at the original refinement level ncells = 4 ** (level - refinement_level) @@ -5303,11 +5308,15 @@ def healpix_increase_refinement_level(self, level, conserve): ) ) - conserve_integral = conserve == "integral" - - dx = f.data.to_dask_array(_force_mask_hardness=False) + # Increase the refinement of the Field data + dx = f.data.to_dask_array( + _force_mask_hardness=False, _force_to_memory=False + ) - if conserve_integral: + if quantity == "extensive": + # Extensive data get divided by 'ncells' in + # `cf_healpix_increase_refinement`, so they end up as + # float64. dtype = np.dtype("float64") else: dtype = dx.dtype @@ -5316,13 +5325,13 @@ def healpix_increase_refinement_level(self, level, conserve): chunks[iaxis] = (np.array(chunks[iaxis]) * ncells).tolist() dx = dx.map_blocks( - cf_healpix_func, + cf_healpix_increase_refinement, chunks=tuple(chunks), dtype=dtype, meta=np.array((), dtype=dtype), ncells=ncells, iaxis=iaxis, - conserve_integral=conserve_integral, + quantity=quantity, ) # Re-size the HEALPix axis @@ -5334,50 +5343,52 @@ def healpix_increase_refinement_level(self, level, conserve): # Increase the refinement level of domain ancillary constructs # that span the HEALPix axis. We're assuming that domain # ancillary data are intensive (i.e. do not depend on the size - # of the cell), so we use conserve_integral=False. + # of the cell), so we use quantity="intensive". meta = np.array((), dtype=dtype) for key, domain_ancillary in f.domain_ancillaries( filter_by_axis=(axis,), axis_mode="and", todict=True ).items(): iaxis = f.get_data_axes(key).index(axis) dx = domain_ancillary.data.to_dask_array( - _force_mask_hardness=False + _force_mask_hardness=False, _force_to_memory=False ) dtype = dx.dtype chunks = list(dx.chunks) chunks[iaxis] = (np.array(chunks[iaxis]) * ncells).tolist() dx = dx.map_blocks( - cf_healpix_func, + cf_healpix_increase_refinement, chunks=tuple(chunks), dtype=dtype, meta=meta, ncells=ncells, iaxis=iaxis, - conserve_integral=False, + quantity="intensive", ) domain_ancillary.set_data(dx, copy=False) # Increase the refinement level of cell measure constructs # that span the HEALPix axis. Cell measure data are extensive # (i.e. depend on the size of the cell), so we use - # conserve_integral=True. + # quantity="extensive". dtype = np.dtype("float64") meta = np.array((), dtype=dtype) for key, cell_measure in f.cell_measures( filter_by_axis=(axis,), axis_mode="and", todict=True ).items(): iaxis = f.get_data_axes(key).index(axis) - dx = cell_measure.data.to_dask_array(_force_mask_hardness=False) + dx = cell_measure.data.to_dask_array( + _force_mask_hardness=False, _force_to_memory=False + ) chunks = list(dx.chunks) chunks[iaxis] = (np.array(chunks[iaxis]) * ncells).tolist() dx = dx.map_blocks( - cf_healpix_func, + cf_healpix_increase_refinement, chunks=tuple(chunks), dtype=dtype, meta=meta, ncells=ncells, iaxis=iaxis, - conserve_integral=True, + quantity="extensive", ) cell_measure.set_data(dx, copy=False) @@ -5402,7 +5413,9 @@ def healpix_increase_refinement_level(self, level, conserve): source=healpix_index, copy=False ) - dx = healpix_index.data.to_dask_array(_force_mask_hardness=False) + dx = healpix_index.data.to_dask_array( + _force_mask_hardness=False, _force_to_memory=False + ) dtype = cfdm.integer_dtype(12 * (4**level) - 1) if dx.dtype != dtype: @@ -5411,7 +5424,7 @@ def healpix_increase_refinement_level(self, level, conserve): chunks = [(np.array(dx.chunks[0]) * ncells).tolist()] dx = dx.map_blocks( - cf_healpix_funcy, + cf_healpix_increase_refinement_indices, chunks=tuple(chunks), dtype=dtype, meta=np.array((), dtype=dtype), diff --git a/cf/healpix.py b/cf/healpix.py index 5b914d504d..af983cdd87 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -473,7 +473,26 @@ def healpix_info(f): def healpix_max_refinement_level(): - """TODOHEALPIX""" + """Return the maxium permitted HEALPix refinement level. + + The maximum refinement level is the highest refiniment level for + which all of its HEALPix indices are representable as double + precision integers. + + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et + al.. HEALPix: A Framework for High-Resolution Discretization and + Fast Analysis of Data Distributed on the Sphere. The Astrophysical + Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + + .. versionadded:: NEXTVERSION + + :Returns: + + `int` + The maxium permitted HEALPix refinement level. + + """ try: import healpix except ImportError as e: From 7498aec20f5d8cc50a364f7995df8ccf6bbdd650 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 31 Jul 2025 10:31:27 +0100 Subject: [PATCH 38/59] dev --- cf/data/dask_utils.py | 18 +-- cf/field.py | 237 +++++++++++++++---------------- cf/healpix.py | 4 +- cf/mixin/fielddomain.py | 32 ++--- cf/mixin/propertiesdata.py | 2 + cf/mixin/propertiesdatabounds.py | 2 + cf/test/test_Field.py | 71 ++++++--- 7 files changed, 194 insertions(+), 172 deletions(-) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index 719af98f65..03bcd6a360 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -748,15 +748,15 @@ def cf_healpix_coordinates( def cf_healpix_increase_refinement(a, ncells, iaxis, quantity): - """Increase the HEALPix refinement level. + """Increase the refinement level of a HEALPix array. - Array elements are broadcast to cells of a higher refinement - level. For an extensive quantity (that depends on the size of the + Data are broadcast to cells of at the higher refinement level. For + an extensive field quantity (that depends on the size of the cells, such as "sea_ice_mass" with units of kg), the new values are reduced so that they are consistent with the new smaller cell - areas. For an intensive quantity (that does not depend on the size - of the cells, such as "sea_ice_amount" with units of kg m-2), the - values are not changed. + areas. For an intensive field quantity (that does not depend on + the size of the cells, such as "sea_ice_amount" with units of kg + m-2), the broadcast values are not changed. K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution Discretization and @@ -788,7 +788,7 @@ def cf_healpix_increase_refinement(a, ncells, iaxis, quantity): :Returns: `numpy.ndarray` - TODOHEALPIX + The array at the new refinement level. """ a = cfdm_to_memory(a) @@ -813,7 +813,7 @@ def cf_healpix_increase_refinement(a, ncells, iaxis, quantity): def cf_healpix_increase_refinement_indices(a, ncells): - """Calculate HEALPix cell bounds.TODOHEALPIX + """Increase the refinement level of HEALPix indices For instance, when going from refinement level 1 to refinement level 2, if *a* is ``(2, 23, 17)`` then it will be transformed to @@ -844,7 +844,7 @@ def cf_healpix_increase_refinement_indices(a, ncells): :Returns: `numpy.ndarray` - TODOHEALPIX + The array at the new refinement level. """ # PERFORMANCE: This function can use a lot of memory when 'a' diff --git a/cf/field.py b/cf/field.py index 824950997e..76e7113854 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4852,17 +4852,10 @@ def healpix_decrease_refinement_level( :Parameters: level: `int` or `None` - Specify the new refinement level. If a non-negative - integer then this will be the new refinement level. If - a negative integer then the new refinement level is - defined by the current refinement level plus - *level*. If `None` then the refinement level is not - changed. - - *Example:* - If the current refinement level is 10 then a new - coarser refinement level of 8 can be specified by - either ``8`` or ``-2``. + Specify the new lower refinement level as a + non-negative integer less than or equal to the current + refinement level, or if `None` then the refinement + level is not changed. reduction: function The function used to calculate the values in the new @@ -4936,19 +4929,12 @@ def healpix_decrease_refinement_level( >>> f.healpix_info()['refinement_level'] 1 - Set the refinement level to 0: + Set the refinement level to 0, showing that every 4 cells + (i.e. the number of cells at the original refinement level + that lie in one cell of the lower refinement leve1) in the + orginal field correspond to one cell at the lower level: >>> g = f.healpix_decrease_refinement_level(0, np.mean) - >>> g - - >>> g.healpix_info()['refinement_level'] - 0 - - Decrease the refinement level by 1, showing that every four - cells in the orginal field correspond to one cell at the lower - level: - - >>> g = f.healpix_decrease_refinement_level(-1, np.mean) >>> g.healpix_info()['refinement_level'] 0 @@ -4968,36 +4954,36 @@ def healpix_decrease_refinement_level( indexing_scheme = hp.get("indexing_scheme") if indexing_scheme is None: raise ValueError( - "Can't decrease HEALPix refinement level: indexing_scheme " - "has not been set in the healpix grid mapping coordinate " - "reference" + f"Can't decrease HEALPix refinement level of {f!r}: " + "indexing_scheme has not been set in the healpix grid " + "mapping coordinate reference" ) refinement_level = hp.get("refinement_level") if refinement_level is None: raise ValueError( - "Can't decrease HEALPix refinement level: refinement_level " - "has not been set in the healpix grid mapping coordinate " - "reference" + f"Can't decrease HEALPix refinement level of {f!r}: " + "refinement_level has not been set in the healpix grid " + "mapping coordinate reference" ) + # Decreasing the refinement level requires the nested indexing + # scheme with ordered HEALPix indices if conform: - # Make sure that we have 'nested' indexing scheme with - # ordered HEALPix indices try: f = f.healpix_indexing_scheme("nested", sort=True) - except ValueError as error: + except ValueError as e: raise ValueError( - f"Can't decrease HEALPix refinement level: {error}" + f"Can't decrease HEALPix refinement level of {f!r}: {e}" ) # Re-get the HEALPix info hp = f.healpix_info() elif indexing_scheme != "nested": raise ValueError( - "Can't decrease HEALPix refinement level: indexing_scheme " - "in the healpix grid mapping coordinate reference is " - f"{indexing_scheme!r}, and not 'nested'. " + f"Can't decrease HEALPix refinement level of {f!r}: " + "indexing_scheme in the healpix grid mapping coordinate " + f"reference is {indexing_scheme!r}, and not 'nested'. " "Consider setting conform=True" ) @@ -5006,45 +4992,41 @@ def healpix_decrease_refinement_level( # No change in refinement level return f - if level >= 0: - if level > refinement_level: - raise ValueError( - "'level' keyword can't be larger than the current " - f"refinement level ({refinement_level}). Got: {level!r}" - ) - - # Convert 'level' to a negative number - level -= refinement_level - elif level < -refinement_level: + if ( + not isinstance(level, Integral) + or level < 0 + or level > refinement_level + ): raise ValueError( - "'level' keyword can't be less than minus the current " - f"refinement level ({-refinement_level}). Got: {level!r}" + f"Can't decrease refinement level of {f!r}: " + "'level' keyword must be a non-negative integer less than " + "or equal to the current refinement level of " + f"{refinement_level}. Got {level!r}" ) - new_refinement_level = refinement_level + level - if new_refinement_level == refinement_level: + if level == refinement_level: # No change in refinement level return f # Get the number of cells at the original refinement level # which are contained in one cell at the coarser refinement # level - ncells = 4**-level + ncells = 4 ** (refinement_level - level) # Get the healpix_index coordinates healpix_index = hp.get("healpix_index") if healpix_index is None: raise ValueError( - "Can't decrease HEALPix refinement level: There are no " - "healpix_index coordinates" + f"Can't decrease HEALPix refinement level of {f!r}: " + "There are no healpix_index coordinates" ) if check_healpix_index: d = healpix_index.data if not (d.diff() > 0).all(): raise ValueError( - "Can't decrease HEALPix refinement level: Nested " - "healpix_index coordinates are not strictly " + f"Can't decrease HEALPix refinement level of {f!r}: " + "Nested healpix_index coordinates are not strictly " "monotonically increasing. Consider setting conform=True" ) @@ -5052,9 +5034,9 @@ def healpix_decrease_refinement_level( d[ncells - 1 :: ncells] - d[::ncells] != ncells - 1 ).any(): raise ValueError( - "Can't decrease HEALPix refinement level: At least one " - "cell at the new coarser refinement level " - f"({new_refinement_level}) contains fewer than {ncells} " + f"Can't decrease HEALPix refinement level of {f!r}: " + "At least one cell at the new coarser refinement level " + f"({level}) contains fewer than {ncells} " "cells at the original finer refinement level " f"({refinement_level})" ) @@ -5138,9 +5120,7 @@ def healpix_decrease_refinement_level( # Update the healpix Coordinate Reference cr = hp.get("grid_mapping_name:healpix") - cr.coordinate_conversion.set_parameter( - "refinement_level", new_refinement_level - ) + cr.coordinate_conversion.set_parameter("refinement_level", level) cr.set_coordinate(hp_key) if create_latlon: @@ -5149,18 +5129,17 @@ def healpix_decrease_refinement_level( return f - # 00000 - def healpix_increase_refinement_level(self, level, quantity): - """Decrease the refinement level of a HEALPix grid. + """Increase the refinement level of a HEALPix grid. - Data are broadcast to cells of a higher refinement level. For - an extensive field quantity (that depends on the size of the - cells, such as "sea_ice_mass" with units of kg), the new - values are reduced so that they are consistent with the new - smaller cell areas. For an intensive field quantity (that does - not depend on the size of the cells, such as "sea_ice_amount" - with units of kg m-2), the values are not changed. + Data are broadcast to cells of at the higher refinement + level. For an extensive field quantity (that depends on the + size of the cells, such as "sea_ice_mass" with units of kg), + the new values are reduced so that they are consistent with + the new smaller cell areas. For an intensive field quantity + (that does not depend on the size of the cells, such as + "sea_ice_amount" with units of kg m-2), the broadcast values + are not changed. K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution @@ -5177,25 +5156,14 @@ def healpix_increase_refinement_level(self, level, quantity): :Parameters: level: `int` or `None` - Specify the new refinement level as an integer greater - than or equal to the current refinement level, or if - `None` then the refinement level is not changed. - - reduction: function - The function used to calculate the values in the new - coarser cells, from the data on the original finer - cells. - - *Example:* - For an intensive field quantity (that does not - depend on the size of the cells, such as - "sea_ice_amount" with units of kg m-2), `np.mean` - might be appropriate. + Specify the new higher refinement level as an integer + greater than or equal to the current refinement level, + or if `None` then the refinement level is not changed. - *Example:* - For an extensive field quantity (that depends on the - size of the cells, such as "sea_ice_mass" with units - of kg), `np.sum` might be appropriate. + quantity: `str` + Whether the data values represent intensive or + extensive quantities, specified with ``'intensive'`` + and ``'extensive'`` respectively. :Returns: @@ -5209,29 +5177,43 @@ def healpix_increase_refinement_level(self, level, quantity): >>> f.healpix_info()['refinement_level'] 1 - - Set the refinement level to 0: - - >>> g = f.healpix_decrease_refinement_level(0, np.mean) + >>> g = f.healpix_increase_refinement_level(3, 'intensive') >>> g - + >>> g.healpix_info()['refinement_level'] - 0 + 3 + >>> g = f.healpix_increase_refinement_level(10, 'intensive') + >>> g + - Decrease the refinement level by 1, showing that every four - cells in the orginal field correspond to one cell at the lower - level: + Set the refinement level to 2, showing that, for an intensive + quantity, every 4 cells at the higher refinement level + (i.e. the number of cells at the higher refinement level that + lie in one cell of the original refinement leve1) have the + same value as a single cell at the original refinement level: - >>> g = f.healpix_decrease_refinement_level(-1, np.mean) - + >>> g = f.healpix_increase_refinement_level(2, 'intensive') + >>> g + >>> g.healpix_info()['refinement_level'] - 0 - >>> np.mean(g.array[0, 0]) - np.float64(289.15) - >>> f.healpix_info()['refinement_level'] - 1 - >>> np.mean(f.array[0, :4]) - np.float64(289.15) + 2 + >>> print(f[0, :2] .array) + [[291.5 293.5]] + >>> print(g[0, :8] .array) + [[291.5 291.5 291.5 291.5 293.5 293.5 293.5 293.5]] + + For an extensive quantity (which ``f`` is not, but we can + assume that it is for demonstration purposes), every four + cells at the higher refinement level have the value of a + single cell at the original refinement level after dividing it + by the number of cells at the higher refinement level that lie + in one cell of the original refinement level (4 in this case): + + >>> g = f.healpix_increase_refinement_level(2, 'extensive') + >>> print(f[0, :2] .array) + [[291.5 293.5]] + >>> print(g[0, :8] .array) + [[72.875 72.875 72.875 72.875 73.375 73.375 73.375 73.375]] """ from .data.dask_utils import ( @@ -5240,11 +5222,13 @@ def healpix_increase_refinement_level(self, level, quantity): ) from .healpix import healpix_max_refinement_level + # Increasing the refinement level requires the nested indexing + # scheme try: f = self.healpix_indexing_scheme("nested", sort=False) - except ValueError as error: + except ValueError as e: raise ValueError( - f"Can't increase HEALPix refinement level: {error}" + f"Can't increase HEALPix refinement level of {self!r}: {e}" ) # Get the HEALPix info @@ -5262,10 +5246,11 @@ def healpix_increase_refinement_level(self, level, quantity): or level > healpix_max_refinement_level() ): raise ValueError( - "Can't increase refinement level: 'level' keyword must be " - "an integer greater than or equal to the current refinement " - f"level of {refinement_level}, and less than or equal to " - f"{healpix_max_refinement_level()}. Got {level!r}" + f"Can't increase refinement level of {f!r}: " + "'level' keyword must be an integer greater than or equal " + f"to the current refinement level of {refinement_level}, and " + f"less than or equal to {healpix_max_refinement_level()}. " + f"Got {level!r}" ) if level == refinement_level: @@ -5276,8 +5261,9 @@ def healpix_increase_refinement_level(self, level, quantity): valid_quantities = ("intensive", "extensive") if quantity not in valid_quantities: raise ValueError( - "Can't increase refinement level: 'quantity' keyword must " - f"be one of {valid_quantities}. Got {quantity!r}" + f"Can't increase refinement level of {f!r}: " + f"'quantity' keyword must be one of {valid_quantities}. " + f"Got {quantity!r}" ) # Get the number of cells at the new refinement level which @@ -5315,12 +5301,13 @@ def healpix_increase_refinement_level(self, level, quantity): if quantity == "extensive": # Extensive data get divided by 'ncells' in - # `cf_healpix_increase_refinement`, so they end up as - # float64. + # `cf_healpix_increase_refinement`, so they end up with a + # data type of float64. dtype = np.dtype("float64") else: dtype = dx.dtype + # Each chunk is going to get larger by a factor of 'ncells' chunks = list(dx.chunks) chunks[iaxis] = (np.array(chunks[iaxis]) * ncells).tolist() @@ -5343,7 +5330,7 @@ def healpix_increase_refinement_level(self, level, quantity): # Increase the refinement level of domain ancillary constructs # that span the HEALPix axis. We're assuming that domain # ancillary data are intensive (i.e. do not depend on the size - # of the cell), so we use quantity="intensive". + # of the cell). meta = np.array((), dtype=dtype) for key, domain_ancillary in f.domain_ancillaries( filter_by_axis=(axis,), axis_mode="and", todict=True @@ -5353,8 +5340,12 @@ def healpix_increase_refinement_level(self, level, quantity): _force_mask_hardness=False, _force_to_memory=False ) dtype = dx.dtype + + # Each chunk is going to get larger by a factor of + # 'ncells' chunks = list(dx.chunks) chunks[iaxis] = (np.array(chunks[iaxis]) * ncells).tolist() + dx = dx.map_blocks( cf_healpix_increase_refinement, chunks=tuple(chunks), @@ -5368,8 +5359,7 @@ def healpix_increase_refinement_level(self, level, quantity): # Increase the refinement level of cell measure constructs # that span the HEALPix axis. Cell measure data are extensive - # (i.e. depend on the size of the cell), so we use - # quantity="extensive". + # (i.e. depend on the size of the cell). dtype = np.dtype("float64") meta = np.array((), dtype=dtype) for key, cell_measure in f.cell_measures( @@ -5379,8 +5369,12 @@ def healpix_increase_refinement_level(self, level, quantity): dx = cell_measure.data.to_dask_array( _force_mask_hardness=False, _force_to_memory=False ) + + # Each chunk is going to get larger by a factor of + # 'ncells' chunks = list(dx.chunks) chunks[iaxis] = (np.array(chunks[iaxis]) * ncells).tolist() + dx = dx.map_blocks( cf_healpix_increase_refinement, chunks=tuple(chunks), @@ -5417,10 +5411,13 @@ def healpix_increase_refinement_level(self, level, quantity): _force_mask_hardness=False, _force_to_memory=False ) + # Set the data type to allow for the largest possible HEALPix + # index at the new refinement level dtype = cfdm.integer_dtype(12 * (4**level) - 1) if dx.dtype != dtype: dx = dx.astype(dtype, copy=False) + # Each chunk is going to get larger by a factor of 'ncells' chunks = [(np.array(dx.chunks[0]) * ncells).tolist()] dx = dx.map_blocks( @@ -5445,8 +5442,6 @@ def healpix_increase_refinement_level(self, level, quantity): return f - # 999999 - def histogram(self, digitized): """Return a multi-dimensional histogram of the data. diff --git a/cf/healpix.py b/cf/healpix.py index af983cdd87..7a023b8868 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -478,7 +478,7 @@ def healpix_max_refinement_level(): The maximum refinement level is the highest refiniment level for which all of its HEALPix indices are representable as double precision integers. - + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution Discretization and Fast Analysis of Data Distributed on the Sphere. The Astrophysical @@ -486,7 +486,7 @@ def healpix_max_refinement_level(): https://dx.doi.org/10.1086/427976 .. versionadded:: NEXTVERSION - + :Returns: `int` diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index aa1c32b184..dc4ba1ec39 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -1970,6 +1970,10 @@ def coordinate_reference_domain_axes(self, identity=None): def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): """Change the indexing scheme of HEALPix indices. + The data are not reordered, only the "healpix_index" + coordinate values are changed, along with parameters of the + "healpix" grid mapping Coordinate reference. + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution Discretization and Fast Analysis of Data Distributed on the @@ -2045,7 +2049,7 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): >>> print(h.coordinate('healpix_index').array) [ 3 7 11 15 2 1 6 5 10 9 14 13 19 0 23 4 27 8 31 12 17 22 21 26 25 30 29 18 16 35 20 39 24 43 28 47 34 33 38 37 42 41 46 45 32 36 40 44] - >>> h = g.healpix_indexing_scheme('nested', sort=True) + >>> h = g.healpix_indexing_scheme(None, sort=True) >>> print(h.coordinate('healpix_index').array) [ 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47] @@ -2072,9 +2076,6 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): "healpix_index coordinates" ) - # Get the healpix_index axis - axis = hp["domain_axis_key"] - indexing_scheme = hp.get("indexing_scheme") if indexing_scheme is None: raise ValueError( @@ -2115,21 +2116,9 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): if new_indexing_scheme == "nested_unique": cr.coordinate_conversion.del_parameter("refinement_level") elif indexing_scheme == "nested_unique": - # Set the refinement level for the new indexing - # scheme. This is the largest integer, N, for which - # 2**(2(N+1)) <= healpix_index[0]. Therefore N = - # int(log2(healpix_index[0]) // 2 - 1) - # - # It doesn't matter if there are in fact multiple - # refinement levels in the grid, as this will get - # trapped as an exception when the lazy calculations - # are computed. - from math import log2 - - cr.coordinate_conversion.set_parameter( - "refinement_level", - int(log2(int(healpix_index.data.first_element())) // 2) - - 1, + raise ValueError( + f"Can't change HEALPix indexing scheme of {f!r} from " + f"{indexing_scheme!r} to {new_indexing_scheme!r}" ) # Change the HEALPix indices @@ -2145,14 +2134,15 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): # Sort the HEALPix axis so that the HEALPix indices are # monotonically increasing. Test for the common case of # already-ordered global nested or ring indices (which is - # a fast test compared to do doing any actual sorting). + # a relatively fast compared to do doing any actual + # sorting and subspacing). h = healpix_index if not ( indexing_scheme in ("nested", "ring") and (h == da.arange(h.size, chunks=h.data.chunks)).all() ): index = h.data.compute() - f = f.subspace(**{axis: np.argsort(index)}) + f = f.subspace(**{hp["domain_axis_key"]: np.argsort(index)}) return f diff --git a/cf/mixin/propertiesdata.py b/cf/mixin/propertiesdata.py index 352aeb7a4f..0bcf545398 100644 --- a/cf/mixin/propertiesdata.py +++ b/cf/mixin/propertiesdata.py @@ -5173,6 +5173,8 @@ def rechunk( {{balance: `bool`, optional}} + {{inplace: `bool`, optional}} + :Returns: `{{class}}` or `None` diff --git a/cf/mixin/propertiesdatabounds.py b/cf/mixin/propertiesdatabounds.py index f1e837a76e..6ee84d19cf 100644 --- a/cf/mixin/propertiesdatabounds.py +++ b/cf/mixin/propertiesdatabounds.py @@ -3681,6 +3681,8 @@ def rechunk( If True (the default) then rechunk an interior ring array, if one exists. + {{inplace: `bool`, optional}} + :Returns: `{{class}}` or `None` diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index baf99afa84..19c7b9fb95 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -3157,17 +3157,9 @@ def test_Field_healpix_indexing_scheme(self): h = g.healpix_indexing_scheme("nested_unique") self.assertTrue(h.equals(g)) - # Can change from 'nested_unique' to 'nested' with a single - # refinement level - g = f.healpix_indexing_scheme("nested_unique") - h = g.healpix_indexing_scheme("nested") - self.assertTrue(h.equals(f, ignore_data_type=True)) - - # Can't change from 'nested_unique' to 'nested' with multiple - # refinement level (error comes at comute time) - g = self.f13.healpix_indexing_scheme("nested") + # Can't change from 'nested_unique' to 'nested' with self.assertRaises(ValueError): - g.auxiliary_coordinate("healpix_index").array + self.f13.healpix_indexing_scheme("nested") # Non-HEALPix field with self.assertRaises(ValueError): @@ -3307,20 +3299,12 @@ def test_Field_healpix_decrease_refinement_level(self): """Test Field.healpix_decrease_refinement_level.""" f = self.f12 g = f.healpix_decrease_refinement_level(0, np.mean) - self.assertTrue(g.shape, (2, 12)) - self.assertTrue( - np.array_equal(g.coord("healpix_index"), np.arange(12)) - ) - - g = f.healpix_decrease_refinement_level(-1, np.mean) - self.assertTrue(g.shape, (2, 12)) self.assertTrue( np.array_equal(g.coord("healpix_index"), np.arange(12)) ) f = f.healpix_indexing_scheme("ring") g = f.healpix_decrease_refinement_level(0, np.mean) - self.assertTrue(g.shape, (2, 12)) self.assertTrue( np.array_equal(g.coord("healpix_index"), np.arange(12)) ) @@ -3331,7 +3315,6 @@ def test_Field_healpix_decrease_refinement_level(self): f = f.healpix_indexing_scheme("nested") g = f.healpix_decrease_refinement_level(0, np.mean, conform=True) - self.assertTrue(g.shape, (2, 12)) self.assertTrue( np.array_equal(g.coord("healpix_index"), np.arange(12)) ) @@ -3354,6 +3337,11 @@ def test_Field_healpix_decrease_refinement_level(self): np.array_equal(h.coord("healpix_index"), np.arange(12)) ) + # Bad 'level' parameter + for level in (-1, 0.785, np.array(1), 2, "string"): + with self.assertRaises(ValueError): + f.healpix_decrease_refinement_level(level, np.mean) + # Can't change refinement level for a 'nested_unique' field with self.assertRaises(ValueError): self.f13.healpix_decrease_refinement_level(0, np.mean) @@ -3362,6 +3350,51 @@ def test_Field_healpix_decrease_refinement_level(self): with self.assertRaises(ValueError): self.f0.healpix_decrease_refinement_level(0, np.mean) + def test_Field_healpix_increase_refinement_level(self): + """Test Field.healpix_increase_refinement_level.""" + f = self.f12.copy() + f.rechunk((-1, 17), inplace=True) + f.coordinate("healpix_index").rechunk(13, inplace=True) + + g = f.healpix_increase_refinement_level(2, "intensive") + self.assertTrue( + np.array_equal(g.coord("healpix_index"), np.arange(192)) + ) + + self.assertEqual(g.shape, (2, 192)) + self.assertEqual(g.array.shape, (2, 192)) + + # Check selected data values for intensive and extensive + # increases + n = 4 ** (2 - 1) + for i in (0, 1, 24, 46, 47): + self.assertTrue( + np.allclose(g[:, i * n : (i + 1) * n], f[:, i : i + 1]) + ) + + g = f.healpix_increase_refinement_level(2, "extensive") + for i in (0, 1, 24, 46, 47): + self.assertTrue( + np.allclose(g[:, i * n : (i + 1) * n], f[:, i : i + 1] / n) + ) + + # Bad 'quantity' parameter + with self.assertRaises(ValueError): + f.healpix_increase_refinement_level(2, "bad quantity") + + # Bad 'level' parameter + for level in (-1, 0, np.array(2), 3.14, 30, "string"): + with self.assertRaises(ValueError): + f.healpix_increase_refinement_level(level, "intensive") + + # Can't change refinement level for a 'nested_unique' field + with self.assertRaises(ValueError): + self.f13.healpix_increase_refinement_level(2, "intensive") + + # Non-HEALPix field + with self.assertRaises(ValueError): + self.f0.healpix_increase_refinement_level(2, "intensive") + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) From 1ef0e68b3bdddaa0068cb9559ce587b0656f4ea1 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 31 Jul 2025 13:35:02 +0100 Subject: [PATCH 39/59] dev --- cf/field.py | 53 ++++++++++++++++++---------------- cf/regrid/regrid.py | 52 ++++++++++++--------------------- cf/regrid/regridoperator.py | 20 +------------ cf/test/test_Data.py | 4 ++- cf/test/test_RegridOperator.py | 1 - cf/test/test_regrid_healpix.py | 15 +++++----- 6 files changed, 59 insertions(+), 86 deletions(-) diff --git a/cf/field.py b/cf/field.py index 76e7113854..a72af7d647 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4823,19 +4823,20 @@ def healpix_decrease_refinement_level( ): """Decrease the refinement level of a HEALPix grid. - Decreasing the refinement level coarsens the horizontal grid - to a lower-level HEALPix grid by combining, using the - *reduction* function, all original cells that lie inside each - larger cell at the new refinement level. + Decreasing the refinement level reduces the resolution of the + HEALPix grid by combining, using the *reduction* function, + data from the original cells that lie inside each larger cell + at the new lower refinement level. - The operation requires that each larger cell at the new + The operation requires that each larger cell at the lower refinement level either contains no original cells (in which case that new cell is not included in the output), or is completely covered by original cells. It is not allowed for a larger cell to be only partially covered by original cells. For instance, if the original refinement level is 10 and the new refinement level is 8, then each output cell will - be the combination of 16 (:math:`=4^(10-8)`) original cells. + be the combination of :math:`16\equiv 4^(10-8)`) original + cells. K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution @@ -4878,11 +4879,10 @@ def healpix_decrease_refinement_level( automatically converted to a form suitable for having its refinement level changed, i.e. the indexing scheme is changed to 'nested' and the HEALPix axis is sorted - so that the nested HEALPix indices are strictly - monotonically increasing. If False then either an - exeption is raised if the HEALPix indexing scheme is - not already 'nested', or else the HEALPix axis is not - sorted. + so that the nested HEALPix indices are monotonically + increasing. If False then either an exeption is raised + if the HEALPix indexing scheme is not already + 'nested', or else the HEALPix axis is not sorted. .. note:: Setting to False will speed up the operation when the HEALPix indexing scheme is already @@ -4892,15 +4892,15 @@ def healpix_decrease_refinement_level( check_healpix_index: `bool`, optional If True (the default) then the following conditions will be checked before the creation of the new Field - (but after the HEALPix grid has been conformed, if + (but after the HEALPix grid has been conformed, when *conform* is True): 1. The nested HEALPix indices are strictly - monotonically increasing + monotonically increasing. - 2. Every cell at the new coarser refinement level + 2. Every cell at the new lower refinement level contains the maximum possible number of cells at - the original finer refinement level. + the original refinement level. If True and any of these conditions is not met, then an exception is raised. @@ -4919,7 +4919,7 @@ def healpix_decrease_refinement_level( :Returns: `Field` - A new Field with a coarsened HEALPix grid. + A new Field with the new HEALPix grid. **Examples** @@ -5132,14 +5132,17 @@ def healpix_decrease_refinement_level( def healpix_increase_refinement_level(self, level, quantity): """Increase the refinement level of a HEALPix grid. - Data are broadcast to cells of at the higher refinement - level. For an extensive field quantity (that depends on the - size of the cells, such as "sea_ice_mass" with units of kg), - the new values are reduced so that they are consistent with - the new smaller cell areas. For an intensive field quantity - (that does not depend on the size of the cells, such as - "sea_ice_amount" with units of kg m-2), the broadcast values - are not changed. + Increasing the refinement level increases the resolution of + the HEALPix grid by broadcasting data from each original cell + to all of the new smaller cells at the new higher refinement + level lie inside it. + + For an extensive field quantity (that depends on the size of + the cells, such as "sea_ice_mass" with units of kg), the + broadcast values are reduced to be consistent with the new + smaller cell areas. For an intensive field quantity (that does + not depend on the size of the cells, such as "sea_ice_amount" + with units of kg m-2), the broadcast values are not changed. K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution @@ -5168,7 +5171,7 @@ def healpix_increase_refinement_level(self, level, quantity): :Returns: `Field` - A new Field with a coarsened TODOHEALPIX HEALPix grid. + A new Field with the new HEALPix grid. **Examples** diff --git a/cf/regrid/regrid.py b/cf/regrid/regrid.py index 11587dd561..29e40c5d85 100644 --- a/cf/regrid/regrid.py +++ b/cf/regrid/regrid.py @@ -83,13 +83,15 @@ class Grid: cyclic: Any = None # The regridding method. method: str = "" - # If True then, for 1-d regridding, the esmpy weights are generated - # for a 2-d grid for which one of the dimensions is a size 2 dummy - # dimension. + # If True then, for 1-d regridding, the esmpy weights are + # generated for a 2-d grid for which one of the dimensions is a + # size 2 dummy dimension. dummy_size_2_dimension: bool = False # Whether or not the grid is a structured grid. is_grid: bool = False - # Whether or not the grid is a UGRID mesh. + # Whether or not the grid is a UGRID mesh (after any + # transformations are applied, such as converting HEALPix to + # UGRID). is_mesh: bool = False # Whether or not the grid is a location stream. is_locstream: bool = False @@ -122,10 +124,8 @@ class Grid: # `None` then there are no vertical coordinates. z_index: Any = None # The original field/domain before any transformations are applied - # (such as creating lat/lon cooridnates, or converting to UGRID). + # (such as creating lat/lon coordinates, or converting to UGRID). domain: Any = None - # Whether or not the field/original domain is a HEALPix grid - healpix: bool = False def regrid( @@ -593,7 +593,8 @@ def regrid( if is_log_level_debug(logger): logger.debug( - f"Source ESMF Grid:\n{src_esmpy_grid}\n\nDestination ESMF Grid:\n{dst_esmpy_grid}\n" + f"Source ESMF Grid:\n{src_esmpy_grid}\n\n" + f"Destination ESMF Grid:\n{dst_esmpy_grid}\n" ) # pragma: no cover esmpy_regrid_operator = [] if return_esmpy_regrid_operator else None @@ -657,7 +658,6 @@ def regrid( src_z=src_grid.z, dst_z=dst_grid.z, ln_z=ln_z, - dst_healpix=dst_grid.healpix, ) else: if weights_file is not None: @@ -846,7 +846,7 @@ def spherical_coords_to_domain( elif coords["lat"].ndim == 2: message = ( "When 'dst' is a sequence containing 2-d latitude and longitude " - "coordinate constructs, 'dst_axes' must be dictionary with at " + "coordinate constructs, 'dst_axes' must be a dictionary with at " "least the keys {'X': 0, 'Y': 1} or {'X': 1, 'Y': 0}. " f"Got: {dst_axes!r}" ) @@ -1133,9 +1133,6 @@ def spherical_grid( """ domain = f.copy() - # Whether or not the original field/domain is a HEALPix grid - healpix = False - # Create any implied lat/lon coordinates in-place try: # Try to convert a HEALPix grid to a UGRID grid (which will @@ -1143,8 +1140,6 @@ def spherical_grid( f.healpix_to_ugrid(inplace=True) except ValueError: f.create_latlon_coordinates(inplace=True) - else: - healpix = True data_axes = f.constructs.data_axes() @@ -1289,7 +1284,7 @@ def spherical_grid( and (x_size == 1 or y_size == 1) ): raise ValueError( - f"Neither the X nor Y dimensions of the {name} field" + f"Neither the X nor Y dimensions of the {name} field " f"{f!r} can be of size 1 for spherical {method!r} regridding." ) @@ -1449,7 +1444,6 @@ def spherical_grid( ln_z=ln_z, z_index=z_index, domain=domain, - healpix=healpix, ) set_grid_type(grid) @@ -2793,7 +2787,6 @@ def update_coordinates(src, dst, src_grid, dst_grid, cr_map): The definition of the destination grid. cr_map `dict` - The mapping of destination coordinate reference identities to source coordinate reference identities, as output by `update_non_coordinates`. @@ -2805,11 +2798,6 @@ def update_coordinates(src, dst, src_grid, dst_grid, cr_map): """ dst = dst_grid.domain - # A HEALPix grid is converted to UGRID for the regridding, but we - # want the original HEALPix metadata to be copied to the regridded - # source field, rather than the UGRID view of it. - dst_grid_is_mesh = dst_grid.is_mesh and not dst_grid.healpix - src_axis_keys = src_grid.axis_keys dst_axis_keys = dst_grid.axis_keys @@ -2875,15 +2863,14 @@ def update_coordinates(src, dst, src_grid, dst_grid, cr_map): # Copy domain topology and cell connectivity constructs from the # destination grid - if dst_grid_is_mesh: - for key, topology in dst.constructs( - filter_by_type=("domain_topology", "cell_connectivity"), - filter_by_axis=dst_axis_keys, - axis_mode="exact", - todict=True, - ).items(): - axes = [axis_map[axis] for axis in dst_data_axes[key]] - src.set_construct(topology, axes=axes) + for key, topology in dst.constructs( + filter_by_type=("domain_topology", "cell_connectivity"), + filter_by_axis=dst_axis_keys, + axis_mode="exact", + todict=True, + ).items(): + axes = [axis_map[axis] for axis in dst_data_axes[key]] + src.set_construct(topology, axes=axes) def update_non_coordinates(src, dst, src_grid, dst_grid, regrid_operator): @@ -2915,7 +2902,6 @@ def update_non_coordinates(src, dst, src_grid, dst_grid, regrid_operator): to source coordinate reference identities. """ - # if dst_grid.domain is not None: dst = dst_grid.domain src_axis_keys = src_grid.axis_keys diff --git a/cf/regrid/regridoperator.py b/cf/regrid/regridoperator.py index 001afe10ae..7621addc7e 100644 --- a/cf/regrid/regridoperator.py +++ b/cf/regrid/regridoperator.py @@ -50,7 +50,6 @@ def __init__( src_z=None, dst_z=None, ln_z=False, - dst_healpix=False, ): """**Initialisation** @@ -125,8 +124,7 @@ def __init__( Use keyword parameters instead. dst: `Field` or `Domain` - The definition of the destination grid from which the - weights were calculated. + The definition of the destination grid. dst_axes: `dict` or sequence or `None`, optional The destination grid axes to be regridded. @@ -193,11 +191,6 @@ def __init__( .. versionadded:: 3.16.2 - dst_healpix: `bool`, optional - Whether or not the destination grid is HEALPix grid. - - .. versionadded:: NEXTVERSION - """ super().__init__() @@ -231,7 +224,6 @@ def __init__( self._set_component("src_z", src_z, copy=False) self._set_component("dst_z", dst_z, copy=False) self._set_component("ln_z", bool(ln_z), copy=False) - self._set_component("dst_healpix", bool(dst_healpix), copy=False) def __repr__(self): """x.__repr__() <==> repr(x)""" @@ -330,15 +322,6 @@ def dst_mesh_location(self): """ return self._get_component("dst_mesh_location") - @property - def dst_healpix(self): - """Whether or not the destination grid is HEALPix grid. - - .. versionadded:: NEXTVERSION - - """ - return self._get_component("dst_healpix") - @property def dst_shape(self): """The shape of the destination grid. @@ -566,7 +549,6 @@ def copy(self): src_z=self.src_z, dst_z=self.dst_z, ln_z=self.ln_z, - dst_healpix=self.dst_healpix, ) @_display_or_return diff --git a/cf/test/test_Data.py b/cf/test/test_Data.py index 4ed649b7f8..7cc2a6df44 100644 --- a/cf/test/test_Data.py +++ b/cf/test/test_Data.py @@ -4592,7 +4592,9 @@ def test_Data_masked_values(self): d = cf.Data(array) e = d.masked_values(1.1) ea = e.array - a = np.ma.masked_values(array, 1.1, rtol=cf.rtol(), atol=cf.atol()) + a = np.ma.masked_values( + array, 1.1, rtol=float(cf.rtol()), atol=float(cf.atol()) + ) self.assertTrue(np.isclose(ea, a).all()) self.assertTrue((ea.mask == a.mask).all()) self.assertIsNone(d.masked_values(1.1, inplace=True)) diff --git a/cf/test/test_RegridOperator.py b/cf/test/test_RegridOperator.py index 94577f56c9..dae6be8ce8 100644 --- a/cf/test/test_RegridOperator.py +++ b/cf/test/test_RegridOperator.py @@ -38,7 +38,6 @@ def test_RegridOperator_attributes(self): self.assertIsNone(self.r.src_z) self.assertIsNone(self.r.dst_z) self.assertFalse(self.r.ln_z) - self.assertFalse(self.r.dst_healpix) def test_RegridOperator_copy(self): self.assertIsInstance(self.r.copy(), self.r.__class__) diff --git a/cf/test/test_regrid_healpix.py b/cf/test/test_regrid_healpix.py index 5f655ec1b6..475cc0ea4e 100644 --- a/cf/test/test_regrid_healpix.py +++ b/cf/test/test_regrid_healpix.py @@ -46,15 +46,16 @@ def setUp(self): """Preparations called immediately before each test method.""" # Disable log messages to silence expected warnings cf.log_level("DISABLE") - # Note: to enable all messages for given methods, lines or calls (those - # without a 'verbose' option to do the same) e.g. to debug them, wrap - # them (for methods, start-to-end internally) as follows: + # Note: to enable all messages for given methods, lines or + # calls (those without a 'verbose' option to do the same) + # e.g. to debug them, wrap them (for methods, start-to-end + # internally) as follows: # cfdm.log_level('DEBUG') # < ... test code ... > # cfdm.log_level('DISABLE') - @unittest.skipUnless(esmpy_imported, "Requires esmpy/ESMF package.") - def test_Field_regrid_to_healpix(self): + @unittest.skipUnless(esmpy_imported, "Requires esmpy package.") + def test_Field_regrid_mesh_to_healpix(self): # Check that UGRID -> healpix is the same as UGRID -> UGRUD self.assertFalse(cf.regrid_logging()) @@ -82,8 +83,8 @@ def test_Field_regrid_to_healpix(self): # Check that the result is a HEALPix grid self.assertTrue(cf.healpix.healpix_info(x)) - @unittest.skipUnless(esmpy_imported, "Requires esmpy/ESMF package.") - def test_Field_regrid_from_healpix(self): + @unittest.skipUnless(esmpy_imported, "Requires esmpy package.") + def test_Field_regrid_healpix_to_mesh(self): # Check that healpix -> UGRID is the same as UGRID -> UGRUD self.assertFalse(cf.regrid_logging()) From 251eee412f36b95c597dcee878ce32a9e4273fcc Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 31 Jul 2025 16:44:42 +0100 Subject: [PATCH 40/59] dev --- Changelog.rst | 4 +- cf/data/dask_utils.py | 60 +++++++++++------- cf/healpix.py | 109 +++++++++++++++++--------------- cf/mixin/fielddomain.py | 25 ++++---- docs/source/class/cf.Data.rst | 1 + docs/source/class/cf.Domain.rst | 24 ++++++- docs/source/class/cf.Field.rst | 17 +++++ 7 files changed, 151 insertions(+), 89 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index fada3516ef..4c22bdbfad 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,7 +8,9 @@ Version NEXTVERSION (https://github.com/NCAS-CMS/cf-python/issues/874) * New method: `cf.Field.create_latlon_coordinates` (https://github.com/NCAS-CMS/cf-python/issues/???) -* New HEALPix methods: `cf.Field.healpix_decrease_refinement_level`, +* New HEALPix methods: `cf.Field.healpix_info`, + `cf.Field.healpix_decrease_refinement_level`, + `cf.Field.healpix_increase_refinement_level`, `cf.Field.healpix_indexing_scheme`, `cf.Field.healpix_to_ugrid`, `cf.Domain.create_healpix` (https://github.com/NCAS-CMS/cf-python/issues/???) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index 03bcd6a360..893b65d1cb 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -641,9 +641,7 @@ def cf_healpix_bounds( def cf_healpix_coordinates( a, indexing_scheme, refinement_level=None, lat=False, lon=False ): - """Calculate HEALPix cell coordinates. - - THe coordinates are the cell centres. + """Calculate HEALPix cell centre coordinates. K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution Discretization and @@ -721,28 +719,36 @@ def cf_healpix_coordinates( elif lon: pos = 0 - if indexing_scheme == "nested_unique": - # Create coordinates for 'nested_unique' cells - c = np.empty(a.shape, dtype="float64") - - nest = True - orders, a = healpix.uniq2pix(a, nest=nest) - for order in np.unique(orders): - nside = healpix.order2nside(order) - indices = np.where(orders == order)[0] - c[indices] = healpix.pix2ang( - nside=nside, ipix=a[indices], nest=nest, lonlat=True + match indexing_scheme: + case "nested_unique": + # Create coordinates for 'nested_unique' cells + c = np.empty(a.shape, dtype="float64") + + nest = True + orders, a = healpix.uniq2pix(a, nest=nest) + for order in np.unique(orders): + nside = healpix.order2nside(order) + indices = np.where(orders == order)[0] + c[indices] = healpix.pix2ang( + nside=nside, ipix=a[indices], nest=nest, lonlat=True + )[pos] + + case "nested" | "ring": + # Create coordinates for 'nested' or 'ring' cells + nest = indexing_scheme == "nested" + nside = healpix.order2nside(refinement_level) + c = healpix.pix2ang( + nside=nside, + ipix=a, + nest=nest, + lonlat=True, )[pos] - else: - # Create coordinates for 'nested' or 'ring' cells - nest = indexing_scheme == "nested" - nside = healpix.order2nside(refinement_level) - c = healpix.pix2ang( - nside=nside, - ipix=a, - nest=nest, - lonlat=True, - )[pos] + + case _: + raise ValueError( + "Can't calculate HEALPix cell coordinates: Unknown " + f"'indexing_scheme': {indexing_scheme!r}" + ) return c @@ -976,6 +982,12 @@ def cf_healpix_indexing_scheme( return a + case _: + raise ValueError( + "Can't calculate HEALPix cell coordinates: Unknown " + f"'indexing_scheme': {indexing_scheme!r}" + ) + raise RuntimeError( "cf_healpix_indexing_scheme: Failed during Dask computation" ) diff --git a/cf/healpix.py b/cf/healpix.py index 7a023b8868..e1529b32eb 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -22,7 +22,8 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): .. versionadded:: NEXTVERSION - .. seealso:: `cf.Field.create_latlon_coordinates` + .. seealso:: `cf.Field.create_latlon_coordinates`, + `cf.Field.healpix_to_ugrid` :Parameters: @@ -31,11 +32,12 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): will be updated in-place. pole_longitude: `None` or number - The longitude of coordinate bounds that lie exactly on the - north (south) pole. If `None` then the longitude of such a - vertex will be the same as the south (north) vertex of the - same cell. If set to a number, then the longitudes of such - vertices will all be given that value. + The longitude of HEALPix coordinate bounds that lie + exactly on the north (south) pole. If `None` then the + longitude of such a vertex will be the same as the south + (north) vertex of the same cell. If set to a number, then + the longitudes of such vertices will all be given that + value. :Returns: @@ -273,58 +275,63 @@ def _healpix_locate(lat, lon, f): ) indexing_scheme = hp.get("indexing_scheme") - if indexing_scheme is None: - raise ValueError( - f"Can't locate HEALPix cells for {f!r}: indexing_scheme has " - "not been set in the healpix grid mapping coordinate reference" - ) + match indexing_scheme: + case "nested_unique": + # nested_unique indexing scheme + index = [] + healpix_index = healpix_index.array + orders = healpix.uniq2pix(healpix_index, nest=True)[0] + orders = np.unique(orders) + for order in orders: + # For this refinement level, find the HEALPix nested + # indices of the cells that contain the lat-lon + # points. + nside = healpix.order2nside(order) + pix = healpix.ang2pix(nside, lon, lat, nest=True, lonlat=True) + # Remove duplicate indices + pix = np.unique(pix) + # Convert back to HEALPix nested_unique indices + pix = healpix._chp.nest2uniq(order, pix, pix) + # Find where these HEALPix indices are located in the + # healpix_index coordinates + index.append(da.where(da.isin(healpix_index, pix))[0]) + + index = da.unique(da.concatenate(index, axis=0)) + + case "nested" | "ring": + # nested or ring indexing scheme + refinement_level = hp.get("refinement_level") + if refinement_level is None: + raise ValueError( + f"Can't locate HEALPix cells for {f!r}: refinement_level " + "has not been set in the healpix grid mapping coordinate " + "reference" + ) - if indexing_scheme == "nested_unique": - # nested_unique indexing scheme - index = [] - healpix_index = healpix_index.array - orders = healpix.uniq2pix(healpix_index, nest=True)[0] - orders = np.unique(orders) - for order in orders: - # For this refinement level, find the HEALPix nested - # indices of the cells that contain the lat-lon points. - nside = healpix.order2nside(order) - pix = healpix.ang2pix(nside, lon, lat, nest=True, lonlat=True) + # Find the HEALPix indices of the cells that contain the + # lat-lon points + nest = indexing_scheme == "nested" + nside = healpix.order2nside(refinement_level) + pix = healpix.ang2pix(nside, lon, lat, nest=nest, lonlat=True) # Remove duplicate indices pix = np.unique(pix) - # Convert back to HEALPix nested_unique indices - pix = healpix._chp.nest2uniq(order, pix, pix) # Find where these HEALPix indices are located in the # healpix_index coordinates - index.append(da.where(da.isin(healpix_index, pix))[0]) + index = da.where(da.isin(healpix_index, pix))[0] - index = da.unique(da.concatenate(index, axis=0)) - elif indexing_scheme in ("nested", "ring"): - # nested or ring indexing scheme - refinement_level = hp.get("refinement_level") - if refinement_level is None: + case None: raise ValueError( - f"Can't locate HEALPix cells for {f!r}: refinement_level " - "has not been set in the healpix grid mapping coordinate " + f"Can't locate HEALPix cells for {f!r}: indexing_scheme has " + "not been set in the healpix grid mapping coordinate " "reference" ) - # Find the HEALPix indices of the cells that contain the - # lat-lon points - nest = indexing_scheme == "nested" - nside = healpix.order2nside(refinement_level) - pix = healpix.ang2pix(nside, lon, lat, nest=nest, lonlat=True) - # Remove duplicate indices - pix = np.unique(pix) - # Find where these HEALPix indices are located in the - # healpix_index coordinates - index = da.where(da.isin(healpix_index, pix))[0] - else: - raise ValueError( - f"Can't locate HEALPix cells for {f!r}: indexing_scheme in the " - "healpix grid mapping coordinate reference must be one of " - f"{HEALPix_indexing_schemes!r}. Got {indexing_scheme!r}" - ) + case _: + raise ValueError( + f"Can't locate HEALPix cells for {f!r}: indexing_scheme in " + "the healpix grid mapping coordinate reference must be one " + f"of {HEALPix_indexing_schemes!r}. Got {indexing_scheme!r}" + ) # Return the cell locations as a numpy array of element indices return index.compute() @@ -333,8 +340,8 @@ def _healpix_locate(lat, lon, f): def del_healpix_coordinate_reference(f): """Remove a healpix grid mapping coordinate reference construct. - A new latitude_longitude grid mapping coordinate reference will be - created in-place, if required, to store any generic coordinate + If required, a new latitude_longitude grid mapping coordinate + reference will be created in-place to store any generic coordinate conversion or datum parameters found in the healpix grid mapping coordinate reference. @@ -498,7 +505,7 @@ def healpix_max_refinement_level(): except ImportError as e: raise ImportError( f"{e}. Must install healpix (https://pypi.org/project/healpix) " - "to find the HEALPix maximum refinement level" + "to find the maximum HEALPix refinement level" ) return healpix.nside2order(healpix._chp.NSIDE_MAX) diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index dc4ba1ec39..c0763e9bce 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -2259,7 +2259,8 @@ def healpix_to_ugrid(self, inplace=False): Auxiliary coords: latitude(ncdim%cell(48)) = [19.47122063449069, ..., -19.47122063449069] degrees_north : longitude(ncdim%cell(48)) = [45.0, ..., 315.0] degrees_east Coord references: grid_mapping_name:latitude_longitude - Topologies : cell:face(ncdim%cell(48), 4) = [[965, ..., 3074]] + Topologies : cell:face(ncdim%cell(48), 4) = [[774, ..., 3267]] + """ from ..healpix import del_healpix_coordinate_reference @@ -2269,8 +2270,8 @@ def healpix_to_ugrid(self, inplace=False): axis = hp.get("domain_axis_key") if axis is None: raise ValueError( - "Can't convert HEALPix to UGRID: There is no HEALPix domain " - "axis" + f"Can't convert {self!r} from HEALPix to UGRID: There is no " + "HEALPix domain axis" ) f = _inplace_enabled_define_and_cleanup(self) @@ -2281,7 +2282,7 @@ def healpix_to_ugrid(self, inplace=False): # that the north (south) polar vertex comes out as a single # node in the domain topology. f.create_latlon_coordinates( - one_d=True, two_d=False, pole_longitude=0, inplace=True + two_d=False, pole_longitude=0, inplace=True ) x_key, x = f.auxiliary_coordinate( @@ -2300,28 +2301,28 @@ def healpix_to_ugrid(self, inplace=False): ) if x is None: raise ValueError( - "Can't convert HEALPix to UGRID: Not able to find (or " - "create) longitude coordinates" + f"Can't convert {f!r} from HEALPix to UGRID: Not able to " + "find (nor create) longitude coordinates" ) if y is None: raise ValueError( - "Can't convert HEALPix to UGRID: Not able to find (or " - "create) latitude coordinates" + f"Can't convert {f!r} from HEALPix to UGRID: Not able to " + "find (or create) latitude coordinates" ) bounds_y = y.get_bounds(None) bounds_x = x.get_bounds(None) if bounds_y is None: raise ValueError( - "Can't convert HEALPix to UGRID: No latitude coordinate " - "bounds" + f"Can't convert {f!r} from HEALPix to UGRID: No latitude " + "coordinate bounds" ) if bounds_x is None: raise ValueError( - "Can't convert HEALPix to UGRID: No longitude coordinate " - "bounds" + f"Can't convert {f!r} from HEALPix to UGRID: No longitude " + "coordinate bounds" ) # Create the Domain Topology construct, by creating a unique diff --git a/docs/source/class/cf.Data.rst b/docs/source/class/cf.Data.rst index b89f262de9..bd4503806b 100644 --- a/docs/source/class/cf.Data.rst +++ b/docs/source/class/cf.Data.rst @@ -134,6 +134,7 @@ Changing data shape ~cf.Data.flatten ~cf.Data.reshape + ~cf.Data.coarsen Transpose-like operations ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/class/cf.Domain.rst b/docs/source/class/cf.Domain.rst index 7f75c0b3ac..9b2a898185 100644 --- a/docs/source/class/cf.Domain.rst +++ b/docs/source/class/cf.Domain.rst @@ -180,7 +180,9 @@ Miscellaneous ~cf.Domain.apply_masking ~cf.Domain.climatological_time_axes ~cf.Domain.copy + ~cf.Domain.create_latlon_coordinates ~cf.Domain.create_regular + ~cf.Domain.create_healpix ~cf.Domain.creation_commands ~cf.Domain.equals ~cf.Domain.fromconstructs @@ -191,7 +193,14 @@ Miscellaneous ~cf.Domain.get_original_filenames ~cf.Domain.close ~cf.Domain.persist - ~cf.Domain.uncompress + ~cf.Domain.uncompresshas_bounds + ~cf.Domain.has_data + ~cf.Domain.has_geometry + ~cf.Domain.apply_masking + ~cf.Domain.get_original_filenames + ~cf.Domain.close + ~cf.Domain.persist + ~cf.Domain.radius Domain axes ----------- @@ -221,6 +230,19 @@ Subspacing ~cf.Domain.indices ~cf.Domain.subspace +HEALPix grids +------------- + +.. autosummary:: + :nosignatures: + :toctree: ../method/ + :template: method.rst + + ~cf.Domain.healpix_info + ~cf.Domain.healpix_indexing_scheme + ~cf.Domain.healpix_to_ugrid + ~cf.Domain.create_healpix + NetCDF ------ diff --git a/docs/source/class/cf.Field.rst b/docs/source/class/cf.Field.rst index 1c307f362e..ede916be8d 100644 --- a/docs/source/class/cf.Field.rst +++ b/docs/source/class/cf.Field.rst @@ -386,6 +386,7 @@ Miscellaneous :template: method.rst ~cf.Field.copy + ~cf.Field.create_latlon_coordinates ~cf.Field.compute_vertical_coordinates ~cf.Field.dataset_compliance ~cf.Field.equals @@ -713,6 +714,22 @@ Regridding operations ~cf.Field.regridc ~cf.Field.regrids + ~cf.Field.healpix_decrease_refinement_level + ~cf.Field.healpix_increase_refinement_level + +HEALPix grids +------------- + +.. autosummary:: + :nosignatures: + :toctree: ../method/ + :template: method.rst + + ~cf.Field.healpix_info + ~cf.Field.healpix_indexing_scheme + ~cf.Field.healpix_to_ugrid + ~cf.Field.healpix_decrease_refinement_level + ~cf.Field.healpix_increase_refinement_level Date-time operations -------------------- From 5d4ffc7261484fe19e77b1baf3a2317c984b5894 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 31 Jul 2025 17:44:31 +0100 Subject: [PATCH 41/59] dev --- cf/field.py | 56 +++++++++++++++++++++-------------------- cf/functions.py | 20 +++++++-------- cf/mixin/fielddomain.py | 12 +++------ cf/test/test_Field.py | 6 +++++ 4 files changed, 48 insertions(+), 46 deletions(-) diff --git a/cf/field.py b/cf/field.py index a72af7d647..2ca4dfe5b3 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4824,9 +4824,9 @@ def healpix_decrease_refinement_level( """Decrease the refinement level of a HEALPix grid. Decreasing the refinement level reduces the resolution of the - HEALPix grid by combining, using the *reduction* function, - data from the original cells that lie inside each larger cell - at the new lower refinement level. + HEALPix grid by combining, using the *reduction* function, the + Field data from the original cells that lie inside each larger + cell at the new lower refinement level. The operation requires that each larger cell at the lower refinement level either contains no original cells (in which @@ -4864,15 +4864,16 @@ def healpix_decrease_refinement_level( cells. *Example:* - For an intensive field quantity (that does not - depend on the size of the cells, such as + For an intensive field quantity (i.e. one that does + not depend on the size of the cells, such as "sea_ice_amount" with units of kg m-2), `np.mean` might be appropriate. *Example:* - For an extensive field quantity (that depends on the - size of the cells, such as "sea_ice_mass" with units - of kg), `np.sum` might be appropriate. + For an extensive field quantity (i.e. one that + depends on the size of the cells, such as + "sea_ice_mass" with units of kg), `np.sum` might be + appropriate. conform: `bool`, optional If True (the default) the HEALPix grid is @@ -4998,8 +4999,8 @@ def healpix_decrease_refinement_level( or level > refinement_level ): raise ValueError( - f"Can't decrease refinement level of {f!r}: " - "'level' keyword must be a non-negative integer less than " + f"Can't decrease HEALPix refinement level of {f!r}: " + "'level' must be a non-negative integer less than " "or equal to the current refinement level of " f"{refinement_level}. Got {level!r}" ) @@ -5009,7 +5010,7 @@ def healpix_decrease_refinement_level( return f # Get the number of cells at the original refinement level - # which are contained in one cell at the coarser refinement + # which are contained in one cell at the lower refinement # level ncells = 4 ** (refinement_level - level) @@ -5133,16 +5134,17 @@ def healpix_increase_refinement_level(self, level, quantity): """Increase the refinement level of a HEALPix grid. Increasing the refinement level increases the resolution of - the HEALPix grid by broadcasting data from each original cell - to all of the new smaller cells at the new higher refinement - level lie inside it. - - For an extensive field quantity (that depends on the size of - the cells, such as "sea_ice_mass" with units of kg), the - broadcast values are reduced to be consistent with the new - smaller cell areas. For an intensive field quantity (that does - not depend on the size of the cells, such as "sea_ice_amount" - with units of kg m-2), the broadcast values are not changed. + the HEALPix grid by broadcasting the Field data from each + original cell to all of the new smaller cells at the new + higher refinement level that lie inside it. + + For an extensive field quantity (i.e. one that depends on the + size of the cells, such as "sea_ice_mass" with units of kg), + the broadcast values are reduced to be consistent with the new + smaller cell areas. For an intensive field quantity (i.e. one + that does not depend on the size of the cells, such as + "sea_ice_amount" with units of kg m-2), the broadcast values + are not changed. K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution @@ -5249,10 +5251,10 @@ def healpix_increase_refinement_level(self, level, quantity): or level > healpix_max_refinement_level() ): raise ValueError( - f"Can't increase refinement level of {f!r}: " - "'level' keyword must be an integer greater than or equal " - f"to the current refinement level of {refinement_level}, and " - f"less than or equal to {healpix_max_refinement_level()}. " + f"Can't increase HEALPix refinement level of {f!r}: " + "'level' must be an integer greater than or equal to the " + f"current refinement level of {refinement_level}, and less " + f"than or equal to {healpix_max_refinement_level()}. " f"Got {level!r}" ) @@ -5264,12 +5266,12 @@ def healpix_increase_refinement_level(self, level, quantity): valid_quantities = ("intensive", "extensive") if quantity not in valid_quantities: raise ValueError( - f"Can't increase refinement level of {f!r}: " + f"Can't increase HEALPix refinement level of {f!r}: " f"'quantity' keyword must be one of {valid_quantities}. " f"Got {quantity!r}" ) - # Get the number of cells at the new refinement level which + # Get the number of cells at the higher refinement level which # are contained in one cell at the original refinement level ncells = 4 ** (level - refinement_level) diff --git a/cf/functions.py b/cf/functions.py index 00172c0a54..e21ab4dfea 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -3324,8 +3324,8 @@ def locate(lat, lon, f=None): """Locate cells containing latitude-longitude locations. The cells must be defined by a discrete axis that has 1-d latitude - and longitude coordinate constructs, or for which it is possible - to create 1-d latitude and longitude coordinate constructs (as is + and longitude coordinate, or for which it is possible to create + 1-d latitude and longitude coordinates from other metadata (as is the case for a HEALPix axis). At present, only a HEALPix axis is supported. @@ -3360,15 +3360,13 @@ def locate(lat, lon, f=None): grid. If `None` (the default) then a callable function is - returned, which when called with an argument of *f* - returns the indices, i.e ``cf.contains(lat, lon, f)`` is - equivalent to ``cf.contains(lat, lon)(f)``. This new - function may be used as a condition in the `subspace` and - `indices` methods of *f*, since such conditions may be - functions that take the calling Field or Domain construct - as an argument. For instance, ``f.subspace(X=cf.locate(0, - 45)`` is equivalent to ``f.subspace(X=cf.locate(0, 45, - f)``. + returned, which when called with A Field as its argument + returns the indices for that field, i.e ``cf.contains(lat, + lon, f)`` is equivalent to ``cf.contains(lat, + lon)(f)``. This returned function may be used as a + condition in a Field's `subspace` and `indices` methods, + since, for instance, ``f.subspace(X=cf.locate(0, 45))`` is + equivalent to ``f.subspace(X=cf.locate(0, 45, f))``. `numpy.ndarray` or function Indices for the discrete axis that contain the diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index c0763e9bce..25c3532559 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -1970,10 +1970,6 @@ def coordinate_reference_domain_axes(self, identity=None): def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): """Change the indexing scheme of HEALPix indices. - The data are not reordered, only the "healpix_index" - coordinate values are changed, along with parameters of the - "healpix" grid mapping Coordinate reference. - K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution Discretization and Fast Analysis of Data Distributed on the @@ -1998,10 +1994,10 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): {{HEALPix indexing schemes}} sort: `bool`, optional - If True then sort the HEALPix axis of the output so - that its HEALPix indices are monotonically increasing, - including when the indexing scheme is unchanged. If - False (the default) then don't do this. + If True then re-order the HEALPix axis of the output + so that its HEALPix indices are monotonically + increasing, including when the indexing scheme is + unchanged. If False (the default) then don't do this. :Returns: diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index 19c7b9fb95..6afcd480d5 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -3295,6 +3295,12 @@ def test_Field_healpix_subspace(self): g = f.subspace(healpix_index=cf.locate(20, [1, 46])) self.assertTrue(np.array_equal(g.coordinate("healpix_index"), [0, 19])) + f = f.healpix_indexing_scheme("ring") + g = f.subspace(healpix_index=cf.locate(20, [1, 46])) + self.assertTrue( + np.array_equal(g.coordinate("healpix_index"), [13, 12]) + ) + def test_Field_healpix_decrease_refinement_level(self): """Test Field.healpix_decrease_refinement_level.""" f = self.f12 From fb4062c513165698062412bc74590f1c11e21c3c Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 1 Aug 2025 15:16:06 +0100 Subject: [PATCH 42/59] dev --- cf/constants.py | 25 ++++ cf/docstring/docstring.py | 19 ++- cf/domain.py | 6 +- cf/field.py | 308 ++++++++++++++++++++++++++++---------- cf/test/test_Field.py | 60 +++++++- 5 files changed, 321 insertions(+), 97 deletions(-) diff --git a/cf/constants.py b/cf/constants.py index aa2bfd0fcd..14c4212f5c 100644 --- a/cf/constants.py +++ b/cf/constants.py @@ -557,6 +557,31 @@ }, } +# -------------------------------------------------------------------- +# CF cell methods +# -------------------------------------------------------------------- +cell_methods = set( + ( + "point", + "sum", + "maximum", + "maximum_absolute_value", + "median", + "mid_range", + "minimum", + "minimum_absolute_value", + "mean", + "mean_absolute_value", + "mean_of_upper_decile", + "mode", + "range", + "root_mean_square", + "standard_deviation", + "sum_of_squares", + "variance", + ) +) + # -------------------------------------------------------------------- # Logging level setup diff --git a/cf/docstring/docstring.py b/cf/docstring/docstring.py index 9bcbe1d44a..25c72539da 100644 --- a/cf/docstring/docstring.py +++ b/cf/docstring/docstring.py @@ -578,14 +578,17 @@ geographical nearest-neighbour operations such as decreasing the refinement level. - For a Multi-Order Coverage (MOC), where pixels with - different refinement levels are stored in the same - array, the indexing scheme has a unique index for each - cell at each refinement level. The "nested_unique" - scheme for an MOC indexes pixels within each - refinement level with the nested scheme, but - geographically close pixels at different refinement - levels do not have close indices.""", + A Multi-Order Coverage (MOC) has pixels with different + refinement levels stored in the same array. The + "nested_unique" scheme for an MOC has a unique index + for each cell at each refinement level, such that + within each refinement level a nested-type scheme is + employed. In the "nested_unique" scheme, pixels at + different refinement levels inside a single coarser + refinement level cell can have widely different + indices. For each refinment level *n*, the + "nested_unique" indices are in the range + :math:`4^{(n+1)}, ..., 4^{(n+2)}-1`.""", # ---------------------------------------------------------------- # Method description substitutions (4 levels of indentation) # ---------------------------------------------------------------- diff --git a/cf/domain.py b/cf/domain.py index 2a9d521a09..8225efd31f 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -278,6 +278,10 @@ def create_healpix( Sphere. The Astrophysical Journal, 2005, 622 (2), pp.759-771. https://dx.doi.org/10.1086/427976 + M. Reinecke and E. Hivon: Efficient data structures for masks + on 2D grids. A&A, 580 (2015) + A132. https://doi.org/10.1051/0004-6361/201526549 + .. versionadded:: NEXTVERSION .. seealso:: `cf.Domain.create_regular`, @@ -339,7 +343,7 @@ def create_healpix( Coordinate conversion:refinement_level = 4 Auxiliary Coordinate: healpix_index - >>> d = cf.Domain.create_healpix(8, "nested_unique", radius=6371000) + >>> d = cf.Domain.create_healpix(4, "nested_unique", radius=6371000) >>> d.dump() -------- Domain: diff --git a/cf/field.py b/cf/field.py index 2ca4dfe5b3..84ddac62c0 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4819,24 +4819,40 @@ def bin( return out def healpix_decrease_refinement_level( - self, level, reduction, conform=True, check_healpix_index=True + self, + level, + method, + reduction=None, + conform=True, + check_healpix_index=True, ): - """Decrease the refinement level of a HEALPix grid. + r"""Decrease the refinement level of a HEALPix grid. Decreasing the refinement level reduces the resolution of the - HEALPix grid by combining, using the *reduction* function, the - Field data from the original cells that lie inside each larger - cell at the new lower refinement level. - - The operation requires that each larger cell at the lower - refinement level either contains no original cells (in which - case that new cell is not included in the output), or is - completely covered by original cells. It is not allowed for a - larger cell to be only partially covered by original - cells. For instance, if the original refinement level is 10 - and the new refinement level is 8, then each output cell will - be the combination of :math:`16\equiv 4^(10-8)`) original - cells. + HEALPix grid by combining, using the given *method*, the Field + data from the original cells that lie inside each larger cell + at the new lower refinement level. + + A new cell method is added, when appropriate, to describe the + reduction. + + The operation requires that a new larger cell at the lower + refinement level either + + * contains no original cells, in which case that larger cell + is not included in the output, + + or + + * is completely covered by original cells. + + It is not allowed for a larger cell to be only partially + covered by original cells. For instance, if the original + refinement level is 10 and the new refinement level is 8, then + each output cell will be the combination of :math:`16\equiv + 4^(10-8)`) original cells, and if a larger cell contains at + least one but fewer than 16 original cells then an exception + is raised (assuming that *check_healpix_index* is True). K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution @@ -4858,64 +4874,92 @@ def healpix_decrease_refinement_level( refinement level, or if `None` then the refinement level is not changed. - reduction: function - The function used to calculate the values in the new - coarser cells, from the data on the original finer - cells. + method: `str` + The method used to calculate the values in the new + larger cells, from the data on the original + cells. Must be one of the CF standardised cell + methods: ``'maximum'``, ``'maximum_absolute_value'``, + ``'mean'``, ``'mean_absolute_value'``, + ``'mean_of_upper_decile'``, ``'median'``, + ``'mid_range'``, ``'minimum'``, + ``'minimum_absolute_value'``, ``'mode'``, ``'range'``, + ``'root_mean_square'``, ``'standard_deviation'``, + ``'sum'``, ``'sum_of_squares'``, ``'variance'``. *Example:* For an intensive field quantity (i.e. one that does not depend on the size of the cells, such as - "sea_ice_amount" with units of kg m-2), `np.mean` - might be appropriate. + "sea_ice_amount" with units of kg m-2), a method of + ``'mean'`` might be appropriate. *Example:* For an extensive field quantity (i.e. one that depends on the size of the cells, such as - "sea_ice_mass" with units of kg), `np.sum` might be - appropriate. + "sea_ice_mass" with units of kg), a method of + ``'sum'`` might be appropriate. + + reduction: function or `None`, optional + The function used to calculate the values in the new + larger cells, from the data on the original cells. The + function must calculate the quantity defined by the + *method* parameter, take an array of values as its + first argument, and have an *axis* keyword that + specifies which axis of the array is the HEALPix axis. + + For some methods there are default *reduction* + functions, which are used when *reduction* is `None` + (the default): + + ======================== =================== + *method* Default *reduction* + ======================== =================== + ``'maximum'`` `np.max` + ``'mean'`` `np.mean` + ``'median'`` `np.median` + ``'minimum'`` `np.min` + ``'standard_deviation'`` `np.std` + ``'sum'`` `np.sum` + ``'variance'`` `np.var` + ======================== =================== conform: `bool`, optional - If True (the default) the HEALPix grid is + If True (the default) then the HEALPix grid is automatically converted to a form suitable for having its refinement level changed, i.e. the indexing scheme is changed to 'nested' and the HEALPix axis is sorted so that the nested HEALPix indices are monotonically - increasing. If False then either an exeption is raised - if the HEALPix indexing scheme is not already + increasing. If False then either an exception is + raised if the HEALPix indexing scheme is not already 'nested', or else the HEALPix axis is not sorted. .. note:: Setting to False will speed up the operation when the HEALPix indexing scheme is already nested and the HEALPix axis is already - sorted montonically. + sorted monotonically. check_healpix_index: `bool`, optional If True (the default) then the following conditions will be checked before the creation of the new Field - (but after the HEALPix grid has been conformed, when + (and after the HEALPix grid has been conformed, when *conform* is True): 1. The nested HEALPix indices are strictly monotonically increasing. 2. Every cell at the new lower refinement level - contains the maximum possible number of cells at - the original refinement level. - - If True and any of these conditions is not met, then an - exception is raised. + contains zero or the maximum possible number of + cells at the original refinement level. If False then these checks are not carried out. .. warning:: Only set to False, which will speed up - the operation, if it is known in advance - that these conditions are already met. If - set to False and any of the conditions is - not met then either an exception will be - raised or, much worse, the operation will - complete and return incorrect data - values. + the operation, when it is known in + advance that these conditions are already + met. If set to False and any of the + conditions are not met then either an + exception will be raised or, much worse, + the operation could complete and return + incorrect data values. :Returns: @@ -4935,20 +4979,89 @@ def healpix_decrease_refinement_level( that lie in one cell of the lower refinement leve1) in the orginal field correspond to one cell at the lower level: - >>> g = f.healpix_decrease_refinement_level(0, np.mean) - + >>> g = f.healpix_decrease_refinement_level(0, 'maximum') + >>> print(g) + Field: air_temperature (ncvar%tas) + ---------------------------------- + Data : air_temperature(time(2), healpix_index(12)) K + Cell methods : time(2): mean area: mean area: maximum + Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian + : height(1) = [1.5] m + Auxiliary coords: healpix_index(healpix_index(12)) = [0, ..., 11] 1 + Coord references: grid_mapping_name:healpix >>> g.healpix_info()['refinement_level'] 0 - >>> np.mean(g.array[0, 0]) - np.float64(289.15) - >>> f.healpix_info()['refinement_level'] - 1 - >>> np.mean(f.array[0, :4]) - np.float64(289.15) + >>> print(f[0, :4].array) + [[291.5 293.5 285.3 286.3]] + >>> print(g[0, 0].array) + [[293.5]] + + >>> import numpy as np + >>> def range_func(a, axis=None): + ... return np.max(a, axis=axis) - np.min(a, axis=axis) + ... + >>> g = f.healpix_decrease_refinement_level(0, 'range', range_func) + >>> print(g) + Field: air_temperature (ncvar%tas) + ---------------------------------- + Data : air_temperature(time(2), healpix_index(12)) K + Cell methods : time(2): mean area: mean area: range + Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian + : height(1) = [1.5] m + Auxiliary coords: healpix_index(healpix_index(12)) = [0, ..., 11] 1 + Coord references: grid_mapping_name:healpix + >>> print(g[0, 0].array) + [[8.2]] """ + from .constants import cell_methods + + # "point" is not a valid cell method after a reduction + cell_methods = cell_methods.copy() + cell_methods.remove("point") + f = self.copy() + # Parse 'method' + if method not in cell_methods: + raise ValueError( + f"Can't decrease HEALPix refinement level of {f!r}: " + "'method' is not one of the standardised CF cell methods " + f"{sorted(cell_methods)}. Got: {method!r}" + ) + + # Parse 'reduction' + if reduction is None: + # Infer 'reduction' function from 'method' + match method: + case "maximum": + reduction = np.max + case "minimum": + reduction = np.min + case "mean": + reduction = np.mean + case "standard_deviation": + reduction = np.std + case "variance": + reduction = np.var + case "sum": + reduction = np.sum + case "median": + reduction = np.median + case _: + raise ValueError( + f"Can't decrease HEALPix refinement level of {f!r}: " + "Must provide a 'reduction' function for " + f"method={method!r}" + ) + + elif not callable(reduction): + raise ValueError( + f"Can't decrease HEALPix refinement level of {f!r}: " + f"'reduction' must be callable. Got: {reduction!r} of type " + f"{type(reduction)}" + ) + # Get the HEALPix info hp = f.healpix_info() @@ -5036,10 +5149,9 @@ def healpix_decrease_refinement_level( ).any(): raise ValueError( f"Can't decrease HEALPix refinement level of {f!r}: " - "At least one cell at the new coarser refinement level " - f"({level}) contains fewer than {ncells} " - "cells at the original finer refinement level " - f"({refinement_level})" + "At least one cell at the new lower refinement level " + f"({level}) contains fewer than {ncells} cells at the " + f"original refinement level ({refinement_level})" ) # Get the HEALPix axis @@ -5059,12 +5171,13 @@ def healpix_decrease_refinement_level( ) ) - # Coarsen the field data using the 'reduction' function. - # - # Note: Using 'Data.coarsen' only works because a) we have - # 'nested' HEALPix ordering, and b) each coarser cell - # contains the maximum possible number of original - # cells. + # ------------------------------------------------------------ + # Chenge the refinement level of the Field's data + # ------------------------------------------------------------ + + # Note: Using 'Data.coarsen' works because a) we have 'nested' + # HEALPix ordering, and b) each coarser cell contains + # the maximum possible number of original cells. f.data.coarsen( reduction, axes={iaxis: ncells}, trim_excess=False, inplace=True ) @@ -5073,6 +5186,10 @@ def healpix_decrease_refinement_level( domain_axis = f.domain_axis(axis) domain_axis.set_size(f.shape[iaxis]) + # ------------------------------------------------------------ + # Change the refinement level of the Field's metadata + # ------------------------------------------------------------ + # Coarsen the domain ancillary constructs that span the # HEALPix axis. We're assuming that domain ancillary data are # intensive (i.e. do not depend on the size of the cell), so @@ -5080,9 +5197,9 @@ def healpix_decrease_refinement_level( for key, domain_ancillary in f.domain_ancillaries( filter_by_axis=(axis,), axis_mode="and", todict=True ).items(): - i = f.get_data_axes(key).index(axis) + iaxis = f.get_data_axes(key).index(axis) domain_ancillary.data.coarsen( - np.mean, axes={i: ncells}, trim_excess=False, inplace=True + np.mean, axes={iaxis: ncells}, trim_excess=False, inplace=True ) # Coarsen the cell measure constructs that span the HEALPix @@ -5092,14 +5209,14 @@ def healpix_decrease_refinement_level( for key, cell_measure in f.cell_measures( filter_by_axis=(axis,), axis_mode="and", todict=True ).items(): - i = f.get_data_axes(key).index(axis) + iaxis = f.get_data_axes(key).index(axis) cell_measure.data.coarsen( - np.sum, axes={i: ncells}, trim_excess=False, inplace=True + np.sum, axes={iaxis: ncells}, trim_excess=False, inplace=True ) # Remove all other metadata constructs that span the HEALPix - # axis, including the original healpix_index coordinate - # construct. + # axis, including the original healpix_index coordinates and + # any lat/lon coordinates. for key in ( f.constructs.filter_by_axis(axis, axis_mode="and") .filter_by_type("cell_measure", "domain_ancillary") @@ -5108,8 +5225,9 @@ def healpix_decrease_refinement_level( ): f.del_construct(key) - # Create the healpix_index coordinates for the new refinement - # level + # ------------------------------------------------------------ + # Change the refinement level of the healpix_index coordinates + # ------------------------------------------------------------ if healpix_index.construct_type == "dimension_coordinate": # Ensure that healpix indices are auxiliary coordinates healpix_index = f._AuxiliaryCoordinate( @@ -5119,13 +5237,26 @@ def healpix_decrease_refinement_level( healpix_index = healpix_index[::ncells] // ncells hp_key = f.set_construct(healpix_index, axes=axis, copy=False) + # ------------------------------------------------------------ # Update the healpix Coordinate Reference + # ------------------------------------------------------------ cr = hp.get("grid_mapping_name:healpix") cr.coordinate_conversion.set_parameter("refinement_level", level) cr.set_coordinate(hp_key) + # ------------------------------------------------------------ + # Update the cell methods + # ------------------------------------------------------------ + f._update_cell_methods( + method=method, + input_axes=("area",), + domain_axes=f.domain_axes(axis, todict=True), + ) + + # ------------------------------------------------------------ + # Create lat/lon coordinates for the new refinement level + # ------------------------------------------------------------ if create_latlon: - # Create lat/lon coordinates for the new refinement level f.create_latlon_coordinates(two_d=False, inplace=True) return f @@ -5140,11 +5271,11 @@ def healpix_increase_refinement_level(self, level, quantity): For an extensive field quantity (i.e. one that depends on the size of the cells, such as "sea_ice_mass" with units of kg), - the broadcast values are reduced to be consistent with the new - smaller cell areas. For an intensive field quantity (i.e. one - that does not depend on the size of the cells, such as - "sea_ice_amount" with units of kg m-2), the broadcast values - are not changed. + the broadcast values are also reduced to be consistent with + the new smaller cell areas. For an intensive field quantity + (i.e. one that does not depend on the size of the cells, such + as "sea_ice_amount" with units of kg m-2), the broadcast + values are not changed. K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution @@ -5166,7 +5297,7 @@ def healpix_increase_refinement_level(self, level, quantity): or if `None` then the refinement level is not changed. quantity: `str` - Whether the data values represent intensive or + Whether the Field data represent intensive or extensive quantities, specified with ``'intensive'`` and ``'extensive'`` respectively. @@ -5207,12 +5338,12 @@ def healpix_increase_refinement_level(self, level, quantity): >>> print(g[0, :8] .array) [[291.5 291.5 291.5 291.5 293.5 293.5 293.5 293.5]] - For an extensive quantity (which ``f`` is not, but we can - assume that it is for demonstration purposes), every four - cells at the higher refinement level have the value of a - single cell at the original refinement level after dividing it - by the number of cells at the higher refinement level that lie - in one cell of the original refinement level (4 in this case): + For an extensive quantity (which ``f`` is not in this example, + but we can assume that it is for demonstration purposes), each + cell at the higher refinement level has the value of a cell at + the original refinement level after dividing it by the number + of cells at the higher refinement level that lie in one cell + of the original refinement level (4 in this case): >>> g = f.healpix_increase_refinement_level(2, 'extensive') >>> print(f[0, :2] .array) @@ -5404,7 +5535,7 @@ def healpix_increase_refinement_level(self, level, quantity): f.del_construct(key) # Create the healpix_index coordinates for the new refinement - # level + # level, and put them into the Field. healpix_index = hp["healpix_index"] if healpix_index.construct_type == "dimension_coordinate": # Ensure that healpix indices are auxiliary coordinates @@ -5412,7 +5543,11 @@ def healpix_increase_refinement_level(self, level, quantity): source=healpix_index, copy=False ) - dx = healpix_index.data.to_dask_array( + # Save any cached data elements + data = healpix_index.data + cached = data._get_cached_elements().copy() + + dx = data.to_dask_array( _force_mask_hardness=False, _force_to_memory=False ) @@ -5434,6 +5569,17 @@ def healpix_increase_refinement_level(self, level, quantity): ) healpix_index.set_data(dx, copy=False) + + # Set new cached data elements + data = healpix_index.data + if 0 in cached: + x = np.array(cached[0], dtype=dtype) * ncells + data._set_cached_elements({0: x, 1: x + 1}) + + if -1 in cached: + x = np.array(cached[-1], dtype=dtype) * ncells + (ncells - 1) + data._set_cached_elements({-1: x}) + hp_key = f.set_construct(healpix_index, axes=axis, copy=False) # Update the healpix Coordinate Reference diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index 6afcd480d5..1719f2d82a 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -3304,23 +3304,56 @@ def test_Field_healpix_subspace(self): def test_Field_healpix_decrease_refinement_level(self): """Test Field.healpix_decrease_refinement_level.""" f = self.f12 - g = f.healpix_decrease_refinement_level(0, np.mean) + + g = f.healpix_decrease_refinement_level(0, "mean") self.assertTrue( np.array_equal(g.coord("healpix_index"), np.arange(12)) ) + for method, first_value in zip( + ( + "maximum", + "mean", + "minimum", + "standard_deviation", + "variance", + "sum", + "median", + ), + (293.5, 289.15, 285.3, 3.44201976, 11.8475, 1156.6, 288.9), + ): + g = f.healpix_decrease_refinement_level(0, method) + self.assertTrue(np.allclose(g[0, 0], first_value)) + + # Bad methods + for method in ("point", "range", "bad method", 3.14): + with self.assertRaises(ValueError): + f.healpix_decrease_refinement_level(0, method) + + def range_func(a, axis=None): + return np.max(a, axis=axis) - np.min(a, axis=axis) + + g = f.healpix_decrease_refinement_level(0, "range", range_func) + self.assertTrue(np.allclose(g[0, 0], 8.2)) + + def my_mean(a, axis=None): + return np.mean(a, axis=axis) + + g = f.healpix_decrease_refinement_level(0, "mean", my_mean) + self.assertTrue(np.allclose(g[0, 0], 289.15)) + f = f.healpix_indexing_scheme("ring") - g = f.healpix_decrease_refinement_level(0, np.mean) + g = f.healpix_decrease_refinement_level(0, "maximum") self.assertTrue( np.array_equal(g.coord("healpix_index"), np.arange(12)) ) with self.assertRaises(ValueError): - f.healpix_decrease_refinement_level(0, np.mean, conform=False) + f.healpix_decrease_refinement_level(0, "maximum", conform=False) f = f.healpix_indexing_scheme("ring", sort=True) f = f.healpix_indexing_scheme("nested") - g = f.healpix_decrease_refinement_level(0, np.mean, conform=True) + g = f.healpix_decrease_refinement_level(0, "mean", conform=True) self.assertTrue( np.array_equal(g.coord("healpix_index"), np.arange(12)) ) @@ -3328,7 +3361,7 @@ def test_Field_healpix_decrease_refinement_level(self): # Check that lat/lon coords get created when they're present # in the original field f = f.create_latlon_coordinates() - g = f.healpix_decrease_refinement_level(0, np.mean) + g = f.healpix_decrease_refinement_level(0, "mean") self.assertEqual(g.auxiliary_coordinate("latitude"), (12,)) self.assertEqual(g.auxiliary_coordinate("longitude"), (12,)) @@ -3337,7 +3370,7 @@ def test_Field_healpix_decrease_refinement_level(self): # Bad results when check_healpix_index=False h = f.healpix_decrease_refinement_level( - 0, np.mean, conform=False, check_healpix_index=False + 0, "mean", conform=False, check_healpix_index=False ) self.assertFalse( np.array_equal(h.coord("healpix_index"), np.arange(12)) @@ -3371,7 +3404,7 @@ def test_Field_healpix_increase_refinement_level(self): self.assertEqual(g.array.shape, (2, 192)) # Check selected data values for intensive and extensive - # increases + # quantities n = 4 ** (2 - 1) for i in (0, 1, 24, 46, 47): self.assertTrue( @@ -3384,6 +3417,19 @@ def test_Field_healpix_increase_refinement_level(self): np.allclose(g[:, i * n : (i + 1) * n], f[:, i : i + 1] / n) ) + # Cached values + f.coordinate("healpix_index").data._del_cached_elements() + g = f.healpix_increase_refinement_level(29, "intensive") + self.assertEqual( + g.coordinate("healpix_index").data._get_cached_elements(), {} + ) + _ = str(f.coordinate("healpix_index").data) # Create cached elements + g = f.healpix_increase_refinement_level(29, "intensive") + self.assertEqual( + g.coordinate("healpix_index").data._get_cached_elements(), + {0: 0, 1: 1, -1: 3458764513820540927}, + ) + # Bad 'quantity' parameter with self.assertRaises(ValueError): f.healpix_increase_refinement_level(2, "bad quantity") From 38cb64d6fd35c4bd685a8671687264b625e934f8 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 1 Aug 2025 19:49:44 +0100 Subject: [PATCH 43/59] dev --- cf/docstring/docstring.py | 14 +++- cf/domain.py | 132 ++++++++++++++++++++------------------ cf/field.py | 91 +++++++++++++------------- cf/mixin/fielddomain.py | 50 +++++---------- cf/test/test_Domain.py | 5 ++ 5 files changed, 150 insertions(+), 142 deletions(-) diff --git a/cf/docstring/docstring.py b/cf/docstring/docstring.py index 25c72539da..4dcfcf1176 100644 --- a/cf/docstring/docstring.py +++ b/cf/docstring/docstring.py @@ -88,7 +88,17 @@ elements will be automatically reduced if including the full amount defined by the halo would extend the subspace beyond the axis limits.""", - # ---------------------------------------------------------------- + # HEALPix references + "{{HEALPix references}}": """K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, + et al.. HEALPix: A Framework for High-Resolution + Discretization and Fast Analysis of Data Distributed on the + Sphere. The Astrophysical Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + + M. Reinecke and E. Hivon: Efficient data structures for masks + on 2D grids. A&A, 580 (2015) + A132. https://doi.org/10.1051/0004-6361/201526549""", + # ---------------------------------------------------------------- # Method description substitutions (3 levels of indentation) # ---------------------------------------------------------------- # i: deprecated at version 3.0.0 @@ -586,7 +596,7 @@ employed. In the "nested_unique" scheme, pixels at different refinement levels inside a single coarser refinement level cell can have widely different - indices. For each refinment level *n*, the + indices. For each refinement level *n*, the "nested_unique" indices are in the range :math:`4^{(n+1)}, ..., 4^{(n+2)}-1`.""", # ---------------------------------------------------------------- diff --git a/cf/domain.py b/cf/domain.py index 8225efd31f..dccc53b5b5 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -267,21 +267,15 @@ def create_regular(cls, x_args, y_args, bounds=True): def create_healpix( cls, refinement_level, indexing_scheme="nested", radius=None ): - """Create a new global HEALPix domain. + r"""Create a new global HEALPix domain. The HEALPix axis of the new Domain is ordered so that the HEALPix indices are monotonically increasing. - K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, - et al.. HEALPix: A Framework for High-Resolution - Discretization and Fast Analysis of Data Distributed on the - Sphere. The Astrophysical Journal, 2005, 622 (2), pp.759-771. - https://dx.doi.org/10.1086/427976 - - M. Reinecke and E. Hivon: Efficient data structures for masks - on 2D grids. A&A, 580 (2015) - A132. https://doi.org/10.1051/0004-6361/201526549 - + **References** + + {{HEALPix references}} + .. versionadded:: NEXTVERSION .. seealso:: `cf.Domain.create_regular`, @@ -293,8 +287,8 @@ def create_healpix( The refinement level of the grid within the HEALPix hierarchy, starting at 0 for the base tessellation with 12 cells. The number of cells in the global - HEALPix grid is :math:`(12 \times - 4^refinement_level)`. + HEALPix grid for refinement level *n* is + :math:`12\times 4^n`. indexing_scheme: `str` The HEALPix indexing scheme. One of ``'nested'`` (the @@ -325,64 +319,78 @@ def create_healpix( **Examples** - >>> d = cf.Domain.create_healpix(4) - >>> d.dump() - -------- - Domain: - -------- - Domain Axis: healpix_index(3072) - - Auxiliary coordinate: healpix_index - standard_name = 'healpix_index' - units = '1' - Data(healpix_index(3072)) = [0, ..., 3071] 1 - - Coordinate reference: grid_mapping_name:healpix - Coordinate conversion:grid_mapping_name = healpix - Coordinate conversion:indexing_scheme = nested - Coordinate conversion:refinement_level = 4 - Auxiliary Coordinate: healpix_index - - >>> d = cf.Domain.create_healpix(4, "nested_unique", radius=6371000) - >>> d.dump() - -------- - Domain: - -------- - Domain Axis: healpix_index(3072) - - Auxiliary coordinate: healpix_index - standard_name = 'healpix_index' - units = '1' - Data(healpix_index(3072)) = [1024, ..., 4095] 1 - - Coordinate reference: grid_mapping_name:healpix - Coordinate conversion:grid_mapping_name = healpix - Coordinate conversion:indexing_scheme = nested_unique - Datum:earth_radius = 6371000.0 - Auxiliary Coordinate: healpix_index - - >>> d.create_latlon_coordinates(inplace=True) - >>> print(d) - Auxiliary coords: healpix_index(ncdim%cell(3072)) = [1024, ..., 4095] 1 - : latitude(ncdim%cell(3072)) = [2.388015463268772, ..., -2.388015463268786] degrees_north - : longitude(ncdim%cell(3072)) = [45.0, ..., 315.0] degrees_east - Coord references: grid_mapping_name:healpix + .. code-block:: python + + >>> d = cf.Domain.create_healpix(4) + >>> d.dump() + -------- + Domain: + -------- + Domain Axis: healpix_index(3072) + + Auxiliary coordinate: healpix_index + standard_name = 'healpix_index' + units = '1' + Data(healpix_index(3072)) = [0, ..., 3071] 1 + + Coordinate reference: grid_mapping_name:healpix + Coordinate conversion:grid_mapping_name = healpix + Coordinate conversion:indexing_scheme = nested + Coordinate conversion:refinement_level = 4 + Auxiliary Coordinate: healpix_index + + .. code-block:: python + + >>> d = cf.Domain.create_healpix(4, "nested_unique", radius=6371000) + >>> d.dump() + -------- + Domain: + -------- + Domain Axis: healpix_index(3072) + + Auxiliary coordinate: healpix_index + standard_name = 'healpix_index' + units = '1' + Data(healpix_index(3072)) = [1024, ..., 4095] 1 + + Coordinate reference: grid_mapping_name:healpix + Coordinate conversion:grid_mapping_name = healpix + Coordinate conversion:indexing_scheme = nested_unique + Datum:earth_radius = 6371000.0 + Auxiliary Coordinate: healpix_index + + .. code-block:: python + + >>> d.create_latlon_coordinates(inplace=True) + >>> print(d) + Auxiliary coords: healpix_index(ncdim%cell(3072)) = [1024, ..., 4095] 1 + : latitude(ncdim%cell(3072)) = [2.388015463268772, ..., -2.388015463268786] degrees_north + : longitude(ncdim%cell(3072)) = [45.0, ..., 315.0] degrees_east + Coord references: grid_mapping_name:healpix """ import dask.array as da - from .healpix import HEALPix_indexing_schemes + from .healpix import ( + HEALPix_indexing_schemes, + healpix_max_refinement_level, + ) - if indexing_scheme not in HEALPix_indexing_schemes: + if ( + not isinstance(refinement_level, Integral) + or refinement_level < 0 + or refinement_level > healpix_max_refinement_level() + ): raise ValueError( - "Can't create HEALPix Domain: 'indexing_scheme' must be one " - f"of {HEALPix_indexing_schemes!r}. Got {indexing_scheme!r}" + "Can't create HEALPix Domain: 'refinement_level' must be a " + "non-negative integer less than or equal to " + f"{healpix_max_refinement_level()}. Got {refinement_level!r}" ) - if not isinstance(refinement_level, Integral) or refinement_level < 0: + if indexing_scheme not in HEALPix_indexing_schemes: raise ValueError( - "Can't create HEALPix Domain: 'refinement_level' must be a " - f"non-negative integer. Got: {refinement_level!r}" + "Can't create HEALPix Domain: 'indexing_scheme' must be one " + f"of {HEALPix_indexing_schemes!r}. Got {indexing_scheme!r}" ) nested_unique = indexing_scheme == "nested_unique" diff --git a/cf/field.py b/cf/field.py index 84ddac62c0..312790dd18 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4820,7 +4820,7 @@ def bin( def healpix_decrease_refinement_level( self, - level, + refinement_level, method, reduction=None, conform=True, @@ -4850,15 +4850,13 @@ def healpix_decrease_refinement_level( covered by original cells. For instance, if the original refinement level is 10 and the new refinement level is 8, then each output cell will be the combination of :math:`16\equiv - 4^(10-8)`) original cells, and if a larger cell contains at + 4^{(10-8)}` original cells, and if a larger cell contains at least one but fewer than 16 original cells then an exception is raised (assuming that *check_healpix_index* is True). - K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, - et al.. HEALPix: A Framework for High-Resolution - Discretization and Fast Analysis of Data Distributed on the - Sphere. The Astrophysical Journal, 2005, 622 (2), pp.759-771. - https://dx.doi.org/10.1086/427976 + **References** + + {{HEALPix references}} .. versionadded:: NEXTVERSION @@ -4868,7 +4866,7 @@ def healpix_decrease_refinement_level( :Parameters: - level: `int` or `None` + refinement_level: `int` or `None` Specify the new lower refinement level as a non-negative integer less than or equal to the current refinement level, or if `None` then the refinement @@ -4954,12 +4952,12 @@ def healpix_decrease_refinement_level( .. warning:: Only set to False, which will speed up the operation, when it is known in - advance that these conditions are already - met. If set to False and any of the + advance that these conditions are + satisfied. If set to False and any of the conditions are not met then either an - exception will be raised or, much worse, - the operation could complete and return - incorrect data values. + exception may be raised or, **much + worse**, the operation could complete and + return incorrect data values. :Returns: @@ -5073,8 +5071,8 @@ def healpix_decrease_refinement_level( "mapping coordinate reference" ) - refinement_level = hp.get("refinement_level") - if refinement_level is None: + old_refinement_level = hp.get("refinement_level") + if old_refinement_level is None: raise ValueError( f"Can't decrease HEALPix refinement level of {f!r}: " "refinement_level has not been set in the healpix grid " @@ -5102,30 +5100,30 @@ def healpix_decrease_refinement_level( ) # Parse 'level' - if level is None: + if refinement_level is None: # No change in refinement level return f if ( - not isinstance(level, Integral) - or level < 0 - or level > refinement_level + not isinstance(refinement_level, Integral) + or refinement_level < 0 + or refinement_level > old_refinement_level ): raise ValueError( f"Can't decrease HEALPix refinement level of {f!r}: " "'level' must be a non-negative integer less than " "or equal to the current refinement level of " - f"{refinement_level}. Got {level!r}" + f"{old_refinement_level}. Got {refinement_level!r}" ) - if level == refinement_level: + if refinement_level == old_refinement_level: # No change in refinement level return f # Get the number of cells at the original refinement level # which are contained in one cell at the lower refinement # level - ncells = 4 ** (refinement_level - level) + ncells = 4 ** (old_refinement_level - refinement_level) # Get the healpix_index coordinates healpix_index = hp.get("healpix_index") @@ -5150,8 +5148,9 @@ def healpix_decrease_refinement_level( raise ValueError( f"Can't decrease HEALPix refinement level of {f!r}: " "At least one cell at the new lower refinement level " - f"({level}) contains fewer than {ncells} cells at the " - f"original refinement level ({refinement_level})" + f"({refinement_level}) contains fewer than {ncells} " + "cells at the original refinement level " + f"({old_refinement_level})" ) # Get the HEALPix axis @@ -5241,7 +5240,9 @@ def healpix_decrease_refinement_level( # Update the healpix Coordinate Reference # ------------------------------------------------------------ cr = hp.get("grid_mapping_name:healpix") - cr.coordinate_conversion.set_parameter("refinement_level", level) + cr.coordinate_conversion.set_parameter( + "refinement_level", refinement_level + ) cr.set_coordinate(hp_key) # ------------------------------------------------------------ @@ -5261,7 +5262,7 @@ def healpix_decrease_refinement_level( return f - def healpix_increase_refinement_level(self, level, quantity): + def healpix_increase_refinement_level(self, refinement_level, quantity): """Increase the refinement level of a HEALPix grid. Increasing the refinement level increases the resolution of @@ -5277,11 +5278,9 @@ def healpix_increase_refinement_level(self, level, quantity): as "sea_ice_amount" with units of kg m-2), the broadcast values are not changed. - K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, - et al.. HEALPix: A Framework for High-Resolution - Discretization and Fast Analysis of Data Distributed on the - Sphere. The Astrophysical Journal, 2005, 622 (2), pp.759-771. - https://dx.doi.org/10.1086/427976 + **References** + + {{HEALPix references}} .. versionadded:: NEXTVERSION @@ -5291,7 +5290,7 @@ def healpix_increase_refinement_level(self, level, quantity): :Parameters: - level: `int` or `None` + refinement_level: `int` or `None` Specify the new higher refinement level as an integer greater than or equal to the current refinement level, or if `None` then the refinement level is not changed. @@ -5369,27 +5368,27 @@ def healpix_increase_refinement_level(self, level, quantity): # Get the HEALPix info hp = f.healpix_info() - refinement_level = hp["refinement_level"] + old_refinement_level = hp["refinement_level"] - # Parse 'level' - if level is None: + # Parse 'refinement_level' + if refinement_level is None: # No change in refinement level return f if ( - not isinstance(level, Integral) - or level < refinement_level - or level > healpix_max_refinement_level() + not isinstance(refinement_level, Integral) + or refinement_level < old_refinement_level + or refinement_level > healpix_max_refinement_level() ): raise ValueError( f"Can't increase HEALPix refinement level of {f!r}: " "'level' must be an integer greater than or equal to the " - f"current refinement level of {refinement_level}, and less " - f"than or equal to {healpix_max_refinement_level()}. " - f"Got {level!r}" + f"current refinement level of {old_refinement_level}, and " + f"less than or equal to {healpix_max_refinement_level()}. " + f"Got {refinement_level!r}" ) - if level == refinement_level: + if refinement_level == old_refinement_level: # No change in refinement level return f @@ -5404,7 +5403,7 @@ def healpix_increase_refinement_level(self, level, quantity): # Get the number of cells at the higher refinement level which # are contained in one cell at the original refinement level - ncells = 4 ** (level - refinement_level) + ncells = 4 ** (refinement_level - old_refinement_level) # Get the HEALPix axis axis = hp["domain_axis_key"] @@ -5553,7 +5552,7 @@ def healpix_increase_refinement_level(self, level, quantity): # Set the data type to allow for the largest possible HEALPix # index at the new refinement level - dtype = cfdm.integer_dtype(12 * (4**level) - 1) + dtype = cfdm.integer_dtype(12 * (4**refinement_level) - 1) if dx.dtype != dtype: dx = dx.astype(dtype, copy=False) @@ -5584,7 +5583,9 @@ def healpix_increase_refinement_level(self, level, quantity): # Update the healpix Coordinate Reference cr = hp.get("grid_mapping_name:healpix") - cr.coordinate_conversion.set_parameter("refinement_level", level) + cr.coordinate_conversion.set_parameter( + "refinement_level", refinement_level + ) cr.set_coordinate(hp_key) if create_latlon: diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 25c3532559..6475ee9f25 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -1968,18 +1968,12 @@ def coordinate_reference_domain_axes(self, identity=None): return set(axes) def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): - """Change the indexing scheme of HEALPix indices. - - K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, - et al.. HEALPix: A Framework for High-Resolution - Discretization and Fast Analysis of Data Distributed on the - Sphere. The Astrophysical Journal, 2005, 622 (2), pp.759-771. - https://dx.doi.org/10.1086/427976 - - M. Reinecke and E. Hivon: Efficient data structures for masks - on 2D grids. A&A, 580 (2015) - A132. https://doi.org/10.1051/0004-6361/201526549 + r"""Change the indexing scheme of HEALPix indices. + **References** + + {{HEALPix references}} + .. versionadded:: NEXTVERSION .. seealso:: `healpix_info`, `healpix_to_ugrid` @@ -2145,11 +2139,9 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): def healpix_info(self): """Get information about the HEALPix grid, if there is one. - K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, - et al.. HEALPix: A Framework for High-Resolution - Discretization and Fast Analysis of Data Distributed on the - Sphere. The Astrophysical Journal, 2005, 622 (2), pp.759-771. - https://dx.doi.org/10.1086/427976 + **References** + + {{HEALPix references}} .. versionadded:: NEXTVERSION @@ -2161,30 +2153,24 @@ def healpix_info(self): all of the following dictionary keys: * ``'coordinate_reference_key'``: The construct key of - the healpix - coordinate reference - construct. + the healpix coordinate reference construct. * ``'grid_mapping_name:healpix'``: The healpix - coordinate - reference - construct. + coordinate reference construct. * ``'indexing_scheme'``: The HEALPix indexing scheme. * ``'refinement_level'``: The refinement level of the - HEALPix grid. + HEALPix grid. * ``'domain_axis_key'``: The construct key of the - HEALPix domain axis - construct. + HEALPix domain axis construct. * ``'coordinate_key'``: The construct key of the - healpix_index coordinate - construct. + healpix_index coordinate construct. * ``'healpix_index'``: The healpix_index coordinate - construct. + construct. The dictionary will be empty if there is no HEALPix axis. @@ -2213,11 +2199,9 @@ def healpix_info(self): def healpix_to_ugrid(self, inplace=False): """Convert a HEALPix domain to a UGRID domain. - K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, - et al.. HEALPix: A Framework for High-Resolution - Discretization and Fast Analysis of Data Distributed on the - Sphere. The Astrophysical Journal, 2005, 622 (2), pp.759-771. - https://dx.doi.org/10.1086/427976 + **References** + + {{HEALPix references}} .. versionadded:: NEXTVERSION diff --git a/cf/test/test_Domain.py b/cf/test/test_Domain.py index f43662bedc..3939c89871 100644 --- a/cf/test/test_Domain.py +++ b/cf/test/test_Domain.py @@ -533,6 +533,11 @@ def test_Domain_create_healpix(self): 1000, ) + # Bad 'refinement_level' + for level in (-1, 3.14, 30, "string"): + with self.assertRaises(ValueError): + cf.Domain.create_healpix(level) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) From 415a0554bfc45d0b508ef8aa27ceb610db80284d Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 4 Aug 2025 11:28:29 +0100 Subject: [PATCH 44/59] dev --- cf/data/dask_utils.py | 32 +++++++++++------------ cf/docstring/docstring.py | 2 +- cf/domain.py | 19 ++++++++------ cf/field.py | 34 ++++++++++++------------ cf/healpix.py | 8 +++--- cf/mixin/fielddomain.py | 54 ++++++++++++++++++++++----------------- cf/test/test_Domain.py | 13 +++++----- cf/test/test_Field.py | 13 ++++------ 8 files changed, 91 insertions(+), 84 deletions(-) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index 893b65d1cb..4cc3b4c5f0 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -469,8 +469,8 @@ def cf_healpix_bounds( a, indexing_scheme, refinement_level=None, - lat=False, - lon=False, + latitude=False, + longitude=False, pole_longitude=None, ): """Calculate HEALPix cell bounds. @@ -511,10 +511,10 @@ def cf_healpix_bounds( ``'nested_unique'`` (in which case *refinement_level* may be `None`). - lat: `bool`, optional + latitude: `bool`, optional If True then return latitude bounds. - lon: `bool`, optional + longitude: `bool`, optional If True then return longitude bounds. pole_longitude: `None` or number @@ -532,21 +532,21 @@ def cf_healpix_bounds( **Examples** >>> cf.data.dask_utils.cf_healpix_bounds( - ... np.array([0, 1, 2, 3]), 'nested', 1, lat=True + ... np.array([0, 1, 2, 3]), 'nested', 1, latitude=True ) array([[41.8103149 , 19.47122063, 0. , 19.47122063], [66.44353569, 41.8103149 , 19.47122063, 41.8103149 ], [66.44353569, 41.8103149 , 19.47122063, 41.8103149 ], [90. , 66.44353569, 41.8103149 , 66.44353569]]) >>> cf.data.dask_utils.cf_healpix_bounds( - ... np.array([0, 1, 2, 3]), 'nested', 1, lon=True + ... np.array([0, 1, 2, 3]), 'nested', 1, longitude=True ) array([[45. , 22.5, 45. , 67.5], [90. , 45. , 67.5, 90. ], [ 0. , 0. , 22.5, 45. ], [45. , 0. , 45. , 90. ]]) >>> cf.data.dask_utils.cf_healpix_bounds( - ... np.array([0, 1, 2, 3]), 'nested', 1, lon=True, + ... np.array([0, 1, 2, 3]), 'nested', 1, longitude=True, ... pole_longitude=3.14159 ) array([[45. , 22.5 , 45. , 67.5 ], @@ -573,9 +573,9 @@ def cf_healpix_bounds( f"healpix_index array has one dimension. Got shape {a.shape}" ) - if lat: + if latitude: pos = 1 - elif lon: + elif longitude: pos = 0 if indexing_scheme == "ring": @@ -639,7 +639,7 @@ def cf_healpix_bounds( def cf_healpix_coordinates( - a, indexing_scheme, refinement_level=None, lat=False, lon=False + a, indexing_scheme, refinement_level=None, latitude=False, longitude=False ): """Calculate HEALPix cell centre coordinates. @@ -674,10 +674,10 @@ def cf_healpix_coordinates( ``'nested_unique'`` (in which case *refinement_level* may be `None`). - lat: `bool`, optional + latitude: `bool`, optional If True then return latitude coordinates. - lon: `bool`, optional + longitude: `bool`, optional If True then return longitude coordinates. :Returns: @@ -688,11 +688,11 @@ def cf_healpix_coordinates( **Examples** >>> cf.data.dask_utils.cf_healpix_coordinates( - ... np.array([0, 1, 2, 3]), 'nested', 1, lat=True + ... np.array([0, 1, 2, 3]), 'nested', 1, latitude=True ) array([19.47122063, 41.8103149 , 41.8103149 , 66.44353569]) >>> cf.data.dask_utils.cf_healpix_coordinates( - ... np.array([0, 1, 2, 3]), 'nested', 1, lon=True + ... np.array([0, 1, 2, 3]), 'nested', 1, longitude=True ) array([45. , 67.5, 22.5, 45. ]) @@ -714,9 +714,9 @@ def cf_healpix_coordinates( f"healpix_index array has one dimension. Got shape {a.shape}" ) - if lat: + if latitude: pos = 1 - elif lon: + elif longitude: pos = 0 match indexing_scheme: diff --git a/cf/docstring/docstring.py b/cf/docstring/docstring.py index 4dcfcf1176..b8b63eaa20 100644 --- a/cf/docstring/docstring.py +++ b/cf/docstring/docstring.py @@ -98,7 +98,7 @@ M. Reinecke and E. Hivon: Efficient data structures for masks on 2D grids. A&A, 580 (2015) A132. https://doi.org/10.1051/0004-6361/201526549""", - # ---------------------------------------------------------------- + # ---------------------------------------------------------------- # Method description substitutions (3 levels of indentation) # ---------------------------------------------------------------- # i: deprecated at version 3.0.0 diff --git a/cf/domain.py b/cf/domain.py index dccc53b5b5..a7463bde9a 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -273,9 +273,9 @@ def create_healpix( HEALPix indices are monotonically increasing. **References** - + {{HEALPix references}} - + .. versionadded:: NEXTVERSION .. seealso:: `cf.Domain.create_regular`, @@ -404,20 +404,23 @@ def create_healpix( axis = domain.set_construct(d, copy=False) # auxiliary_coordinate: healpix_index - c = domain._AuxiliaryCoordinate() + c = domain._DimensionCoordinate() c.set_properties({"standard_name": "healpix_index"}) c.nc_set_variable("healpix_index") # Create the healpix_index data if nested_unique: - index0 = 4 ** (refinement_level + 1) + start = 4 ** (refinement_level + 1) else: - index0 = 0 + start = 0 - c.set_data( - Data(da.arange(index0, index0 + ncells), units="1"), copy=False - ) + stop = start + ncells + data = Data(da.arange(start, stop), units="1") + + # Set cached data elements + data._set_cached_elements({0: start, -1: stop - 1}) + c.set_data(data, copy=False) key = domain.set_construct(c, axes=axis, copy=False) # coordinate_reference: grid_mapping_name:healpix diff --git a/cf/field.py b/cf/field.py index 312790dd18..51d71d742a 100644 --- a/cf/field.py +++ b/cf/field.py @@ -180,8 +180,6 @@ # -------------------------------------------------------------------- _collapse_ddof_methods = set(("sd", "var")) -# _earth_radius = Data(6371229.0, "m") - _relational_methods = ( "__eq__", "__ne__", @@ -4855,7 +4853,7 @@ def healpix_decrease_refinement_level( is raised (assuming that *check_healpix_index* is True). **References** - + {{HEALPix references}} .. versionadded:: NEXTVERSION @@ -5227,12 +5225,6 @@ def healpix_decrease_refinement_level( # ------------------------------------------------------------ # Change the refinement level of the healpix_index coordinates # ------------------------------------------------------------ - if healpix_index.construct_type == "dimension_coordinate": - # Ensure that healpix indices are auxiliary coordinates - healpix_index = f._AuxiliaryCoordinate( - source=healpix_index, copy=False - ) - healpix_index = healpix_index[::ncells] // ncells hp_key = f.set_construct(healpix_index, axes=axis, copy=False) @@ -5279,7 +5271,7 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): values are not changed. **References** - + {{HEALPix references}} .. versionadded:: NEXTVERSION @@ -5536,11 +5528,6 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): # Create the healpix_index coordinates for the new refinement # level, and put them into the Field. healpix_index = hp["healpix_index"] - if healpix_index.construct_type == "dimension_coordinate": - # Ensure that healpix indices are auxiliary coordinates - healpix_index = f._AuxiliaryCoordinate( - source=healpix_index, copy=False - ) # Save any cached data elements data = healpix_index.data @@ -5589,7 +5576,8 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): cr.set_coordinate(hp_key) if create_latlon: - # Create lat/lon coordinates for the new refinement level + # Create 1-d lat/lon coordinates for the new refinement + # level f.create_latlon_coordinates(two_d=False, inplace=True) return f @@ -6025,7 +6013,9 @@ def collapse( .. versionadded:: 1.0 .. seealso:: `bin`, `cell_area`, `convolution_filter`, - `moving_window`, `radius`, `weights` + `moving_window`, + `healpix_decrease_refinement_level`, `radius`, + `weights` :Parameters: @@ -7621,7 +7611,9 @@ def collapse( f.del_coordinate_reference(ref_key) # -------------------------------------------------------- - # Remove a HEALPix coordinate reference + # Remove a HEALPix Coordinate Reference and Dimension + # Coordinate. (A HEALPix Auxiliary Coordinate gets removed + # later with the other 1-d Auxiliary Coordinates.) # -------------------------------------------------------- healpix_axis = f.domain_axis( "healpix_index", key=True, default=None @@ -7632,6 +7624,12 @@ def collapse( del_healpix_coordinate_reference(f) + key = f.dimension_coordinate( + "healpix_index", key=True, default=None + ) + if key is not None: + f.del_construct(key) + # --------------------------------------------------------- # Update dimension coordinates, auxiliary coordinates, # cell measures and domain ancillaries diff --git a/cf/healpix.py b/cf/healpix.py index e1529b32eb..37655f2a4f 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -96,7 +96,7 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): meta=meta, indexing_scheme=indexing_scheme, refinement_level=refinement_level, - lat=True, + latitude=True, ) lat = f._AuxiliaryCoordinate( data=f._Data(dy, "degrees_north", copy=False), @@ -110,7 +110,7 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): meta=meta, indexing_scheme=indexing_scheme, refinement_level=refinement_level, - lon=True, + longitude=True, ) lon = f._AuxiliaryCoordinate( data=f._Data(dy, "degrees_east", copy=False), @@ -128,7 +128,7 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): meta=meta, indexing_scheme=indexing_scheme, refinement_level=refinement_level, - lat=True, + latitude=True, ) bounds = f._Bounds(data=dy) lat.set_bounds(bounds) @@ -143,7 +143,7 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): meta=meta, indexing_scheme=indexing_scheme, refinement_level=refinement_level, - lon=True, + longitude=True, pole_longitude=pole_longitude, ) bounds = f._Bounds(data=dy) diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 6475ee9f25..fdc92f4b86 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -1,7 +1,6 @@ import logging from numbers import Integral -import dask.array as da import numpy as np from cfdm import is_log_level_debug, is_log_level_info from dask.array.slicing import normalize_index @@ -1971,9 +1970,9 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): r"""Change the indexing scheme of HEALPix indices. **References** - + {{HEALPix references}} - + .. versionadded:: NEXTVERSION .. seealso:: `healpix_info`, `healpix_to_ugrid` @@ -2116,31 +2115,40 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): _healpix_indexing_scheme(healpix_index, hp, new_indexing_scheme) - # Ensure that healpix indices are auxiliary coordinates - if healpix_index.construct_type == "dimension_coordinate": - f.dimension_to_auxiliary(hp["coordinate_key"], inplace=True) + # # Ensure that healpix indices are auxiliary coordinates + # if healpix_index.construct_type == "dimension_coordinate": + # f.dimension_to_auxiliary(hp["coordinate_key"], inplace=True) if sort: # Sort the HEALPix axis so that the HEALPix indices are - # monotonically increasing. Test for the common case of - # already-ordered global nested or ring indices (which is - # a relatively fast compared to do doing any actual - # sorting and subspacing). - h = healpix_index - if not ( - indexing_scheme in ("nested", "ring") - and (h == da.arange(h.size, chunks=h.data.chunks)).all() - ): - index = h.data.compute() + # monotonically increasing. + + # # Test for the common case of + # # already-ordered global nested or ring indices (which is + # # a relatively fast compared to do doing any actual + # # sorting and subspacing). + # h = healpix_index + # if not ( + # indexing_scheme in ("nested", "ring") + # and (h == da.arange(h.size, chunks=h.data.chunks)).all() + # ): + d = healpix_index.data + if (d.diff() < 0).any(): + index = d.compute() f = f.subspace(**{hp["domain_axis_key"]: np.argsort(index)}) + # Now that the HEALPix indices are ordered, make sure that + # they're stored in a Dimension Coordinate. + if healpix_index.construct_type == "auxiliary_coordinate": + f.auxiliary_to_dimension(hp["coordinate_key"], inplace=True) + return f def healpix_info(self): """Get information about the HEALPix grid, if there is one. **References** - + {{HEALPix references}} .. versionadded:: NEXTVERSION @@ -2200,7 +2208,7 @@ def healpix_to_ugrid(self, inplace=False): """Convert a HEALPix domain to a UGRID domain. **References** - + {{HEALPix references}} .. versionadded:: NEXTVERSION @@ -2502,9 +2510,9 @@ def create_latlon_coordinates( # Initialize the flag that tells us if any new coordinates # have been created - new_coords = False + coords_created = False - if one_d: + if one_d and not coords_created: # -------------------------------------------------------- # 1-d lat/lon coordinates # -------------------------------------------------------- @@ -2517,15 +2525,15 @@ def create_latlon_coordinates( lat_key, lon_key = _healpix_create_latlon_coordinates( f, pole_longitude ) - new_coords = lat_key is not None + coords_created = lat_key is not None - elif two_d: + if two_d and not coords_created: # -------------------------------------------------------- # 2-d lat/lon coordinates # -------------------------------------------------------- pass # Add some code here! - if new_coords: + if coords_created: # -------------------------------------------------------- # Update the approrpriate coordinate reference with the # new coordinate keys diff --git a/cf/test/test_Domain.py b/cf/test/test_Domain.py index 3939c89871..202691b9d3 100644 --- a/cf/test/test_Domain.py +++ b/cf/test/test_Domain.py @@ -499,12 +499,13 @@ def test_Domain_create_healpix(self): d = cf.Domain.create_healpix(0) self.assertEqual(len(d.constructs), 3) self.assertEqual(len(d.domain_axes()), 1) - self.assertEqual(len(d.auxiliary_coordinates()), 1) + self.assertEqual(len(d.dimension_coordinates()), 1) + self.assertEqual(len(d.auxiliary_coordinates()), 0) self.assertEqual(len(d.coordinate_references()), 1) - self.assertTrue( - (d.auxiliary_coordinate().array == np.arange(12)).all() - ) + hp = d.dimension_coordinate() + self.assertEqual(hp.data._get_cached_elements(), {0: 0, -1: 11}) + self.assertTrue(np.array_equal(hp, np.arange(12))) self.assertEqual( d.coordinate_reference().coordinate_conversion.get_parameter( @@ -515,7 +516,7 @@ def test_Domain_create_healpix(self): d = cf.Domain.create_healpix(0, "nested_unique") self.assertTrue( - (d.auxiliary_coordinate().array == np.arange(4, 16)).all() + np.array_equal(d.dimension_coordinate(), np.arange(4, 16)) ) self.assertIsNone( d.coordinate_reference().datum.get_parameter("earth_radius", None) @@ -533,7 +534,7 @@ def test_Domain_create_healpix(self): 1000, ) - # Bad 'refinement_level' + # Bad refinement_level for level in (-1, 3.14, 30, "string"): with self.assertRaises(ValueError): cf.Domain.create_healpix(level) diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index 1719f2d82a..c84740b393 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -2970,8 +2970,8 @@ def test_Field_auxiliary_to_dimension_to_auxiliary(self): self.assertIsNone(g.auxiliary_to_dimension("Y", inplace=True)) f = self.f12.copy() - g = f.auxiliary_to_dimension("healpix_index") - h = g.dimension_to_auxiliary("healpix_index") + g = f.dimension_to_auxiliary("healpix_index") + h = g.auxiliary_to_dimension("healpix_index") self.assertFalse(f.equals(g)) self.assertTrue(f.equals(h)) @@ -3211,14 +3211,11 @@ def test_Field_create_latlon_coordinates(self): # HEALPix field # ------------------------------------------------------------ f = self.f12.copy() - self.assertEqual(len(f.auxiliary_coordinates()), 1) - self.assertEqual(len(f.auxiliary_coordinates("healpix_index")), 1) + self.assertEqual(len(f.auxiliary_coordinates()), 0) + self.assertEqual(len(f.dimension_coordinates("healpix_index")), 1) g = f.create_latlon_coordinates() - self.assertEqual(len(g.auxiliary_coordinates()), 3) - self.assertEqual( - len(g.auxiliary_coordinates("healpix_index", "X", "Y")), 3 - ) + self.assertEqual(len(g.auxiliary_coordinates("X", "Y")), 2) self.assertIsNone(f.create_latlon_coordinates(inplace=True)) self.assertTrue(f.equals(g)) From b4caa8a98df03a4d53f726a2907fe85d95e47815 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 4 Aug 2025 21:21:47 +0100 Subject: [PATCH 45/59] dev --- Changelog.rst | 3 ++ cf/field.py | 2 + cf/mixin/fielddomain.py | 90 +++++++++++++++++------------------------ cf/test/test_Field.py | 1 - 4 files changed, 43 insertions(+), 53 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 968dc2bbf2..58abe8a51a 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -6,6 +6,9 @@ Version NEXTVERSION * New methods to allow changing units in a chain: `cf.Field.to_units`, `cf.Data.to_units` (https://github.com/NCAS-CMS/cf-python/issues/874) +* Allow multiple conditions for the same axis in `cf.Field.subspace` + and `cf.Field.indices` + (https://github.com/NCAS-CMS/cf-python/issues/881) * New method: `cf.Field.create_latlon_coordinates` (https://github.com/NCAS-CMS/cf-python/issues/???) * New HEALPix methods: `cf.Field.healpix_info`, diff --git a/cf/field.py b/cf/field.py index 51d71d742a..562e711d5c 100644 --- a/cf/field.py +++ b/cf/field.py @@ -13475,6 +13475,8 @@ def subspace(self): * Multiple domain axes may be subspaced simultaneously, and it doesn't matter which order they are specified in. + * Multiple criteria may be specified for the same domain axis. + * Subspace criteria may be provided for size 1 domain axes that are not spanned by the field construct's data. diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index fdc92f4b86..cecc123b1c 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -1640,66 +1640,66 @@ def del_coordinate_reference( """Remove a coordinate reference construct and all of its domain ancillary constructs. - .. versionadded:: 3.0.0 + .. versionadded:: 3.0.0 - .. seealso:: `del_construct` + .. seealso:: `del_construct` - :Parameters: + :Parameters: - identity: optional - Select the coordinate reference construct by one of: + identity: optional + Select the coordinate reference construct by one of: - * The identity of a coordinate reference construct. + * The identity of a coordinate reference construct. - {{construct selection identity}} + {{construct selection identity}} - * The key of a coordinate reference construct + * The key of a coordinate reference construct - * `None`. This is the default, which selects the - coordinate reference construct when there is only - one of them. + * `None`. This is the default, which selects the + coordinate reference construct when there is only + one of them. - *Parameter example:* - ``identity='standard_name:atmosphere_hybrid_height_coordinate'`` + *Parameter example:* + ``identity='standard_name:atmosphere_hybrid_height_coordinate'`` - *Parameter example:* - ``identity='grid_mapping_name:rotated_latitude_longitude'`` + *Parameter example:* + ``identity='grid_mapping_name:rotated_latitude_longitude'`` - *Parameter example:* - ``identity='transverse_mercator'`` + *Parameter example:* + ``identity='transverse_mercator'`` - *Parameter example:* - ``identity='coordinatereference1'`` + *Parameter example:* + ``identity='coordinatereference1'`` - *Parameter example:* - ``identity='key%coordinatereference1'`` + *Parameter example:* + ``identity='key%coordinatereference1'`` - *Parameter example:* - ``identity='ncvar%lat_lon'`` + *Parameter example:* + ``identity='ncvar%lat_lon'`` - *Parameter example:* - ``identity=cf.eq('rotated_pole')'`` + *Parameter example:* + ``identity=cf.eq('rotated_pole')'`` - *Parameter example:* - ``identity=re.compile('^rotated')`` + *Parameter example:* + ``identity=re.compile('^rotated')`` - construct: optional - TODO + construct: optional + TODO - default: optional - Return the value of the *default* parameter if the - construct can not be removed, or does not exist. + default: optional + Return the value of the *default* parameter if the + construct can not be removed, or does not exist. - {{default Exception}} + {{default Exception}} - :Returns: + :Returns: - The removed coordinate reference construct. + The removed coordinate reference construct. - **Examples** + **Examples** - >>> f.del_coordinate_reference('rotated_latitude_longitude') - + >>> f.del_coordinate_reference('rotated_latitude_longitude') + """ if construct is None: @@ -2115,23 +2115,9 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): _healpix_indexing_scheme(healpix_index, hp, new_indexing_scheme) - # # Ensure that healpix indices are auxiliary coordinates - # if healpix_index.construct_type == "dimension_coordinate": - # f.dimension_to_auxiliary(hp["coordinate_key"], inplace=True) - if sort: # Sort the HEALPix axis so that the HEALPix indices are # monotonically increasing. - - # # Test for the common case of - # # already-ordered global nested or ring indices (which is - # # a relatively fast compared to do doing any actual - # # sorting and subspacing). - # h = healpix_index - # if not ( - # indexing_scheme in ("nested", "ring") - # and (h == da.arange(h.size, chunks=h.data.chunks)).all() - # ): d = healpix_index.data if (d.diff() < 0).any(): index = d.compute() diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index c84740b393..e734a2a050 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -1491,7 +1491,6 @@ def test_Field_indices(self): self.assertEqual(g.shape, (1, 10, 3)) self.assertTrue((x == [120, 200, 280]).all()) - cf.log_level(1) # 2-d lon = f.construct("longitude").array lon = np.transpose(lon) From 2ca65bd8c539eaf9ea2791565e0fee8f0bac2297 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 5 Aug 2025 09:11:00 +0100 Subject: [PATCH 46/59] dev --- cf/field.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/cf/field.py b/cf/field.py index 562e711d5c..63ae304e43 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4826,10 +4826,10 @@ def healpix_decrease_refinement_level( ): r"""Decrease the refinement level of a HEALPix grid. - Decreasing the refinement level reduces the resolution of the - HEALPix grid by combining, using the given *method*, the Field - data from the original cells that lie inside each larger cell - at the new lower refinement level. + Decreasing the HEALPix refinement level reduces the resolution + of the HEALPix grid by combining, using the given *method*, + the Field data from the original cells that lie inside each + larger cell at the new lower refinement level. A new cell method is added, when appropriate, to describe the reduction. @@ -4860,7 +4860,7 @@ def healpix_decrease_refinement_level( .. seealso:: `healpix_increase_refinement_level`, `healpix_info`, `healpix_indexing_scheme`, - `healpix_to_ugrid` + `healpix_to_ugrid`, `collapse` :Parameters: @@ -4992,6 +4992,9 @@ def healpix_decrease_refinement_level( >>> print(g[0, 0].array) [[293.5]] + Set the refinement level to 0 using the ``'`range'`` *method*, + which requires a new *reduction* function to be defined: + >>> import numpy as np >>> def range_func(a, axis=None): ... return np.max(a, axis=axis) - np.min(a, axis=axis) @@ -5012,7 +5015,7 @@ def healpix_decrease_refinement_level( """ from .constants import cell_methods - # "point" is not a valid cell method after a reduction + # "point" is not a reduction method cell_methods = cell_methods.copy() cell_methods.remove("point") @@ -5028,7 +5031,7 @@ def healpix_decrease_refinement_level( # Parse 'reduction' if reduction is None: - # Infer 'reduction' function from 'method' + # Use a default reduction function for selected methods match method: case "maximum": reduction = np.max @@ -5054,8 +5057,8 @@ def healpix_decrease_refinement_level( elif not callable(reduction): raise ValueError( f"Can't decrease HEALPix refinement level of {f!r}: " - f"'reduction' must be callable. Got: {reduction!r} of type " - f"{type(reduction)}" + f"'reduction' must be a callable. Got: {reduction!r} of " + f"type {type(reduction)}" ) # Get the HEALPix info From 63ca9cec2db92ead3d47500c7a72cb58d351e544 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 5 Aug 2025 13:25:57 +0100 Subject: [PATCH 47/59] dev --- cf/data/dask_utils.py | 31 ++++++---- cf/domain.py | 5 +- cf/field.py | 135 ++++++++++++------------------------------ cf/healpix.py | 72 +++++++++++++++++++++- cf/test/test_Field.py | 31 +++++++--- 5 files changed, 153 insertions(+), 121 deletions(-) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index 4cc3b4c5f0..d0ca3b8485 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -754,15 +754,18 @@ def cf_healpix_coordinates( def cf_healpix_increase_refinement(a, ncells, iaxis, quantity): - """Increase the refinement level of a HEALPix array. + """Increase the refinement level of a HEALPix data. Data are broadcast to cells of at the higher refinement level. For - an extensive field quantity (that depends on the size of the - cells, such as "sea_ice_mass" with units of kg), the new values - are reduced so that they are consistent with the new smaller cell - areas. For an intensive field quantity (that does not depend on - the size of the cells, such as "sea_ice_amount" with units of kg - m-2), the broadcast values are not changed. + an "extensive" quantity only, the broadcast values are reduced to + be consistent with the new smaller cell areas. + + .. warning:: The returned `numpy` array will take up at least + *ncells* times more memory than the input array + *a*. For instance, if *a* has shape ``(3600, 48)`` + (1.3 MiB), the HEALPix axis is ``1``, and *ncells* is + ``4096``, then the returned array will have shape + ``(3600, 196608)`` (5.3 GiB), where 196608=48*4096. K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution Discretization and @@ -781,7 +784,7 @@ def cf_healpix_increase_refinement(a, ncells, iaxis, quantity): ncells: `int` The number of cells at the new refinement level which are - contained in one cell at the original refinement level + contained in one cell at the original refinement level. iaxis: `int` The position of the HEALPix axis in the array dimensions. @@ -819,7 +822,7 @@ def cf_healpix_increase_refinement(a, ncells, iaxis, quantity): def cf_healpix_increase_refinement_indices(a, ncells): - """Increase the refinement level of HEALPix indices + """Increase the refinement level of HEALPix indices. For instance, when going from refinement level 1 to refinement level 2, if *a* is ``(2, 23, 17)`` then it will be transformed to @@ -828,6 +831,13 @@ def cf_healpix_increase_refinement_indices(a, ncells): ``ncells`` is the number of cells at refinement level 2 that lie inside one cell at refinement level 1, i.e. ``ncells=4**(2-1)=4``. + .. warning:: The returned `numpy` array will take up at least + *ncells* times more memory than the input array + *a*. For instance, if *a* has shape ``(48,)`` (384 + B), and *ncells* is ``262144``, then the returned + array will have shape ``(12582912,)`` (96 MiB), where + 12582912=48*262144. + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution Discretization and Fast Analysis of Data Distributed on the Sphere. The Astrophysical @@ -853,9 +863,6 @@ def cf_healpix_increase_refinement_indices(a, ncells): The array at the new refinement level. """ - # PERFORMANCE: This function can use a lot of memory when 'a' - # and/or 'ncells' are large. - a = cfdm_to_memory(a) a = a * ncells diff --git a/cf/domain.py b/cf/domain.py index a7463bde9a..942399fe3b 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -415,10 +415,11 @@ def create_healpix( start = 0 stop = start + ncells - data = Data(da.arange(start, stop), units="1") + dtype = cfdm.integer_dtype(stop - 1) + data = Data(da.arange(start, stop, dtype=dtype), units="1") # Set cached data elements - data._set_cached_elements({0: start, -1: stop - 1}) + data._set_cached_elements({0: start, 1: start + 1, -1: stop - 1}) c.set_data(data, copy=False) key = domain.set_construct(c, axes=axis, copy=False) diff --git a/cf/field.py b/cf/field.py index 63ae304e43..1d0ead43df 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4994,7 +4994,7 @@ def healpix_decrease_refinement_level( Set the refinement level to 0 using the ``'`range'`` *method*, which requires a new *reduction* function to be defined: - + >>> import numpy as np >>> def range_func(a, axis=None): ... return np.max(a, axis=axis) - np.min(a, axis=axis) @@ -5114,16 +5114,17 @@ def healpix_decrease_refinement_level( f"Can't decrease HEALPix refinement level of {f!r}: " "'level' must be a non-negative integer less than " "or equal to the current refinement level of " - f"{old_refinement_level}. Got {refinement_level!r}" + f"{old_refinement_level}. Got: {refinement_level!r}" ) + # Parse 'level' if refinement_level == old_refinement_level: # No change in refinement level return f # Get the number of cells at the original refinement level - # which are contained in one cell at the lower refinement - # level + # which are contained in one larger cell at the new lower + # refinement level ncells = 4 ** (old_refinement_level - refinement_level) # Get the healpix_index coordinates @@ -5260,18 +5261,20 @@ def healpix_decrease_refinement_level( def healpix_increase_refinement_level(self, refinement_level, quantity): """Increase the refinement level of a HEALPix grid. - Increasing the refinement level increases the resolution of - the HEALPix grid by broadcasting the Field data from each - original cell to all of the new smaller cells at the new - higher refinement level that lie inside it. - - For an extensive field quantity (i.e. one that depends on the - size of the cells, such as "sea_ice_mass" with units of kg), - the broadcast values are also reduced to be consistent with - the new smaller cell areas. For an intensive field quantity - (i.e. one that does not depend on the size of the cells, such - as "sea_ice_amount" with units of kg m-2), the broadcast - values are not changed. + Increasing the HEALPix refinement level increases the + resolution of the HEALPix grid by broadcasting the Field data + from each original cell to all of the new smaller cells at the + new higher refinement level that lie inside it. + + It must be specified whether the field data contains an + extensive or intensive quantity. An extensive quantity depends + on the size of the cells (such as "sea_ice_mass" with units of + kg, or "cell_area" with units of m2), and an intensive + quantity does not depend on the size of the cells (such as + "sea_ice_amount" with units of kg m-2, or "air_temperature" + with units of K). For an extensive quantity only, the + broadcast values are reduced to be consistent with the new + smaller cell areas. **References** @@ -5332,12 +5335,12 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): >>> print(g[0, :8] .array) [[291.5 291.5 291.5 291.5 293.5 293.5 293.5 293.5]] - For an extensive quantity (which ``f`` is not in this example, - but we can assume that it is for demonstration purposes), each - cell at the higher refinement level has the value of a cell at - the original refinement level after dividing it by the number - of cells at the higher refinement level that lie in one cell - of the original refinement level (4 in this case): + For an extensive quantity (which the ``f`` is this example is + not, but we can assume that it is for demonstration purposes), + each cell at the higher refinement level has the value of a + cell at the original refinement level after dividing it by the + number of cells at the higher refinement level that lie in one + cell of the original refinement level (4 in this case): >>> g = f.healpix_increase_refinement_level(2, 'extensive') >>> print(f[0, :2] .array) @@ -5346,11 +5349,11 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): [[72.875 72.875 72.875 72.875 73.375 73.375 73.375 73.375]] """ - from .data.dask_utils import ( - cf_healpix_increase_refinement, - cf_healpix_increase_refinement_indices, + from .data.dask_utils import cf_healpix_increase_refinement_indices + from .healpix import ( + _healpix_increase_refinement_level, + healpix_max_refinement_level, ) - from .healpix import healpix_max_refinement_level # Increasing the refinement level requires the nested indexing # scheme @@ -5380,7 +5383,7 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): "'level' must be an integer greater than or equal to the " f"current refinement level of {old_refinement_level}, and " f"less than or equal to {healpix_max_refinement_level()}. " - f"Got {refinement_level!r}" + f"Got: {refinement_level!r}" ) if refinement_level == old_refinement_level: @@ -5393,7 +5396,7 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): raise ValueError( f"Can't increase HEALPix refinement level of {f!r}: " f"'quantity' keyword must be one of {valid_quantities}. " - f"Got {quantity!r}" + f"Got: {quantity!r}" ) # Get the number of cells at the higher refinement level which @@ -5424,97 +5427,35 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): ) ) - # Increase the refinement of the Field data - dx = f.data.to_dask_array( - _force_mask_hardness=False, _force_to_memory=False - ) - - if quantity == "extensive": - # Extensive data get divided by 'ncells' in - # `cf_healpix_increase_refinement`, so they end up with a - # data type of float64. - dtype = np.dtype("float64") - else: - dtype = dx.dtype - - # Each chunk is going to get larger by a factor of 'ncells' - chunks = list(dx.chunks) - chunks[iaxis] = (np.array(chunks[iaxis]) * ncells).tolist() - - dx = dx.map_blocks( - cf_healpix_increase_refinement, - chunks=tuple(chunks), - dtype=dtype, - meta=np.array((), dtype=dtype), - ncells=ncells, - iaxis=iaxis, - quantity=quantity, - ) - # Re-size the HEALPix axis domain_axis = f.domain_axis(axis) - domain_axis.set_size(dx.shape[iaxis]) + domain_axis.set_size(f.shape[iaxis] * ncells) - f.set_data(dx, copy=False) + # Increase the refinement of the Field data + _healpix_increase_refinement_level(f, ncells, iaxis, quantity) # Increase the refinement level of domain ancillary constructs # that span the HEALPix axis. We're assuming that domain # ancillary data are intensive (i.e. do not depend on the size # of the cell). - meta = np.array((), dtype=dtype) for key, domain_ancillary in f.domain_ancillaries( filter_by_axis=(axis,), axis_mode="and", todict=True ).items(): iaxis = f.get_data_axes(key).index(axis) - dx = domain_ancillary.data.to_dask_array( - _force_mask_hardness=False, _force_to_memory=False - ) - dtype = dx.dtype - - # Each chunk is going to get larger by a factor of - # 'ncells' - chunks = list(dx.chunks) - chunks[iaxis] = (np.array(chunks[iaxis]) * ncells).tolist() - - dx = dx.map_blocks( - cf_healpix_increase_refinement, - chunks=tuple(chunks), - dtype=dtype, - meta=meta, - ncells=ncells, - iaxis=iaxis, - quantity="intensive", + _healpix_increase_refinement_level( + domain_ancillary, ncells, iaxis, "intensive" ) - domain_ancillary.set_data(dx, copy=False) # Increase the refinement level of cell measure constructs # that span the HEALPix axis. Cell measure data are extensive # (i.e. depend on the size of the cell). - dtype = np.dtype("float64") - meta = np.array((), dtype=dtype) for key, cell_measure in f.cell_measures( filter_by_axis=(axis,), axis_mode="and", todict=True ).items(): iaxis = f.get_data_axes(key).index(axis) - dx = cell_measure.data.to_dask_array( - _force_mask_hardness=False, _force_to_memory=False - ) - - # Each chunk is going to get larger by a factor of - # 'ncells' - chunks = list(dx.chunks) - chunks[iaxis] = (np.array(chunks[iaxis]) * ncells).tolist() - - dx = dx.map_blocks( - cf_healpix_increase_refinement, - chunks=tuple(chunks), - dtype=dtype, - meta=meta, - ncells=ncells, - iaxis=iaxis, - quantity="extensive", + _healpix_increase_refinement_level( + cell_measure, ncells, iaxis, "extensive" ) - cell_measure.set_data(dx, copy=False) # Remove all other metadata constructs that span the HEALPix # axis (including the original healpix_index coordinate diff --git a/cf/healpix.py b/cf/healpix.py index 37655f2a4f..abf8df3394 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -157,8 +157,78 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): return lat_key, lon_key +def _healpix_increase_refinement_level(x, ncells, iaxis, quantity): + """Increase the HEALPix refinement level in-place. + + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et + al.. HEALPix: A Framework for High-Resolution Discretization and + Fast Analysis of Data Distributed on the Sphere. The Astrophysical + Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + + .. versionadded:: NEXTVERSION + + .. seealso:: `cf.Field.healpix_increase_refinement_level` + + :Parameters: + + x: construct + The construct containing data that is to be changed. + + ncells: `int` + The number of cells at the new refinement level which are + contained in one cell at the original refinement level. + + iaxis: `int` + The position of the HEALPix axis in the construct's data + dimensions. + + quantity: `str` + Whether the data represent intensive or extensive + quantities, specified with ``'intensive'`` and + ``'extensive'`` respectively. + + :Returns: + + `None` + + """ + from .data.dask_utils import cf_healpix_increase_refinement + + # Get the Dask array. + # + # `cf_healpix_increase_refinement` has its own call to + # `cfdm_to_memory`, so we can set _force_to_memory=False. + dx = x.data.to_dask_array( + _force_mask_hardness=False, _force_to_memory=False + ) + + if quantity == "extensive": + # Extensive data get divided by 'ncells' in + # `cf_healpix_increase_refinement`, so they end up with a + # data type of float64. + dtype = np.dtype("float64") + else: + dtype = dx.dtype + + # Each chunk is going to get larger by a factor of 'ncells' + chunks = list(dx.chunks) + chunks[iaxis] = (np.array(chunks[iaxis]) * ncells).tolist() + + dx = dx.map_blocks( + cf_healpix_increase_refinement, + chunks=tuple(chunks), + dtype=dtype, + meta=np.array((), dtype=dtype), + ncells=ncells, + iaxis=iaxis, + quantity=quantity, + ) + x.set_data(dx, copy=False) + + def _healpix_indexing_scheme(healpix_index, hp, new_indexing_scheme): - """Change the indexing scheme of HEALPix indices. + """Change the indexing scheme of HEALPix indices in-place. K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution Discretization and diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index e734a2a050..f18e88e86f 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -3301,6 +3301,16 @@ def test_Field_healpix_decrease_refinement_level(self): """Test Field.healpix_decrease_refinement_level.""" f = self.f12 + # No change + for level in (None, 1): + g = f.healpix_decrease_refinement_level(level, "mean") + self.assertTrue(g.equals(f)) + + # Can't change refinement level when a larger cell is only + # partially covered by original cells + with self.assertRaises(ValueError): + f[:, 1:].healpix_decrease_refinement_level(0, "mean") + g = f.healpix_decrease_refinement_level(0, "mean") self.assertTrue( np.array_equal(g.coord("healpix_index"), np.arange(12)) @@ -3319,7 +3329,7 @@ def test_Field_healpix_decrease_refinement_level(self): (293.5, 289.15, 285.3, 3.44201976, 11.8475, 1156.6, 288.9), ): g = f.healpix_decrease_refinement_level(0, method) - self.assertTrue(np.allclose(g[0, 0], first_value)) + self.assertTrue(np.isclose(g[0, 0], first_value)) # Bad methods for method in ("point", "range", "bad method", 3.14): @@ -3330,13 +3340,13 @@ def range_func(a, axis=None): return np.max(a, axis=axis) - np.min(a, axis=axis) g = f.healpix_decrease_refinement_level(0, "range", range_func) - self.assertTrue(np.allclose(g[0, 0], 8.2)) + self.assertTrue(np.isclose(g[0, 0], 8.2)) def my_mean(a, axis=None): return np.mean(a, axis=axis) g = f.healpix_decrease_refinement_level(0, "mean", my_mean) - self.assertTrue(np.allclose(g[0, 0], 289.15)) + self.assertTrue(np.isclose(g[0, 0], 289.15)) f = f.healpix_indexing_scheme("ring") g = f.healpix_decrease_refinement_level(0, "maximum") @@ -3361,8 +3371,10 @@ def my_mean(a, axis=None): self.assertEqual(g.auxiliary_coordinate("latitude"), (12,)) self.assertEqual(g.auxiliary_coordinate("longitude"), (12,)) + # Can't change refinement level when the HEALPix indices are + # not strictly monotonically increasing with self.assertRaises(ValueError): - f.healpix_decrease_refinement_level(0, np.mean, conform=False) + f.healpix_decrease_refinement_level(0, "mean", conform=False) # Bad results when check_healpix_index=False h = f.healpix_decrease_refinement_level( @@ -3375,20 +3387,21 @@ def my_mean(a, axis=None): # Bad 'level' parameter for level in (-1, 0.785, np.array(1), 2, "string"): with self.assertRaises(ValueError): - f.healpix_decrease_refinement_level(level, np.mean) + f.healpix_decrease_refinement_level(level, "mean") # Can't change refinement level for a 'nested_unique' field + # TODOHEALPIX with self.assertRaises(ValueError): - self.f13.healpix_decrease_refinement_level(0, np.mean) + self.f13.healpix_decrease_refinement_level(0, "mean") # Non-HEALPix field with self.assertRaises(ValueError): - self.f0.healpix_decrease_refinement_level(0, np.mean) + self.f0.healpix_decrease_refinement_level(0, "mean") def test_Field_healpix_increase_refinement_level(self): """Test Field.healpix_increase_refinement_level.""" - f = self.f12.copy() - f.rechunk((-1, 17), inplace=True) + f = self.f12 + f = f.rechunk((None, 17)) f.coordinate("healpix_index").rechunk(13, inplace=True) g = f.healpix_increase_refinement_level(2, "intensive") From bb7ac5b73099b51b20452c00126dd07352d587d1 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 6 Aug 2025 15:27:25 +0100 Subject: [PATCH 48/59] dev --- cf/domain.py | 10 ++++-- cf/field.py | 59 ++++++++---------------------- cf/healpix.py | 79 +++++++++++++++++++++++++++++++++++++++++ cf/mixin/fielddomain.py | 11 +++++- cf/test/test_Domain.py | 2 +- 5 files changed, 113 insertions(+), 48 deletions(-) diff --git a/cf/domain.py b/cf/domain.py index 942399fe3b..618dbcbeb1 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -737,7 +737,10 @@ def indices(self, *config, **kwargs): Metadata constructs are selected conditions are specified on their data. Indices for subspacing are then automatically - inferred from where the conditions are met. + inferred from where the conditions are met. If a condition is + a callable function then if is automateically replaced with + the result of calling that function with the Domain as its + only argument. Metadata constructs and the conditions on their data are defined by keyword parameters. @@ -1092,7 +1095,10 @@ def subspace(self, *config, **kwargs): Subspacing by metadata selects metadata constructs and specifies conditions on their data. Indices for subspacing are - then automatically inferred from where the conditions are met. + then automatically inferred from where the conditions are + met. If a condition is a callable function then if is + automateically replaced with the result of calling that + function with the Domain as its only argument. Metadata constructs and the conditions on their data are defined by keyword parameters. diff --git a/cf/field.py b/cf/field.py index 1d0ead43df..760ffb29ef 100644 --- a/cf/field.py +++ b/cf/field.py @@ -5349,9 +5349,9 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): [[72.875 72.875 72.875 72.875 73.375 73.375 73.375 73.375]] """ - from .data.dask_utils import cf_healpix_increase_refinement_indices from .healpix import ( _healpix_increase_refinement_level, + _healpix_increase_refinement_level_indices, healpix_max_refinement_level, ) @@ -5470,46 +5470,11 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): f.del_construct(key) # Create the healpix_index coordinates for the new refinement - # level, and put them into the Field. + # level healpix_index = hp["healpix_index"] - - # Save any cached data elements - data = healpix_index.data - cached = data._get_cached_elements().copy() - - dx = data.to_dask_array( - _force_mask_hardness=False, _force_to_memory=False - ) - - # Set the data type to allow for the largest possible HEALPix - # index at the new refinement level - dtype = cfdm.integer_dtype(12 * (4**refinement_level) - 1) - if dx.dtype != dtype: - dx = dx.astype(dtype, copy=False) - - # Each chunk is going to get larger by a factor of 'ncells' - chunks = [(np.array(dx.chunks[0]) * ncells).tolist()] - - dx = dx.map_blocks( - cf_healpix_increase_refinement_indices, - chunks=tuple(chunks), - dtype=dtype, - meta=np.array((), dtype=dtype), - ncells=ncells, + _healpix_increase_refinement_level_indices( + healpix_index, ncells, refinement_level ) - - healpix_index.set_data(dx, copy=False) - - # Set new cached data elements - data = healpix_index.data - if 0 in cached: - x = np.array(cached[0], dtype=dtype) * ncells - data._set_cached_elements({0: x, 1: x + 1}) - - if -1 in cached: - x = np.array(cached[-1], dtype=dtype) * ncells + (ncells - 1) - data._set_cached_elements({-1: x}) - hp_key = f.set_construct(healpix_index, axes=axis, copy=False) # Update the healpix Coordinate Reference @@ -9221,7 +9186,10 @@ def indices(self, *config, **kwargs): Metadata constructs are selected by conditions specified on their data. Indices for subspacing are then automatically - inferred from where the conditions are met. + inferred from where the conditions are met. If a condition is + a callable function then if is automateically replaced with + the result of calling that function with the Field as its only + argument. The returned tuple of indices may be used to created a subspace by indexing the original field construct with them. @@ -13406,10 +13374,13 @@ def subspace(self): **Subspacing by metadata** - Subspacing by metadata, signified by the use of round brackets, - selects metadata constructs and specifies conditions on their - data. Indices for subspacing are then automatically inferred from - where the conditions are met. + Subspacing by metadata, signified by the use of round + brackets, selects metadata constructs and specifies conditions + on their data. Indices for subspacing are then automatically + inferred from where the conditions are met. If a condition is + a callable function then if is automateically replaced with + the result of calling that function with the Field as its only + argument. Metadata constructs and the conditions on their data are defined by keyword parameters. diff --git a/cf/healpix.py b/cf/healpix.py index abf8df3394..f034c4c7b7 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -227,6 +227,85 @@ def _healpix_increase_refinement_level(x, ncells, iaxis, quantity): x.set_data(dx, copy=False) +def _healpix_increase_refinement_level_indices( + healpix_index, ncells, refinement_level +): + """Increase the refinement level of HEALPix indices in-place. + + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et + al.. HEALPix: A Framework for High-Resolution Discretization and + Fast Analysis of Data Distributed on the Sphere. The Astrophysical + Journal, 2005, 622 (2), pp.759-771. + https://dx.doi.org/10.1086/427976 + + .. versionadded:: NEXTVERSION + + .. seealso:: `cf.Field.healpix_increase_refinement_level` + + :Parameters: + + healpix_index: `Coordinate` + The HEALPix indices to be changed. It is assumed they use + the "nested" indexing scheme. + + ncells: `int` + The number of cells at the new higher refinement level + which are contained in one cell at the original refinement + level. + + refinement_level: `int` + The new higher refinement level. + + :Returns: + + `None` + + """ + from cfdm import integer_dtype + + from .data.dask_utils import cf_healpix_increase_refinement_indices + + # Save any cached data elements + cached = healpix_index.data._get_cached_elements().copy() + + # Get the Dask array. + # + # `cf_healpix_increase_refinement_indices` has its own call to + # `cfdm_to_memory`, so we can set _force_to_memory=False. + dx = healpix_index.data.to_dask_array( + _force_mask_hardness=False, _force_to_memory=False + ) + + # Set the data type to allow for the largest possible HEALPix + # index at the new refinement level + dtype = integer_dtype(12 * (4**refinement_level) - 1) + if dx.dtype != dtype: + dx = dx.astype(dtype, copy=False) + + # Each chunk is going to get larger by a factor of 'ncells' + chunks = [(np.array(dx.chunks[0]) * ncells).tolist()] + + dx = dx.map_blocks( + cf_healpix_increase_refinement_indices, + chunks=tuple(chunks), + dtype=dtype, + meta=np.array((), dtype=dtype), + ncells=ncells, + ) + + healpix_index.set_data(dx, copy=False) + + # Set new cached data elements + data = healpix_index.data + if 0 in cached: + x = np.array(cached[0], dtype=dtype) * ncells + data._set_cached_elements({0: x, 1: x + 1}) + + if -1 in cached: + x = np.array(cached[-1], dtype=dtype) * ncells + (ncells - 1) + data._set_cached_elements({-1: x}) + + def _healpix_indexing_scheme(healpix_index, hp, new_indexing_scheme): """Change the indexing scheme of HEALPix indices in-place. diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index cecc123b1c..117b822cd3 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -332,8 +332,17 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): f"defined by {identity!r}" ) + # If the condition is a callable function, then call it + # with 'self' as the only argument and replace the + # condition with the result. if callable(value): - value = value(self) + try: + value = value(self) + except Exception as e: + raise RuntimeError( + f"Error encountered when calling condition " + f"{identity}={value}: {e}" + ) if axes in parsed: # The axes are the same as an existing key diff --git a/cf/test/test_Domain.py b/cf/test/test_Domain.py index 202691b9d3..f1a7b533d5 100644 --- a/cf/test/test_Domain.py +++ b/cf/test/test_Domain.py @@ -504,7 +504,7 @@ def test_Domain_create_healpix(self): self.assertEqual(len(d.coordinate_references()), 1) hp = d.dimension_coordinate() - self.assertEqual(hp.data._get_cached_elements(), {0: 0, -1: 11}) + self.assertEqual(hp.data._get_cached_elements(), {0: 0, 1: 1, -1: 11}) self.assertTrue(np.array_equal(hp, np.arange(12))) self.assertEqual( From 34652f4981974f7824f7a12f4f73f083969c0e17 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 7 Aug 2025 10:26:03 +0100 Subject: [PATCH 49/59] dev --- cf/data/dask_utils.py | 122 ++++++++++++++++++++++++++++-------------- cf/healpix.py | 3 +- 2 files changed, 83 insertions(+), 42 deletions(-) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index d0ca3b8485..f5b3d9299f 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -8,6 +8,7 @@ from functools import partial import numpy as np +from cfdm import integer_dtype from cfdm.data.dask_utils import cfdm_to_memory from scipy.ndimage import convolve1d @@ -508,21 +509,21 @@ def cf_healpix_bounds( hierarchy, starting at 0 for the base tessellation with 12 cells. Must be an `int` for *indexing_scheme* ``'nested'`` or ``'ring'``, but is ignored for *indexing_scheme* - ``'nested_unique'`` (in which case *refinement_level* may - be `None`). + ``'nested_unique'``, in which case *refinement_level* may + be `None`. latitude: `bool`, optional - If True then return latitude bounds. + If True then return the bounds' latitudes. longitude: `bool`, optional - If True then return longitude bounds. + If True then return the bounds' longitudes. pole_longitude: `None` or number - The longitude of coordinate bounds that lie exactly on the - north (south) pole. If `None` then the longitude of such a - vertex will be the same as the south (north) vertex of the - same cell. If set to a number, then the longitudes of such - vertices will all be given that value. + The longitude of bounds that lie exactly on the north + (south) pole. If `None` (the default) then the longitude + of such a vertex will be the same as the south (north) + vertex of the same cell. If set to a number, then the + longitudes of all such vertices will be given that value. :Returns: @@ -560,8 +561,8 @@ def cf_healpix_bounds( except ImportError as e: raise ImportError( f"{e}. Must install healpix (https://pypi.org/project/healpix) " - "to allow the calculation of latitude/longitude coordinate " - "bounds for a HEALPix grid" + "for the calculation of latitude/longitude coordinate bounds " + "of a HEALPix grid" ) a = cfdm_to_memory(a) @@ -570,7 +571,7 @@ def cf_healpix_bounds( if a.ndim != 1: raise ValueError( "Can only calculate HEALPix cell bounds when the " - f"healpix_index array has one dimension. Got shape {a.shape}" + f"healpix_index array has one dimension. Got shape: {a.shape}" ) if latitude: @@ -578,6 +579,8 @@ def cf_healpix_bounds( elif longitude: pos = 0 + # Define the function that's going to calculate the bounds from + # the HEALPix indices if indexing_scheme == "ring": bounds_func = healpix._chp.ring2ang_uv else: @@ -596,7 +599,7 @@ def cf_healpix_bounds( b = np.empty((a.size, 4), dtype="float64") if indexing_scheme == "nested_unique": - # Create bounds for 'nested_unique' cells + # Create bounds for 'nested_unique' indices orders, a = healpix.uniq2pix(a, nest=True) for order in np.unique(orders): nside = healpix.order2nside(order) @@ -604,18 +607,23 @@ def cf_healpix_bounds( for j, (u, v) in enumerate(vertices): thetaphi = bounds_func(nside, a[indices], u, v) b[indices, j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] + + del orders, indices else: - # Create bounds for 'nested' or 'ring' cells + # Create bounds for 'nested' or 'ring' indices nside = healpix.order2nside(refinement_level) for j, (u, v) in enumerate(vertices): thetaphi = bounds_func(nside, a, u, v) - b[..., j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] + b[:, j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] + + del thetaphi, a if not pos: # Ensure that longitude bounds are less than 360 where_ge_360 = np.where(b >= 360) if where_ge_360[0].size: b[where_ge_360] -= 360.0 + del where_ge_360 # Vertices on the north or south pole come out with a # longitude of NaN, so replace these with sensible values: @@ -671,14 +679,14 @@ def cf_healpix_coordinates( hierarchy, starting at 0 for the base tessellation with 12 cells. Must be an `int` for *indexing_scheme* ``'nested'`` or ``'ring'``, but is ignored for *indexing_scheme* - ``'nested_unique'`` (in which case *refinement_level* may - be `None`). + ``'nested_unique'``, in which case *refinement_level* may + be `None`. latitude: `bool`, optional - If True then return latitude coordinates. + If True then return the coordinate latitudes. longitude: `bool`, optional - If True then return longitude coordinates. + If True then return the coordinate longitudes. :Returns: @@ -702,8 +710,8 @@ def cf_healpix_coordinates( except ImportError as e: raise ImportError( f"{e}. Must install healpix (https://pypi.org/project/healpix) " - "to allow the calculation of latitude/longitude coordinates " - "for a HEALPix grid" + "for the calculation of latitude/longitude coordinates of a " + "HEALPix grid" ) a = cfdm_to_memory(a) @@ -711,7 +719,7 @@ def cf_healpix_coordinates( if a.ndim != 1: raise ValueError( "Can only calculate HEALPix cell coordinates when the " - f"healpix_index array has one dimension. Got shape {a.shape}" + f"healpix_index array has one dimension. Got shape: {a.shape}" ) if latitude: @@ -721,7 +729,7 @@ def cf_healpix_coordinates( match indexing_scheme: case "nested_unique": - # Create coordinates for 'nested_unique' cells + # Create coordinates for 'nested_unique' indices c = np.empty(a.shape, dtype="float64") nest = True @@ -734,7 +742,7 @@ def cf_healpix_coordinates( )[pos] case "nested" | "ring": - # Create coordinates for 'nested' or 'ring' cells + # Create coordinates for 'nested' or 'ring' indices nest = indexing_scheme == "nested" nside = healpix.order2nside(refinement_level) c = healpix.pix2ang( @@ -760,12 +768,14 @@ def cf_healpix_increase_refinement(a, ncells, iaxis, quantity): an "extensive" quantity only, the broadcast values are reduced to be consistent with the new smaller cell areas. - .. warning:: The returned `numpy` array will take up at least - *ncells* times more memory than the input array - *a*. For instance, if *a* has shape ``(3600, 48)`` - (1.3 MiB), the HEALPix axis is ``1``, and *ncells* is - ``4096``, then the returned array will have shape - ``(3600, 196608)`` (5.3 GiB), where 196608=48*4096. + .. warning:: The returned `numpy` array will take up *ncells* + times more memory than the input array *a* (or two + times *ncells* more memory, if *a* contains 32-bit + integers and *quantity* is ``'extensive'``). For + instance, if 64-bit *a* has shape ``(3600, 48)`` + (1.318 MiB), the HEALPix axis is ``1``, and *ncells* + is ``4096``, then the returned array will have shape + ``(3600, 196608)`` (5.4 GiB), where 196608=48*4096. K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution Discretization and @@ -805,13 +815,19 @@ def cf_healpix_increase_refinement(a, ncells, iaxis, quantity): if quantity == "extensive": a = a / ncells + # Create a new dimension over which the original HEALPix axis will + # be broadcast iaxis = iaxis + 1 a = np.expand_dims(a, iaxis) + # Define the shape of the new array, and broadcast 'a' to it. + # shape shape = list(a.shape) shape[iaxis] = ncells - a = np.broadcast_to(a, shape, subok=True) + + # Reshape the new array to combine the original HEALPix and + # broadcast dimensions into a single new HEALPix dimension a = a.reshape( shape[: iaxis - 1] + [shape[iaxis - 1] * shape[iaxis]] @@ -821,7 +837,7 @@ def cf_healpix_increase_refinement(a, ncells, iaxis, quantity): return a -def cf_healpix_increase_refinement_indices(a, ncells): +def cf_healpix_increase_refinement_indices(a, refinement_level, ncells): """Increase the refinement level of HEALPix indices. For instance, when going from refinement level 1 to refinement @@ -831,12 +847,14 @@ def cf_healpix_increase_refinement_indices(a, ncells): ``ncells`` is the number of cells at refinement level 2 that lie inside one cell at refinement level 1, i.e. ``ncells=4**(2-1)=4``. - .. warning:: The returned `numpy` array will take up at least - *ncells* times more memory than the input array - *a*. For instance, if *a* has shape ``(48,)`` (384 - B), and *ncells* is ``262144``, then the returned - array will have shape ``(12582912,)`` (96 MiB), where - 12582912=48*262144. + .. warning:: The returned `numpy` array will take up *ncells* + times more memory than the input array *a* (or two + times *ncells* more memory, if *a* is 32-bit and the + output requires 64-bit integers). For instance, if + 32-bit *a* has shape ``(48,)`` (192 B), and *ncells* + is ``67108864``, then the returned 64-bit array will + have shape ``(3221225472,)`` (24 GiB), where + 3221225472=48*67108864. K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution Discretization and @@ -853,6 +871,9 @@ def cf_healpix_increase_refinement_indices(a, ncells): a: `numpy.ndarray` The array of HEALPix nested indices. + refinement_level: `int` + The new higher HEALPix refinement level. + ncells: `int` The number of cells at the new refinement level which are contained in one cell at the original refinement level @@ -865,10 +886,31 @@ def cf_healpix_increase_refinement_indices(a, ncells): """ a = cfdm_to_memory(a) - a = a * ncells + # Set the data type to allow for the largest possible HEALPix + # index at the new refinement level + dtype = integer_dtype(12 * (4**refinement_level) - 1) + if a.dtype != dtype: + a = a.astype(dtype, copy=True) + a *= ncells + else: + a = a * ncells + + # Create a new dimension over which the original HEALPix axis will + # be broadcast a = np.expand_dims(a, -1) - a = np.broadcast_to(a, (a.size, ncells), subok=True).copy() + + # Define the shape of the new array, and broadcast 'a' to it. + shape = (a.size, ncells) + a = np.broadcast_to(a, shape, subok=True).copy() + + # Increment the broadcast values along the new dimension, so that + # a[i, :] contains all of the nested indices at the higher + # refinement level that correspond to index i at the original + # refinement level. a += np.arange(ncells) + + # Reshape the new array to combine the original HEALPix and + # broadcast dimensions into a single new HEALPix dimension a = a.flatten() return a diff --git a/cf/healpix.py b/cf/healpix.py index f034c4c7b7..e08962ffa5 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -279,8 +279,6 @@ def _healpix_increase_refinement_level_indices( # Set the data type to allow for the largest possible HEALPix # index at the new refinement level dtype = integer_dtype(12 * (4**refinement_level) - 1) - if dx.dtype != dtype: - dx = dx.astype(dtype, copy=False) # Each chunk is going to get larger by a factor of 'ncells' chunks = [(np.array(dx.chunks[0]) * ncells).tolist()] @@ -290,6 +288,7 @@ def _healpix_increase_refinement_level_indices( chunks=tuple(chunks), dtype=dtype, meta=np.array((), dtype=dtype), + refinement_level=refinement_level, ncells=ncells, ) From da72ae59096dba216c73e72a7b43c198742f5ed2 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 7 Aug 2025 10:56:11 +0100 Subject: [PATCH 50/59] dev --- cf/data/dask_utils.py | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index f5b3d9299f..f427dca4e4 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -919,10 +919,7 @@ def cf_healpix_increase_refinement_indices(a, refinement_level, ncells): def cf_healpix_indexing_scheme( a, indexing_scheme, new_indexing_scheme, refinement_level=None ): - """Change the ordering of HEALPix indices. - - Does not change the position of each cell in the array, but - redefines their indices according to the new ordering scheme. + """Change the indexing scheme of HEALPix indices. K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution Discretization and @@ -989,7 +986,7 @@ def cf_healpix_indexing_scheme( except ImportError as e: raise ImportError( f"{e}. Must install healpix (https://pypi.org/project/healpix) " - "to allow the changing of the HEALPix index scheme" + "for changing the HEALPix indexing scheme" ) a = cfdm_to_memory(a) @@ -1094,18 +1091,18 @@ def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): 1.06263432e+13, 1.06263432e+13]) """ + if indexing_scheme != "nested_unique": + raise ValueError( + "cf_healpix_weights: Can only calulate weights for the " + "'nested_unique' indexing scheme" + ) + try: import healpix except ImportError as e: raise ImportError( f"{e}. Must install healpix (https://pypi.org/project/healpix) " - "to allow the calculation of cell area weights for a HEALPix grid" - ) - - if indexing_scheme != "nested_unique": - raise ValueError( - "cf_healpix_weights: Can only calulate weights for the " - "'nested_unique' indexing scheme" + "for the calculation of cell area weights of a HEALPix grid" ) a = cfdm_to_memory(a) @@ -1113,15 +1110,17 @@ def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): if a.ndim != 1: raise ValueError( "Can only calculate HEALPix cell area weights when the " - f"healpix_index array has one dimension. Got shape {a.shape}" + f"healpix_index array has one dimension. Got shape: {a.shape}" ) + # Each cell at refinement level N has weight x/(4**N), where ... if measure: - # Surface area of sphere is 4*pi*(r**2) - # Number of HEALPix cells at refinement level N is 12*(4**N) - # => Area of one cell is pi*(r**2)/(3* (4**N)) + # Cell weights equal cell areas. Surface area of a sphere is + # 4*pi*(r**2), number of HEALPix cells at refinement level N + # is 12*(4**N) => Area of each cell is (pi*(r**2)/3.0)/(4**N) x = np.pi * (radius**2) / 3.0 else: + # Normalised weights x = 1.0 orders = healpix.uniq2pix(a, nest=True)[0] @@ -1133,8 +1132,8 @@ def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): w = np.empty(a.shape, dtype="float64") # For each refinement level N, put the weights (= x/4**N) into 'w' - # at the correct locations - for order, i in zip(orders, index): - w = np.where(inverse == inverse[i], x / (4**order), w) + # at the correct locations. + for N, i in zip(orders, index): + w = np.where(inverse == inverse[i], x / (4**N), w) return w From d4feb647ed89615bb919cd1d25ab0ddb4fad6e12 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 11 Aug 2025 17:49:44 +0100 Subject: [PATCH 51/59] dev --- cf/data/dask_utils.py | 171 +++------------------------------------- cf/field.py | 31 ++++---- cf/healpix.py | 123 +++++++++++++++++------------ cf/mixin/fielddomain.py | 39 ++++----- cf/test/test_Field.py | 24 +++--- 5 files changed, 133 insertions(+), 255 deletions(-) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index f427dca4e4..baada9b028 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -8,7 +8,6 @@ from functools import partial import numpy as np -from cfdm import integer_dtype from cfdm.data.dask_utils import cfdm_to_memory from scipy.ndimage import convolve1d @@ -761,161 +760,6 @@ def cf_healpix_coordinates( return c -def cf_healpix_increase_refinement(a, ncells, iaxis, quantity): - """Increase the refinement level of a HEALPix data. - - Data are broadcast to cells of at the higher refinement level. For - an "extensive" quantity only, the broadcast values are reduced to - be consistent with the new smaller cell areas. - - .. warning:: The returned `numpy` array will take up *ncells* - times more memory than the input array *a* (or two - times *ncells* more memory, if *a* contains 32-bit - integers and *quantity* is ``'extensive'``). For - instance, if 64-bit *a* has shape ``(3600, 48)`` - (1.318 MiB), the HEALPix axis is ``1``, and *ncells* - is ``4096``, then the returned array will have shape - ``(3600, 196608)`` (5.4 GiB), where 196608=48*4096. - - K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et - al.. HEALPix: A Framework for High-Resolution Discretization and - Fast Analysis of Data Distributed on the Sphere. The Astrophysical - Journal, 2005, 622 (2), pp.759-771. - https://dx.doi.org/10.1086/427976 - - .. versionadded:: NEXTVERSION - - .. seealso:: `cf.Field.healpix_increase_refinement_level` - - :Parameters: - - a: `numpy.ndarray` - The array. - - ncells: `int` - The number of cells at the new refinement level which are - contained in one cell at the original refinement level. - - iaxis: `int` - The position of the HEALPix axis in the array dimensions. - - quantity: `str` - Whether the array values represent intensive or extensive - quantities, specified with ``'intensive'`` and - ``'extensive'`` respectively. - - :Returns: - - `numpy.ndarray` - The array at the new refinement level. - - """ - a = cfdm_to_memory(a) - - if quantity == "extensive": - a = a / ncells - - # Create a new dimension over which the original HEALPix axis will - # be broadcast - iaxis = iaxis + 1 - a = np.expand_dims(a, iaxis) - - # Define the shape of the new array, and broadcast 'a' to it. - # shape - shape = list(a.shape) - shape[iaxis] = ncells - a = np.broadcast_to(a, shape, subok=True) - - # Reshape the new array to combine the original HEALPix and - # broadcast dimensions into a single new HEALPix dimension - a = a.reshape( - shape[: iaxis - 1] - + [shape[iaxis - 1] * shape[iaxis]] - + shape[iaxis + 1 :] - ) - - return a - - -def cf_healpix_increase_refinement_indices(a, refinement_level, ncells): - """Increase the refinement level of HEALPix indices. - - For instance, when going from refinement level 1 to refinement - level 2, if *a* is ``(2, 23, 17)`` then it will be transformed to - ``(8, 9, 10, 11, 92, 93, 94, 95, 68, 69, 70, 71)`` where - ``8=2*ncells, 9=2*ncells+1, ..., 71=17*ncells+3``, and where - ``ncells`` is the number of cells at refinement level 2 that lie - inside one cell at refinement level 1, i.e. ``ncells=4**(2-1)=4``. - - .. warning:: The returned `numpy` array will take up *ncells* - times more memory than the input array *a* (or two - times *ncells* more memory, if *a* is 32-bit and the - output requires 64-bit integers). For instance, if - 32-bit *a* has shape ``(48,)`` (192 B), and *ncells* - is ``67108864``, then the returned 64-bit array will - have shape ``(3221225472,)`` (24 GiB), where - 3221225472=48*67108864. - - K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et - al.. HEALPix: A Framework for High-Resolution Discretization and - Fast Analysis of Data Distributed on the Sphere. The Astrophysical - Journal, 2005, 622 (2), pp.759-771. - https://dx.doi.org/10.1086/427976 - - .. versionadded:: NEXTVERSION - - .. seealso:: `cf.Field.healpix_increase_refinement_level` - - :Parameters: - - a: `numpy.ndarray` - The array of HEALPix nested indices. - - refinement_level: `int` - The new higher HEALPix refinement level. - - ncells: `int` - The number of cells at the new refinement level which are - contained in one cell at the original refinement level - - :Returns: - - `numpy.ndarray` - The array at the new refinement level. - - """ - a = cfdm_to_memory(a) - - # Set the data type to allow for the largest possible HEALPix - # index at the new refinement level - dtype = integer_dtype(12 * (4**refinement_level) - 1) - if a.dtype != dtype: - a = a.astype(dtype, copy=True) - a *= ncells - else: - a = a * ncells - - # Create a new dimension over which the original HEALPix axis will - # be broadcast - a = np.expand_dims(a, -1) - - # Define the shape of the new array, and broadcast 'a' to it. - shape = (a.size, ncells) - a = np.broadcast_to(a, shape, subok=True).copy() - - # Increment the broadcast values along the new dimension, so that - # a[i, :] contains all of the nested indices at the higher - # refinement level that correspond to index i at the original - # refinement level. - a += np.arange(ncells) - - # Reshape the new array to combine the original HEALPix and - # broadcast dimensions into a single new HEALPix dimension - a = a.flatten() - - return a - - def cf_healpix_indexing_scheme( a, indexing_scheme, new_indexing_scheme, refinement_level=None ): @@ -972,7 +816,7 @@ def cf_healpix_indexing_scheme( ) array([16, 17, 18, 19]) >>> cf.data.dask_utils.cf_healpix_indexing_scheme( - ... [16, 17, 18, 19], 'nested_unique', 'nest', None + ... [16, 17, 18, 19], 'nested_unique', 'nested', None ) array([0, 1, 2, 3]) @@ -1030,12 +874,15 @@ def cf_healpix_indexing_scheme( case _: raise ValueError( - "Can't calculate HEALPix cell coordinates: Unknown " - f"'indexing_scheme': {indexing_scheme!r}" + "Can't change HEALPix indexing scheme: Unknown " + f"'indexing_scheme' in cf_healpix_indexing_scheme: " + f"{indexing_scheme!r}" ) - raise RuntimeError( - "cf_healpix_indexing_scheme: Failed during Dask computation" + raise ValueError( + "Can't change HEALPix indexing scheme: Unknown " + f"'new_indexing_scheme in cf_healpix_indexing_scheme: " + f"{new_indexing_scheme!r}" ) @@ -1117,7 +964,7 @@ def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): if measure: # Cell weights equal cell areas. Surface area of a sphere is # 4*pi*(r**2), number of HEALPix cells at refinement level N - # is 12*(4**N) => Area of each cell is (pi*(r**2)/3.0)/(4**N) + # is 12*(4**N) => Area of each cell is (pi*(r**2)/3)/(4**N) x = np.pi * (radius**2) / 3.0 else: # Normalised weights diff --git a/cf/field.py b/cf/field.py index bee172a655..92ede3fe79 100644 --- a/cf/field.py +++ b/cf/field.py @@ -5279,7 +5279,8 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): "sea_ice_amount" with units of kg m-2, or "air_temperature" with units of K). For an extensive quantity only, the broadcast values are reduced to be consistent with the new - smaller cell areas. + smaller cell areas x(by dividing them by the number of new + cells per original cell). **References** @@ -5410,14 +5411,6 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): # Get the HEALPix axis axis = hp["domain_axis_key"] - try: - iaxis = f.get_data_axes().index(axis) - except ValueError: - # Field data doesn't span the HEALPix axis, so insert it - # (note that it must be size 1, given that the Field data - # doen't span it). - f.insert_dimension(axis, -1, inplace=True) - iaxis = f.get_data_axes().index(axis) # Whether or not to create lat/lon coordinates for the new # refinement level. Only do so if the original grid has @@ -5432,17 +5425,26 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): ) ) + # Find the position of the HEALPix axis in the Field data + try: + iaxis = f.get_data_axes().index(axis) + except ValueError: + # The Field data doesn't span the HEALPix axis, so insert + # it. + f.insert_dimension(axis, -1, inplace=True) + iaxis = f.get_data_axes().index(axis) + # Re-size the HEALPix axis domain_axis = f.domain_axis(axis) domain_axis.set_size(f.shape[iaxis] * ncells) - # Increase the refinement of the Field data + # Increase the refinement level of the Field data _healpix_increase_refinement_level(f, ncells, iaxis, quantity) # Increase the refinement level of domain ancillary constructs # that span the HEALPix axis. We're assuming that domain - # ancillary data are intensive (i.e. do not depend on the size - # of the cell). + # ancillary data are intensive (i.e. they do not depend on the + # size of the cell). for key, domain_ancillary in f.domain_ancillaries( filter_by_axis=(axis,), axis_mode="and", todict=True ).items(): @@ -5453,7 +5455,7 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): # Increase the refinement level of cell measure constructs # that span the HEALPix axis. Cell measure data are extensive - # (i.e. depend on the size of the cell). + # (i.e. they depend on the size of the cell). for key, cell_measure in f.cell_measures( filter_by_axis=(axis,), axis_mode="and", todict=True ).items(): @@ -5465,12 +5467,11 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): # Remove all other metadata constructs that span the HEALPix # axis (including the original healpix_index coordinate # construct, and any lat/lon coordinate constructs that span - # the HEALPix axis) + # the HEALPix axis). for key in ( f.constructs.filter_by_axis(axis, axis_mode="and") .filter_by_type("cell_measure", "domain_ancillary") .inverse_filter(1) - .todict() ): f.del_construct(key) diff --git a/cf/healpix.py b/cf/healpix.py index e08962ffa5..b0f54c64e2 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -84,7 +84,10 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): return (None, None) - # Get the Dask array of HEALPix indices + # Get the Dask array of HEALPix indices. + # + # `cf_healpix_coordinates` anad `cf_healpix_bounds` have their own + # calls to `cfdm_to_memory`, so we can set _force_to_memory=False. dx = healpix_index.data.to_dask_array( _force_mask_hardness=False, _force_to_memory=False ) @@ -131,7 +134,7 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): latitude=True, ) bounds = f._Bounds(data=dy) - lat.set_bounds(bounds) + lat.set_bounds(bounds, copy=False) # Create longitude bounds dy = da.blockwise( @@ -147,7 +150,7 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): pole_longitude=pole_longitude, ) bounds = f._Bounds(data=dy) - lon.set_bounds(bounds) + lon.set_bounds(bounds, copy=False) # Set the new latitude and longitude coordinates axis = hp["domain_axis_key"] @@ -173,7 +176,8 @@ def _healpix_increase_refinement_level(x, ncells, iaxis, quantity): :Parameters: x: construct - The construct containing data that is to be changed. + The construct containing data that is to be changed + in-place. ncells: `int` The number of cells at the new refinement level which are @@ -193,37 +197,38 @@ def _healpix_increase_refinement_level(x, ncells, iaxis, quantity): `None` """ - from .data.dask_utils import cf_healpix_increase_refinement + from dask.array.core import normalize_chunks # Get the Dask array. - # - # `cf_healpix_increase_refinement` has its own call to - # `cfdm_to_memory`, so we can set _force_to_memory=False. - dx = x.data.to_dask_array( - _force_mask_hardness=False, _force_to_memory=False - ) + dx = x.data.to_dask_array(_force_mask_hardness=False) + # Divide extensive data by the number of new cells if quantity == "extensive": - # Extensive data get divided by 'ncells' in - # `cf_healpix_increase_refinement`, so they end up with a - # data type of float64. - dtype = np.dtype("float64") - else: - dtype = dx.dtype - - # Each chunk is going to get larger by a factor of 'ncells' - chunks = list(dx.chunks) - chunks[iaxis] = (np.array(chunks[iaxis]) * ncells).tolist() + dx = dx / ncells - dx = dx.map_blocks( - cf_healpix_increase_refinement, - chunks=tuple(chunks), - dtype=dtype, - meta=np.array((), dtype=dtype), - ncells=ncells, - iaxis=iaxis, - quantity=quantity, + # Add a new size dimension just after the HEALPix dimension + new_axis = iaxis + 1 + dx = da.expand_dims(dx, new_axis) + + # Work out what the chunks should be for the new dimension + shape = list(dx.shape) + shape[new_axis] = ncells + + chunks = list(dx.chunks) + chunks[new_axis] = "auto" + chunks = normalize_chunks(chunks, shape, dtype=dx.dtype) + + # Broadcast the data along the new dimension + dx = da.broadcast_to(dx, shape, chunks=chunks) + + # Reshape the array so that it has a single, larger HEALPix + # dimension + dx = dx.reshape( + shape[:iaxis] + + [shape[iaxis] * shape[new_axis]] + + shape[new_axis + 1 :] ) + x.set_data(dx, copy=False) @@ -262,35 +267,50 @@ def _healpix_increase_refinement_level_indices( """ from cfdm import integer_dtype + from dask.array.core import normalize_chunks - from .data.dask_utils import cf_healpix_increase_refinement_indices - - # Save any cached data elements + # Get any cached data values cached = healpix_index.data._get_cached_elements().copy() - # Get the Dask array. - # - # `cf_healpix_increase_refinement_indices` has its own call to - # `cfdm_to_memory`, so we can set _force_to_memory=False. - dx = healpix_index.data.to_dask_array( - _force_mask_hardness=False, _force_to_memory=False - ) + # Get the Dask array + dx = healpix_index.data.to_dask_array(_force_mask_hardness=False) # Set the data type to allow for the largest possible HEALPix # index at the new refinement level dtype = integer_dtype(12 * (4**refinement_level) - 1) + if dx.dtype != dtype: + dx = dx.astype(dtype) - # Each chunk is going to get larger by a factor of 'ncells' - chunks = [(np.array(dx.chunks[0]) * ncells).tolist()] + # Change each original HEALpix index to the smallest new HEALPix + # index that the larger cell contains + dx = dx * ncells - dx = dx.map_blocks( - cf_healpix_increase_refinement_indices, - chunks=tuple(chunks), - dtype=dtype, - meta=np.array((), dtype=dtype), - refinement_level=refinement_level, - ncells=ncells, - ) + # Add a new size dimension just after the HEALPix dimension + new_axis = 1 + dx = da.expand_dims(dx, new_axis) + + # Work out what the chunks should be for the new dimension + shape = list(dx.shape) + shape[new_axis] = ncells + + chunks = list(dx.chunks) + chunks[new_axis] = "auto" + chunks = normalize_chunks(chunks, shape, dtype=dx.dtype) + + # Broadcast the data along the new dimension + dx = da.broadcast_to(dx, shape, chunks=chunks) + + # Increment the broadcast values along the new dimension, so that + # dx[i, :] contains all of the nested indices at the higher + # refinement level that correspond to HEALPix index value i at the + # original refinement level. + new_shape = [1] * dx.ndim + new_shape[new_axis] = shape[new_axis] + dx += da.arange(ncells, chunks=chunks[new_axis]).reshape(new_shape) + + # Reshape the new array to combine the original HEALPix and + # broadcast dimensions into a single new HEALPix dimension + dx = dx.reshape(shape[0] * shape[new_axis]) healpix_index.set_data(dx, copy=False) @@ -304,6 +324,8 @@ def _healpix_increase_refinement_level_indices( x = np.array(cached[-1], dtype=dtype) * ncells + (ncells - 1) data._set_cached_elements({-1: x}) + return + def _healpix_indexing_scheme(healpix_index, hp, new_indexing_scheme): """Change the indexing scheme of HEALPix indices in-place. @@ -345,6 +367,9 @@ def _healpix_indexing_scheme(healpix_index, hp, new_indexing_scheme): refinement_level = hp.get("refinement_level") # Change the HEALPix indices + # + # `cf_healpix_indexing_scheme` has its own call to + # `cfdm_to_memory`, so we can set _force_to_memory=False. dx = healpix_index.data.to_dask_array( _force_mask_hardness=False, _force_to_memory=False ) diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 117b822cd3..468af92fe6 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -2354,8 +2354,8 @@ def create_latlon_coordinates( When it is not possible to create latitude and longitude coordinates, the reason why will be reported if the log level - at ``2``/``'INFO'`` or higher (as set by `cf.log_level` or the - *verbose* parameter). + is at ``2``/``'INFO'`` or higher (as set by `cf.log_level` or + the *verbose* parameter). .. versionadded:: NEXTVERSION @@ -2365,13 +2365,13 @@ def create_latlon_coordinates( one_d: `bool`, optional` If True (the default) then attempt to create 1-d - latitude and longitude coordinates, if - possible. Otherwise do not attempt this. + latitude and longitude coordinates. Otherwise do not + attempt this. two_d: `bool`, optional` If True (the default) then attempt to create 2-d - latitude and longitude coordinates, if - possible. Otherwise do not attempt this. + latitude and longitude coordinates. Otherwise do not + attempt this. pole_longitude: `None` or number The longitude of coordinates, or coordinate bounds, @@ -2396,9 +2396,8 @@ def create_latlon_coordinates( `{{class}}` or `None` The {{class}} with new latitude and longitude - constructs, if any could be created. If none could be - created then a new, identical field is returned. If - the operation was in-place then `None` is returned. + constructs, if any could be created. If the operation + was in-place then `None` is returned. **Examples** @@ -2453,11 +2452,12 @@ def create_latlon_coordinates( # Store all of the grid mapping Coordinate References in a # dictionary - identities = { - cr.identity(""): cr + coordinate_references = { + cr.identity(None): cr for cr in f.coordinate_references(todict=True).values() } - if not identities: + coordinate_references.pop(None, None) + if not coordinate_references: if is_log_level_info(logger): logger.info( "Can't create latitude and longitude coordinates: " @@ -2466,18 +2466,18 @@ def create_latlon_coordinates( return f - identities = { + coordinate_references = { identity: cr - for identity, cr in identities.items() + for identity, cr in coordinate_references.items() if identity.startswith("grid_mapping_name:") } # Remove a 'latitude_longitude' grid mapping (if there is one) # from the dictionary, saving it for later. - latlon_cr = identities.pop( + latlon_cr = coordinate_references.pop( "grid_mapping_name:latitude_longitude", None ) - if not identities: + if not coordinate_references: if is_log_level_info(logger): logger.info( "Can't create latitude and longitude coordinates: There " @@ -2487,12 +2487,13 @@ def create_latlon_coordinates( return f - if len(identities) > 1: + if len(coordinate_references) > 1: if is_log_level_info(logger): logger.info( "Can't create latitude and longitude coordinates: There " "is more than one non-latitude_longitude grid mapping " - "coordinate reference" + "coordinate reference: " + f"{', '.join(map(repr, coordinate_references.values()))}" ) # pragma: no cover return f @@ -2501,7 +2502,7 @@ def create_latlon_coordinates( # Still here? Then get the unique non-latitude_longitude grid # mapping, and use it to calculate the lat/lon coordinates. # ------------------------------------------------------------ - identity, cr = identities.popitem() + identity, cr = coordinate_references.popitem() # Initialize the flag that tells us if any new coordinates # have been created diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index f18e88e86f..9fee5cc39f 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -3428,16 +3428,20 @@ def test_Field_healpix_increase_refinement_level(self): # Cached values f.coordinate("healpix_index").data._del_cached_elements() - g = f.healpix_increase_refinement_level(29, "intensive") - self.assertEqual( - g.coordinate("healpix_index").data._get_cached_elements(), {} - ) - _ = str(f.coordinate("healpix_index").data) # Create cached elements - g = f.healpix_increase_refinement_level(29, "intensive") - self.assertEqual( - g.coordinate("healpix_index").data._get_cached_elements(), - {0: 0, 1: 1, -1: 3458764513820540927}, - ) + with cf.chunksize(256 * (2**30)): + g = f.healpix_increase_refinement_level(16, "intensive") + self.assertEqual(g.data.npartitions, 6) + self.assertEqual( + g.coordinate("healpix_index").data._get_cached_elements(), {} + ) + _ = str( + f.coordinate("healpix_index").data + ) # Create cached elements + g = f.healpix_increase_refinement_level(16, "intensive") + self.assertEqual( + g.coordinate("healpix_index").data._get_cached_elements(), + {0: 0, 1: 1, -1: 51539607551}, + ) # Bad 'quantity' parameter with self.assertRaises(ValueError): From aa0f70cbc9b12122d8f43b0ce5458003dd436edb Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 12 Aug 2025 09:57:17 +0100 Subject: [PATCH 52/59] dev --- cf/data/dask_utils.py | 12 +++++++----- cf/healpix.py | 28 +++++++++++++++++++--------- cf/mixin/fielddomain.py | 21 +++++++++++++-------- cf/test/test_Field.py | 7 +++---- 4 files changed, 42 insertions(+), 26 deletions(-) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index baada9b028..1cd0e1da53 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -518,11 +518,13 @@ def cf_healpix_bounds( If True then return the bounds' longitudes. pole_longitude: `None` or number - The longitude of bounds that lie exactly on the north - (south) pole. If `None` (the default) then the longitude - of such a vertex will be the same as the south (north) - vertex of the same cell. If set to a number, then the - longitudes of all such vertices will be given that value. + Define the longitudes of vertices that lie exactly on the + north or south pole. If `None` (the default) then the + longitude of such a vertex on the north (south) pole will + be the same as the longitude of the south (north) vertex + of the same cell. If set to a number, then the longitudes + of all vertices on the north or south pole will be given + the value *pole_longitude*. :Returns: diff --git a/cf/healpix.py b/cf/healpix.py index b0f54c64e2..e5b7c1a2b4 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -199,7 +199,7 @@ def _healpix_increase_refinement_level(x, ncells, iaxis, quantity): """ from dask.array.core import normalize_chunks - # Get the Dask array. + # Get the Dask array (e.g. dx.shape is (12, 19, 48)) dx = x.data.to_dask_array(_force_mask_hardness=False) # Divide extensive data by the number of new cells @@ -207,22 +207,27 @@ def _healpix_increase_refinement_level(x, ncells, iaxis, quantity): dx = dx / ncells # Add a new size dimension just after the HEALPix dimension + # (e.g. .shape becomes (12, 19, 48, 1)) new_axis = iaxis + 1 dx = da.expand_dims(dx, new_axis) - # Work out what the chunks should be for the new dimension + # Modify the size of the new dimension to be the number of cells + # at the new refinement level which are contained in one cell at + # the original refinement level (e.g. size becomes 16) shape = list(dx.shape) shape[new_axis] = ncells + # Work out what the chunks should be for the new dimension chunks = list(dx.chunks) chunks[new_axis] = "auto" chunks = normalize_chunks(chunks, shape, dtype=dx.dtype) - # Broadcast the data along the new dimension + # Broadcast the data along the new dimension (e.g. dx.shape + # becomes (12, 19, 48, 16)) dx = da.broadcast_to(dx, shape, chunks=chunks) # Reshape the array so that it has a single, larger HEALPix - # dimension + # dimension (e.g. dx.shape becomes (12, 19, 768)) dx = dx.reshape( shape[:iaxis] + [shape[iaxis] * shape[new_axis]] @@ -272,7 +277,7 @@ def _healpix_increase_refinement_level_indices( # Get any cached data values cached = healpix_index.data._get_cached_elements().copy() - # Get the Dask array + # Get the Dask array (e.g. dx.shape is (48,)) dx = healpix_index.data.to_dask_array(_force_mask_hardness=False) # Set the data type to allow for the largest possible HEALPix @@ -286,18 +291,23 @@ def _healpix_increase_refinement_level_indices( dx = dx * ncells # Add a new size dimension just after the HEALPix dimension + # (e.g. .shape becomes (48, 1)) new_axis = 1 dx = da.expand_dims(dx, new_axis) - # Work out what the chunks should be for the new dimension + # Modify the size of the new dimension to be the number of cells + # at the new refinement level which are contained in one cell at + # the original refinement level (e.g. size becomes 16) shape = list(dx.shape) shape[new_axis] = ncells + # Work out what the chunks should be for the new dimension chunks = list(dx.chunks) chunks[new_axis] = "auto" chunks = normalize_chunks(chunks, shape, dtype=dx.dtype) - # Broadcast the data along the new dimension + # Broadcast the data along the new dimension (e.g. dx.shape + # becomes (48, 16)) dx = da.broadcast_to(dx, shape, chunks=chunks) # Increment the broadcast values along the new dimension, so that @@ -306,10 +316,12 @@ def _healpix_increase_refinement_level_indices( # original refinement level. new_shape = [1] * dx.ndim new_shape[new_axis] = shape[new_axis] + dx += da.arange(ncells, chunks=chunks[new_axis]).reshape(new_shape) # Reshape the new array to combine the original HEALPix and # broadcast dimensions into a single new HEALPix dimension + # (e.g. dx.shape becomes (768,)) dx = dx.reshape(shape[0] * shape[new_axis]) healpix_index.set_data(dx, copy=False) @@ -324,8 +336,6 @@ def _healpix_increase_refinement_level_indices( x = np.array(cached[-1], dtype=dtype) * ncells + (ncells - 1) data._set_cached_elements({-1: x}) - return - def _healpix_indexing_scheme(healpix_index, hp, new_indexing_scheme): """Change the indexing scheme of HEALPix indices in-place. diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 468af92fe6..dd4c2a9b20 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -2268,6 +2268,7 @@ def healpix_to_ugrid(self, inplace=False): two_d=False, pole_longitude=0, inplace=True ) + # Get the lat/lon coordinates x_key, x = f.auxiliary_coordinate( "Y", filter_by_axis=(axis,), @@ -2308,8 +2309,9 @@ def healpix_to_ugrid(self, inplace=False): "coordinate bounds" ) - # Create the Domain Topology construct, by creating a unique - # integer identifer for each node location. + # Create the UGRID Domain Topology construct, by creating an + # arbitrary unique integer identifer for each unique node + # location. bounds_y = bounds_y.data.to_dask_array(_force_mask_hardness=False) bounds_x = bounds_x.data.to_dask_array(_force_mask_hardness=False) @@ -2374,12 +2376,15 @@ def create_latlon_coordinates( attempt this. pole_longitude: `None` or number - The longitude of coordinates, or coordinate bounds, - that lie exactly on the north or south pole. If `None` - (the default) then the longitudes of such points will - vary according to the algorithm being used to create - them. If set to a number, then the longitudes of such - points will all be given that value. + Define the longitudes of coordinates or coordinate + bounds that lie exactly on the north or south pole. If + `None` (the default) then the longitudes of such + points are determined by whatever algorithm was used + to create the coordinates, which will likely result in + different points on a pole having different + longitudes. If set to a number, then the longitudes of + all points on the north or south pole will be given + the value *pole_longitude*. overwrite: `bool`, optional If True then remove any existing latitude and diff --git a/cf/test/test_Field.py b/cf/test/test_Field.py index 9fee5cc39f..cbfec2b144 100644 --- a/cf/test/test_Field.py +++ b/cf/test/test_Field.py @@ -3173,7 +3173,7 @@ def test_Field_healpix_to_ugrid(self): self.assertEqual(len(u.domain_topologies()), 1) self.assertEqual(len(u.auxiliary_coordinates()), 2) - topology = u.domain_topology().normalise().array + topology = u.domain_topology().normalise() self.assertEqual(np.unique(topology).size, 53) self.assertTrue( np.array_equal( @@ -3434,9 +3434,8 @@ def test_Field_healpix_increase_refinement_level(self): self.assertEqual( g.coordinate("healpix_index").data._get_cached_elements(), {} ) - _ = str( - f.coordinate("healpix_index").data - ) # Create cached elements + # Create cached elements + _ = str(f.coordinate("healpix_index").data) g = f.healpix_increase_refinement_level(16, "intensive") self.assertEqual( g.coordinate("healpix_index").data._get_cached_elements(), From 181141c878198a10ebf3b25bae7b9394b02921cf Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 13 Aug 2025 23:48:36 +0100 Subject: [PATCH 53/59] dev --- cf/mixin/fielddomain.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index dd4c2a9b20..fa07e11fa5 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -1978,6 +1978,11 @@ def coordinate_reference_domain_axes(self, identity=None): def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): r"""Change the indexing scheme of HEALPix indices. + Note that the Field data values are not changed, nor is the + Field Data array reordered. Only the "healpix_index" + coordinate values are changed, along with the corresponding + "healpix" grid mapping Coordinate Reference. + **References** {{HEALPix references}} From bf76284f6fe52866bdbe9fc910881559f7fb98e5 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 21 Aug 2025 10:51:49 +0100 Subject: [PATCH 54/59] dev --- cf/constants.py | 7 +++++++ cf/data/dask_utils.py | 39 +++++++++++++++++++++++++++------------ cf/domain.py | 10 ++++------ cf/field.py | 36 +++++++++++++++++++++++++++--------- cf/healpix.py | 25 ++++++++++++++----------- cf/mixin/fielddomain.py | 10 +++++----- 6 files changed, 84 insertions(+), 43 deletions(-) diff --git a/cf/constants.py b/cf/constants.py index 14c4212f5c..f29b44bb70 100644 --- a/cf/constants.py +++ b/cf/constants.py @@ -583,6 +583,12 @@ ) +# -------------------------------------------------------------------- +# CF HEALPix indexing schemes +# -------------------------------------------------------------------- +healpix_indexing_schemes = ("nested", "ring", "nested_unique") + + # -------------------------------------------------------------------- # Logging level setup # -------------------------------------------------------------------- @@ -604,3 +610,4 @@ class OperandBoundsCombination(Enum): OR = auto() XOR = auto() NONE = auto() + diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index 1cd0e1da53..9d9d711f75 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -763,7 +763,11 @@ def cf_healpix_coordinates( def cf_healpix_indexing_scheme( - a, indexing_scheme, new_indexing_scheme, refinement_level=None + a, + indexing_scheme, + new_indexing_scheme, + healpix_index_dtype, + refinement_level=None, ): """Change the indexing scheme of HEALPix indices. @@ -794,6 +798,12 @@ def cf_healpix_indexing_scheme( The new HEALPix indexing scheme to change to. One of ``'nested'``, ``'ring'``, or ``'nested_unique'``. + healpix_index_dtype: `str` or `numpy.dtype` + Typecode or data-type to which the new indices will be + cast. This should be the smallest data type needed for + storing the largest possible index value at the refinement + level. + refinement_level: `int` or `None`, optional The refinement level of the grid within the HEALPix hierarchy, starting at 0 for the base tessellation with 12 @@ -827,6 +837,15 @@ def cf_healpix_indexing_scheme( # Null operation return a + from ..constants import healpix_indexing_schemes + + if new_indexing_scheme not in healpix_indexing_schemes: + raise ValueError( + "Can't change HEALPix indexing scheme: Unknown " + f"'new_indexing_scheme' in cf_healpix_indexing_scheme: " + f"{new_indexing_scheme!r}" + ) + try: import healpix except ImportError as e: @@ -842,19 +861,19 @@ def cf_healpix_indexing_scheme( match new_indexing_scheme: case "ring": nside = healpix.order2nside(refinement_level) - return healpix.nest2ring(nside, a) + a = healpix.nest2ring(nside, a) case "nested_unique": - return healpix.pix2uniq(refinement_level, a, nest=True) + a = healpix.pix2uniq(refinement_level, a, nest=True) case "ring": match new_indexing_scheme: case "nested": nside = healpix.order2nside(refinement_level) - return healpix.ring2nest(nside, a) + a = healpix.ring2nest(nside, a) case "nested_unique": - return healpix.pix2uniq(refinement_level, a, nest=False) + a = healpix.pix2uniq(refinement_level, a, nest=False) case "nested_unique": match new_indexing_scheme: @@ -872,8 +891,6 @@ def cf_healpix_indexing_scheme( f"{refinement_levels.tolist()})" ) - return a - case _: raise ValueError( "Can't change HEALPix indexing scheme: Unknown " @@ -881,11 +898,9 @@ def cf_healpix_indexing_scheme( f"{indexing_scheme!r}" ) - raise ValueError( - "Can't change HEALPix indexing scheme: Unknown " - f"'new_indexing_scheme in cf_healpix_indexing_scheme: " - f"{new_indexing_scheme!r}" - ) + # Cast the new indices to the given data type + a = a.astype(healpix_index_dtype, copy=False) + return a def cf_healpix_weights(a, indexing_scheme, measure=False, radius=None): diff --git a/cf/domain.py b/cf/domain.py index 71f93c4629..b19e340282 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -370,10 +370,8 @@ def create_healpix( """ import dask.array as da - from .healpix import ( - HEALPix_indexing_schemes, - healpix_max_refinement_level, - ) + from .constants import healpix_indexing_schemes + from .healpix import healpix_max_refinement_level if ( not isinstance(refinement_level, Integral) @@ -386,10 +384,10 @@ def create_healpix( f"{healpix_max_refinement_level()}. Got {refinement_level!r}" ) - if indexing_scheme not in HEALPix_indexing_schemes: + if indexing_scheme not in healpix_indexing_schemes: raise ValueError( "Can't create HEALPix Domain: 'indexing_scheme' must be one " - f"of {HEALPix_indexing_schemes!r}. Got {indexing_scheme!r}" + f"of {healpix_indexing_schemes!r}. Got {indexing_scheme!r}" ) nested_unique = indexing_scheme == "nested_unique" diff --git a/cf/field.py b/cf/field.py index 92ede3fe79..948de55e18 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4923,6 +4923,9 @@ def healpix_decrease_refinement_level( ``'variance'`` `np.var` ======================== =================== + Note that these methods may be also calculated by any + function provided by the *reduction* parameter. + conform: `bool`, optional If True (the default) then the HEALPix grid is automatically converted to a form suitable for having @@ -4970,8 +4973,15 @@ def healpix_decrease_refinement_level( **Examples** >>> f = cf.example_field(12) - >>> f - + >>> print(f) + Field: air_temperature (ncvar%tas) + ---------------------------------- + Data : air_temperature(time(2), healpix_index(48)) K + Cell methods : time(2): mean area: mean + Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian + : healpix_index(48) = [0, ..., 47] + : height(1) = [1.5] m + Coord references: grid_mapping_name:healpix >>> f.healpix_info()['refinement_level'] 1 @@ -4988,7 +4998,7 @@ def healpix_decrease_refinement_level( Cell methods : time(2): mean area: mean area: maximum Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian : height(1) = [1.5] m - Auxiliary coords: healpix_index(healpix_index(12)) = [0, ..., 11] 1 + : healpix_index(healpix_index(12)) = [0, ..., 11] Coord references: grid_mapping_name:healpix >>> g.healpix_info()['refinement_level'] 0 @@ -5012,7 +5022,7 @@ def healpix_decrease_refinement_level( Cell methods : time(2): mean area: mean area: range Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian : height(1) = [1.5] m - Auxiliary coords: healpix_index(healpix_index(12)) = [0, ..., 11] 1 + : healpix_index(healpix_index(12)) = [0, ..., 11] Coord references: grid_mapping_name:healpix >>> print(g[0, 0].array) [[8.2]] @@ -5181,9 +5191,10 @@ def healpix_decrease_refinement_level( # Chenge the refinement level of the Field's data # ------------------------------------------------------------ - # Note: Using 'Data.coarsen' works because a) we have 'nested' - # HEALPix ordering, and b) each coarser cell contains - # the maximum possible number of original cells. + # Note: Using 'Data.coarsen' only works because a) the HEALPix + # indexing scheme is "nested", and b) each new coarser + # cell contains the maximum possible number + # (i.e. 'ncells') of original cells. f.data.coarsen( reduction, axes={iaxis: ncells}, trim_excess=False, inplace=True ) @@ -5312,8 +5323,15 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): **Examples** >>> f = cf.example_field(12) - >>> f - + >>> print(f) + Field: air_temperature (ncvar%tas) + ---------------------------------- + Data : air_temperature(time(2), healpix_index(48)) K + Cell methods : time(2): mean area: mean + Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian + : healpix_index(48) = [0, ..., 47] + : height(1) = [1.5] m + Coord references: grid_mapping_name:healpix >>> f.healpix_info()['refinement_level'] 1 >>> g = f.healpix_increase_refinement_level(3, 'intensive') diff --git a/cf/healpix.py b/cf/healpix.py index 183bf0135b..c2ee05d982 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -8,8 +8,6 @@ logger = logging.getLogger(__name__) -HEALPix_indexing_schemes = ("nested", "ring", "nested_unique") - def _healpix_create_latlon_coordinates(f, pole_longitude): """Create latitude and longitude coordinates for a HEALPix grid. @@ -47,18 +45,19 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): created. """ + from .constants import healpix_indexing_schemes from .data.dask_utils import cf_healpix_bounds, cf_healpix_coordinates hp = f.healpix_info() indexing_scheme = hp.get("indexing_scheme") - if indexing_scheme not in HEALPix_indexing_schemes: + if indexing_scheme not in healpix_indexing_schemes: if is_log_level_info(logger): logger.info( "Can't create 1-d latitude and longitude coordinates for " f"{f!r}: indexing_scheme in the healpix grid mapping " "coordinate reference must be one of " - f"{HEALPix_indexing_schemes!r}. Got {indexing_scheme!r}" + f"{healpix_indexing_schemes!r}. Got {indexing_scheme!r}" ) # pragma: no cover return (None, None) @@ -356,12 +355,13 @@ def _healpix_indexing_scheme(f, hp, new_indexing_scheme): :Parameters: - healpix_index: `Coordinate` TODOHEALPIX - The healpix_index coordinates, which will be updated TODOHEALPIX - in-place. + f: `Field` or `Domain` + The Field or Domain that contains the healpix_index + coordinates. *f* will be updated with the new HEALPix + indices in-place. hp: `dict` - The HEALPix info dictionary. + The HEALPix info dictionary for *f*. new_indexing_scheme: `str` The new indexing scheme. @@ -390,10 +390,12 @@ def _healpix_indexing_scheme(f, hp, new_indexing_scheme): # Find the datatype for the largest possible index at this # refinement level if new_indexing_scheme == "nested_unique": - # 16*(4**n) - 1 = 4*(4**n) + 12*(4**n) - 1 + # The largest possible "nested_unique" index at refinement + # level N is 16*(4**N) - 1 = 4*(4**N) + 12*(4**N) - 1 dtype = integer_dtype(16 * (4**refinement_level) - 1) else: - # nested or ring + # The largest possible "nested" or "ring" index at refinement + # level N is 12*(4**N) - 1 dtype = integer_dtype(12 * (4**refinement_level) - 1) dx = dx.map_blocks( @@ -402,6 +404,7 @@ def _healpix_indexing_scheme(f, hp, new_indexing_scheme): meta=np.array((), dtype=dtype), indexing_scheme=indexing_scheme, new_indexing_scheme=new_indexing_scheme, + healpix_index_dtype=dtype, refinement_level=refinement_level, ) healpix_index.set_data(dx, copy=False) @@ -534,7 +537,7 @@ def _healpix_locate(lat, lon, f): raise ValueError( f"Can't locate HEALPix cells for {f!r}: indexing_scheme in " "the healpix grid mapping coordinate reference must be one " - f"of {HEALPix_indexing_schemes!r}. Got {indexing_scheme!r}" + f"of {healpix_indexing_schemes!r}. Got {indexing_scheme!r}" ) # Return the cell locations as a numpy array of element indices diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index fdfc408727..d020a684e7 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -2058,17 +2058,17 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47] """ - from ..healpix import HEALPix_indexing_schemes + from ..constants import healpix_indexing_schemes f = self.copy() hp = f.healpix_info() - if new_indexing_scheme not in HEALPix_indexing_schemes + (None,): + if new_indexing_scheme not in healpix_indexing_schemes + (None,): raise ValueError( f"Can't change HEALPix index scheme of {f!r}: " "new_indexing_scheme keyword must be None or one of " - f"{HEALPix_indexing_schemes!r}. Got {new_indexing_scheme!r}" + f"{healpix_indexing_schemes!r}. Got {new_indexing_scheme!r}" ) # Get the healpix_index coordinates @@ -2087,12 +2087,12 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): "mapping coordinate reference" ) - if indexing_scheme not in HEALPix_indexing_schemes: + if indexing_scheme not in healpix_indexing_schemes: raise ValueError( f"Can't change HEALPix indexing scheme of {f!r}: " "indexing_scheme in the healpix grid mapping coordinate " "reference must be one of " - f"{HEALPix_indexing_schemes!r}. Got {indexing_scheme!r}" + f"{healpix_indexing_schemes!r}. Got {new_indexing_scheme!r}" ) if ( From bb08431ef2672f4452d64b666670acb95a1ec304 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 21 Aug 2025 10:53:43 +0100 Subject: [PATCH 55/59] dev --- cf/constants.py | 1 - cf/domain.py | 2 +- cf/healpix.py | 2 ++ 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cf/constants.py b/cf/constants.py index f29b44bb70..8e42431740 100644 --- a/cf/constants.py +++ b/cf/constants.py @@ -610,4 +610,3 @@ class OperandBoundsCombination(Enum): OR = auto() XOR = auto() NONE = auto() - diff --git a/cf/domain.py b/cf/domain.py index b19e340282..82ce722f84 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -371,7 +371,7 @@ def create_healpix( import dask.array as da from .constants import healpix_indexing_schemes - from .healpix import healpix_max_refinement_level + from .healpix import healpix_max_refinement_level if ( not isinstance(refinement_level, Integral) diff --git a/cf/healpix.py b/cf/healpix.py index c2ee05d982..a8538a2e51 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -534,6 +534,8 @@ def _healpix_locate(lat, lon, f): ) case _: + from .constants import healpix_indexing_schemes + raise ValueError( f"Can't locate HEALPix cells for {f!r}: indexing_scheme in " "the healpix grid mapping coordinate reference must be one " From d81a735dd5eec6e1d2a54da34416d06ee5426f46 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 21 Aug 2025 17:37:50 +0100 Subject: [PATCH 56/59] dev --- cf/data/dask_utils.py | 6 ++++ cf/domain.py | 45 +++++++++++++++++++-------- cf/field.py | 67 +++++++++++++++++++---------------------- cf/healpix.py | 8 +++-- cf/mixin/fielddomain.py | 61 ++++++++++++++++++++----------------- 5 files changed, 108 insertions(+), 79 deletions(-) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index 9d9d711f75..76874a9679 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -723,6 +723,12 @@ def cf_healpix_coordinates( f"healpix_index array has one dimension. Got shape: {a.shape}" ) + if latitude == longitude: + raise ValueError( + "Can't calculate HEALPix cell coordinates: " + f"latitude={latitude!r} and longitude={longitude!r}" + ) + if latitude: pos = 1 elif longitude: diff --git a/cf/domain.py b/cf/domain.py index 82ce722f84..dec83a76ef 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -267,11 +267,31 @@ def create_regular(cls, x_args, y_args, bounds=True): def create_healpix( cls, refinement_level, indexing_scheme="nested", radius=None ): - r"""Create a new global HEALPix domain. + r"""Create a new global HEALPix grid. The HEALPix axis of the new Domain is ordered so that the HEALPix indices are monotonically increasing. + **Performance** + + Very high refinement levels may require the setting of a very + large Dask chunksize, to prevent a possible run-time failure + resulting from an attempt to create an excessive amount of + chunks for the healpix_index coordinates. For instance, + healpix_index coordinates at refinement level 29 would need + ~206e9 chunks with the default Dask chunksize of 128 MiB, but + with a chunksize of 1 PiB, only 24576 chunks are required:: + + >>> cf.chunksize() + >>> 134217728 + >>> d = cf.Domain.create_healpix(10) + >>> assert d.coord('healpix_index').data.npartitions == 1 + >>> d = cf.Domain.create_healpix(15) + >>> assert d.coord('healpix_index').data.npartitions == 768 + >>> with cf.chunksize('1 PiB'): + ... d = cf.Domain.create_healpix(29) + ... assert d.coord('healpix_index').data.npartitions == 24576 + **References** {{HEALPix references}} @@ -324,34 +344,33 @@ def create_healpix( >>> d = cf.Domain.create_healpix(4) >>> d.dump() -------- - Domain: + Domain: -------- Domain Axis: healpix_index(3072) - + Dimension coordinate: healpix_index standard_name = 'healpix_index' - units = '1' Data(healpix_index(3072)) = [0, ..., 3071] - + Coordinate reference: grid_mapping_name:healpix Coordinate conversion:grid_mapping_name = healpix Coordinate conversion:indexing_scheme = nested Coordinate conversion:refinement_level = 4 - Dimension Coordinate: healpix_index + Dimension Coordinate: healpix_index .. code-block:: python >>> d = cf.Domain.create_healpix(4, "nested_unique", radius=6371000) >>> d.dump() -------- - Domain: + Domain: -------- Domain Axis: healpix_index(3072) - + Dimension coordinate: healpix_index standard_name = 'healpix_index' Data(healpix_index(3072)) = [1024, ..., 4095] - + Coordinate reference: grid_mapping_name:healpix Coordinate conversion:grid_mapping_name = healpix Coordinate conversion:indexing_scheme = nested_unique @@ -362,9 +381,9 @@ def create_healpix( >>> d.create_latlon_coordinates(inplace=True) >>> print(d) - Dimension coords: healpix_index(ncdim%cell(3072)) = [1024, ..., 4095] - Auxiliary coords: latitude(ncdim%cell(3072)) = [2.388015463268772, ..., -2.388015463268786] degrees_north - : longitude(ncdim%cell(3072)) = [45.0, ..., 315.0] degrees_east + Dimension coords: healpix_index(3072) = [1024, ..., 4095] + Auxiliary coords: latitude(healpix_index(3072)) = [2.388015463268772, ..., -2.388015463268786] degrees_north + : longitude(healpix_index(3072)) = [45.0, ..., 315.0] degrees_east Coord references: grid_mapping_name:healpix """ @@ -413,7 +432,7 @@ def create_healpix( stop = start + ncells dtype = cfdm.integer_dtype(stop - 1) - data = Data(da.arange(start, stop, dtype=dtype), units="1") + data = Data(da.arange(start, stop, dtype=dtype)) # Set cached data elements data._set_cached_elements({0: start, 1: start + 1, -1: stop - 1}) diff --git a/cf/field.py b/cf/field.py index 948de55e18..f194ef71df 100644 --- a/cf/field.py +++ b/cf/field.py @@ -4853,9 +4853,10 @@ def healpix_decrease_refinement_level( covered by original cells. For instance, if the original refinement level is 10 and the new refinement level is 8, then each output cell will be the combination of :math:`16\equiv - 4^{(10-8)}` original cells, and if a larger cell contains at - least one but fewer than 16 original cells then an exception - is raised (assuming that *check_healpix_index* is True). + 4^2\equiv 4^{(10-8)}` original cells, and if a larger cell + contains at least one but fewer than 16 original cells then an + exception is raised (assuming that *check_healpix_index* is + True). **References** @@ -4878,7 +4879,7 @@ def healpix_decrease_refinement_level( method: `str` The method used to calculate the values in the new larger cells, from the data on the original - cells. Must be one of the CF standardised cell + cells. Must be one of these CF standardised cell methods: ``'maximum'``, ``'maximum_absolute_value'``, ``'mean'``, ``'mean_absolute_value'``, ``'mean_of_upper_decile'``, ``'median'``, @@ -4887,17 +4888,12 @@ def healpix_decrease_refinement_level( ``'root_mean_square'``, ``'standard_deviation'``, ``'sum'``, ``'sum_of_squares'``, ``'variance'``. - *Example:* - For an intensive field quantity (i.e. one that does - not depend on the size of the cells, such as - "sea_ice_amount" with units of kg m-2), a method of - ``'mean'`` might be appropriate. - - *Example:* - For an extensive field quantity (i.e. one that - depends on the size of the cells, such as - "sea_ice_mass" with units of kg), a method of - ``'sum'`` might be appropriate. + The method should be appropriate to nature of the + Field quantity, which is either intensive (i.e. that + does not depend on the size of the cells, such as + "sea_ice_amount" with units of kg m-2), or extensive + (i.e. that depends on the size of the cells, such as + "sea_ice_mass" with units of kg). reduction: function or `None`, optional The function used to calculate the values in the new @@ -4905,11 +4901,11 @@ def healpix_decrease_refinement_level( function must calculate the quantity defined by the *method* parameter, take an array of values as its first argument, and have an *axis* keyword that - specifies which axis of the array is the HEALPix axis. + specifies which of the array axes is the HEALPix axis. For some methods there are default *reduction* - functions, which are used when *reduction* is `None` - (the default): + functions, which are only used when *reduction* is + `None` (the default): ======================== =================== *method* Default *reduction* @@ -4924,7 +4920,7 @@ def healpix_decrease_refinement_level( ======================== =================== Note that these methods may be also calculated by any - function provided by the *reduction* parameter. + other function provided by the *reduction* parameter. conform: `bool`, optional If True (the default) then the HEALPix grid is @@ -4997,8 +4993,8 @@ def healpix_decrease_refinement_level( Data : air_temperature(time(2), healpix_index(12)) K Cell methods : time(2): mean area: mean area: maximum Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian + : healpix_index(12) = [0, ..., 11] : height(1) = [1.5] m - : healpix_index(healpix_index(12)) = [0, ..., 11] Coord references: grid_mapping_name:healpix >>> g.healpix_info()['refinement_level'] 0 @@ -5007,7 +5003,7 @@ def healpix_decrease_refinement_level( >>> print(g[0, 0].array) [[293.5]] - Set the refinement level to 0 using the ``'`range'`` *method*, + Set the refinement level to 0 using the ``'range'`` *method*, which requires a new *reduction* function to be defined: >>> import numpy as np @@ -5021,8 +5017,8 @@ def healpix_decrease_refinement_level( Data : air_temperature(time(2), healpix_index(12)) K Cell methods : time(2): mean area: mean area: range Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian + : healpix_index(12) = [0, ..., 11] : height(1) = [1.5] m - : healpix_index(healpix_index(12)) = [0, ..., 11] Coord references: grid_mapping_name:healpix >>> print(g[0, 0].array) [[8.2]] @@ -5191,7 +5187,7 @@ def healpix_decrease_refinement_level( # Chenge the refinement level of the Field's data # ------------------------------------------------------------ - # Note: Using 'Data.coarsen' only works because a) the HEALPix + # Note: Using 'Data.coarsen' works because a) the HEALPix # indexing scheme is "nested", and b) each new coarser # cell contains the maximum possible number # (i.e. 'ncells') of original cells. @@ -5283,15 +5279,15 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): new higher refinement level that lie inside it. It must be specified whether the field data contains an - extensive or intensive quantity. An extensive quantity depends - on the size of the cells (such as "sea_ice_mass" with units of - kg, or "cell_area" with units of m2), and an intensive - quantity does not depend on the size of the cells (such as - "sea_ice_amount" with units of kg m-2, or "air_temperature" - with units of K). For an extensive quantity only, the - broadcast values are reduced to be consistent with the new - smaller cell areas x(by dividing them by the number of new - cells per original cell). + extensive or intensive quantity. An intensive quantity does + not depend on the size of the cells (such as "sea_ice_amount" + with units of kg m-2, or "air_temperature" with units of K), + and an extensive quantity depends on the size of the cells + (such as "sea_ice_mass" with units of kg, or "cell_area" with + units of m2). For an extensive quantity only, the broadcast + values are reduced to be consistent with the new smaller cell + areas (by dividing them by the number of new cells per + original cell). **References** @@ -5361,10 +5357,9 @@ def healpix_increase_refinement_level(self, refinement_level, quantity): For an extensive quantity (which the ``f`` is this example is not, but we can assume that it is for demonstration purposes), - each cell at the higher refinement level has the value of a - cell at the original refinement level after dividing it by the - number of cells at the higher refinement level that lie in one - cell of the original refinement level (4 in this case): + each output cell has the value of the original cell in which + it lies, divided by the ratio of the cells' areas (4 in this + case): >>> g = f.healpix_increase_refinement_level(2, 'extensive') >>> print(f[0, :2] .array) diff --git a/cf/healpix.py b/cf/healpix.py index a8538a2e51..e62222658c 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -12,6 +12,10 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): """Create latitude and longitude coordinates for a HEALPix grid. + When it is not possible to create latitude and longitude + coordinates, the reason why will be reported if the log level is + at ``2``/``'INFO'`` or higher. + K. Gorski, Eric Hivon, A. Banday, B. Wandelt, M. Bartelmann, et al.. HEALPix: A Framework for High-Resolution Discretization and Fast Analysis of Data Distributed on the Sphere. The Astrophysical @@ -41,8 +45,8 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): (`str`, `str`) or (`None`, `None`) The keys of the new latitude and longitude coordinate - constructs, or `None` if the coordinates could not be - created. + constructs, in that order, or two `None`s if the + coordinates could not be created. """ from .constants import healpix_indexing_schemes diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index d020a684e7..7ff1c04eb9 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -2021,8 +2021,8 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): Data : air_temperature(time(2), healpix_index(48)) K Cell methods : time(2): mean area: mean Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian + : healpix_index(48) = [0, ..., 47] : height(1) = [1.5] m - Dimension coords: healpix_index(healpix_index(48)) = [0, ..., 47] Coord references: grid_mapping_name:healpix >>> f.healpix_info()['indexing_scheme'] 'nested' @@ -2234,8 +2234,8 @@ def healpix_to_ugrid(self, inplace=False): Data : air_temperature(time(2), healpix_index(48)) K Cell methods : time(2): mean area: mean Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian + : healpix_index(48) = [0, ..., 47] : height(1) = [1.5] m - Dimension coords: healpix_index(healpix_index(48)) = [0, ..., 47] Coord references: grid_mapping_name:healpix >>> print(f.healpix_to_ugrid()) Field: air_temperature (ncvar%tas) @@ -2249,7 +2249,6 @@ def healpix_to_ugrid(self, inplace=False): Coord references: grid_mapping_name:latitude_longitude Topologies : cell:face(ncdim%cell(48), 4) = [[774, ..., 3267]] - """ from ..healpix import del_healpix_coordinate_reference @@ -2356,8 +2355,8 @@ def create_latlon_coordinates( Creates 1-d or 2-d latitude and longitude coordinate constructs that are implied by the {{class}} coordinate reference constructs. By default (or if *overwrite* is False), - new coordinates are only created if the {{class}} metadata - doesn't already include any latitude or longitude coordinates. + new coordinates are only created if the {{class}} doesn't + already include any latitude or longitude coordinates. When it is not possible to create latitude and longitude coordinates, the reason why will be reported if the log level @@ -2371,20 +2370,21 @@ def create_latlon_coordinates( :Parameters: one_d: `bool`, optional` - If True (the default) then attempt to create 1-d - latitude and longitude coordinates. Otherwise do not - attempt this. + + If True (the default) then consider creating 1-d + latitude and longitude coordinates. If False then 1-d + coordinates will not be created. two_d: `bool`, optional` - If True (the default) then attempt to create 2-d - latitude and longitude coordinates. Otherwise do not - attempt this. + If True (the default) then consider creating 2-d + latitude and longitude coordinates. If False then 2-d + coordinates will not be created. pole_longitude: `None` or number Define the longitudes of coordinates or coordinate bounds that lie exactly on the north or south pole. If `None` (the default) then the longitudes of such - points are determined by whatever algorithm was used + points are determined by whichever algorithm was used to create the coordinates, which will likely result in different points on a pole having different longitudes. If set to a number, then the longitudes of @@ -2395,7 +2395,7 @@ def create_latlon_coordinates( If True then remove any existing latitude and longitude coordinates, prior to attempting to create new ones. If False (the default) then if any latitude - and longitude coordinates already exist, new ones will + or longitude coordinates already exist, new ones will not be created. {{inplace: `bool`, optional}} @@ -2405,8 +2405,8 @@ def create_latlon_coordinates( :Returns: `{{class}}` or `None` - The {{class}} with new latitude and longitude - constructs, if any could be created. If the operation + A new {{class}}, with new latitude and longitude + constructs if any could be created. If the operation was in-place then `None` is returned. **Examples** @@ -2418,25 +2418,29 @@ def create_latlon_coordinates( Data : air_temperature(time(2), healpix_index(48)) K Cell methods : time(2): mean area: mean Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian + : healpix_index(healpix_index(48)) = [0, ..., 47] : height(1) = [1.5] m - Auxiliary coords: healpix_index(healpix_index(48)) = [0, ..., 47] 1 Coord references: grid_mapping_name:healpix >>> g = f.create_latlon_coordinates() >>> print(g) Field: air_temperature (ncvar%tas) ---------------------------------- - Data : air_temperature(time(2), ncdim%cell(48)) K + Data : air_temperature(time(2), healpix_index(48)) K Cell methods : time(2): mean area: mean Dimension coords: time(2) = [2025-06-16 00:00:00, 2025-07-16 12:00:00] proleptic_gregorian + : healpix_index(48) = [0, ..., 47] : height(1) = [1.5] m - Auxiliary coords: healpix_index(ncdim%cell(48)) = [0, ..., 47] 1 - : latitude(ncdim%cell(48)) = [19.47122063449069, ..., -19.47122063449069] degrees_north - : longitude(ncdim%cell(48)) = [45.0, ..., 315.0] degrees_east + Auxiliary coords: latitude(healpix_index(48)) = [19.47122063449069, ..., -19.47122063449069] degrees_north + : longitude(healpix_index(48)) = [45.0, ..., 315.0] degrees_east Coord references: grid_mapping_name:healpix """ f = _inplace_enabled_define_and_cleanup(self) + # ------------------------------------------------------------ + # See if any lat/lon coordinates should be created + # ------------------------------------------------------------ + # See if there are any existing latitude/longutude coordinates latlon_coordinates = { key: c @@ -2509,15 +2513,16 @@ def create_latlon_coordinates( return f # ------------------------------------------------------------ - # Still here? Then get the unique non-latitude_longitude grid - # mapping, and use it to calculate the lat/lon coordinates. + # Still here? Then try to calculate some lat/lon coordinates. # ------------------------------------------------------------ - identity, cr = coordinate_references.popitem() # Initialize the flag that tells us if any new coordinates # have been created coords_created = False + # Get the unique non-latitude_longitude grid mapping + identity, cr = coordinate_references.popitem() + if one_d and not coords_created: # -------------------------------------------------------- # 1-d lat/lon coordinates @@ -2537,13 +2542,13 @@ def create_latlon_coordinates( # -------------------------------------------------------- # 2-d lat/lon coordinates # -------------------------------------------------------- - pass # Add some code here! + pass # For now ... + # ------------------------------------------------------------ + # Update the appropriate coordinate reference with any new + # coordinate keys + # ------------------------------------------------------------ if coords_created: - # -------------------------------------------------------- - # Update the approrpriate coordinate reference with the - # new coordinate keys - # -------------------------------------------------------- if latlon_cr is not None: latlon_cr.set_coordinates((lat_key, lon_key)) else: From 9b7dbf7657793298720970cc4d855e06a0f6b442 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 22 Aug 2025 12:25:51 +0100 Subject: [PATCH 57/59] dev --- cf/data/dask_utils.py | 22 +++++++------- cf/domain.py | 16 +++++----- cf/field.py | 2 +- cf/healpix.py | 66 +++++++++++++++++++++++++++++++++++++---- cf/mixin/fielddomain.py | 34 +++++++++++++++++---- cf/regrid/regrid.py | 2 +- 6 files changed, 109 insertions(+), 33 deletions(-) diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index 76874a9679..df635f74a0 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -517,14 +517,15 @@ def cf_healpix_bounds( longitude: `bool`, optional If True then return the bounds' longitudes. - pole_longitude: `None` or number + pole_longitude: `None` or number, optional Define the longitudes of vertices that lie exactly on the north or south pole. If `None` (the default) then the longitude of such a vertex on the north (south) pole will be the same as the longitude of the south (north) vertex of the same cell. If set to a number, then the longitudes of all vertices on the north or south pole will be given - the value *pole_longitude*. + the value *pole_longitude*. Ignored if *longitude* is + False. :Returns: @@ -569,7 +570,7 @@ def cf_healpix_bounds( a = cfdm_to_memory(a) # Keep an eye on https://github.com/ntessore/healpix/issues/66 - if a.ndim != 1: + if a.ndim > 1: raise ValueError( "Can only calculate HEALPix cell bounds when the " f"healpix_index array has one dimension. Got shape: {a.shape}" @@ -619,7 +620,7 @@ def cf_healpix_bounds( del thetaphi, a - if not pos: + if longitude: # Ensure that longitude bounds are less than 360 where_ge_360 = np.where(b >= 360) if where_ge_360[0].size: @@ -717,7 +718,7 @@ def cf_healpix_coordinates( a = cfdm_to_memory(a) - if a.ndim != 1: + if a.ndim > 1: raise ValueError( "Can only calculate HEALPix cell coordinates when the " f"healpix_index array has one dimension. Got shape: {a.shape}" @@ -887,14 +888,13 @@ def cf_healpix_indexing_scheme( nest = new_indexing_scheme == "nested" order, a = healpix.uniq2pix(a, nest=nest) - refinement_levels = np.unique(order) - if refinement_levels.size > 1: + order = np.unique(order) + if order.size > 1: raise ValueError( "Can't change HEALPix indexing scheme from " - f"'nested_unique' to {new_indexing_scheme!r} " - "when the HEALPix indices span multiple " - "refinement levels (at least levels " - f"{refinement_levels.tolist()})" + f"'nested_unique' to {new_indexing_scheme!r}: " + "HEALPix indices span multiple refinement levels " + f"(at least levels {order.tolist()})" ) case _: diff --git a/cf/domain.py b/cf/domain.py index dec83a76ef..0043a252ec 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -273,7 +273,7 @@ def create_healpix( HEALPix indices are monotonically increasing. **Performance** - + Very high refinement levels may require the setting of a very large Dask chunksize, to prevent a possible run-time failure resulting from an attempt to create an excessive amount of @@ -344,33 +344,33 @@ def create_healpix( >>> d = cf.Domain.create_healpix(4) >>> d.dump() -------- - Domain: + Domain: -------- Domain Axis: healpix_index(3072) - + Dimension coordinate: healpix_index standard_name = 'healpix_index' Data(healpix_index(3072)) = [0, ..., 3071] - + Coordinate reference: grid_mapping_name:healpix Coordinate conversion:grid_mapping_name = healpix Coordinate conversion:indexing_scheme = nested Coordinate conversion:refinement_level = 4 - Dimension Coordinate: healpix_index + Dimension Coordinate: healpix_index .. code-block:: python >>> d = cf.Domain.create_healpix(4, "nested_unique", radius=6371000) >>> d.dump() -------- - Domain: + Domain: -------- Domain Axis: healpix_index(3072) - + Dimension coordinate: healpix_index standard_name = 'healpix_index' Data(healpix_index(3072)) = [1024, ..., 4095] - + Coordinate reference: grid_mapping_name:healpix Coordinate conversion:grid_mapping_name = healpix Coordinate conversion:indexing_scheme = nested_unique diff --git a/cf/field.py b/cf/field.py index f194ef71df..cdead08b39 100644 --- a/cf/field.py +++ b/cf/field.py @@ -7168,7 +7168,7 @@ def collapse( if f_latlon is None: # Temporarily create any implied lat/lon coordinates - f_latlon = f.create_latlon_coordinates() + f_latlon = f.create_latlon_coordinates(cache=False) axes2 = [] for axis in axes: diff --git a/cf/healpix.py b/cf/healpix.py index e62222658c..68a0f5654b 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -9,8 +9,8 @@ logger = logging.getLogger(__name__) -def _healpix_create_latlon_coordinates(f, pole_longitude): - """Create latitude and longitude coordinates for a HEALPix grid. +def _healpix_create_latlon_coordinates(f, pole_longitude, cache=True): + """Create HEALPIx latitude and longitude coordinates and bounds. When it is not possible to create latitude and longitude coordinates, the reason why will be reported if the log level is @@ -41,6 +41,16 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): the longitudes of such vertices will all be given that value. + cache: `bool`, optional + If True (the default) then cache in memory the first and + last of any newly-created coordinates and bounds. This + will slightly slow down the coordinate creation process, + but will greatly speed up, and reduce the memory + requirement of, a future inspection of the coordinates and + bounds. Even when *cache* is True, new cached values can + only be created if the existing source coordinates + themselves have cached first and last values. + :Returns: (`str`, `str`) or (`None`, `None`) @@ -87,6 +97,34 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): return (None, None) + # Create latitude and longitude cached elements + if cache: + cache = healpix_index.data._get_cached_elements() + cached_lon_coords = {} + cached_lat_coords = {} + cached_lon_bounds = {} + cached_lat_bounds = {} + for i, value in cache.items(): + if i not in (0, -1): + continue + + cached_lon_coords[i] = cf_healpix_coordinates( + value, indexing_scheme, refinement_level, longitude=True + ) + cached_lat_coords[i] = cf_healpix_coordinates( + value, indexing_scheme, refinement_level, latitude=True + ) + cached_lon_bounds[i] = cf_healpix_bounds( + value, + indexing_scheme, + refinement_level, + longitude=True, + pole_longitude=pole_longitude, + )[0, i] + cached_lat_bounds[i] = cf_healpix_bounds( + value, indexing_scheme, refinement_level, latitude=True + )[0, i] + # Get the Dask array of HEALPix indices. # # `cf_healpix_coordinates` anad `cf_healpix_bounds` have their own @@ -104,8 +142,12 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): refinement_level=refinement_level, latitude=True, ) + data = f._Data(dy, "degrees_north") + if cache: + data._set_cached_elements(cached_lat_coords) + lat = f._AuxiliaryCoordinate( - data=f._Data(dy, "degrees_north", copy=False), + data=data, properties={"standard_name": "latitude"}, copy=False, ) @@ -118,8 +160,12 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): refinement_level=refinement_level, longitude=True, ) + data = f._Data(dy, "degrees_east") + if cache: + data._set_cached_elements(cached_lon_coords) + lon = f._AuxiliaryCoordinate( - data=f._Data(dy, "degrees_east", copy=False), + data=data, properties={"standard_name": "longitude"}, copy=False, ) @@ -136,7 +182,11 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): refinement_level=refinement_level, latitude=True, ) - bounds = f._Bounds(data=dy) + data = f._Data(dy, "degrees_north") + if cache: + data._set_cached_elements(cached_lat_bounds) + + bounds = f._Bounds(data=data) lat.set_bounds(bounds, copy=False) # Create longitude bounds @@ -152,7 +202,11 @@ def _healpix_create_latlon_coordinates(f, pole_longitude): longitude=True, pole_longitude=pole_longitude, ) - bounds = f._Bounds(data=dy) + data = f._Data(dy, "degrees_east") + if cache: + data._set_cached_elements(cached_lon_bounds) + + bounds = f._Bounds(data=data) lon.set_bounds(bounds, copy=False) # Set the new latitude and longitude coordinates diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index 7ff1c04eb9..fec3b3a593 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -303,7 +303,7 @@ def _indices(self, config, data_axes, ancillary_mask, kwargs): # metadata. # # Do not do this in-place. - self = self.create_latlon_coordinates() + self = self.create_latlon_coordinates(cache=False) domain_axes = self.domain_axes(todict=True) @@ -2204,7 +2204,7 @@ def healpix_info(self): return healpix_info(self) @_inplace_enabled(default=False) - def healpix_to_ugrid(self, inplace=False): + def healpix_to_ugrid(self, cache=True, inplace=False): """Convert a HEALPix domain to a UGRID domain. **References** @@ -2217,6 +2217,17 @@ def healpix_to_ugrid(self, inplace=False): :Parameters: + cache: `bool`, optional + If True (the default) then cache in memory the first + and last of any newly-created coordinates and + bounds. This will slightly slow down the coordinate + creation process, but will greatly speed up, and + reduce the memory requirement of, a future inspection + of the coordinates and bounds. Even when *cache* is + True, new cached values can only be created if the + existing source coordinates themselves have cached + first and last values. + {{inplace: `bool`, optional}} :Returns: @@ -2269,7 +2280,7 @@ def healpix_to_ugrid(self, inplace=False): # that the north (south) polar vertex comes out as a single # node in the domain topology. f.create_latlon_coordinates( - two_d=False, pole_longitude=0, inplace=True + two_d=False, pole_longitude=0, cache=cache, inplace=True ) # Get the lat/lon coordinates @@ -2347,6 +2358,7 @@ def create_latlon_coordinates( two_d=True, pole_longitude=None, overwrite=False, + cache=True, inplace=False, verbose=None, ): @@ -2370,7 +2382,6 @@ def create_latlon_coordinates( :Parameters: one_d: `bool`, optional` - If True (the default) then consider creating 1-d latitude and longitude coordinates. If False then 1-d coordinates will not be created. @@ -2398,6 +2409,17 @@ def create_latlon_coordinates( or longitude coordinates already exist, new ones will not be created. + cache: `bool`, optional + If True (the default) then cache in memory the first + and last of any newly-created coordinates and + bounds. This will slightly slow down the coordinate + creation process, but will greatly speed up, and + reduce the memory requirement of, a future inspection + of the coordinates and bounds. Even when *cache* is + True, new cached values can only be created if the + existing source coordinates themselves have cached + first and last values. + {{inplace: `bool`, optional}} {{verbose: `int` or `str` or `None`, optional}} @@ -2440,7 +2462,7 @@ def create_latlon_coordinates( # ------------------------------------------------------------ # See if any lat/lon coordinates should be created # ------------------------------------------------------------ - + # See if there are any existing latitude/longutude coordinates latlon_coordinates = { key: c @@ -2534,7 +2556,7 @@ def create_latlon_coordinates( from ..healpix import _healpix_create_latlon_coordinates lat_key, lon_key = _healpix_create_latlon_coordinates( - f, pole_longitude + f, pole_longitude, cache ) coords_created = lat_key is not None diff --git a/cf/regrid/regrid.py b/cf/regrid/regrid.py index 29e40c5d85..015653445c 100644 --- a/cf/regrid/regrid.py +++ b/cf/regrid/regrid.py @@ -1139,7 +1139,7 @@ def spherical_grid( # create 1-d lat/lon coordinates) f.healpix_to_ugrid(inplace=True) except ValueError: - f.create_latlon_coordinates(inplace=True) + f.create_latlon_coordinates(cache=False, inplace=True) data_axes = f.constructs.data_axes() From 70fce9740bbe5e2e7c7fca0ccf67a7a8620963a1 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 19 Sep 2025 14:58:08 +0100 Subject: [PATCH 58/59] dev --- Changelog.rst | 4 ++-- cf/data/dask_utils.py | 51 +++++++++++++++++++++++++---------------- cf/healpix.py | 2 +- cf/mixin/fielddomain.py | 4 ++-- 4 files changed, 36 insertions(+), 25 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 61813d2276..0a8231377a 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -3,14 +3,14 @@ Version NEXTVERSION **2025-??-??** -* New method: `cf.Field.create_latlon_coordinates` - (https://github.com/NCAS-CMS/cf-python/issues/???) * New HEALPix methods: `cf.Field.healpix_info`, `cf.Field.healpix_decrease_refinement_level`, `cf.Field.healpix_increase_refinement_level`, `cf.Field.healpix_indexing_scheme`, `cf.Field.healpix_to_ugrid`, `cf.Domain.create_healpix` (https://github.com/NCAS-CMS/cf-python/issues/???) +* New method: `cf.Field.create_latlon_coordinates` + (https://github.com/NCAS-CMS/cf-python/issues/???) * New method: `cf.Data.coarsen` (https://github.com/NCAS-CMS/cf-python/issues/???) * New function: `cf.locate` diff --git a/cf/data/dask_utils.py b/cf/data/dask_utils.py index df635f74a0..e1a7870805 100644 --- a/cf/data/dask_utils.py +++ b/cf/data/dask_utils.py @@ -583,10 +583,17 @@ def cf_healpix_bounds( # Define the function that's going to calculate the bounds from # the HEALPix indices - if indexing_scheme == "ring": - bounds_func = healpix._chp.ring2ang_uv - else: - bounds_func = healpix._chp.nest2ang_uv + match indexing_scheme: + case "ring": + bounds_func = healpix._chp.ring2ang_uv + case "nested" | "nested_unique": + bounds_func = healpix._chp.nest2ang_uv + case _: + raise ValueError( + "Can't calculate HEALPix cell bounds: Unknown " + f"'indexing_scheme' in cf_healpix_bounds: " + f"{indexing_scheme!r}" + ) # Define the cell vertices in an anticlockwise direction, as seen # from above, starting with the northern-most vertex. Each vertex @@ -600,23 +607,27 @@ def cf_healpix_bounds( # Initialise the output bounds array b = np.empty((a.size, 4), dtype="float64") - if indexing_scheme == "nested_unique": - # Create bounds for 'nested_unique' indices - orders, a = healpix.uniq2pix(a, nest=True) - for order in np.unique(orders): - nside = healpix.order2nside(order) - indices = np.where(orders == order)[0] - for j, (u, v) in enumerate(vertices): - thetaphi = bounds_func(nside, a[indices], u, v) - b[indices, j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] + match indexing_scheme: + case "nested_unique": + # Create bounds for 'nested_unique' indices + orders, a = healpix.uniq2pix(a, nest=True) + for order in np.unique(orders): + nside = healpix.order2nside(order) + indices = np.where(orders == order)[0] + for j, (u, v) in enumerate(vertices): + thetaphi = bounds_func(nside, a[indices], u, v) + b[indices, j] = healpix.lonlat_from_thetaphi(*thetaphi)[ + pos + ] - del orders, indices - else: - # Create bounds for 'nested' or 'ring' indices - nside = healpix.order2nside(refinement_level) - for j, (u, v) in enumerate(vertices): - thetaphi = bounds_func(nside, a, u, v) - b[:, j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] + del orders, indices + + case "nested" | "ring": + # Create bounds for 'nested' or 'ring' indices + nside = healpix.order2nside(refinement_level) + for j, (u, v) in enumerate(vertices): + thetaphi = bounds_func(nside, a, u, v) + b[:, j] = healpix.lonlat_from_thetaphi(*thetaphi)[pos] del thetaphi, a diff --git a/cf/healpix.py b/cf/healpix.py index 68a0f5654b..571ebcbc52 100644 --- a/cf/healpix.py +++ b/cf/healpix.py @@ -107,7 +107,7 @@ def _healpix_create_latlon_coordinates(f, pole_longitude, cache=True): for i, value in cache.items(): if i not in (0, -1): continue - + cached_lon_coords[i] = cf_healpix_coordinates( value, indexing_scheme, refinement_level, longitude=True ) diff --git a/cf/mixin/fielddomain.py b/cf/mixin/fielddomain.py index fec3b3a593..0dd6e7c406 100644 --- a/cf/mixin/fielddomain.py +++ b/cf/mixin/fielddomain.py @@ -2137,8 +2137,8 @@ def healpix_indexing_scheme(self, new_indexing_scheme, sort=False): index = d.compute() f = f.subspace(**{hp["domain_axis_key"]: np.argsort(index)}) - # Now that the HEALPix indices are ordered, make sure that - # they're stored in a Dimension Coordinate. + # Now that the HEALPix indices are ordered, store them in + # a Dimension Coordinate. if healpix_index.construct_type == "auxiliary_coordinate": f.auxiliary_to_dimension(hp["coordinate_key"], inplace=True) From 715ee829fd567cf255268e26d4bcaa362ec9e5b3 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 27 Nov 2025 17:02:34 +0000 Subject: [PATCH 59/59] dev --- Changelog.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 888bfec1dd..ebd09a5130 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -3,18 +3,20 @@ Version NEXTVERSION **2025-??-??** +* Support for HEALPix grids + (https://github.com/NCAS-CMS/cf-python/issues/909) * New HEALPix methods: `cf.Field.healpix_info`, `cf.Field.healpix_decrease_refinement_level`, `cf.Field.healpix_increase_refinement_level`, `cf.Field.healpix_indexing_scheme`, `cf.Field.healpix_to_ugrid`, `cf.Domain.create_healpix` - (https://github.com/NCAS-CMS/cf-python/issues/???) + (https://github.com/NCAS-CMS/cf-python/issues/909) * New method: `cf.Field.create_latlon_coordinates` - (https://github.com/NCAS-CMS/cf-python/issues/???) + (https://github.com/NCAS-CMS/cf-python/issues/909) * New method: `cf.Data.coarsen` - (https://github.com/NCAS-CMS/cf-python/issues/???) + (https://github.com/NCAS-CMS/cf-python/issues/909) * New function: `cf.locate` - (https://github.com/NCAS-CMS/cf-python/issues/???) + (https://github.com/NCAS-CMS/cf-python/issues/909) * Reduce the time taken to import `cf` (https://github.com/NCAS-CMS/cf-python/issues/902) * New optional dependency: ``healpix>=2025.1``