From f4633bdce4d263620631aa4d3582a3dba76267ea Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 3 Jul 2024 16:50:01 +0200 Subject: [PATCH 001/240] transfer functionality from notebook (1). --- model/common/pyproject.toml | 2 +- .../model/common/decomposition/halo.py | 159 ++++++++++++++++++ .../tests/decomposition_tests/test_halo.py | 0 3 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 model/common/src/icon4py/model/common/decomposition/halo.py create mode 100644 model/common/tests/decomposition_tests/test_halo.py diff --git a/model/common/pyproject.toml b/model/common/pyproject.toml index c9dfc455da..0ae6ee8dfa 100644 --- a/model/common/pyproject.toml +++ b/model/common/pyproject.toml @@ -31,7 +31,7 @@ requires-python = ">=3.10" [project.optional-dependencies] all = ["icon4py-common[ghex,netcdf]"] -ghex = ["ghex", "mpi4py"] +ghex = ["ghex", "mpi4py", "xugrid","pymetis>2022.1", "scipy"] netcdf = ["netcdf4>=1.6.0"] [project.urls] diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py new file mode 100644 index 0000000000..de8e5e62ce --- /dev/null +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -0,0 +1,159 @@ +import enum + +import scipy as sp +import xugrid.ugrid.ugrid2d as ux + +import icon4py.model.common.decomposition.definitions as defs +from icon4py.model.common import dimension as dims +from icon4py.model.common.settings import xp + + +#TODO do we need three of those? +class DecompositionFlag(enum.IntEnum): + #: cell is owned by this rank + OWNED = 0, + #: cell is in the first halo line: that is cells that share and edge with an owned cell + FIRST_HALO_LINE = 1, + #: cell is in the second halo line: that is cells that share only a vertex with an owned cell (and at least an edge with a FIRST_HALO_LINE cell) + SECOND_HALO_LINE = 2 + + + +class HaloGenerator: + """Creates necessary halo information for a given rank.""" + def __init__(self, rank_info:defs.ProcessorProperties, rank_mapping: xp.ndarray, ugrid:ux.Ugrid2d, num_lev:int): + """ + + Args: + rank_info: contains information on the communicator and local compute node. + rank_mapping: array with shape (global_num_cells) mapping of global cell indices to their rank in the distribution + """ + self._props = rank_info + self._mapping = rank_mapping + self._global_grid = ugrid + self._num_lev = num_lev + + + + def _validate(self): + assert self._mapping.ndim == 1 + # the decomposition should match the communicator size + assert xp.max(self._mapping) == self._props.comm_size - 1 + + + + def _post_init(self): + self._validate() + + + def next_halo_line(self, cell_line: xp.ndarray, depot=None): + """Returns the global indices of the next halo line. + + Args: + cell_line: global indices of cells we want to find the neighbors of + depot: global indices that have already been collected + Returns: + next_halo_cells: global indices of the next halo line + """ + cell_neighbors = self._face_face_connectivity(cell_line) + + if depot is not None: + cells_so_far = xp.hstack((depot, cell_line)) + else: + cells_so_far = cell_line + + next_halo_cells = xp.setdiff1d(xp.unique(cell_neighbors), cells_so_far, assume_unique=True) + # assert next_halo_cells.shape[0] == next_halo_size + return next_halo_cells + + + def _face_face_connectivity(self, cells: xp.ndarray): + """ In xugrid face-face connectivity is a scipy spars matrix, so we reduce it to the our regular sparse matrix format: (n_cells, 3) + """ + conn = self._global_grid.face_face_connectivity + _, c2e2c, _ = sp.sparse.find(conn[cells, :]) + return c2e2c + + + def _find_neighbors(self, cell_line:xp.ndarray, connectivity: xp.ndarray)->xp.ndarray: + """ Get a flattened list of all (unique) neighbors to a given global index list""" + neighbors = connectivity[cell_line, :] + shp = neighbors.shape + unique_neighbors = xp.unique(neighbors.reshape(shp[0] * shp[1])) + return unique_neighbors + + + def find_edge_neighbors_for_cells(self, cell_line: xp.ndarray) -> xp.ndarray: + return self._find_neighbors(self, cell_line, connectivity=self._global_grid.face_edge_connectivity) + def find_vertex_neighbors_for_cells(self, cell_line:xp.ndarray)->xp.ndarray: + return self._find_neighbors(self, cell_line, connectivity=self._global_grid.face_node_connectivity) + + + def owned_cells(self)->xp.ndarray: + """Returns the global indices of the cells owned by this rank""" + owned_cells = self._mapping == self._props.rank + return xp.asarray(owned_cells).nonzero()[0] + + def construct_decomposition_info(self): + """Constructs the decomposition info for the current rank""" + + #: cells + owned_cells = self.owned_cells() # global indices of owned cells + first_halo_cells = self.next_halo_line(owned_cells) + second_halo_cells = self.next_halo_line(first_halo_cells, owned_cells) + + total_halo_cells = xp.hstack((first_halo_cells, second_halo_cells)) + global_cell_index = xp.hstack((owned_cells, total_halo_cells)) + + c_owner_mask = xp.isin(global_cell_index, owned_cells) + + decomp_info = defs.DecompositionInfo(klevels=self._num_lev).with_dimension(dims.CellDim, + global_cell_index, + c_owner_mask) + + #: edges + edges_on_owned_cells = self.find_edge_neighbors_for_cells(owned_cells) + edges_on_first_halo_line = self.find_edge_neighbors_for_cells(first_halo_cells) + edges_on_second_halo_line = self.find_edge_neighbors_for_cells(second_halo_cells) + # reduce overlap + + all_edges = xp.hstack((edges_on_owned_cells, xp.setdiff1d(edges_on_owned_cells, edges_on_first_halo_line), xp.setdiff1d(edges_on_first_halo_line, edges_on_second_halo_line))) + """ + We need to reduce the overlap: + + `edges_on_owned_cells` and `edges_on_first_halo_line` contain the edges on the cutting line. + In order to have unique ownership of edges (and vertices) among nodes there needs to be a convention as to where + those elements on the cutting line go: according to a remark in `mo_decomposition_tools.f90` ICON puts them to the node with the higher rank. + """ + edge_owner_mask = xp.isin(all_edges, edges_on_owned_cells) + intersect_owned_first_line = xp.intersect1d(edges_on_owned_cells, edges_on_first_halo_line) + + for edge in intersect_owned_first_line: + local_index = xp.where(all_edges == edge)[0][0] + owning_ranks = self._mapping[self._global_grid.edge_face_connectivity[edge]] + assert owning_ranks.shape[0] == 2 + assert owning_ranks[0] != owning_ranks[1], "both neighboring cells are owned by the same rank" + assert self._props.rank in owning_ranks, "neither of the neighboring cells is owned by this rank" + # assign the edge to the rank with the higher rank + if max(owning_ranks) > self._props.rank: + edge_owner_mask[local_index] = False + else: + edge_owner_mask[local_index] = True + + + decomp_info.with_dimension(dims.EdgeDim, all_edges, edge_owner_mask) + + # vertices + vertices_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) + vertices_on_first_halo_line = self.find_vertex_neighbors_for_cells(first_halo_cells) + vertices_on_second_halo_line = self.find_vertex_neighbors_for_cells(second_halo_cells) #TODO: do we need that? + unique_vertices_on_halo_cells = xp.setdiff1d(vertices_on_first_halo_line, vertices_on_owned_cells) + vertices_on_owned_edges = xp.unique(self._global_grid.edge_node_connectivity[all_edges[edge_owner_mask]]) + + # create decomposition_info + all_vertices = xp.hstack((vertices_on_owned_cells, unique_vertices_on_halo_cells)) + v_owner_mask = xp.isin(all_vertices, vertices_on_owned_edges) + decomp_info.with_dimension(dims.VertexDim, all_vertices, v_owner_mask) + return decomp_info + + \ No newline at end of file diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py new file mode 100644 index 0000000000..e69de29bb2 From 38299d93ab065cd8f3923530aa1b721b50595b8f Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 5 Jul 2024 12:12:09 +0200 Subject: [PATCH 002/240] add tests against simple grid extract conventional ownership assignment pass connectivities explicitly in order to avoid sparse matrix 0 - based index business. (preliminary fix) --- .../model/common/decomposition/halo.py | 135 ++++++++++------ .../common/decomposition/mpi_decomposition.py | 4 +- .../tests/decomposition_tests/test_halo.py | 149 ++++++++++++++++++ 3 files changed, 241 insertions(+), 47 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index de8e5e62ce..97fd43ded4 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -1,4 +1,6 @@ import enum +import functools +from typing import Union import scipy as sp import xugrid.ugrid.ugrid2d as ux @@ -21,18 +23,27 @@ class DecompositionFlag(enum.IntEnum): class HaloGenerator: """Creates necessary halo information for a given rank.""" - def __init__(self, rank_info:defs.ProcessorProperties, rank_mapping: xp.ndarray, ugrid:ux.Ugrid2d, num_lev:int): + def __init__(self, rank_info:defs.ProcessProperties, rank_mapping: xp.ndarray, ugrid:ux.Ugrid2d, num_lev:int, face_face_connectivity:xp.ndarray = None, node_face_connectivity = None): """ Args: rank_info: contains information on the communicator and local compute node. rank_mapping: array with shape (global_num_cells) mapping of global cell indices to their rank in the distribution + ugrid: the global grid + num_lev: number of vertical levels + face_face_connectivity: face-face connectivity matrix: (n_cells, 3) xugrid uses a + scipy.sparse matrix for this which causes problems with zero based indices so we + allow to pass it directly as a workaround + node_face_connectivity: node-face connectivity matrix: (n_vertex, 6) xugrid uses a + sparse matrix for this which causes problems with zero based indices so wes """ self._props = rank_info self._mapping = rank_mapping self._global_grid = ugrid self._num_lev = num_lev - + self._c2e2c = face_face_connectivity + self._v2c = node_face_connectivity + def _validate(self): @@ -55,7 +66,7 @@ def next_halo_line(self, cell_line: xp.ndarray, depot=None): Returns: next_halo_cells: global indices of the next halo line """ - cell_neighbors = self._face_face_connectivity(cell_line) + cell_neighbors = self._cell_neighbors(cell_line) if depot is not None: cells_so_far = xp.hstack((depot, cell_line)) @@ -67,12 +78,28 @@ def next_halo_line(self, cell_line: xp.ndarray, depot=None): return next_halo_cells - def _face_face_connectivity(self, cells: xp.ndarray): - """ In xugrid face-face connectivity is a scipy spars matrix, so we reduce it to the our regular sparse matrix format: (n_cells, 3) + def _cell_neighbors(self, cells: xp.ndarray): + if self._c2e2c is not None: + return xp.unique(self._c2e2c[cells, :]) + else: + return self._cell_neighbors_from_sparse(cells) + @functools.cached_property + def _node_face_connectivity(self)->Union[xp.ndarray, sp.sparse.csr_matrix]: + if self._v2c is not None: + return self._v2c + else: + return self._global_grid._node_face_connectivity + + def _cell_neighbors_from_sparse(self, cells: xp.ndarray): + """ In xugrid face-face connectivity is a scipy spars matrix, so we reduce it to the regular sparse matrix format: (n_cells, 3) """ - conn = self._global_grid.face_face_connectivity - _, c2e2c, _ = sp.sparse.find(conn[cells, :]) - return c2e2c + conn = self._c2e2c.face_face_connectivity + + neighbors = conn[cells, :] + # There is an issue with explicit 0 (for zero based indices) since sparse.find projects them out... + i, j, vals = sp.sparse.find(neighbors) + + return j def _find_neighbors(self, cell_line:xp.ndarray, connectivity: xp.ndarray)->xp.ndarray: @@ -84,9 +111,9 @@ def _find_neighbors(self, cell_line:xp.ndarray, connectivity: xp.ndarray)->xp.nd def find_edge_neighbors_for_cells(self, cell_line: xp.ndarray) -> xp.ndarray: - return self._find_neighbors(self, cell_line, connectivity=self._global_grid.face_edge_connectivity) + return self._find_neighbors(cell_line, connectivity=self._global_grid.face_edge_connectivity) def find_vertex_neighbors_for_cells(self, cell_line:xp.ndarray)->xp.ndarray: - return self._find_neighbors(self, cell_line, connectivity=self._global_grid.face_node_connectivity) + return self._find_neighbors(cell_line, connectivity=self._global_grid.face_node_connectivity) def owned_cells(self)->xp.ndarray: @@ -94,65 +121,83 @@ def owned_cells(self)->xp.ndarray: owned_cells = self._mapping == self._props.rank return xp.asarray(owned_cells).nonzero()[0] - def construct_decomposition_info(self): - """Constructs the decomposition info for the current rank""" + def construct_decomposition_info(self)->defs.DecompositionInfo: + """ + Constructs the DecompositionInfo for the current rank. + The DecompositionInfo object is constructed for all horizontal dimension starting from the + cell distribution. Edges and vertices are then handled through their connectivity to the distributed cells. + """ + #: cells owned_cells = self.owned_cells() # global indices of owned cells first_halo_cells = self.next_halo_line(owned_cells) second_halo_cells = self.next_halo_line(first_halo_cells, owned_cells) total_halo_cells = xp.hstack((first_halo_cells, second_halo_cells)) - global_cell_index = xp.hstack((owned_cells, total_halo_cells)) + all_cells = xp.hstack((owned_cells, total_halo_cells)) - c_owner_mask = xp.isin(global_cell_index, owned_cells) + c_owner_mask = xp.isin(all_cells, owned_cells) decomp_info = defs.DecompositionInfo(klevels=self._num_lev).with_dimension(dims.CellDim, - global_cell_index, + all_cells, c_owner_mask) #: edges edges_on_owned_cells = self.find_edge_neighbors_for_cells(owned_cells) edges_on_first_halo_line = self.find_edge_neighbors_for_cells(first_halo_cells) edges_on_second_halo_line = self.find_edge_neighbors_for_cells(second_halo_cells) - # reduce overlap - all_edges = xp.hstack((edges_on_owned_cells, xp.setdiff1d(edges_on_owned_cells, edges_on_first_halo_line), xp.setdiff1d(edges_on_first_halo_line, edges_on_second_halo_line))) - """ - We need to reduce the overlap: - - `edges_on_owned_cells` and `edges_on_first_halo_line` contain the edges on the cutting line. - In order to have unique ownership of edges (and vertices) among nodes there needs to be a convention as to where - those elements on the cutting line go: according to a remark in `mo_decomposition_tools.f90` ICON puts them to the node with the higher rank. - """ - edge_owner_mask = xp.isin(all_edges, edges_on_owned_cells) + all_edges = xp.hstack((edges_on_owned_cells, xp.setdiff1d(edges_on_first_halo_line, edges_on_owned_cells), xp.setdiff1d(edges_on_second_halo_line, edges_on_first_halo_line))) + all_edges = xp.unique(all_edges) + # We need to reduce the overlap: + # `edges_on_owned_cells` and `edges_on_first_halo_line` both contain the edges on the cutting line. intersect_owned_first_line = xp.intersect1d(edges_on_owned_cells, edges_on_first_halo_line) + + def _update_owner_mask_by_max_rank_convention(owner_mask, all_indices, indices_on_cutting_line, target_connectivity): + """ + In order to have unique ownership of edges (and vertices) among nodes there needs to be + a convention as to where those elements on the cutting line go: + according to a remark in `mo_decomposition_tools.f90` ICON puts them to the node + with the higher rank. + + # TODO (@halungge): can we add an assert for the target dimension of the connectivity being cells. + Args: + owner_mask: owner mask for the dimension + all_indices: (global) indices of the dimension + indices_on_cutting_line: global indices of the elements on the cutting line + target_connectivity: connectivity matrix mapping the dimension d to faces + Returns: + updated owner mask + """ + for index in indices_on_cutting_line: + local_index = xp.nonzero(all_indices == index)[0][0] + owning_ranks = self._mapping[target_connectivity[index]] + assert xp.unique(owning_ranks).size > 1, f"rank {self._props.rank}: all neighboring cells are owned by the same rank" + assert self._props.rank in owning_ranks, f"rank {self._props.rank}: neither of the neighboring cells: {owning_ranks} is owned by me" + # assign the index to the rank with the higher rank + if max(owning_ranks) > self._props.rank: + owner_mask[local_index] = False + else: + owner_mask[local_index] = True + return owner_mask + - for edge in intersect_owned_first_line: - local_index = xp.where(all_edges == edge)[0][0] - owning_ranks = self._mapping[self._global_grid.edge_face_connectivity[edge]] - assert owning_ranks.shape[0] == 2 - assert owning_ranks[0] != owning_ranks[1], "both neighboring cells are owned by the same rank" - assert self._props.rank in owning_ranks, "neither of the neighboring cells is owned by this rank" - # assign the edge to the rank with the higher rank - if max(owning_ranks) > self._props.rank: - edge_owner_mask[local_index] = False - else: - edge_owner_mask[local_index] = True - - + # construct the owner mask + edge_owner_mask = xp.isin(all_edges, edges_on_owned_cells) + edge_owner_mask = _update_owner_mask_by_max_rank_convention(edge_owner_mask, all_edges, intersect_owned_first_line, self._global_grid.edge_face_connectivity) decomp_info.with_dimension(dims.EdgeDim, all_edges, edge_owner_mask) # vertices vertices_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) vertices_on_first_halo_line = self.find_vertex_neighbors_for_cells(first_halo_cells) - vertices_on_second_halo_line = self.find_vertex_neighbors_for_cells(second_halo_cells) #TODO: do we need that? - unique_vertices_on_halo_cells = xp.setdiff1d(vertices_on_first_halo_line, vertices_on_owned_cells) - vertices_on_owned_edges = xp.unique(self._global_grid.edge_node_connectivity[all_edges[edge_owner_mask]]) - - # create decomposition_info - all_vertices = xp.hstack((vertices_on_owned_cells, unique_vertices_on_halo_cells)) - v_owner_mask = xp.isin(all_vertices, vertices_on_owned_edges) + vertices_on_second_halo_line = self.find_vertex_neighbors_for_cells(second_halo_cells) #TODO (@halungge): do we need that? + intersect_owned_first_line = xp.intersect1d(vertices_on_owned_cells, vertices_on_first_halo_line) + + # create decomposition_info for vertices + all_vertices = xp.unique(xp.hstack((vertices_on_owned_cells, vertices_on_first_halo_line))) + v_owner_mask = xp.isin(all_vertices, vertices_on_owned_cells) + v_owner_mask = _update_owner_mask_by_max_rank_convention(v_owner_mask, all_vertices, intersect_owned_first_line, self._node_face_connectivity) decomp_info.with_dimension(dims.VertexDim, all_vertices, v_owner_mask) return decomp_info diff --git a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py index d6fb764ee0..04eca68f15 100644 --- a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py +++ b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py @@ -103,8 +103,8 @@ def filter(self, record: logging.LogRecord) -> bool: def get_multinode_properties(s: definitions.MultiNodeRun) -> definitions.ProcessProperties: return _get_processor_properties(with_mpi=True) - -@dataclass(frozen=True) +# TODO (@halungge) changed for dev/testing set back to frozen +@dataclass(frozen=False) class MPICommProcessProperties(definitions.ProcessProperties): comm: mpi4py.MPI.Comm = None diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index e69de29bb2..5d1f07a3a1 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -0,0 +1,149 @@ +import mpi4py +import mpi4py.MPI +import pytest +import xugrid as xu + +import icon4py.model.common.dimension as dims +from icon4py.model.common.decomposition import definitions as defs +from icon4py.model.common.decomposition.halo import HaloGenerator +from icon4py.model.common.grid import simple +from icon4py.model.common.settings import xp +from icon4py.model.common.test_utils.parallel_helpers import ( # noqa: F401 # import fixtures from test_utils package + check_comm_size, + processor_props, +) + + +simple_distribution = xp.asarray([0, # 0c + 1, # 1c + 1, # 2c + 0, # 3c + 0, # 4c + 1, # 5c + 0, # 6c + 0, # 7c + 2, # 8c + 2, # 9c + 0, # 10c + 2, # 11c + 3, # 12c + 3, # 13c + 1, # 14c + 3, # 15c + 3, # 16c + 1, #17c + ]) +cell_own = {0:[0,3,4,6,7,10], + 1:[1,2,5,14,17], + 2:[8,9,11], + 3:[12,13,15,16]} +cell_halos = {0:[2,15, 1, 11, 13, 9,17, 5, 12, 14, 8,16], + 1:[4, 16, 3, 8, 15, 11, 13, 0,7,6,9,10,12], + 2:[5, 6, 12,14,7, 2,1,4,3, 10, 15, 16, 17], + 3:[9, 10, 17, 14, 0, 1,6, 7,8,2, 3, 4, 5,11]} + + +edge_own = {0:[1,5,12,13,14,9], + 1:[8,7,6,25,4, 2], + 2:[16,11,15,17,10,24], + 3:[19,23,22,26,0,3,20,18,21]} + +edge_halos = {0:[0,4,21,10,2, 3,8,6,7,19,20,17,16,11,18,26,25,15,23,24,22], + 1:[5,12,22,23,3,1,9,15,16,11,19,20,0,17,24,21,26,13,10,14,18,], + 2:[7,6,9,8,14,18,19,23,25,20,12,13,2,3,4,5,1,21,22,0,26], + 3:[10,11,13,14,25,6,24,1,5,4,8,9,17,12,15,16,2,7]} + +vertex_own = {0:[4], + 1:[], + 2:[3,5], + 3:[0,1,2,6,7,8,]} +vertex_halo = {0:[0,1,2,3,5,6,7,8], + 1:[1,2,0,5,3,8,6, 7, 4], + 2:[8,6,7,4, 0,2,1 ], + 3:[3,4,5]} + +owned ={dims.CellDim:cell_own, dims.EdgeDim:edge_own, dims.VertexDim:vertex_own} +halos = {dims.CellDim:cell_halos, dims.EdgeDim:edge_halos, dims.VertexDim:vertex_halo} +def test_halo_constructor_owned_cells(processor_props, simple_ugrid): # noqa: F811 # fixture + + grid = simple_ugrid + halo_generator = HaloGenerator(ugrid=grid, + rank_info=processor_props, + rank_mapping=simple_distribution, + num_lev=1, + face_face_connectivity=simple.SimpleGridData.c2e2c_table) + my_owned_cells = halo_generator.owned_cells() + + print(f"rank {processor_props.rank} owns {my_owned_cells} ") + assert my_owned_cells.size == len(cell_own[processor_props.rank]) + assert xp.setdiff1d(my_owned_cells, cell_own[processor_props.rank]).size == 0 + +@pytest.mark.skip(reason="mpi.GATHER ???") +@pytest.mark.mpi(min_size=4) +def test_cell_ownership_is_unique(processor_props, simple_ugrid): + grid = simple_ugrid + num_cells = simple.SimpleGrid._CELLS + halo_generator = HaloGenerator(ugrid=grid, + rank_info=processor_props, + rank_mapping=simple_distribution, + num_lev=1, face_face_connectivity=simple.SimpleGridData.c2e2c_table) + my_owned_cells = halo_generator.owned_cells() + print(f"rank {processor_props.rank} owns {my_owned_cells} ") + # assert that each cell is only owned by one rank + if not mpi4py.MPI.Is_initialized(): + mpi4py.MPI.Init() + if processor_props.rank == 0: + gathered = -1 *xp.ones([processor_props.comm_size, num_cells], dtype=xp.int64) + else: + gathered = None + processor_props.comm.Gather(my_owned_cells, gathered, root=0) + if processor_props.rank == 0: + print(gathered.shape) + print(gathered) + # gathered = xp.where(gathered.reshape(processor_props.comm_size * 18) > 0) + # assert gathered == simple_distribution.size - 1 + # assert gathered.size == len(xp.unique(gathered)) +@pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) +def test_halo_constructor_decomposition_info(processor_props, simple_ugrid, dim): # noqa: F811 # fixture + grid = simple_ugrid + halo_generator = HaloGenerator(ugrid=grid, + rank_info=processor_props, + rank_mapping=simple_distribution, + num_lev=1, face_face_connectivity=simple.SimpleGridData.c2e2c_table, + node_face_connectivity=simple.SimpleGridData.v2c_table) + + decomp_info = halo_generator.construct_decomposition_info() + my_halo = decomp_info.global_index(dim, defs.DecompositionInfo.EntryType.HALO) + print(f"rank {processor_props.rank} has halo {dim} : {my_halo}") + assert my_halo.size == len(halos[dim][processor_props.rank]) + assert xp.setdiff1d(my_halo, halos[dim][processor_props.rank], assume_unique=True).size == 0 + my_owned = decomp_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED) + print(f"rank {processor_props.rank} owns {dim} : {my_owned} ") + assert my_owned.size == len(owned[dim][processor_props.rank]) + assert xp.setdiff1d(my_owned, owned[dim][processor_props.rank], assume_unique=True).size == 0 + + + + + +@pytest.fixture +def simple_ugrid()->xu.Ugrid2d: + """ + Programmatically construct a xugrid.ugrid.ugrid2d.Ugrid2d object + + Returns: a Ugrid2d object base on the SimpleGrid + + """ + simple_mesh = simple.SimpleGrid() + fill_value = -1 + node_x = xp.arange(simple_mesh.num_vertices, dtype=xp.float64) + node_y = xp.arange(simple_mesh.num_vertices, dtype=xp.float64) + grid = xu.Ugrid2d(node_x, node_y, fill_value, + projected=True, + face_node_connectivity=simple_mesh.connectivities[dims.C2VDim], + edge_node_connectivity=simple_mesh.connectivities[dims.E2VDim], ) + + return grid + + + \ No newline at end of file From ab696bb2f01ede3b7428d1c1adbfb7568050d72b Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 5 Jul 2024 14:05:28 +0200 Subject: [PATCH 003/240] pre-commit fixes --- model/common/pyproject.toml | 2 +- .../model/common/decomposition/halo.py | 174 ++++++++----- .../common/decomposition/mpi_decomposition.py | 1 + .../tests/decomposition_tests/test_halo.py | 236 +++++++++++------- 4 files changed, 256 insertions(+), 157 deletions(-) diff --git a/model/common/pyproject.toml b/model/common/pyproject.toml index 0ae6ee8dfa..a8ba2f3eeb 100644 --- a/model/common/pyproject.toml +++ b/model/common/pyproject.toml @@ -31,7 +31,7 @@ requires-python = ">=3.10" [project.optional-dependencies] all = ["icon4py-common[ghex,netcdf]"] -ghex = ["ghex", "mpi4py", "xugrid","pymetis>2022.1", "scipy"] +ghex = ["ghex", "mpi4py", "xugrid", "pymetis>2022.1", "scipy"] netcdf = ["netcdf4>=1.6.0"] [project.urls] diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 97fd43ded4..2747d6d321 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -1,3 +1,16 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# This file is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or any later +# version. See the LICENSE.txt file at the top-level directory of this +# distribution for a copy of the license or check . +# +# SPDX-License-Identifier: GPL-3.0-or-later + import enum import functools from typing import Union @@ -10,32 +23,40 @@ from icon4py.model.common.settings import xp -#TODO do we need three of those? +# TODO do we need three of those? class DecompositionFlag(enum.IntEnum): #: cell is owned by this rank - OWNED = 0, + OWNED = (0,) #: cell is in the first halo line: that is cells that share and edge with an owned cell - FIRST_HALO_LINE = 1, + FIRST_HALO_LINE = (1,) #: cell is in the second halo line: that is cells that share only a vertex with an owned cell (and at least an edge with a FIRST_HALO_LINE cell) SECOND_HALO_LINE = 2 - class HaloGenerator: """Creates necessary halo information for a given rank.""" - def __init__(self, rank_info:defs.ProcessProperties, rank_mapping: xp.ndarray, ugrid:ux.Ugrid2d, num_lev:int, face_face_connectivity:xp.ndarray = None, node_face_connectivity = None): + + def __init__( + self, + rank_info: defs.ProcessProperties, + rank_mapping: xp.ndarray, + ugrid: ux.Ugrid2d, + num_lev: int, + face_face_connectivity: xp.ndarray = None, + node_face_connectivity=None, + ): """ - + Args: rank_info: contains information on the communicator and local compute node. rank_mapping: array with shape (global_num_cells) mapping of global cell indices to their rank in the distribution ugrid: the global grid num_lev: number of vertical levels - face_face_connectivity: face-face connectivity matrix: (n_cells, 3) xugrid uses a - scipy.sparse matrix for this which causes problems with zero based indices so we - allow to pass it directly as a workaround - node_face_connectivity: node-face connectivity matrix: (n_vertex, 6) xugrid uses a - sparse matrix for this which causes problems with zero based indices so wes + face_face_connectivity: face-face connectivity matrix: (n_cells, 3) xugrid uses a + scipy.sparse matrix for this which causes problems with zero based indices so we + allow to pass it directly as a workaround + node_face_connectivity: node-face connectivity matrix: (n_vertex, 6) xugrid uses a + sparse matrix for this which causes problems with zero based indices so wes """ self._props = rank_info self._mapping = rank_mapping @@ -43,23 +64,18 @@ def __init__(self, rank_info:defs.ProcessProperties, rank_mapping: xp.ndarray, u self._num_lev = num_lev self._c2e2c = face_face_connectivity self._v2c = node_face_connectivity - - - + def _validate(self): assert self._mapping.ndim == 1 # the decomposition should match the communicator size assert xp.max(self._mapping) == self._props.comm_size - 1 - - - + def _post_init(self): self._validate() - def next_halo_line(self, cell_line: xp.ndarray, depot=None): """Returns the global indices of the next halo line. - + Args: cell_line: global indices of cells we want to find the neighbors of depot: global indices that have already been collected @@ -74,25 +90,23 @@ def next_halo_line(self, cell_line: xp.ndarray, depot=None): cells_so_far = cell_line next_halo_cells = xp.setdiff1d(xp.unique(cell_neighbors), cells_so_far, assume_unique=True) - # assert next_halo_cells.shape[0] == next_halo_size return next_halo_cells - def _cell_neighbors(self, cells: xp.ndarray): if self._c2e2c is not None: return xp.unique(self._c2e2c[cells, :]) else: return self._cell_neighbors_from_sparse(cells) + @functools.cached_property - def _node_face_connectivity(self)->Union[xp.ndarray, sp.sparse.csr_matrix]: + def _node_face_connectivity(self) -> Union[xp.ndarray, sp.sparse.csr_matrix]: if self._v2c is not None: return self._v2c else: return self._global_grid._node_face_connectivity - + def _cell_neighbors_from_sparse(self, cells: xp.ndarray): - """ In xugrid face-face connectivity is a scipy spars matrix, so we reduce it to the regular sparse matrix format: (n_cells, 3) - """ + """In xugrid face-face connectivity is a scipy spars matrix, so we reduce it to the regular sparse matrix format: (n_cells, 3)""" conn = self._c2e2c.face_face_connectivity neighbors = conn[cells, :] @@ -101,68 +115,78 @@ def _cell_neighbors_from_sparse(self, cells: xp.ndarray): return j - - def _find_neighbors(self, cell_line:xp.ndarray, connectivity: xp.ndarray)->xp.ndarray: - """ Get a flattened list of all (unique) neighbors to a given global index list""" + def _find_neighbors(self, cell_line: xp.ndarray, connectivity: xp.ndarray) -> xp.ndarray: + """Get a flattened list of all (unique) neighbors to a given global index list""" neighbors = connectivity[cell_line, :] shp = neighbors.shape unique_neighbors = xp.unique(neighbors.reshape(shp[0] * shp[1])) return unique_neighbors - def find_edge_neighbors_for_cells(self, cell_line: xp.ndarray) -> xp.ndarray: - return self._find_neighbors(cell_line, connectivity=self._global_grid.face_edge_connectivity) - def find_vertex_neighbors_for_cells(self, cell_line:xp.ndarray)->xp.ndarray: - return self._find_neighbors(cell_line, connectivity=self._global_grid.face_node_connectivity) - - - def owned_cells(self)->xp.ndarray: + return self._find_neighbors( + cell_line, connectivity=self._global_grid.face_edge_connectivity + ) + + def find_vertex_neighbors_for_cells(self, cell_line: xp.ndarray) -> xp.ndarray: + return self._find_neighbors( + cell_line, connectivity=self._global_grid.face_node_connectivity + ) + + def owned_cells(self) -> xp.ndarray: """Returns the global indices of the cells owned by this rank""" owned_cells = self._mapping == self._props.rank return xp.asarray(owned_cells).nonzero()[0] - - def construct_decomposition_info(self)->defs.DecompositionInfo: + + def construct_decomposition_info(self) -> defs.DecompositionInfo: """ Constructs the DecompositionInfo for the current rank. - - The DecompositionInfo object is constructed for all horizontal dimension starting from the + + The DecompositionInfo object is constructed for all horizontal dimension starting from the cell distribution. Edges and vertices are then handled through their connectivity to the distributed cells. """ - + #: cells - owned_cells = self.owned_cells() # global indices of owned cells + owned_cells = self.owned_cells() # global indices of owned cells first_halo_cells = self.next_halo_line(owned_cells) second_halo_cells = self.next_halo_line(first_halo_cells, owned_cells) - + total_halo_cells = xp.hstack((first_halo_cells, second_halo_cells)) all_cells = xp.hstack((owned_cells, total_halo_cells)) c_owner_mask = xp.isin(all_cells, owned_cells) - - decomp_info = defs.DecompositionInfo(klevels=self._num_lev).with_dimension(dims.CellDim, - all_cells, - c_owner_mask) - + + decomp_info = defs.DecompositionInfo(klevels=self._num_lev).with_dimension( + dims.CellDim, all_cells, c_owner_mask + ) + #: edges edges_on_owned_cells = self.find_edge_neighbors_for_cells(owned_cells) edges_on_first_halo_line = self.find_edge_neighbors_for_cells(first_halo_cells) edges_on_second_halo_line = self.find_edge_neighbors_for_cells(second_halo_cells) - - all_edges = xp.hstack((edges_on_owned_cells, xp.setdiff1d(edges_on_first_halo_line, edges_on_owned_cells), xp.setdiff1d(edges_on_second_halo_line, edges_on_first_halo_line))) + + all_edges = xp.hstack( + ( + edges_on_owned_cells, + xp.setdiff1d(edges_on_first_halo_line, edges_on_owned_cells), + xp.setdiff1d(edges_on_second_halo_line, edges_on_first_halo_line), + ) + ) all_edges = xp.unique(all_edges) # We need to reduce the overlap: - # `edges_on_owned_cells` and `edges_on_first_halo_line` both contain the edges on the cutting line. + # `edges_on_owned_cells` and `edges_on_first_halo_line` both contain the edges on the cutting line. intersect_owned_first_line = xp.intersect1d(edges_on_owned_cells, edges_on_first_halo_line) - - def _update_owner_mask_by_max_rank_convention(owner_mask, all_indices, indices_on_cutting_line, target_connectivity): + + def _update_owner_mask_by_max_rank_convention( + owner_mask, all_indices, indices_on_cutting_line, target_connectivity + ): """ - In order to have unique ownership of edges (and vertices) among nodes there needs to be - a convention as to where those elements on the cutting line go: - according to a remark in `mo_decomposition_tools.f90` ICON puts them to the node + In order to have unique ownership of edges (and vertices) among nodes there needs to be + a convention as to where those elements on the cutting line go: + according to a remark in `mo_decomposition_tools.f90` ICON puts them to the node with the higher rank. - - # TODO (@halungge): can we add an assert for the target dimension of the connectivity being cells. - Args: + + # TODO (@halungge): can we add an assert for the target dimension of the connectivity being cells. + Args: owner_mask: owner mask for the dimension all_indices: (global) indices of the dimension indices_on_cutting_line: global indices of the elements on the cutting line @@ -173,32 +197,44 @@ def _update_owner_mask_by_max_rank_convention(owner_mask, all_indices, indices_o for index in indices_on_cutting_line: local_index = xp.nonzero(all_indices == index)[0][0] owning_ranks = self._mapping[target_connectivity[index]] - assert xp.unique(owning_ranks).size > 1, f"rank {self._props.rank}: all neighboring cells are owned by the same rank" - assert self._props.rank in owning_ranks, f"rank {self._props.rank}: neither of the neighboring cells: {owning_ranks} is owned by me" + assert ( + xp.unique(owning_ranks).size > 1 + ), f"rank {self._props.rank}: all neighboring cells are owned by the same rank" + assert ( + self._props.rank in owning_ranks + ), f"rank {self._props.rank}: neither of the neighboring cells: {owning_ranks} is owned by me" # assign the index to the rank with the higher rank if max(owning_ranks) > self._props.rank: owner_mask[local_index] = False else: owner_mask[local_index] = True return owner_mask - # construct the owner mask edge_owner_mask = xp.isin(all_edges, edges_on_owned_cells) - edge_owner_mask = _update_owner_mask_by_max_rank_convention(edge_owner_mask, all_edges, intersect_owned_first_line, self._global_grid.edge_face_connectivity) + edge_owner_mask = _update_owner_mask_by_max_rank_convention( + edge_owner_mask, + all_edges, + intersect_owned_first_line, + self._global_grid.edge_face_connectivity, + ) decomp_info.with_dimension(dims.EdgeDim, all_edges, edge_owner_mask) - + # vertices vertices_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) vertices_on_first_halo_line = self.find_vertex_neighbors_for_cells(first_halo_cells) - vertices_on_second_halo_line = self.find_vertex_neighbors_for_cells(second_halo_cells) #TODO (@halungge): do we need that? - intersect_owned_first_line = xp.intersect1d(vertices_on_owned_cells, vertices_on_first_halo_line) - + vertices_on_second_halo_line = self.find_vertex_neighbors_for_cells( + second_halo_cells + ) # TODO (@halungge): do we need that? + intersect_owned_first_line = xp.intersect1d( + vertices_on_owned_cells, vertices_on_first_halo_line + ) + # create decomposition_info for vertices all_vertices = xp.unique(xp.hstack((vertices_on_owned_cells, vertices_on_first_halo_line))) v_owner_mask = xp.isin(all_vertices, vertices_on_owned_cells) - v_owner_mask = _update_owner_mask_by_max_rank_convention(v_owner_mask, all_vertices, intersect_owned_first_line, self._node_face_connectivity) + v_owner_mask = _update_owner_mask_by_max_rank_convention( + v_owner_mask, all_vertices, intersect_owned_first_line, self._node_face_connectivity + ) decomp_info.with_dimension(dims.VertexDim, all_vertices, v_owner_mask) return decomp_info - - \ No newline at end of file diff --git a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py index 04eca68f15..5e1429391c 100644 --- a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py +++ b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py @@ -103,6 +103,7 @@ def filter(self, record: logging.LogRecord) -> bool: def get_multinode_properties(s: definitions.MultiNodeRun) -> definitions.ProcessProperties: return _get_processor_properties(with_mpi=True) + # TODO (@halungge) changed for dev/testing set back to frozen @dataclass(frozen=False) class MPICommProcessProperties(definitions.ProcessProperties): diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index 5d1f07a3a1..01d8a79e33 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -1,3 +1,16 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# This file is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or any later +# version. See the LICENSE.txt file at the top-level directory of this +# distribution for a copy of the license or check . +# +# SPDX-License-Identifier: GPL-3.0-or-later + import mpi4py import mpi4py.MPI import pytest @@ -14,106 +27,157 @@ ) -simple_distribution = xp.asarray([0, # 0c - 1, # 1c - 1, # 2c - 0, # 3c - 0, # 4c - 1, # 5c - 0, # 6c - 0, # 7c - 2, # 8c - 2, # 9c - 0, # 10c - 2, # 11c - 3, # 12c - 3, # 13c - 1, # 14c - 3, # 15c - 3, # 16c - 1, #17c - ]) -cell_own = {0:[0,3,4,6,7,10], - 1:[1,2,5,14,17], - 2:[8,9,11], - 3:[12,13,15,16]} -cell_halos = {0:[2,15, 1, 11, 13, 9,17, 5, 12, 14, 8,16], - 1:[4, 16, 3, 8, 15, 11, 13, 0,7,6,9,10,12], - 2:[5, 6, 12,14,7, 2,1,4,3, 10, 15, 16, 17], - 3:[9, 10, 17, 14, 0, 1,6, 7,8,2, 3, 4, 5,11]} - - -edge_own = {0:[1,5,12,13,14,9], - 1:[8,7,6,25,4, 2], - 2:[16,11,15,17,10,24], - 3:[19,23,22,26,0,3,20,18,21]} - -edge_halos = {0:[0,4,21,10,2, 3,8,6,7,19,20,17,16,11,18,26,25,15,23,24,22], - 1:[5,12,22,23,3,1,9,15,16,11,19,20,0,17,24,21,26,13,10,14,18,], - 2:[7,6,9,8,14,18,19,23,25,20,12,13,2,3,4,5,1,21,22,0,26], - 3:[10,11,13,14,25,6,24,1,5,4,8,9,17,12,15,16,2,7]} - -vertex_own = {0:[4], - 1:[], - 2:[3,5], - 3:[0,1,2,6,7,8,]} -vertex_halo = {0:[0,1,2,3,5,6,7,8], - 1:[1,2,0,5,3,8,6, 7, 4], - 2:[8,6,7,4, 0,2,1 ], - 3:[3,4,5]} - -owned ={dims.CellDim:cell_own, dims.EdgeDim:edge_own, dims.VertexDim:vertex_own} -halos = {dims.CellDim:cell_halos, dims.EdgeDim:edge_halos, dims.VertexDim:vertex_halo} -def test_halo_constructor_owned_cells(processor_props, simple_ugrid): # noqa: F811 # fixture - +simple_distribution = xp.asarray( + [ + 0, # 0c + 1, # 1c + 1, # 2c + 0, # 3c + 0, # 4c + 1, # 5c + 0, # 6c + 0, # 7c + 2, # 8c + 2, # 9c + 0, # 10c + 2, # 11c + 3, # 12c + 3, # 13c + 1, # 14c + 3, # 15c + 3, # 16c + 1, # 17c + ] +) +cell_own = {0: [0, 3, 4, 6, 7, 10], 1: [1, 2, 5, 14, 17], 2: [8, 9, 11], 3: [12, 13, 15, 16]} +cell_halos = { + 0: [2, 15, 1, 11, 13, 9, 17, 5, 12, 14, 8, 16], + 1: [4, 16, 3, 8, 15, 11, 13, 0, 7, 6, 9, 10, 12], + 2: [5, 6, 12, 14, 7, 2, 1, 4, 3, 10, 15, 16, 17], + 3: [9, 10, 17, 14, 0, 1, 6, 7, 8, 2, 3, 4, 5, 11], +} + + +edge_own = { + 0: [1, 5, 12, 13, 14, 9], + 1: [8, 7, 6, 25, 4, 2], + 2: [16, 11, 15, 17, 10, 24], + 3: [19, 23, 22, 26, 0, 3, 20, 18, 21], +} + +edge_halos = { + 0: [0, 4, 21, 10, 2, 3, 8, 6, 7, 19, 20, 17, 16, 11, 18, 26, 25, 15, 23, 24, 22], + 1: [ + 5, + 12, + 22, + 23, + 3, + 1, + 9, + 15, + 16, + 11, + 19, + 20, + 0, + 17, + 24, + 21, + 26, + 13, + 10, + 14, + 18, + ], + 2: [7, 6, 9, 8, 14, 18, 19, 23, 25, 20, 12, 13, 2, 3, 4, 5, 1, 21, 22, 0, 26], + 3: [10, 11, 13, 14, 25, 6, 24, 1, 5, 4, 8, 9, 17, 12, 15, 16, 2, 7], +} + +vertex_own = { + 0: [4], + 1: [], + 2: [3, 5], + 3: [ + 0, + 1, + 2, + 6, + 7, + 8, + ], +} +vertex_halo = { + 0: [0, 1, 2, 3, 5, 6, 7, 8], + 1: [1, 2, 0, 5, 3, 8, 6, 7, 4], + 2: [8, 6, 7, 4, 0, 2, 1], + 3: [3, 4, 5], +} + +owned = {dims.CellDim: cell_own, dims.EdgeDim: edge_own, dims.VertexDim: vertex_own} +halos = {dims.CellDim: cell_halos, dims.EdgeDim: edge_halos, dims.VertexDim: vertex_halo} + + +def test_halo_constructor_owned_cells(processor_props, simple_ugrid): # noqa: F811 # fixture grid = simple_ugrid - halo_generator = HaloGenerator(ugrid=grid, - rank_info=processor_props, - rank_mapping=simple_distribution, - num_lev=1, - face_face_connectivity=simple.SimpleGridData.c2e2c_table) + halo_generator = HaloGenerator( + ugrid=grid, + rank_info=processor_props, + rank_mapping=simple_distribution, + num_lev=1, + face_face_connectivity=simple.SimpleGridData.c2e2c_table, + ) my_owned_cells = halo_generator.owned_cells() - + print(f"rank {processor_props.rank} owns {my_owned_cells} ") assert my_owned_cells.size == len(cell_own[processor_props.rank]) assert xp.setdiff1d(my_owned_cells, cell_own[processor_props.rank]).size == 0 + @pytest.mark.skip(reason="mpi.GATHER ???") @pytest.mark.mpi(min_size=4) -def test_cell_ownership_is_unique(processor_props, simple_ugrid): +def test_cell_ownership_is_unique(processor_props, simple_ugrid): # noqa: F811 # fixture grid = simple_ugrid num_cells = simple.SimpleGrid._CELLS - halo_generator = HaloGenerator(ugrid=grid, - rank_info=processor_props, - rank_mapping=simple_distribution, - num_lev=1, face_face_connectivity=simple.SimpleGridData.c2e2c_table) + halo_generator = HaloGenerator( + ugrid=grid, + rank_info=processor_props, + rank_mapping=simple_distribution, + num_lev=1, + face_face_connectivity=simple.SimpleGridData.c2e2c_table, + ) my_owned_cells = halo_generator.owned_cells() print(f"rank {processor_props.rank} owns {my_owned_cells} ") # assert that each cell is only owned by one rank if not mpi4py.MPI.Is_initialized(): mpi4py.MPI.Init() if processor_props.rank == 0: - gathered = -1 *xp.ones([processor_props.comm_size, num_cells], dtype=xp.int64) + gathered = -1 * xp.ones([processor_props.comm_size, num_cells], dtype=xp.int64) else: gathered = None - processor_props.comm.Gather(my_owned_cells, gathered, root=0) + processor_props.comm.Gather(my_owned_cells, gathered, root=0) if processor_props.rank == 0: print(gathered.shape) print(gathered) - # gathered = xp.where(gathered.reshape(processor_props.comm_size * 18) > 0) - # assert gathered == simple_distribution.size - 1 - # assert gathered.size == len(xp.unique(gathered)) + gathered = xp.where(gathered.reshape(processor_props.comm_size * 18) > 0) + assert gathered == simple_distribution.size - 1 + assert gathered.size == len(xp.unique(gathered)) + + @pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) -def test_halo_constructor_decomposition_info(processor_props, simple_ugrid, dim): # noqa: F811 # fixture +def test_halo_constructor_decomposition_info(processor_props, simple_ugrid, dim): # noqa: F811 # fixture grid = simple_ugrid - halo_generator = HaloGenerator(ugrid=grid, - rank_info=processor_props, - rank_mapping=simple_distribution, - num_lev=1, face_face_connectivity=simple.SimpleGridData.c2e2c_table, - node_face_connectivity=simple.SimpleGridData.v2c_table) + halo_generator = HaloGenerator( + ugrid=grid, + rank_info=processor_props, + rank_mapping=simple_distribution, + num_lev=1, + face_face_connectivity=simple.SimpleGridData.c2e2c_table, + node_face_connectivity=simple.SimpleGridData.v2c_table, + ) decomp_info = halo_generator.construct_decomposition_info() - my_halo = decomp_info.global_index(dim, defs.DecompositionInfo.EntryType.HALO) + my_halo = decomp_info.global_index(dim, defs.DecompositionInfo.EntryType.HALO) print(f"rank {processor_props.rank} has halo {dim} : {my_halo}") assert my_halo.size == len(halos[dim][processor_props.rank]) assert xp.setdiff1d(my_halo, halos[dim][processor_props.rank], assume_unique=True).size == 0 @@ -121,16 +185,13 @@ def test_halo_constructor_decomposition_info(processor_props, simple_ugrid, dim) print(f"rank {processor_props.rank} owns {dim} : {my_owned} ") assert my_owned.size == len(owned[dim][processor_props.rank]) assert xp.setdiff1d(my_owned, owned[dim][processor_props.rank], assume_unique=True).size == 0 - - - @pytest.fixture -def simple_ugrid()->xu.Ugrid2d: +def simple_ugrid() -> xu.Ugrid2d: """ Programmatically construct a xugrid.ugrid.ugrid2d.Ugrid2d object - + Returns: a Ugrid2d object base on the SimpleGrid """ @@ -138,12 +199,13 @@ def simple_ugrid()->xu.Ugrid2d: fill_value = -1 node_x = xp.arange(simple_mesh.num_vertices, dtype=xp.float64) node_y = xp.arange(simple_mesh.num_vertices, dtype=xp.float64) - grid = xu.Ugrid2d(node_x, node_y, fill_value, - projected=True, - face_node_connectivity=simple_mesh.connectivities[dims.C2VDim], - edge_node_connectivity=simple_mesh.connectivities[dims.E2VDim], ) + grid = xu.Ugrid2d( + node_x, + node_y, + fill_value, + projected=True, + face_node_connectivity=simple_mesh.connectivities[dims.C2VDim], + edge_node_connectivity=simple_mesh.connectivities[dims.E2VDim], + ) return grid - - - \ No newline at end of file From 3ff819112147a5158e2696ee95f383fb19305016 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 5 Jul 2024 15:50:34 +0200 Subject: [PATCH 004/240] test for uniqueness of owned indices --- model/common/pyproject.toml | 2 +- .../tests/decomposition_tests/test_halo.py | 41 +++++++++++++------ 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/model/common/pyproject.toml b/model/common/pyproject.toml index a8ba2f3eeb..611eacc7cd 100644 --- a/model/common/pyproject.toml +++ b/model/common/pyproject.toml @@ -31,7 +31,7 @@ requires-python = ">=3.10" [project.optional-dependencies] all = ["icon4py-common[ghex,netcdf]"] -ghex = ["ghex", "mpi4py", "xugrid", "pymetis>2022.1", "scipy"] +ghex = ["ghex", "mpi4py>=3.1.4", "xugrid", "pymetis>2022.1", "scipy"] netcdf = ["netcdf4>=1.6.0"] [project.urls] diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index 01d8a79e33..324b918747 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -126,6 +126,7 @@ def test_halo_constructor_owned_cells(processor_props, simple_ugrid): # noqa: F rank_mapping=simple_distribution, num_lev=1, face_face_connectivity=simple.SimpleGridData.c2e2c_table, + node_face_connectivity=simple.SimpleGridData.v2c_table, ) my_owned_cells = halo_generator.owned_cells() @@ -134,36 +135,45 @@ def test_halo_constructor_owned_cells(processor_props, simple_ugrid): # noqa: F assert xp.setdiff1d(my_owned_cells, cell_own[processor_props.rank]).size == 0 -@pytest.mark.skip(reason="mpi.GATHER ???") +@pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) @pytest.mark.mpi(min_size=4) -def test_cell_ownership_is_unique(processor_props, simple_ugrid): # noqa: F811 # fixture +def test_cell_ownership_is_unique(dim, processor_props, simple_ugrid): # noqa: F811 # fixture grid = simple_ugrid - num_cells = simple.SimpleGrid._CELLS halo_generator = HaloGenerator( ugrid=grid, rank_info=processor_props, rank_mapping=simple_distribution, num_lev=1, face_face_connectivity=simple.SimpleGridData.c2e2c_table, + node_face_connectivity=simple.SimpleGridData.v2c_table, ) - my_owned_cells = halo_generator.owned_cells() - print(f"rank {processor_props.rank} owns {my_owned_cells} ") - # assert that each cell is only owned by one rank + assert processor_props.comm.Get_size() == 4, "This test requires 4 MPI ranks." + + decomposition_info = halo_generator.construct_decomposition_info() + owned = decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED) if not mpi4py.MPI.Is_initialized(): mpi4py.MPI.Init() + # assert that each cell is only owned by one rank + comm = processor_props.comm + + my_size = owned.shape[0] + local_sizes = xp.array(comm.gather(my_size, root=0)) + if processor_props.rank == 0: - gathered = -1 * xp.ones([processor_props.comm_size, num_cells], dtype=xp.int64) + gathered = xp.empty(sum(local_sizes), dtype=int) + else: gathered = None - processor_props.comm.Gather(my_owned_cells, gathered, root=0) + comm.Gatherv(sendbuf=owned, recvbuf=(gathered, local_sizes), root=0) if processor_props.rank == 0: - print(gathered.shape) - print(gathered) - gathered = xp.where(gathered.reshape(processor_props.comm_size * 18) > 0) - assert gathered == simple_distribution.size - 1 - assert gathered.size == len(xp.unique(gathered)) + # check there are no duplicates + assert gathered.size == len(xp.unique(gathered)) + # check the buffer has all global indices + assert xp.all(xp.sort(gathered) == global_indices(dim)) +# TODO (@halungge) this test can be run on 4MPI ranks or should we rather switch to a single node +# that parametrizes the rank number @pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) def test_halo_constructor_decomposition_info(processor_props, simple_ugrid, dim): # noqa: F811 # fixture grid = simple_ugrid @@ -209,3 +219,8 @@ def simple_ugrid() -> xu.Ugrid2d: ) return grid + + +def global_indices(dim: dims.Dimension) -> int: + mesh = simple.SimpleGrid() + return xp.arange(mesh.size[dim], dtype=xp.int32) From 36d120e058d5d607274a83c62107a0777bc2ca86 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 9 Jul 2024 13:01:20 +0200 Subject: [PATCH 005/240] local connectivities WIP (1) --- .../icon4py/model/common/decomposition/halo.py | 10 ++++++++++ .../src/icon4py/model/common/grid/grid_manager.py | 15 ++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 2747d6d321..3cb4e50632 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -238,3 +238,13 @@ def _update_owner_mask_by_max_rank_convention( ) decomp_info.with_dimension(dims.VertexDim, all_vertices, v_owner_mask) return decomp_info + + def construct_local_connectivities(self, decom_info: defs.DecompositionInfo): + #ugrid required connectivity + #self._props + global_node_idx = decom_info.global_index(dims.VertexDim, defs.DecompositionInfo.EntryType.ALL) + local_node_idx = decom_info.local_index(dims.VertexDim, defs.DecompositionInfo.EntryType.ALL) + sliced_connectivity = self._global_grid.face_node_connectivity[decom_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.ALL)] + + + \ No newline at end of file diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index fedf61f254..9784f08e5f 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -377,18 +377,19 @@ def _from_grid_dataset( ) c2e = self._get_index_field(reader, GridFile.OffsetName.C2E) - e2c = self._get_index_field(reader, GridFile.OffsetName.E2C) - c2v = self._get_index_field(reader, GridFile.OffsetName.C2V) - e2v = self._get_index_field(reader, GridFile.OffsetName.E2V) + e2c = self._get_index_field(reader, GridFile.OffsetName.E2C) # edge_face_connectivity (optional) + c2v = self._get_index_field(reader, GridFile.OffsetName.C2V) # face_node_connectivity (required) + v2c = self._get_index_field(reader, GridFile.OffsetName.V2C) # node_face_connectivity -- (pentagon/hexagon) + e2v = self._get_index_field(reader, GridFile.OffsetName.E2V) # edge_node_connectivity (optionally required) + v2e = self._get_index_field(reader, GridFile.OffsetName.V2E) # node_edge_connectivity -- (pentagon/hexagon) + v2e2v = self._get_index_field(reader, GridFile.OffsetName.V2E2V) #node_node_connectivity -- ((pentagon/hexagon)) + c2e2c = self._get_index_field(reader, GridFile.OffsetName.C2E2C) # face_face_connectivity (optional) e2c2v = self._construct_diamond_vertices(e2v, c2v, e2c) e2c2e = self._construct_diamond_edges(e2c, c2e) e2c2e0 = np.column_stack((np.asarray(range(e2c2e.shape[0])), e2c2e)) - v2c = self._get_index_field(reader, GridFile.OffsetName.V2C) - v2e = self._get_index_field(reader, GridFile.OffsetName.V2E) - v2e2v = self._get_index_field(reader, GridFile.OffsetName.V2E2V) - c2e2c = self._get_index_field(reader, GridFile.OffsetName.C2E2C) + c2e2c2e = self._construct_triangle_edges(c2e2c, c2e) c2e2c0 = np.column_stack((np.asarray(range(c2e2c.shape[0])), c2e2c)) ( From 0df329dc7052b12cf14a4c6d2c852caa9391cde7 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 10 Jul 2024 14:13:40 +0200 Subject: [PATCH 006/240] add construction of local connectivities --- model/common/pyproject.toml | 2 + .../model/common/decomposition/halo.py | 133 ++++++++++++----- .../tests/decomposition_tests/test_halo.py | 136 ++++++++++++++++-- 3 files changed, 227 insertions(+), 44 deletions(-) diff --git a/model/common/pyproject.toml b/model/common/pyproject.toml index 5e7620fb95..c0c1d60903 100644 --- a/model/common/pyproject.toml +++ b/model/common/pyproject.toml @@ -85,9 +85,11 @@ warn_unused_configs = true warn_unused_ignores = true [tool.pytest] +log_cli = true [tool.pytest.ini_options] addopts = ['-p icon4py.model.common.test_utils.pytest_config'] +log_cli = true markers = [ "datatest: test depending on serialized data generated by a full model run", "with_netcdf: test depending on a compatible version of netCDF4" diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 3cb4e50632..4ababf3373 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -12,17 +12,21 @@ # SPDX-License-Identifier: GPL-3.0-or-later import enum -import functools -from typing import Union +import logging +import uuid +import gt4py.next as gtx import scipy as sp import xugrid.ugrid.ugrid2d as ux import icon4py.model.common.decomposition.definitions as defs from icon4py.model.common import dimension as dims +from icon4py.model.common.grid import base as base_grid, icon as icon_grid from icon4py.model.common.settings import xp +log = logging.getLogger(__name__) + # TODO do we need three of those? class DecompositionFlag(enum.IntEnum): #: cell is owned by this rank @@ -44,6 +48,7 @@ def __init__( num_lev: int, face_face_connectivity: xp.ndarray = None, node_face_connectivity=None, + node_edge_connectivity=None, ): """ @@ -62,8 +67,21 @@ def __init__( self._mapping = rank_mapping self._global_grid = ugrid self._num_lev = num_lev - self._c2e2c = face_face_connectivity - self._v2c = node_face_connectivity + self._connectivities = { + dims.C2E2CDim: face_face_connectivity + if face_face_connectivity is not None + else self._global_grid.face_face_connectivity, + dims.C2EDim: self._global_grid.face_edge_connectivity, + dims.C2VDim: self._global_grid.face_node_connectivity, + dims.E2CDim: self._global_grid.edge_face_connectivity, + dims.E2VDim: self._global_grid.edge_node_connectivity, + dims.V2CDim: node_face_connectivity + if node_face_connectivity is not None + else self._global_grid.node_face_connectivity, + dims.V2EDim: node_edge_connectivity + if node_edge_connectivity is not None + else self._global_grid.node_edge_connectivity, + } def _validate(self): assert self._mapping.ndim == 1 @@ -72,6 +90,19 @@ def _validate(self): def _post_init(self): self._validate() + # TODO handle sparse neibhbors tables? + + def connectivity(self, dim) -> xp.ndarray: + try: + conn_table = self._connectivities[dim] + if sp.sparse.issparse(conn_table): + raise NotImplementedError("Scipy sparse for connectivity tables are not supported") + # TODO this is not tested/working. It returns a (N x N) matrix not a (N x sparse_dim) matrix + # _and_ we cannot make the difference between a valid "0" and a missing-value 0 + # return conn_table.toarray(order="C") + return conn_table + except KeyError as err: + raise (f"Connectivity for dimension {dim} is not available") from err def next_halo_line(self, cell_line: xp.ndarray, depot=None): """Returns the global indices of the next halo line. @@ -93,21 +124,11 @@ def next_halo_line(self, cell_line: xp.ndarray, depot=None): return next_halo_cells def _cell_neighbors(self, cells: xp.ndarray): - if self._c2e2c is not None: - return xp.unique(self._c2e2c[cells, :]) - else: - return self._cell_neighbors_from_sparse(cells) - - @functools.cached_property - def _node_face_connectivity(self) -> Union[xp.ndarray, sp.sparse.csr_matrix]: - if self._v2c is not None: - return self._v2c - else: - return self._global_grid._node_face_connectivity + return xp.unique(self.connectivity(dims.C2E2CDim)[cells, :]) def _cell_neighbors_from_sparse(self, cells: xp.ndarray): """In xugrid face-face connectivity is a scipy spars matrix, so we reduce it to the regular sparse matrix format: (n_cells, 3)""" - conn = self._c2e2c.face_face_connectivity + conn = self.connectivity(dims.C2E2CDim) neighbors = conn[cells, :] # There is an issue with explicit 0 (for zero based indices) since sparse.find projects them out... @@ -123,20 +144,17 @@ def _find_neighbors(self, cell_line: xp.ndarray, connectivity: xp.ndarray) -> xp return unique_neighbors def find_edge_neighbors_for_cells(self, cell_line: xp.ndarray) -> xp.ndarray: - return self._find_neighbors( - cell_line, connectivity=self._global_grid.face_edge_connectivity - ) + return self._find_neighbors(cell_line, connectivity=self.connectivity(dims.C2EDim)) def find_vertex_neighbors_for_cells(self, cell_line: xp.ndarray) -> xp.ndarray: - return self._find_neighbors( - cell_line, connectivity=self._global_grid.face_node_connectivity - ) + return self._find_neighbors(cell_line, connectivity=self.connectivity(dims.C2VDim)) def owned_cells(self) -> xp.ndarray: """Returns the global indices of the cells owned by this rank""" owned_cells = self._mapping == self._props.rank return xp.asarray(owned_cells).nonzero()[0] + # TODO (@halungge): move out of halo generator def construct_decomposition_info(self) -> defs.DecompositionInfo: """ Constructs the DecompositionInfo for the current rank. @@ -234,17 +252,68 @@ def _update_owner_mask_by_max_rank_convention( all_vertices = xp.unique(xp.hstack((vertices_on_owned_cells, vertices_on_first_halo_line))) v_owner_mask = xp.isin(all_vertices, vertices_on_owned_cells) v_owner_mask = _update_owner_mask_by_max_rank_convention( - v_owner_mask, all_vertices, intersect_owned_first_line, self._node_face_connectivity + v_owner_mask, all_vertices, intersect_owned_first_line, self.connectivity(dims.V2CDim) ) decomp_info.with_dimension(dims.VertexDim, all_vertices, v_owner_mask) return decomp_info - - def construct_local_connectivities(self, decom_info: defs.DecompositionInfo): - #ugrid required connectivity - #self._props - global_node_idx = decom_info.global_index(dims.VertexDim, defs.DecompositionInfo.EntryType.ALL) - local_node_idx = decom_info.local_index(dims.VertexDim, defs.DecompositionInfo.EntryType.ALL) - sliced_connectivity = self._global_grid.face_node_connectivity[decom_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.ALL)] - + + def construct_local_connectivity(self, field_offset: gtx.FieldOffset, + decom_info: defs.DecompositionInfo) -> xp.ndarray: + """ + Construct a connectivity table for use on a given rank: it maps from source to target dimension in local indices. - \ No newline at end of file + Args: + field_offset: FieldOffset for which we want to construct the offset table + decom_info: DecompositionInfo for the current rank + + Returns: array, containt the connectivity table for the field_offset with rank-local indices + + """ + source_dim = field_offset.source + target_dim = field_offset.target[0] + local_dim = field_offset.target[1] + connectivity = self.connectivity(local_dim) + global_node_idx = decom_info.global_index(source_dim, defs.DecompositionInfo.EntryType.ALL) + global_node_sorted = xp.argsort(global_node_idx) + sliced_connectivity = connectivity[ + decom_info.global_index(target_dim, defs.DecompositionInfo.EntryType.ALL) + ] + log.debug(f"rank {self._props.rank} has local connectivity f: {sliced_connectivity}") + for i in xp.arange(sliced_connectivity.shape[0]): + positions = xp.searchsorted( + global_node_idx[global_node_sorted], sliced_connectivity[i, :] + ) + indices = global_node_sorted[positions] + sliced_connectivity[i, :] = indices + log.debug(f"rank {self._props.rank} has local connectivity f: {sliced_connectivity}") + return sliced_connectivity + +# should be done in grid manager! +def local_grid(props: defs.ProcessProperties, + decomp_info:defs.DecompositionInfo, + global_params:icon_grid.GlobalGridParams, + num_lev:int, + limited_area:bool = False, on_gpu:bool=False, )->base_grid.BaseGrid: + """ + Constructs a local grid for this rank based on the decomposition info. + TODO (@halungge): for now only returning base grid as we have not start/end indices implementation yet + Args: + decomp_info: the decomposition info for this rank + Returns: + local_grid: the local grid + """ + num_vertices = decomp_info.global_index(dims.VertexDim, defs.DecompositionInfo.EntryType.ALL).size + num_edges = decomp_info.global_index(dims.EdgeDim, defs.DecompositionInfo.EntryType.ALL).size + num_cells = decomp_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.ALL).size + grid_size = base_grid.HorizontalGridSize( + num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells + ) + config = base_grid.GridConfig(horizontal_config = grid_size, + vertical_size = num_lev, + on_gpu=on_gpu, + limited_area=limited_area) + + local_grid = icon_grid.IconGrid(uuid.uuid4()).with_config(config).with_global_params(global_params) + # add connectivities + + return local_grid \ No newline at end of file diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index 324b918747..9a888a799d 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -10,6 +10,8 @@ # distribution for a copy of the license or check . # # SPDX-License-Identifier: GPL-3.0-or-later +import logging +import pathlib import mpi4py import mpi4py.MPI @@ -17,10 +19,13 @@ import xugrid as xu import icon4py.model.common.dimension as dims +import icon4py.model.common.grid.grid_manager as gm +import icon4py.model.common.grid.vertical as v_grid from icon4py.model.common.decomposition import definitions as defs from icon4py.model.common.decomposition.halo import HaloGenerator -from icon4py.model.common.grid import simple +from icon4py.model.common.grid import icon, simple from icon4py.model.common.settings import xp +from icon4py.model.common.test_utils import datatest_utils as dt_utils from icon4py.model.common.test_utils.parallel_helpers import ( # noqa: F401 # import fixtures from test_utils package check_comm_size, processor_props, @@ -137,7 +142,7 @@ def test_halo_constructor_owned_cells(processor_props, simple_ugrid): # noqa: F @pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) @pytest.mark.mpi(min_size=4) -def test_cell_ownership_is_unique(dim, processor_props, simple_ugrid): # noqa: F811 # fixture +def test_element_ownership_is_unique(dim, processor_props, simple_ugrid): # noqa: F811 # fixture grid = simple_ugrid halo_generator = HaloGenerator( ugrid=grid, @@ -147,10 +152,11 @@ def test_cell_ownership_is_unique(dim, processor_props, simple_ugrid): # noqa: face_face_connectivity=simple.SimpleGridData.c2e2c_table, node_face_connectivity=simple.SimpleGridData.v2c_table, ) - assert processor_props.comm.Get_size() == 4, "This test requires 4 MPI ranks." + assert processor_props.comm_size == 4, "This test requires 4 MPI ranks." decomposition_info = halo_generator.construct_decomposition_info() owned = decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED) + print(f"\nrank {processor_props.rank} owns {dim} : {owned} ") if not mpi4py.MPI.Is_initialized(): mpi4py.MPI.Init() # assert that each cell is only owned by one rank @@ -158,22 +164,31 @@ def test_cell_ownership_is_unique(dim, processor_props, simple_ugrid): # noqa: my_size = owned.shape[0] local_sizes = xp.array(comm.gather(my_size, root=0)) - + buffer_size = 27 + send_buf = -1 * xp.ones(buffer_size, dtype=int) + send_buf[:my_size] = owned + print(f"rank {processor_props.rank} send_buf: {send_buf}") if processor_props.rank == 0: - gathered = xp.empty(sum(local_sizes), dtype=int) - + print(f"local_sizes: {local_sizes}") + # recv_buffer = xp.empty(sum(local_sizes), dtype=int) + recv_buffer = -1 * xp.ones((4, buffer_size), dtype=int) + print(f"{recv_buffer.shape}") else: - gathered = None - comm.Gatherv(sendbuf=owned, recvbuf=(gathered, local_sizes), root=0) + recv_buffer = None + # TODO (@halungge) Gatherv does not work if one of the buffers has size-0 (VertexDim) + # comm.Gatherv(sendbuf=owned, recvbuf=(recv_buffer, local_sizes), root=0) + comm.Gather(sendbuf=send_buf, recvbuf=recv_buffer, root=0) if processor_props.rank == 0: + print(f"global indices: {recv_buffer}") # check there are no duplicates - assert gathered.size == len(xp.unique(gathered)) + values = recv_buffer[recv_buffer != -1] + assert values.size == len(xp.unique(values)) # check the buffer has all global indices - assert xp.all(xp.sort(gathered) == global_indices(dim)) + assert xp.all(xp.sort(values) == global_indices(dim)) -# TODO (@halungge) this test can be run on 4MPI ranks or should we rather switch to a single node -# that parametrizes the rank number +# TODO (@halungge) this test can be run on 4 MPI ranks or should we rather switch to a single node, +# and parametrizes the rank number @pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) def test_halo_constructor_decomposition_info(processor_props, simple_ugrid, dim): # noqa: F811 # fixture grid = simple_ugrid @@ -197,6 +212,41 @@ def test_halo_constructor_decomposition_info(processor_props, simple_ugrid, dim) assert xp.setdiff1d(my_owned, owned[dim][processor_props.rank], assume_unique=True).size == 0 +# TODO V2E2V (from grid file vertices_of_vertex) do we use that at all? +@pytest.mark.parametrize( + "field_offset", [dims.C2V, dims.E2V, dims.V2C, dims.E2C, dims.V2E, dims.C2E2C] +) +def test_local_connectivities(processor_props, caplog, field_offset): # noqa: F811 # fixture + caplog.set_level(logging.INFO) + grid = as_ugrid2d(UGRID_FILE) + icon_grid = as_icon_grid(GRID_FILE) + distributed_grids = grid.partition(n_part=4) + labels = grid.label_partitions(n_part=4) + halo_generator = HaloGenerator( + ugrid=grid, + rank_info=processor_props, + rank_mapping=labels, + num_lev=1, + face_face_connectivity=icon_grid.connectivities[dims.C2E2CDim], + node_face_connectivity=icon_grid.connectivities[dims.V2CDim], + node_edge_connectivity=icon_grid.connectivities[dims.V2EDim], + ) + + decomposition_info = halo_generator.construct_decomposition_info() + + connectivity = halo_generator.construct_local_connectivity(field_offset, decomposition_info) + # TODO (@halungge): think of more valuable assertions + assert ( + connectivity.shape[0] + == decomposition_info.global_index( + field_offset.target[0], defs.DecompositionInfo.EntryType.ALL + ).size + ) + assert xp.max(connectivity) == xp.max( + decomposition_info.local_index(field_offset.source, defs.DecompositionInfo.EntryType.ALL) + ) + + @pytest.fixture def simple_ugrid() -> xu.Ugrid2d: """ @@ -221,6 +271,68 @@ def simple_ugrid() -> xu.Ugrid2d: return grid +UGRID_FILE = dt_utils.GRIDS_PATH.joinpath(dt_utils.R02B04_GLOBAL).joinpath( + "icon_grid_0013_R02B04_R_ugrid.nc" +) +GRID_FILE = dt_utils.GRIDS_PATH.joinpath(dt_utils.R02B04_GLOBAL).joinpath( + "icon_grid_0013_R02B04_R.nc" +) + + +def as_icon_grid(file: pathlib.Path) -> icon.IconGrid: + manager = gm.GridManager( + gm.ToGt4PyTransformation(), file, v_grid.VerticalGridConfig(num_levels=1) + ) + manager() + return manager.grid + + +def as_ugrid2d(file: pathlib.Path) -> xu.Ugrid2d: + xu_dataset = xu.open_dataset(file.as_posix()) + return xu_dataset.grid + + def global_indices(dim: dims.Dimension) -> int: mesh = simple.SimpleGrid() return xp.arange(mesh.size[dim], dtype=xp.int32) + +def icon_distribution(props:defs.ProcessProperties, decomposition_info:defs.DecompositionInfo) -> xp.ndarray: + cell_index = decomposition_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.OWNED) + comm = props.comm + local_sizes = xp.array(comm.gather(cell_index.size, root=0)) + if comm.rank == 0: + recv_buffer = xp.empty(sum(local_sizes), dtype=int) + else: + recv_buffer = None + comm.Gatherv(sendbuf=cell_index, recvbuf=(recv_buffer, local_sizes), root=0) + distribution = xp.empty((sum(local_sizes)), dtype=int) + if comm.rank == 0: + start_index = 0 + for s in comm.size: + end_index = local_sizes[s] + distribution[recv_buffer[start_index:end_index]] = s + start_index = end_index + + comm.Bcast(distribution, root=0) + return distribution + +def test_local_grid(processor_props, caplog): # noqa: F811 # fixture + caplog.set_level(logging.INFO) + grid = as_ugrid2d(UGRID_FILE) + icon_grid = as_icon_grid(GRID_FILE) + distributed_grids = grid.partition(n_part=4) + # TODO (@halungge): replace with the icon 4 nodes distribution from serialbox data. + labels = grid.label_partitions(n_part=4) + halo_generator = HaloGenerator( + ugrid=grid, + rank_info=processor_props, + rank_mapping=labels, + num_lev=1, + face_face_connectivity=icon_grid.connectivities[dims.C2E2CDim], + node_face_connectivity=icon_grid.connectivities[dims.V2CDim], + node_edge_connectivity=icon_grid.connectivities[dims.V2EDim], + ) + decomposition_info = halo_generator.construct_decomposition_info() + local_grid = halo_generator.local_grid(decomposition_info) + + assert local_grid.num_cells == decomposition_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.All).size \ No newline at end of file From bd16dc97ec1adcf3925fcca2285bffa874a58f2f Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 11 Jul 2024 08:58:16 +0200 Subject: [PATCH 007/240] WIP: test decomposition (1) - read data fields from grid file (only selected indices) --- .../model/common/decomposition/halo.py | 5 ++- .../icon4py/model/common/grid/grid_manager.py | 41 ++++++++++++++++++- .../tests/decomposition_tests/test_halo.py | 19 ++++++--- 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 4ababf3373..257b983f04 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -296,7 +296,10 @@ def local_grid(props: defs.ProcessProperties, limited_area:bool = False, on_gpu:bool=False, )->base_grid.BaseGrid: """ Constructs a local grid for this rank based on the decomposition info. - TODO (@halungge): for now only returning base grid as we have not start/end indices implementation yet + TODO (@halungge): for now only returning BaseGrid as we have not start/end indices implementation yet + TODO (@halungge): make sure the INVALID_INDEX is set correctly: - when set in the original (global index) connectivity it should remain + TODO (@halungge): how to handle the (source) indices of last halo line: their (global) neighbors are not all present on the local grid, set INVALID_INDEX (that is what xugrid does) + check what ICON does, (they probably duplicate the valid indices...) Args: decomp_info: the decomposition info for this rank Returns: diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 9784f08e5f..4255320107 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -18,6 +18,8 @@ import gt4py.next as gtx import numpy as np +from icon4py.model.common.decomposition import definitions as defs + try: from netCDF4 import Dataset @@ -180,6 +182,18 @@ class GridRefinementName(GridFileName): #: end indices of horizontal grid zones for vertex fields END_INDEX_VERTICES = "end_idx_v" + + class GeometryName(GridFileName): + CELL_AREA = "cell_area" + EDGE_LENGTH = "edge_length" + + class CoordinateName(GridFileName): + CELL_LONGITUDE = "clon" + CELL_LATITUDE = "clat" + EDGE_LONGITUDE = "elon" + EDGE_LATITUDE = "elat" + VERTEX_LONGITUDE = "vlon" + VERTEX_LATITUDE = "vlat" def __init__(self, dataset: Dataset): self._dataset = dataset self._log = logging.getLogger(__name__) @@ -192,6 +206,7 @@ def int_field(self, name: GridFileName, transpose=True, dtype=gtx.int32) -> np.n nc_variable = self._dataset.variables[name] self._log.debug(f"reading {name}: {nc_variable}") + data = nc_variable[:] data = np.array(data, dtype=dtype) return np.transpose(data) if transpose else data @@ -200,6 +215,19 @@ def int_field(self, name: GridFileName, transpose=True, dtype=gtx.int32) -> np.n self._log.warning(msg) raise IconGridError(msg) from err + def float_field(self, name: GridFileName, indices:np.ndarray = [], dtype=gtx.float64) -> np.ndarray: + + try: + # use python slice? + nc_variable = self._dataset.variables[name] + self._log.debug(f"reading {name}: {nc_variable}") + data = nc_variable[:] if not indices else nc_variable[indices] + data = np.array(data, dtype=dtype) + return data + except KeyError as err: + msg = f"{name} does not exist in dataset" + self._log.warning(msg) + raise IconGridError(msg) from err class IconGridError(RuntimeError): pass @@ -243,12 +271,21 @@ def __init__( self._config = config self._grid: Optional[icon_grid.IconGrid] = None self._file_name = grid_file + self._dataset = None def __call__(self, on_gpu: bool = False, limited_area=True): - dataset = self._read_gridfile(self._file_name) - grid = self._construct_grid(dataset, on_gpu=on_gpu, limited_area=limited_area) + self._dataset = self._read_gridfile(self._file_name) + grid = self._construct_grid(self._dataset, on_gpu=on_gpu, limited_area=limited_area) self._grid = grid + + def read_geometry(self, decomposition_info:defs.DecompositionInfo): + reader = GridFile(self._dataset) + cell_area = reader.int_field(GridFile.GeometryName.CELL_AREA) + edge_length = reader.int_field(GridFile.GeometryName.EDGE_LENGTH) + return cell_area, edge_length + + def _read_gridfile(self, fname: str) -> Dataset: try: dataset = Dataset(self._file_name, "r", format="NETCDF4") diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index 9a888a799d..2bcbc06d4a 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -219,7 +219,7 @@ def test_halo_constructor_decomposition_info(processor_props, simple_ugrid, dim) def test_local_connectivities(processor_props, caplog, field_offset): # noqa: F811 # fixture caplog.set_level(logging.INFO) grid = as_ugrid2d(UGRID_FILE) - icon_grid = as_icon_grid(GRID_FILE) + icon_grid = grid_file_manager(GRID_FILE).grid distributed_grids = grid.partition(n_part=4) labels = grid.label_partitions(n_part=4) halo_generator = HaloGenerator( @@ -279,13 +279,13 @@ def simple_ugrid() -> xu.Ugrid2d: ) -def as_icon_grid(file: pathlib.Path) -> icon.IconGrid: +def grid_file_manager(file: pathlib.Path) -> icon.IconGrid: manager = gm.GridManager( gm.ToGt4PyTransformation(), file, v_grid.VerticalGridConfig(num_levels=1) ) manager() - return manager.grid - + return manager + def as_ugrid2d(file: pathlib.Path) -> xu.Ugrid2d: xu_dataset = xu.open_dataset(file.as_posix()) @@ -316,10 +316,11 @@ def icon_distribution(props:defs.ProcessProperties, decomposition_info:defs.Deco comm.Bcast(distribution, root=0) return distribution +@pytest.mark.xfail(reason="This test is not yet implemented") def test_local_grid(processor_props, caplog): # noqa: F811 # fixture caplog.set_level(logging.INFO) grid = as_ugrid2d(UGRID_FILE) - icon_grid = as_icon_grid(GRID_FILE) + icon_grid = grid_file_manager(GRID_FILE).grid distributed_grids = grid.partition(n_part=4) # TODO (@halungge): replace with the icon 4 nodes distribution from serialbox data. labels = grid.label_partitions(n_part=4) @@ -335,4 +336,10 @@ def test_local_grid(processor_props, caplog): # noqa: F811 # fixture decomposition_info = halo_generator.construct_decomposition_info() local_grid = halo_generator.local_grid(decomposition_info) - assert local_grid.num_cells == decomposition_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.All).size \ No newline at end of file + assert local_grid.num_cells == decomposition_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.All).size + + + +def test_distributed_fields(): + pass + \ No newline at end of file From 810e184f499a1af95173cfd6ccdf07f3893ff7e7 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 24 Jul 2024 10:39:35 +0200 Subject: [PATCH 008/240] add distribute and gather test case (1) --- .../icon4py/model/common/grid/grid_manager.py | 63 ++++++++++++------- .../tests/decomposition_tests/test_halo.py | 58 ++++++++++++----- .../tests/grid_tests/test_grid_manager.py | 26 ++++---- 3 files changed, 98 insertions(+), 49 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index b4c934fab4..57bcbef30e 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -32,6 +32,7 @@ def __init__(self, *args, **kwargs): raise ModuleNotFoundError("NetCDF4 is not installed.") +import icon4py.model.common.dimension as dims from icon4py.model.common.dimension import ( C2E2C2EDim, C2E2CDim, @@ -89,7 +90,7 @@ class PropertyName(GridFileName): LEVEL = "grid_level" ROOT = "grid_root" - class OffsetName(GridFileName): + class ConnectivityName(GridFileName): """Names for connectivities used in the grid file.""" # e2c2e/e2c2eO: diamond edges (including origin) not present in grid file-> construct @@ -216,13 +217,19 @@ def int_field(self, name: GridFileName, transpose=True, dtype=gtx.int32) -> np.n self._log.warning(msg) raise IconGridError(msg) from err - def float_field(self, name: GridFileName, indices:np.ndarray = [], dtype=gtx.float64) -> np.ndarray: - + def float_field(self, name: GridFileName, indices:np.ndarray = None, dtype=gtx.float64) -> np.ndarray: + """ Read a float field from the grid file. + + If a index array is given it only reads the values at those positions. + TODO (@halungge): currently only supports 1D fields + """ try: - # use python slice? - nc_variable = self._dataset.variables[name] - self._log.debug(f"reading {name}: {nc_variable}") - data = nc_variable[:] if not indices else nc_variable[indices] + # use python slice? 2D fields + # (horizontal, vertical) not present in grid file + # (sparse, horizontal) + variable = self._dataset.variables[name] + self._log.debug(f"reading {name}: {variable}") + data = variable[:] if indices is None else variable[indices] data = np.array(data, dtype=dtype) return data except KeyError as err: @@ -264,7 +271,7 @@ class GridManager: def __init__( self, transformation: IndexTransformation, - grid_file: str, + grid_file: str, # TODO use pathlib.Path? config: v_grid.VerticalGridConfig, ): self._log = logging.getLogger(__name__) @@ -279,12 +286,7 @@ def __call__(self, on_gpu: bool = False, limited_area=True): grid = self._construct_grid(self._dataset, on_gpu=on_gpu, limited_area=limited_area) self._grid = grid - - def read_geometry(self, decomposition_info:defs.DecompositionInfo): - reader = GridFile(self._dataset) - cell_area = reader.int_field(GridFile.GeometryName.CELL_AREA) - edge_length = reader.int_field(GridFile.GeometryName.EDGE_LENGTH) - return cell_area, edge_length + def _read_gridfile(self, fname: str) -> Dataset: @@ -346,6 +348,21 @@ def _read_grid_refinement_information(self, dataset): return start_indices, end_indices, refin_ctrl, refin_ctrl_max + def _read_geometry(self, dataset, decomposition_info:defs.DecompositionInfo): + reader = GridFile(dataset) + cells_on_node = decomposition_info.global_index(dims.CellDim, + defs.DecompositionInfo.EntryType.ALL) + edges_on_node = decomposition_info.global_index(dims.EdgeDim, + defs.DecompositionInfo.EntryType.ALL) + vertices_on_node = decomposition_info.global_index(dims.VertexDim, + defs.DecompositionInfo.EntryType.ALL) + cell_area = reader.float_field(GridFile.GeometryName.CELL_AREA, cells_on_node) + edge_length = reader.float_field(GridFile.GeometryName.EDGE_LENGTH, edges_on_node) + return cell_area, edge_length + + def read_geometry(self, decomposition_info:defs.DecompositionInfo): + return self._read_geometry(self._dataset, decomposition_info) + @property def grid(self): return self._grid @@ -406,15 +423,15 @@ def _from_grid_dataset( grid_size = grid_def.HorizontalGridSize( num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells ) - c2e = self._get_index_field(reader, GridFile.OffsetName.C2E) - - e2c = self._get_index_field(reader, GridFile.OffsetName.E2C) # edge_face_connectivity (optional) - c2v = self._get_index_field(reader, GridFile.OffsetName.C2V) # face_node_connectivity (required) - v2c = self._get_index_field(reader, GridFile.OffsetName.V2C) # node_face_connectivity -- (pentagon/hexagon) - e2v = self._get_index_field(reader, GridFile.OffsetName.E2V) # edge_node_connectivity (optionally required) - v2e = self._get_index_field(reader, GridFile.OffsetName.V2E) # node_edge_connectivity -- (pentagon/hexagon) - v2e2v = self._get_index_field(reader, GridFile.OffsetName.V2E2V) #node_node_connectivity -- ((pentagon/hexagon)) - c2e2c = self._get_index_field(reader, GridFile.OffsetName.C2E2C) # face_face_connectivity (optional) + c2e = self._get_index_field(reader, GridFile.ConnectivityName.C2E) + + e2c = self._get_index_field(reader, GridFile.ConnectivityName.E2C) # edge_face_connectivity (optional) + c2v = self._get_index_field(reader, GridFile.ConnectivityName.C2V) # face_node_connectivity (required) + v2c = self._get_index_field(reader, GridFile.ConnectivityName.V2C) # node_face_connectivity -- (pentagon/hexagon) + e2v = self._get_index_field(reader, GridFile.ConnectivityName.E2V) # edge_node_connectivity (optionally required) + v2e = self._get_index_field(reader, GridFile.ConnectivityName.V2E) # node_edge_connectivity -- (pentagon/hexagon) + v2e2v = self._get_index_field(reader, GridFile.ConnectivityName.V2E2V) #node_node_connectivity -- ((pentagon/hexagon)) + c2e2c = self._get_index_field(reader, GridFile.ConnectivityName.C2E2C) # face_face_connectivity (optional) e2c2v = self._construct_diamond_vertices(e2v, c2v, e2c) e2c2e = self._construct_diamond_edges(e2c, c2e) diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index 2bcbc06d4a..38d8692c81 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -15,6 +15,7 @@ import mpi4py import mpi4py.MPI +import numpy as np import pytest import xugrid as xu @@ -23,7 +24,7 @@ import icon4py.model.common.grid.vertical as v_grid from icon4py.model.common.decomposition import definitions as defs from icon4py.model.common.decomposition.halo import HaloGenerator -from icon4py.model.common.grid import icon, simple +from icon4py.model.common.grid import simple from icon4py.model.common.settings import xp from icon4py.model.common.test_utils import datatest_utils as dt_utils from icon4py.model.common.test_utils.parallel_helpers import ( # noqa: F401 # import fixtures from test_utils package @@ -123,7 +124,7 @@ halos = {dims.CellDim: cell_halos, dims.EdgeDim: edge_halos, dims.VertexDim: vertex_halo} -def test_halo_constructor_owned_cells(processor_props, simple_ugrid): # noqa: F811 # fixture +def test_halo_constructor_owned_cells(processor_props, simple_ugrid): # fixture grid = simple_ugrid halo_generator = HaloGenerator( ugrid=grid, @@ -142,7 +143,7 @@ def test_halo_constructor_owned_cells(processor_props, simple_ugrid): # noqa: F @pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) @pytest.mark.mpi(min_size=4) -def test_element_ownership_is_unique(dim, processor_props, simple_ugrid): # noqa: F811 # fixture +def test_element_ownership_is_unique(dim, processor_props, simple_ugrid): # fixture grid = simple_ugrid halo_generator = HaloGenerator( ugrid=grid, @@ -190,7 +191,7 @@ def test_element_ownership_is_unique(dim, processor_props, simple_ugrid): # noq # TODO (@halungge) this test can be run on 4 MPI ranks or should we rather switch to a single node, # and parametrizes the rank number @pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) -def test_halo_constructor_decomposition_info(processor_props, simple_ugrid, dim): # noqa: F811 # fixture +def test_halo_constructor_decomposition_info(processor_props, simple_ugrid, dim): # fixture grid = simple_ugrid halo_generator = HaloGenerator( ugrid=grid, @@ -216,7 +217,7 @@ def test_halo_constructor_decomposition_info(processor_props, simple_ugrid, dim) @pytest.mark.parametrize( "field_offset", [dims.C2V, dims.E2V, dims.V2C, dims.E2C, dims.V2E, dims.C2E2C] ) -def test_local_connectivities(processor_props, caplog, field_offset): # noqa: F811 # fixture +def test_local_connectivities(processor_props, caplog, field_offset): # fixture caplog.set_level(logging.INFO) grid = as_ugrid2d(UGRID_FILE) icon_grid = grid_file_manager(GRID_FILE).grid @@ -279,18 +280,19 @@ def simple_ugrid() -> xu.Ugrid2d: ) -def grid_file_manager(file: pathlib.Path) -> icon.IconGrid: +def grid_file_manager(file: pathlib.Path) -> gm.GridManager: manager = gm.GridManager( - gm.ToGt4PyTransformation(), file, v_grid.VerticalGridConfig(num_levels=1) + gm.ToGt4PyTransformation(), str(file), v_grid.VerticalGridConfig(num_levels=1) ) manager() return manager def as_ugrid2d(file: pathlib.Path) -> xu.Ugrid2d: - xu_dataset = xu.open_dataset(file.as_posix()) - return xu_dataset.grid + return as_xudataset(file).grid +def as_xudataset(file: pathlib.Path) -> xu.UgridDataset: + return xu.open_dataset(file.as_posix()) def global_indices(dim: dims.Dimension) -> int: mesh = simple.SimpleGrid() @@ -317,7 +319,7 @@ def icon_distribution(props:defs.ProcessProperties, decomposition_info:defs.Deco return distribution @pytest.mark.xfail(reason="This test is not yet implemented") -def test_local_grid(processor_props, caplog): # noqa: F811 # fixture +def test_local_grid(processor_props, caplog): # fixture caplog.set_level(logging.INFO) grid = as_ugrid2d(UGRID_FILE) icon_grid = grid_file_manager(GRID_FILE).grid @@ -339,7 +341,35 @@ def test_local_grid(processor_props, caplog): # noqa: F811 # fixture assert local_grid.num_cells == decomposition_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.All).size - -def test_distributed_fields(): - pass - \ No newline at end of file +#@pytest.mark.with_mpi(min_size=4) +def test_distributed_fields(processor_props): # fixture + #if processor_props.comm_size != 4: + # pytest.skip("This test requires 4 MPI ranks.") + grid_manager = grid_file_manager(GRID_FILE) + processor_props.rank = 2 + ugrid = as_ugrid2d(UGRID_FILE) + labels = ugrid.label_partitions(n_part=4) + local_patches = ugrid.partition(n_part=4) + halo_generator = HaloGenerator( + ugrid=ugrid, + rank_info=processor_props, + rank_mapping=labels, + num_lev=1, + face_face_connectivity=grid_manager.grid.connectivities[dims.C2E2CDim], + node_face_connectivity=grid_manager.grid.connectivities[dims.V2CDim], + ) + decomposition_info = halo_generator.construct_decomposition_info() + cell_area, edge_length = grid_manager.read_geometry(decomposition_info) + print(f"rank = {processor_props.rank} has size(cell_area): {cell_area.shape}, has size(edge_length): {edge_length.shape}") + assert cell_area.size == decomposition_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.ALL).size + assert cell_area.size <= grid_manager.grid.global_properties.num_cells + assert edge_length.size == decomposition_info.global_index(dims.EdgeDim, + defs.DecompositionInfo.EntryType.ALL).size + owned_cell_area = cell_area[decomposition_info.local_index(dims.CellDim, defs.DecompositionInfo.EntryType.OWNED)] + assert np.allclose(local_patches[processor_props.rank]["cell_area"] , cell_area) + merged = xu.merge_partitions(local_patches) + + + + + diff --git a/model/common/tests/grid_tests/test_grid_manager.py b/model/common/tests/grid_tests/test_grid_manager.py index 3c62ca5fa9..aa07619168 100644 --- a/model/common/tests/grid_tests/test_grid_manager.py +++ b/model/common/tests/grid_tests/test_grid_manager.py @@ -158,7 +158,7 @@ def simple_grid_gridfile(tmp_path): _add_to_dataset( dataset, grid.connectivities[C2EDim], - GridFile.OffsetName.C2E, + GridFile.ConnectivityName.C2E, ( GridFile.DimensionName.NEIGHBORS_TO_CELL_SIZE, GridFile.DimensionName.CELL_NAME, @@ -168,7 +168,7 @@ def simple_grid_gridfile(tmp_path): _add_to_dataset( dataset, grid.connectivities[E2CDim], - GridFile.OffsetName.E2C, + GridFile.ConnectivityName.E2C, ( GridFile.DimensionName.NEIGHBORS_TO_EDGE_SIZE, GridFile.DimensionName.EDGE_NAME, @@ -177,7 +177,7 @@ def simple_grid_gridfile(tmp_path): _add_to_dataset( dataset, grid.connectivities[E2VDim], - GridFile.OffsetName.E2V, + GridFile.ConnectivityName.E2V, ( GridFile.DimensionName.NEIGHBORS_TO_EDGE_SIZE, GridFile.DimensionName.EDGE_NAME, @@ -187,7 +187,7 @@ def simple_grid_gridfile(tmp_path): _add_to_dataset( dataset, grid.connectivities[V2CDim], - GridFile.OffsetName.V2C, + GridFile.ConnectivityName.V2C, ( GridFile.DimensionName.NEIGHBORS_TO_VERTEX_SIZE, GridFile.DimensionName.VERTEX_NAME, @@ -197,7 +197,7 @@ def simple_grid_gridfile(tmp_path): _add_to_dataset( dataset, grid.connectivities[C2VDim], - GridFile.OffsetName.C2V, + GridFile.ConnectivityName.C2V, ( GridFile.DimensionName.NEIGHBORS_TO_CELL_SIZE, GridFile.DimensionName.CELL_NAME, @@ -206,13 +206,13 @@ def simple_grid_gridfile(tmp_path): _add_to_dataset( dataset, np.zeros((grid.num_vertices, 4), dtype=np.int32), - GridFile.OffsetName.V2E2V, + GridFile.ConnectivityName.V2E2V, (GridFile.DimensionName.DIAMOND_EDGE_SIZE, GridFile.DimensionName.VERTEX_NAME), ) _add_to_dataset( dataset, grid.connectivities[V2EDim], - GridFile.OffsetName.V2E, + GridFile.ConnectivityName.V2E, ( GridFile.DimensionName.NEIGHBORS_TO_VERTEX_SIZE, GridFile.DimensionName.VERTEX_NAME, @@ -221,7 +221,7 @@ def simple_grid_gridfile(tmp_path): _add_to_dataset( dataset, grid.connectivities[C2E2CDim], - GridFile.OffsetName.C2E2C, + GridFile.ConnectivityName.C2E2C, ( GridFile.DimensionName.NEIGHBORS_TO_CELL_SIZE, GridFile.DimensionName.CELL_NAME, @@ -312,10 +312,10 @@ def test_grid_parser_index_fields(simple_grid_gridfile, caplog): grid = SimpleGrid() grid_parser = GridFile(data) - assert np.allclose(grid_parser.int_field(GridFile.OffsetName.C2E), grid.connectivities[C2EDim]) - assert np.allclose(grid_parser.int_field(GridFile.OffsetName.E2C), grid.connectivities[E2CDim]) - assert np.allclose(grid_parser.int_field(GridFile.OffsetName.V2E), grid.connectivities[V2EDim]) - assert np.allclose(grid_parser.int_field(GridFile.OffsetName.V2C), grid.connectivities[V2CDim]) + assert np.allclose(grid_parser.int_field(GridFile.ConnectivityName.C2E), grid.connectivities[C2EDim]) + assert np.allclose(grid_parser.int_field(GridFile.ConnectivityName.E2C), grid.connectivities[E2CDim]) + assert np.allclose(grid_parser.int_field(GridFile.ConnectivityName.V2E), grid.connectivities[V2EDim]) + assert np.allclose(grid_parser.int_field(GridFile.ConnectivityName.V2C), grid.connectivities[V2CDim]) # TODO @magdalena add test cases for hexagon vertices v2e2v @@ -984,3 +984,5 @@ def test_gridmanager_eval_c2e2c2e(caplog, grid_savepoint, grid_file): serialized_grid.get_offset_provider("C2E2C2E").table, ) assert grid.get_offset_provider("C2E2C2E").table.shape == (grid.num_cells, 9) + + From 67192f4bcc00150a7cb64999363979d95ec91e53 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 24 Jul 2024 18:37:59 +0200 Subject: [PATCH 009/240] add distribute and gather test case (2) --- .../model/common/decomposition/halo.py | 43 +++--- .../icon4py/model/common/grid/grid_manager.py | 136 +++++++++++++----- .../tests/decomposition_tests/test_halo.py | 130 ++++++++++++----- .../tests/grid_tests/test_grid_manager.py | 18 ++- 4 files changed, 228 insertions(+), 99 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 257b983f04..ead0619294 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -27,6 +27,7 @@ log = logging.getLogger(__name__) + # TODO do we need three of those? class DecompositionFlag(enum.IntEnum): #: cell is owned by this rank @@ -257,13 +258,14 @@ def _update_owner_mask_by_max_rank_convention( decomp_info.with_dimension(dims.VertexDim, all_vertices, v_owner_mask) return decomp_info - def construct_local_connectivity(self, field_offset: gtx.FieldOffset, - decom_info: defs.DecompositionInfo) -> xp.ndarray: + def construct_local_connectivity( + self, field_offset: gtx.FieldOffset, decom_info: defs.DecompositionInfo + ) -> xp.ndarray: """ Construct a connectivity table for use on a given rank: it maps from source to target dimension in local indices. - + Args: - field_offset: FieldOffset for which we want to construct the offset table + field_offset: FieldOffset for which we want to construct the offset table decom_info: DecompositionInfo for the current rank Returns: array, containt the connectivity table for the field_offset with rank-local indices @@ -288,12 +290,16 @@ def construct_local_connectivity(self, field_offset: gtx.FieldOffset, log.debug(f"rank {self._props.rank} has local connectivity f: {sliced_connectivity}") return sliced_connectivity + # should be done in grid manager! -def local_grid(props: defs.ProcessProperties, - decomp_info:defs.DecompositionInfo, - global_params:icon_grid.GlobalGridParams, - num_lev:int, - limited_area:bool = False, on_gpu:bool=False, )->base_grid.BaseGrid: +def local_grid( + props: defs.ProcessProperties, + decomp_info: defs.DecompositionInfo, + global_params: icon_grid.GlobalGridParams, + num_lev: int, + limited_area: bool = False, + on_gpu: bool = False, +) -> base_grid.BaseGrid: """ Constructs a local grid for this rank based on the decomposition info. TODO (@halungge): for now only returning BaseGrid as we have not start/end indices implementation yet @@ -305,18 +311,21 @@ def local_grid(props: defs.ProcessProperties, Returns: local_grid: the local grid """ - num_vertices = decomp_info.global_index(dims.VertexDim, defs.DecompositionInfo.EntryType.ALL).size + num_vertices = decomp_info.global_index( + dims.VertexDim, defs.DecompositionInfo.EntryType.ALL + ).size num_edges = decomp_info.global_index(dims.EdgeDim, defs.DecompositionInfo.EntryType.ALL).size num_cells = decomp_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.ALL).size grid_size = base_grid.HorizontalGridSize( num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells ) - config = base_grid.GridConfig(horizontal_config = grid_size, - vertical_size = num_lev, - on_gpu=on_gpu, - limited_area=limited_area) - - local_grid = icon_grid.IconGrid(uuid.uuid4()).with_config(config).with_global_params(global_params) + config = base_grid.GridConfig( + horizontal_config=grid_size, vertical_size=num_lev, on_gpu=on_gpu, limited_area=limited_area + ) + + local_grid = ( + icon_grid.IconGrid(uuid.uuid4()).with_config(config).with_global_params(global_params) + ) # add connectivities - return local_grid \ No newline at end of file + return local_grid diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 57bcbef30e..ec03eca278 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -13,7 +13,7 @@ import dataclasses import enum import logging -from typing import Optional +from typing import Optional, Sequence import gt4py.next as gtx import numpy as np @@ -184,11 +184,10 @@ class GridRefinementName(GridFileName): #: end indices of horizontal grid zones for vertex fields END_INDEX_VERTICES = "end_idx_v" - class GeometryName(GridFileName): CELL_AREA = "cell_area" EDGE_LENGTH = "edge_length" - + class CoordinateName(GridFileName): CELL_LONGITUDE = "clon" CELL_LATITUDE = "clat" @@ -196,6 +195,7 @@ class CoordinateName(GridFileName): EDGE_LATITUDE = "elat" VERTEX_LONGITUDE = "vlon" VERTEX_LATITUDE = "vlat" + def __init__(self, dataset: Dataset): self._dataset = dataset self._log = logging.getLogger(__name__) @@ -217,16 +217,18 @@ def int_field(self, name: GridFileName, transpose=True, dtype=gtx.int32) -> np.n self._log.warning(msg) raise IconGridError(msg) from err - def float_field(self, name: GridFileName, indices:np.ndarray = None, dtype=gtx.float64) -> np.ndarray: - """ Read a float field from the grid file. - + def float_field( + self, name: GridFileName, indices: np.ndarray = None, dtype=gtx.float64 + ) -> np.ndarray: + """Read a float field from the grid file. + If a index array is given it only reads the values at those positions. TODO (@halungge): currently only supports 1D fields """ try: - # use python slice? 2D fields + # use python slice? 2D fields # (horizontal, vertical) not present in grid file - # (sparse, horizontal) + # (sparse, horizontal) variable = self._dataset.variables[name] self._log.debug(f"reading {name}: {variable}") data = variable[:] if indices is None else variable[indices] @@ -237,6 +239,7 @@ def float_field(self, name: GridFileName, indices:np.ndarray = None, dtype=gtx.f self._log.warning(msg) raise IconGridError(msg) from err + class IconGridError(RuntimeError): pass @@ -271,7 +274,7 @@ class GridManager: def __init__( self, transformation: IndexTransformation, - grid_file: str, # TODO use pathlib.Path? + grid_file: str, # TODO use pathlib.Path? config: v_grid.VerticalGridConfig, ): self._log = logging.getLogger(__name__) @@ -280,20 +283,20 @@ def __init__( self._grid: Optional[icon_grid.IconGrid] = None self._file_name = grid_file self._dataset = None + self._reader = None def __call__(self, on_gpu: bool = False, limited_area=True): - self._dataset = self._read_gridfile(self._file_name) + self._read_gridfile(self._file_name) + grid = self._construct_grid(self._dataset, on_gpu=on_gpu, limited_area=limited_area) self._grid = grid - - - - def _read_gridfile(self, fname: str) -> Dataset: + def _read_gridfile(self, fname: str) -> None: try: dataset = Dataset(self._file_name, "r", format="NETCDF4") self._log.debug(dataset) - return dataset + self._dataset = dataset + self._reader = GridFile(dataset) except FileNotFoundError: self._log.error(f"gridfile {fname} not found, aborting") exit(1) @@ -348,21 +351,65 @@ def _read_grid_refinement_information(self, dataset): return start_indices, end_indices, refin_ctrl, refin_ctrl_max - def _read_geometry(self, dataset, decomposition_info:defs.DecompositionInfo): - reader = GridFile(dataset) - cells_on_node = decomposition_info.global_index(dims.CellDim, - defs.DecompositionInfo.EntryType.ALL) - edges_on_node = decomposition_info.global_index(dims.EdgeDim, - defs.DecompositionInfo.EntryType.ALL) - vertices_on_node = decomposition_info.global_index(dims.VertexDim, - defs.DecompositionInfo.EntryType.ALL) - cell_area = reader.float_field(GridFile.GeometryName.CELL_AREA, cells_on_node) - edge_length = reader.float_field(GridFile.GeometryName.EDGE_LENGTH, edges_on_node) - return cell_area, edge_length - - def read_geometry(self, decomposition_info:defs.DecompositionInfo): - return self._read_geometry(self._dataset, decomposition_info) - + def _read( + self, + decomposition_info: defs.DecompositionInfo, + fields: dict[dims.Dimension, Sequence[GridFileName]], + ): + (cells_on_node, edges_on_node, vertices_on_node) = ( + ( + decomposition_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.ALL), + decomposition_info.global_index(dims.EdgeDim, defs.DecompositionInfo.EntryType.ALL), + decomposition_info.global_index( + dims.VertexDim, defs.DecompositionInfo.EntryType.ALL + ), + ) + if decomposition_info is not None + else (None, None, None) + ) + + def _read_local(fields: dict[dims.Dimension, Sequence[GridFileName]]): + cell_fields = fields.get(dims.CellDim, []) + edge_fields = fields.get(dims.EdgeDim, []) + vertex_fields = fields.get(dims.VertexDim, []) + vals = ( + {name: self._reader.float_field(name, cells_on_node) for name in cell_fields} + | {name: self._reader.float_field(name, edges_on_node) for name in edge_fields} + | {name: self._reader.float_field(name, vertices_on_node) for name in vertex_fields} + ) + + return vals + + return _read_local(fields) + + def read_geometry(self, decomposition_info: Optional[defs.DecompositionInfo] = None): + return self._read( + decomposition_info, + { + CellDim: [GridFile.GeometryName.CELL_AREA], + EdgeDim: [GridFile.GeometryName.EDGE_LENGTH], + }, + ) + + def read_coordinates(self, decomposition_info: Optional[defs.DecompositionInfo] = None): + return self._read( + decomposition_info, + { + CellDim: [ + GridFile.CoordinateName.CELL_LONGITUDE, + GridFile.CoordinateName.CELL_LATITUDE, + ], + EdgeDim: [ + GridFile.CoordinateName.EDGE_LONGITUDE, + GridFile.CoordinateName.EDGE_LATITUDE, + ], + VertexDim: [ + GridFile.CoordinateName.VERTEX_LONGITUDE, + GridFile.CoordinateName.VERTEX_LATITUDE, + ], + }, + ) + @property def grid(self): return self._grid @@ -425,19 +472,32 @@ def _from_grid_dataset( ) c2e = self._get_index_field(reader, GridFile.ConnectivityName.C2E) - e2c = self._get_index_field(reader, GridFile.ConnectivityName.E2C) # edge_face_connectivity (optional) - c2v = self._get_index_field(reader, GridFile.ConnectivityName.C2V) # face_node_connectivity (required) - v2c = self._get_index_field(reader, GridFile.ConnectivityName.V2C) # node_face_connectivity -- (pentagon/hexagon) - e2v = self._get_index_field(reader, GridFile.ConnectivityName.E2V) # edge_node_connectivity (optionally required) - v2e = self._get_index_field(reader, GridFile.ConnectivityName.V2E) # node_edge_connectivity -- (pentagon/hexagon) - v2e2v = self._get_index_field(reader, GridFile.ConnectivityName.V2E2V) #node_node_connectivity -- ((pentagon/hexagon)) - c2e2c = self._get_index_field(reader, GridFile.ConnectivityName.C2E2C) # face_face_connectivity (optional) + e2c = self._get_index_field( + reader, GridFile.ConnectivityName.E2C + ) # edge_face_connectivity (optional) + c2v = self._get_index_field( + reader, GridFile.ConnectivityName.C2V + ) # face_node_connectivity (required) + v2c = self._get_index_field( + reader, GridFile.ConnectivityName.V2C + ) # node_face_connectivity -- (pentagon/hexagon) + e2v = self._get_index_field( + reader, GridFile.ConnectivityName.E2V + ) # edge_node_connectivity (optionally required) + v2e = self._get_index_field( + reader, GridFile.ConnectivityName.V2E + ) # node_edge_connectivity -- (pentagon/hexagon) + v2e2v = self._get_index_field( + reader, GridFile.ConnectivityName.V2E2V + ) # node_node_connectivity -- ((pentagon/hexagon)) + c2e2c = self._get_index_field( + reader, GridFile.ConnectivityName.C2E2C + ) # face_face_connectivity (optional) e2c2v = self._construct_diamond_vertices(e2v, c2v, e2c) e2c2e = self._construct_diamond_edges(e2c, c2e) e2c2e0 = np.column_stack((np.asarray(range(e2c2e.shape[0])), e2c2e)) - c2e2c2e = self._construct_triangle_edges(c2e2c, c2e) c2e2c0 = np.column_stack((np.asarray(range(c2e2c.shape[0])), c2e2c)) ( diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index 38d8692c81..5b957195ba 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -15,7 +15,6 @@ import mpi4py import mpi4py.MPI -import numpy as np import pytest import xugrid as xu @@ -26,7 +25,7 @@ from icon4py.model.common.decomposition.halo import HaloGenerator from icon4py.model.common.grid import simple from icon4py.model.common.settings import xp -from icon4py.model.common.test_utils import datatest_utils as dt_utils +from icon4py.model.common.test_utils import datatest_utils as dt_utils, helpers from icon4py.model.common.test_utils.parallel_helpers import ( # noqa: F401 # import fixtures from test_utils package check_comm_size, processor_props, @@ -286,27 +285,29 @@ def grid_file_manager(file: pathlib.Path) -> gm.GridManager: ) manager() return manager - + def as_ugrid2d(file: pathlib.Path) -> xu.Ugrid2d: return as_xudataset(file).grid + def as_xudataset(file: pathlib.Path) -> xu.UgridDataset: return xu.open_dataset(file.as_posix()) + def global_indices(dim: dims.Dimension) -> int: mesh = simple.SimpleGrid() return xp.arange(mesh.size[dim], dtype=xp.int32) -def icon_distribution(props:defs.ProcessProperties, decomposition_info:defs.DecompositionInfo) -> xp.ndarray: - cell_index = decomposition_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.OWNED) + +def icon_distribution( + props: defs.ProcessProperties, decomposition_info: defs.DecompositionInfo +) -> xp.ndarray: + cell_index = decomposition_info.global_index( + dims.CellDim, defs.DecompositionInfo.EntryType.OWNED + ) comm = props.comm - local_sizes = xp.array(comm.gather(cell_index.size, root=0)) - if comm.rank == 0: - recv_buffer = xp.empty(sum(local_sizes), dtype=int) - else: - recv_buffer = None - comm.Gatherv(sendbuf=cell_index, recvbuf=(recv_buffer, local_sizes), root=0) + local_sizes, recv_buffer = gather_field(cell_index, comm) distribution = xp.empty((sum(local_sizes)), dtype=int) if comm.rank == 0: start_index = 0 @@ -314,17 +315,26 @@ def icon_distribution(props:defs.ProcessProperties, decomposition_info:defs.Deco end_index = local_sizes[s] distribution[recv_buffer[start_index:end_index]] = s start_index = end_index - + comm.Bcast(distribution, root=0) return distribution + +def gather_field(field: xp.ndarray, comm: mpi4py.MPI.Comm, dtype=int) -> tuple: + local_sizes = xp.array(comm.gather(field.size, root=0)) + if comm.rank == 0: + recv_buffer = xp.empty(sum(local_sizes), dtype=dtype) + else: + recv_buffer = None + comm.Gatherv(sendbuf=field, recvbuf=(recv_buffer, local_sizes), root=0) + return local_sizes, recv_buffer + + @pytest.mark.xfail(reason="This test is not yet implemented") def test_local_grid(processor_props, caplog): # fixture caplog.set_level(logging.INFO) grid = as_ugrid2d(UGRID_FILE) icon_grid = grid_file_manager(GRID_FILE).grid - distributed_grids = grid.partition(n_part=4) - # TODO (@halungge): replace with the icon 4 nodes distribution from serialbox data. labels = grid.label_partitions(n_part=4) halo_generator = HaloGenerator( ugrid=grid, @@ -337,19 +347,20 @@ def test_local_grid(processor_props, caplog): # fixture ) decomposition_info = halo_generator.construct_decomposition_info() local_grid = halo_generator.local_grid(decomposition_info) - - assert local_grid.num_cells == decomposition_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.All).size - - -#@pytest.mark.with_mpi(min_size=4) -def test_distributed_fields(processor_props): # fixture - #if processor_props.comm_size != 4: - # pytest.skip("This test requires 4 MPI ranks.") + + assert ( + local_grid.num_cells + == decomposition_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.All).size + ) + + +@pytest.mark.with_mpi +def test_distributed_fields(processor_props): # fixture grid_manager = grid_file_manager(GRID_FILE) - processor_props.rank = 2 + ugrid = as_ugrid2d(UGRID_FILE) - labels = ugrid.label_partitions(n_part=4) - local_patches = ugrid.partition(n_part=4) + labels = ugrid.label_partitions(n_part=processor_props.comm_size) + halo_generator = HaloGenerator( ugrid=ugrid, rank_info=processor_props, @@ -359,17 +370,60 @@ def test_distributed_fields(processor_props): # fixture node_face_connectivity=grid_manager.grid.connectivities[dims.V2CDim], ) decomposition_info = halo_generator.construct_decomposition_info() - cell_area, edge_length = grid_manager.read_geometry(decomposition_info) - print(f"rank = {processor_props.rank} has size(cell_area): {cell_area.shape}, has size(edge_length): {edge_length.shape}") - assert cell_area.size == decomposition_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.ALL).size - assert cell_area.size <= grid_manager.grid.global_properties.num_cells - assert edge_length.size == decomposition_info.global_index(dims.EdgeDim, - defs.DecompositionInfo.EntryType.ALL).size - owned_cell_area = cell_area[decomposition_info.local_index(dims.CellDim, defs.DecompositionInfo.EntryType.OWNED)] - assert np.allclose(local_patches[processor_props.rank]["cell_area"] , cell_area) - merged = xu.merge_partitions(local_patches) - - - - + # distributed read: read one field per dimension + local_geometry_fields = grid_manager.read_geometry(decomposition_info) + local_cell_area = local_geometry_fields[gm.GridFile.GeometryName.CELL_AREA] + local_edge_length = local_geometry_fields[gm.GridFile.GeometryName.EDGE_LENGTH] + local_vlon = grid_manager.read_coordinates(decomposition_info)[ + gm.GridFile.CoordinateName.VERTEX_LONGITUDE + ] + print( + f"rank = {processor_props.rank} has size(cell_area): {local_cell_area.shape}, has size(edge_length): {local_edge_length.shape}" + ) + # the local number of cells must be at most the global number of cells (analytically computed) + assert local_cell_area.size <= grid_manager.grid.global_properties.num_cells + # global read: read the same (global fields) + global_geometry_fields = grid_manager.read_geometry() + global_vlon = grid_manager.read_coordinates()[gm.GridFile.CoordinateName.VERTEX_LONGITUDE] + + global_cell_area = global_geometry_fields[gm.GridFile.GeometryName.CELL_AREA] + global_edge_length = global_geometry_fields[gm.GridFile.GeometryName.EDGE_LENGTH] + assert_gathered_field_against_global( + decomposition_info, processor_props, dims.CellDim, global_cell_area, local_cell_area + ) + + assert_gathered_field_against_global( + decomposition_info, processor_props, dims.EdgeDim, global_edge_length, local_edge_length + ) + assert_gathered_field_against_global( + decomposition_info, processor_props, dims.VertexDim, global_vlon, local_vlon + ) + +def assert_gathered_field_against_global( + decomposition_info: defs.DecompositionInfo, + processor_props: defs.ProcessProperties, + dim: dims.Dimension, + global_reference_field: xp.ndarray, + local_field: xp.ndarray, +): + assert ( + local_field.size + == decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.ALL).size + ) + owned_entries = local_field[ + decomposition_info.local_index(dim, defs.DecompositionInfo.EntryType.OWNED) + ] + gathered_sizes, gathered_field = gather_field( + owned_entries, processor_props.comm, dtype=xp.float64 + ) + global_index_sizes, gathered_global_indices = gather_field( + decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED), + processor_props.comm, + dtype=int, + ) + if processor_props.rank == 0: + assert xp.all(gathered_sizes == global_index_sizes) + sorted = xp.zeros(global_reference_field.shape, dtype=xp.float64) + sorted[gathered_global_indices] = gathered_field + assert helpers.dallclose(sorted, global_reference_field) diff --git a/model/common/tests/grid_tests/test_grid_manager.py b/model/common/tests/grid_tests/test_grid_manager.py index aa07619168..b47df17f55 100644 --- a/model/common/tests/grid_tests/test_grid_manager.py +++ b/model/common/tests/grid_tests/test_grid_manager.py @@ -312,10 +312,18 @@ def test_grid_parser_index_fields(simple_grid_gridfile, caplog): grid = SimpleGrid() grid_parser = GridFile(data) - assert np.allclose(grid_parser.int_field(GridFile.ConnectivityName.C2E), grid.connectivities[C2EDim]) - assert np.allclose(grid_parser.int_field(GridFile.ConnectivityName.E2C), grid.connectivities[E2CDim]) - assert np.allclose(grid_parser.int_field(GridFile.ConnectivityName.V2E), grid.connectivities[V2EDim]) - assert np.allclose(grid_parser.int_field(GridFile.ConnectivityName.V2C), grid.connectivities[V2CDim]) + assert np.allclose( + grid_parser.int_field(GridFile.ConnectivityName.C2E), grid.connectivities[C2EDim] + ) + assert np.allclose( + grid_parser.int_field(GridFile.ConnectivityName.E2C), grid.connectivities[E2CDim] + ) + assert np.allclose( + grid_parser.int_field(GridFile.ConnectivityName.V2E), grid.connectivities[V2EDim] + ) + assert np.allclose( + grid_parser.int_field(GridFile.ConnectivityName.V2C), grid.connectivities[V2CDim] + ) # TODO @magdalena add test cases for hexagon vertices v2e2v @@ -984,5 +992,3 @@ def test_gridmanager_eval_c2e2c2e(caplog, grid_savepoint, grid_file): serialized_grid.get_offset_provider("C2E2C2E").table, ) assert grid.get_offset_provider("C2E2C2E").table.shape == (grid.num_cells, 9) - - From 9b4b08f264a71120a79f90c9fb5bdf50c8c67a78 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 26 Jul 2024 09:52:38 +0200 Subject: [PATCH 010/240] remove xugrid dependency --- model/common/pyproject.toml | 2 +- .../model/common/decomposition/halo.py | 145 ++++++++++-------- .../tests/decomposition_tests/test_halo.py | 117 +++++--------- 3 files changed, 118 insertions(+), 146 deletions(-) diff --git a/model/common/pyproject.toml b/model/common/pyproject.toml index c0c1d60903..91425e7770 100644 --- a/model/common/pyproject.toml +++ b/model/common/pyproject.toml @@ -30,7 +30,7 @@ requires-python = ">=3.10" [project.optional-dependencies] all = ["icon4py-common[ghex,io]"] -ghex = ["ghex", "mpi4py>=3.1.4", "xugrid", "pymetis>2022.1", "scipy"] +ghex = ["ghex", "mpi4py>=3.1.4", "pymetis>2022.1",] io = [ "icon4py-common[netcdf]", "xarray[complete]>=2024.3.0", diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index ead0619294..ad09f9b946 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -14,21 +14,20 @@ import enum import logging import uuid +from typing import Protocol import gt4py.next as gtx -import scipy as sp -import xugrid.ugrid.ugrid2d as ux import icon4py.model.common.decomposition.definitions as defs from icon4py.model.common import dimension as dims -from icon4py.model.common.grid import base as base_grid, icon as icon_grid +from icon4py.model.common.grid import base as base_grid, grid_manager as gm, icon as icon_grid from icon4py.model.common.settings import xp log = logging.getLogger(__name__) -# TODO do we need three of those? +# TODO (@halungge) do we need three of those: one for each dimension? class DecompositionFlag(enum.IntEnum): #: cell is owned by this rank OWNED = (0,) @@ -37,53 +36,42 @@ class DecompositionFlag(enum.IntEnum): #: cell is in the second halo line: that is cells that share only a vertex with an owned cell (and at least an edge with a FIRST_HALO_LINE cell) SECOND_HALO_LINE = 2 - +SKIP_VALUE = gm.GridFile.INVALID_INDEX class HaloGenerator: """Creates necessary halo information for a given rank.""" def __init__( self, - rank_info: defs.ProcessProperties, + run_properties: defs.ProcessProperties, rank_mapping: xp.ndarray, - ugrid: ux.Ugrid2d, - num_lev: int, - face_face_connectivity: xp.ndarray = None, - node_face_connectivity=None, - node_edge_connectivity=None, + ugrid: base_grid.BaseGrid, ): """ Args: - rank_info: contains information on the communicator and local compute node. - rank_mapping: array with shape (global_num_cells) mapping of global cell indices to their rank in the distribution + run_properties: contains information on the communicator and local compute node. + rank_mapping: array with shape (global_num_cells,): mapping of global cell indices to their rank in the distribution ugrid: the global grid - num_lev: number of vertical levels - face_face_connectivity: face-face connectivity matrix: (n_cells, 3) xugrid uses a - scipy.sparse matrix for this which causes problems with zero based indices so we - allow to pass it directly as a workaround - node_face_connectivity: node-face connectivity matrix: (n_vertex, 6) xugrid uses a - sparse matrix for this which causes problems with zero based indices so wes """ - self._props = rank_info + self._props = run_properties self._mapping = rank_mapping self._global_grid = ugrid - self._num_lev = num_lev - self._connectivities = { - dims.C2E2CDim: face_face_connectivity - if face_face_connectivity is not None - else self._global_grid.face_face_connectivity, - dims.C2EDim: self._global_grid.face_edge_connectivity, - dims.C2VDim: self._global_grid.face_node_connectivity, - dims.E2CDim: self._global_grid.edge_face_connectivity, - dims.E2VDim: self._global_grid.edge_node_connectivity, - dims.V2CDim: node_face_connectivity - if node_face_connectivity is not None - else self._global_grid.node_face_connectivity, - dims.V2EDim: node_edge_connectivity - if node_edge_connectivity is not None - else self._global_grid.node_edge_connectivity, - } - + self._connectivities = self._global_grid.connectivities + + + @property + def edge_face_connectivity(self): + return self.connectivity(dims.E2CDim) + + @property + def face_face_connectivity(self): + return self.connectivity(dims.C2E2CDim) + + + @property + def node_edge_connectivity(self): + return self.connectivity(dims.V2EDim) + def _validate(self): assert self._mapping.ndim == 1 # the decomposition should match the communicator size @@ -91,20 +79,14 @@ def _validate(self): def _post_init(self): self._validate() - # TODO handle sparse neibhbors tables? def connectivity(self, dim) -> xp.ndarray: try: conn_table = self._connectivities[dim] - if sp.sparse.issparse(conn_table): - raise NotImplementedError("Scipy sparse for connectivity tables are not supported") - # TODO this is not tested/working. It returns a (N x N) matrix not a (N x sparse_dim) matrix - # _and_ we cannot make the difference between a valid "0" and a missing-value 0 - # return conn_table.toarray(order="C") return conn_table except KeyError as err: raise (f"Connectivity for dimension {dim} is not available") from err - + def next_halo_line(self, cell_line: xp.ndarray, depot=None): """Returns the global indices of the next halo line. @@ -127,16 +109,6 @@ def next_halo_line(self, cell_line: xp.ndarray, depot=None): def _cell_neighbors(self, cells: xp.ndarray): return xp.unique(self.connectivity(dims.C2E2CDim)[cells, :]) - def _cell_neighbors_from_sparse(self, cells: xp.ndarray): - """In xugrid face-face connectivity is a scipy spars matrix, so we reduce it to the regular sparse matrix format: (n_cells, 3)""" - conn = self.connectivity(dims.C2E2CDim) - - neighbors = conn[cells, :] - # There is an issue with explicit 0 (for zero based indices) since sparse.find projects them out... - i, j, vals = sp.sparse.find(neighbors) - - return j - def _find_neighbors(self, cell_line: xp.ndarray, connectivity: xp.ndarray) -> xp.ndarray: """Get a flattened list of all (unique) neighbors to a given global index list""" neighbors = connectivity[cell_line, :] @@ -174,7 +146,7 @@ def construct_decomposition_info(self) -> defs.DecompositionInfo: c_owner_mask = xp.isin(all_cells, owned_cells) - decomp_info = defs.DecompositionInfo(klevels=self._num_lev).with_dimension( + decomp_info = defs.DecompositionInfo(klevels=self._global_grid.num_levels).with_dimension( dims.CellDim, all_cells, c_owner_mask ) @@ -235,7 +207,7 @@ def _update_owner_mask_by_max_rank_convention( edge_owner_mask, all_edges, intersect_owned_first_line, - self._global_grid.edge_face_connectivity, + self.edge_face_connectivity, ) decomp_info.with_dimension(dims.EdgeDim, all_edges, edge_owner_mask) @@ -268,30 +240,37 @@ def construct_local_connectivity( field_offset: FieldOffset for which we want to construct the offset table decom_info: DecompositionInfo for the current rank - Returns: array, containt the connectivity table for the field_offset with rank-local indices - + Returns: array, containing the connectivity table for the field_offset with rank-local indices + # TODO (@halungge): this does not properly work for outermost halo points: they have neighbors that are not present in the local decomposition_info.global_index list!! + # those should have an SKIP_VALUE entry in the local connectivity matrix -> revise the `global_to_local` handling! """ source_dim = field_offset.source target_dim = field_offset.target[0] local_dim = field_offset.target[1] connectivity = self.connectivity(local_dim) - global_node_idx = decom_info.global_index(source_dim, defs.DecompositionInfo.EntryType.ALL) - global_node_sorted = xp.argsort(global_node_idx) + global_idx = decom_info.global_index(source_dim, defs.DecompositionInfo.EntryType.ALL) + global_idx_sorted = xp.argsort(global_idx) sliced_connectivity = connectivity[ decom_info.global_index(target_dim, defs.DecompositionInfo.EntryType.ALL) ] log.debug(f"rank {self._props.rank} has local connectivity f: {sliced_connectivity}") for i in xp.arange(sliced_connectivity.shape[0]): + valid_neighbor_mask = sliced_connectivity[i, :] != SKIP_VALUE + positions = xp.searchsorted( - global_node_idx[global_node_sorted], sliced_connectivity[i, :] + global_idx[global_idx_sorted], sliced_connectivity[i, valid_neighbor_mask] ) - indices = global_node_sorted[positions] - sliced_connectivity[i, :] = indices + # outer most halo points have neighbors that do not exist in the local + # decomposition_info.global_index list. + # those should have an SKIP_VALUE entry in the local connectivity matrix + global_idx_sorted = xp.append(global_idx_sorted,SKIP_VALUE) + indices = global_idx_sorted[positions] + sliced_connectivity[i, valid_neighbor_mask] = indices log.debug(f"rank {self._props.rank} has local connectivity f: {sliced_connectivity}") return sliced_connectivity -# should be done in grid manager! +# should be done in grid manager!sor def local_grid( props: defs.ProcessProperties, decomp_info: defs.DecompositionInfo, @@ -329,3 +308,41 @@ def local_grid( # add connectivities return local_grid + +# TODO (@halungge): refine type hints: adjacency_matrix should be a connectivity matrix of C2E2C and +# the return value an array of shape (n_cells,) + + +class Decomposer(Protocol): + def __call__(self, adjacency_matrix, n_part: int) -> xp.ndarray: + ... + +class SimpleMetisDecomposer(Decomposer): + """ + A simple decomposer using METIS for partitioning a grid topology. + + We use the simple pythonic interface to pymetis: just passing the adjacency matrix + if more control is needed (for example by using weights we need to switch to the C like interface) + https://documen.tician.de/pymetis/functionality.html + """ + + + def __call__(self, adjacency_matrix, n_part: int) -> xp.ndarray: + """ + Generate partition labesl for this grid topology using METIS: + https://github.com/KarypisLab/METIS + + This method utilizes the pymetis Python bindings: + https://github.com/inducer/pymetis + + Args: + n_part: int, number of partitions to create + Returns: np.ndarray: array with partition label (int, rank number) for each cell + """ + + import pymetis + cut_count, partition_index = pymetis.part_graph( + nparts=n_part, + adjacency=adjacency_matrix + ) + return xp.array(partition_index) \ No newline at end of file diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index 5b957195ba..cc7207f58f 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -16,13 +16,12 @@ import mpi4py import mpi4py.MPI import pytest -import xugrid as xu import icon4py.model.common.dimension as dims import icon4py.model.common.grid.grid_manager as gm import icon4py.model.common.grid.vertical as v_grid from icon4py.model.common.decomposition import definitions as defs -from icon4py.model.common.decomposition.halo import HaloGenerator +from icon4py.model.common.decomposition.halo import HaloGenerator, SimpleMetisDecomposer from icon4py.model.common.grid import simple from icon4py.model.common.settings import xp from icon4py.model.common.test_utils import datatest_utils as dt_utils, helpers @@ -123,15 +122,12 @@ halos = {dims.CellDim: cell_halos, dims.EdgeDim: edge_halos, dims.VertexDim: vertex_halo} -def test_halo_constructor_owned_cells(processor_props, simple_ugrid): # fixture - grid = simple_ugrid +def test_halo_constructor_owned_cells(processor_props): # fixture + grid = simple.SimpleGrid() halo_generator = HaloGenerator( ugrid=grid, - rank_info=processor_props, + run_properties=processor_props, rank_mapping=simple_distribution, - num_lev=1, - face_face_connectivity=simple.SimpleGridData.c2e2c_table, - node_face_connectivity=simple.SimpleGridData.v2c_table, ) my_owned_cells = halo_generator.owned_cells() @@ -142,18 +138,16 @@ def test_halo_constructor_owned_cells(processor_props, simple_ugrid): # fixture @pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) @pytest.mark.mpi(min_size=4) -def test_element_ownership_is_unique(dim, processor_props, simple_ugrid): # fixture - grid = simple_ugrid +def test_element_ownership_is_unique(dim, processor_props): # fixture + if processor_props.comm_size != 4: + pytest.skip("This test requires exactly 4 MPI ranks.") + grid = simple.SimpleGrid() halo_generator = HaloGenerator( ugrid=grid, - rank_info=processor_props, + run_properties=processor_props, rank_mapping=simple_distribution, - num_lev=1, - face_face_connectivity=simple.SimpleGridData.c2e2c_table, - node_face_connectivity=simple.SimpleGridData.v2c_table, ) - assert processor_props.comm_size == 4, "This test requires 4 MPI ranks." - + decomposition_info = halo_generator.construct_decomposition_info() owned = decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED) print(f"\nrank {processor_props.rank} owns {dim} : {owned} ") @@ -186,19 +180,17 @@ def test_element_ownership_is_unique(dim, processor_props, simple_ugrid): # fix # check the buffer has all global indices assert xp.all(xp.sort(values) == global_indices(dim)) - -# TODO (@halungge) this test can be run on 4 MPI ranks or should we rather switch to a single node, -# and parametrizes the rank number +@pytest.mark.with_mpi(min_size=4) @pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) -def test_halo_constructor_decomposition_info(processor_props, simple_ugrid, dim): # fixture - grid = simple_ugrid +def test_halo_constructor_decomposition_info(processor_props, dim): # fixture + if processor_props.comm_size != 4: + pytest.skip("This test requires exactly 4 MPI ranks.") + + grid = simple.SimpleGrid() halo_generator = HaloGenerator( ugrid=grid, - rank_info=processor_props, + run_properties=processor_props, rank_mapping=simple_distribution, - num_lev=1, - face_face_connectivity=simple.SimpleGridData.c2e2c_table, - node_face_connectivity=simple.SimpleGridData.v2c_table, ) decomp_info = halo_generator.construct_decomposition_info() @@ -214,28 +206,25 @@ def test_halo_constructor_decomposition_info(processor_props, simple_ugrid, dim) # TODO V2E2V (from grid file vertices_of_vertex) do we use that at all? @pytest.mark.parametrize( - "field_offset", [dims.C2V, dims.E2V, dims.V2C, dims.E2C, dims.V2E, dims.C2E2C] + "field_offset", [dims.C2V, dims.E2V, dims.V2C, dims.E2C, dims.C2E, dims.V2E, dims.C2E2C] ) def test_local_connectivities(processor_props, caplog, field_offset): # fixture caplog.set_level(logging.INFO) - grid = as_ugrid2d(UGRID_FILE) - icon_grid = grid_file_manager(GRID_FILE).grid - distributed_grids = grid.partition(n_part=4) - labels = grid.label_partitions(n_part=4) + grid = grid_file_manager(GRID_FILE).grid + partitioner = SimpleMetisDecomposer() + processor_props.comm_size = 4 + processor_props.rank = 3 + labels = partitioner(grid.connectivities[dims.C2E2CDim], processor_props.comm_size) halo_generator = HaloGenerator( ugrid=grid, - rank_info=processor_props, + run_properties=processor_props, rank_mapping=labels, - num_lev=1, - face_face_connectivity=icon_grid.connectivities[dims.C2E2CDim], - node_face_connectivity=icon_grid.connectivities[dims.V2CDim], - node_edge_connectivity=icon_grid.connectivities[dims.V2EDim], ) decomposition_info = halo_generator.construct_decomposition_info() connectivity = halo_generator.construct_local_connectivity(field_offset, decomposition_info) - # TODO (@halungge): think of more valuable assertions + # TODO (@halungge): think of more valuable assertions... assert ( connectivity.shape[0] == decomposition_info.global_index( @@ -247,29 +236,6 @@ def test_local_connectivities(processor_props, caplog, field_offset): # fixture ) -@pytest.fixture -def simple_ugrid() -> xu.Ugrid2d: - """ - Programmatically construct a xugrid.ugrid.ugrid2d.Ugrid2d object - - Returns: a Ugrid2d object base on the SimpleGrid - - """ - simple_mesh = simple.SimpleGrid() - fill_value = -1 - node_x = xp.arange(simple_mesh.num_vertices, dtype=xp.float64) - node_y = xp.arange(simple_mesh.num_vertices, dtype=xp.float64) - grid = xu.Ugrid2d( - node_x, - node_y, - fill_value, - projected=True, - face_node_connectivity=simple_mesh.connectivities[dims.C2VDim], - edge_node_connectivity=simple_mesh.connectivities[dims.E2VDim], - ) - - return grid - UGRID_FILE = dt_utils.GRIDS_PATH.joinpath(dt_utils.R02B04_GLOBAL).joinpath( "icon_grid_0013_R02B04_R_ugrid.nc" @@ -287,12 +253,6 @@ def grid_file_manager(file: pathlib.Path) -> gm.GridManager: return manager -def as_ugrid2d(file: pathlib.Path) -> xu.Ugrid2d: - return as_xudataset(file).grid - - -def as_xudataset(file: pathlib.Path) -> xu.UgridDataset: - return xu.open_dataset(file.as_posix()) def global_indices(dim: dims.Dimension) -> int: @@ -333,17 +293,15 @@ def gather_field(field: xp.ndarray, comm: mpi4py.MPI.Comm, dtype=int) -> tuple: @pytest.mark.xfail(reason="This test is not yet implemented") def test_local_grid(processor_props, caplog): # fixture caplog.set_level(logging.INFO) - grid = as_ugrid2d(UGRID_FILE) - icon_grid = grid_file_manager(GRID_FILE).grid - labels = grid.label_partitions(n_part=4) + + grid = grid_file_manager(GRID_FILE).grid + partitioner = SimpleMetisDecomposer() + labels = partitioner(grid.connectivities[dims.C2E2CDim], + n_part=processor_props.comm_size) halo_generator = HaloGenerator( ugrid=grid, - rank_info=processor_props, + run_properties=processor_props, rank_mapping=labels, - num_lev=1, - face_face_connectivity=icon_grid.connectivities[dims.C2E2CDim], - node_face_connectivity=icon_grid.connectivities[dims.V2CDim], - node_edge_connectivity=icon_grid.connectivities[dims.V2EDim], ) decomposition_info = halo_generator.construct_decomposition_info() local_grid = halo_generator.local_grid(decomposition_info) @@ -358,16 +316,13 @@ def test_local_grid(processor_props, caplog): # fixture def test_distributed_fields(processor_props): # fixture grid_manager = grid_file_manager(GRID_FILE) - ugrid = as_ugrid2d(UGRID_FILE) - labels = ugrid.label_partitions(n_part=processor_props.comm_size) - + partitioner = SimpleMetisDecomposer() + labels = partitioner(grid_manager.grid.connectivities[dims.C2E2CDim], n_part=processor_props.comm_size) + halo_generator = HaloGenerator( - ugrid=ugrid, - rank_info=processor_props, + ugrid=grid_manager.grid, + run_properties=processor_props, rank_mapping=labels, - num_lev=1, - face_face_connectivity=grid_manager.grid.connectivities[dims.C2E2CDim], - node_face_connectivity=grid_manager.grid.connectivities[dims.V2CDim], ) decomposition_info = halo_generator.construct_decomposition_info() # distributed read: read one field per dimension From fb495391056171f1615803bb39b917253c30412d Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 26 Jul 2024 11:53:01 +0200 Subject: [PATCH 011/240] fix dtype issue in gather field tests --- model/common/pyproject.toml | 2 +- .../model/common/decomposition/halo.py | 91 ++++++++++--------- .../tests/decomposition_tests/test_halo.py | 81 +++++++++-------- 3 files changed, 92 insertions(+), 82 deletions(-) diff --git a/model/common/pyproject.toml b/model/common/pyproject.toml index 91425e7770..d9bbdfd6cf 100644 --- a/model/common/pyproject.toml +++ b/model/common/pyproject.toml @@ -30,7 +30,7 @@ requires-python = ">=3.10" [project.optional-dependencies] all = ["icon4py-common[ghex,io]"] -ghex = ["ghex", "mpi4py>=3.1.4", "pymetis>2022.1",] +ghex = ["ghex", "mpi4py>=3.1.4", "pymetis>2022.1"] io = [ "icon4py-common[netcdf]", "xarray[complete]>=2024.3.0", diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index ad09f9b946..3c329c9155 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -36,7 +36,10 @@ class DecompositionFlag(enum.IntEnum): #: cell is in the second halo line: that is cells that share only a vertex with an owned cell (and at least an edge with a FIRST_HALO_LINE cell) SECOND_HALO_LINE = 2 + SKIP_VALUE = gm.GridFile.INVALID_INDEX + + class HaloGenerator: """Creates necessary halo information for a given rank.""" @@ -58,20 +61,18 @@ def __init__( self._global_grid = ugrid self._connectivities = self._global_grid.connectivities - @property def edge_face_connectivity(self): return self.connectivity(dims.E2CDim) - + @property def face_face_connectivity(self): return self.connectivity(dims.C2E2CDim) - - + @property def node_edge_connectivity(self): return self.connectivity(dims.V2EDim) - + def _validate(self): assert self._mapping.ndim == 1 # the decomposition should match the communicator size @@ -86,7 +87,7 @@ def connectivity(self, dim) -> xp.ndarray: return conn_table except KeyError as err: raise (f"Connectivity for dimension {dim} is not available") from err - + def next_halo_line(self, cell_line: xp.ndarray, depot=None): """Returns the global indices of the next halo line. @@ -127,7 +128,7 @@ def owned_cells(self) -> xp.ndarray: owned_cells = self._mapping == self._props.rank return xp.asarray(owned_cells).nonzero()[0] - # TODO (@halungge): move out of halo generator + # TODO (@halungge): move out of halo generator? def construct_decomposition_info(self) -> defs.DecompositionInfo: """ Constructs the DecompositionInfo for the current rank. @@ -231,43 +232,52 @@ def _update_owner_mask_by_max_rank_convention( return decomp_info def construct_local_connectivity( - self, field_offset: gtx.FieldOffset, decom_info: defs.DecompositionInfo + self, field_offset: gtx.FieldOffset, decomposition_info: defs.DecompositionInfo ) -> xp.ndarray: """ - Construct a connectivity table for use on a given rank: it maps from source to target dimension in local indices. + Construct a connectivity table for use on a given rank: it maps from source to target dimension in _local_ indices. + + Starting from the connectivity table on the global grid + - we reduce it to the lines for the locally present entries of the the target dimension + - the reduced connectivity then still maps to global source dimension indices: + we replace those source dimension indices not present on the node to SKIP_VALUE and replace the rest with the local indices Args: - field_offset: FieldOffset for which we want to construct the offset table - decom_info: DecompositionInfo for the current rank + field_offset: FieldOffset for which we want to construct the local connectivity table + decomposition_info: DecompositionInfo for the current rank. - Returns: array, containing the connectivity table for the field_offset with rank-local indices - # TODO (@halungge): this does not properly work for outermost halo points: they have neighbors that are not present in the local decomposition_info.global_index list!! - # those should have an SKIP_VALUE entry in the local connectivity matrix -> revise the `global_to_local` handling! + Returns: + connectivity are for the same FieldOffset but mapping from local target dimension indices to local source dimension indices. """ source_dim = field_offset.source target_dim = field_offset.target[0] local_dim = field_offset.target[1] - connectivity = self.connectivity(local_dim) - global_idx = decom_info.global_index(source_dim, defs.DecompositionInfo.EntryType.ALL) - global_idx_sorted = xp.argsort(global_idx) - sliced_connectivity = connectivity[ - decom_info.global_index(target_dim, defs.DecompositionInfo.EntryType.ALL) + sliced_connectivity = self.connectivity(local_dim)[ + decomposition_info.global_index(target_dim, defs.DecompositionInfo.EntryType.ALL) ] log.debug(f"rank {self._props.rank} has local connectivity f: {sliced_connectivity}") - for i in xp.arange(sliced_connectivity.shape[0]): - valid_neighbor_mask = sliced_connectivity[i, :] != SKIP_VALUE - + + global_idx = decomposition_info.global_index( + source_dim, defs.DecompositionInfo.EntryType.ALL + ) + + # replace indices in the connectivity that do not exist on the local node by the SKIP_VALUE (those are for example neighbors of the outermost halo points) + local_connectivity = xp.where( + xp.isin(sliced_connectivity, global_idx), sliced_connectivity, SKIP_VALUE + ) + + # map to local source indices + sorted_index_of_global_idx = xp.argsort(global_idx) + global_idx_sorted = global_idx[sorted_index_of_global_idx] + for i in xp.arange(local_connectivity.shape[0]): + valid_neighbor_mask = local_connectivity[i, :] != SKIP_VALUE positions = xp.searchsorted( - global_idx[global_idx_sorted], sliced_connectivity[i, valid_neighbor_mask] + global_idx_sorted, local_connectivity[i, valid_neighbor_mask] ) - # outer most halo points have neighbors that do not exist in the local - # decomposition_info.global_index list. - # those should have an SKIP_VALUE entry in the local connectivity matrix - global_idx_sorted = xp.append(global_idx_sorted,SKIP_VALUE) - indices = global_idx_sorted[positions] - sliced_connectivity[i, valid_neighbor_mask] = indices - log.debug(f"rank {self._props.rank} has local connectivity f: {sliced_connectivity}") - return sliced_connectivity + indices = sorted_index_of_global_idx[positions] + local_connectivity[i, valid_neighbor_mask] = indices + log.debug(f"rank {self._props.rank} has local connectivity f: {local_connectivity}") + return local_connectivity # should be done in grid manager!sor @@ -309,7 +319,8 @@ def local_grid( return local_grid -# TODO (@halungge): refine type hints: adjacency_matrix should be a connectivity matrix of C2E2C and + +# TODO (@halungge): refine type hints: adjacency_matrix should be a connectivity matrix of C2E2C and # the return value an array of shape (n_cells,) @@ -317,18 +328,18 @@ class Decomposer(Protocol): def __call__(self, adjacency_matrix, n_part: int) -> xp.ndarray: ... + class SimpleMetisDecomposer(Decomposer): """ A simple decomposer using METIS for partitioning a grid topology. - + We use the simple pythonic interface to pymetis: just passing the adjacency matrix if more control is needed (for example by using weights we need to switch to the C like interface) https://documen.tician.de/pymetis/functionality.html """ - def __call__(self, adjacency_matrix, n_part: int) -> xp.ndarray: - """ + """ Generate partition labesl for this grid topology using METIS: https://github.com/KarypisLab/METIS @@ -337,12 +348,10 @@ def __call__(self, adjacency_matrix, n_part: int) -> xp.ndarray: Args: n_part: int, number of partitions to create - Returns: np.ndarray: array with partition label (int, rank number) for each cell + Returns: np.ndarray: array with partition label (int, rank number) for each cell """ import pymetis - cut_count, partition_index = pymetis.part_graph( - nparts=n_part, - adjacency=adjacency_matrix - ) - return xp.array(partition_index) \ No newline at end of file + + cut_count, partition_index = pymetis.part_graph(nparts=n_part, adjacency=adjacency_matrix) + return xp.array(partition_index) diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index cc7207f58f..8cf659defc 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -19,10 +19,9 @@ import icon4py.model.common.dimension as dims import icon4py.model.common.grid.grid_manager as gm -import icon4py.model.common.grid.vertical as v_grid from icon4py.model.common.decomposition import definitions as defs from icon4py.model.common.decomposition.halo import HaloGenerator, SimpleMetisDecomposer -from icon4py.model.common.grid import simple +from icon4py.model.common.grid import base as base_grid, simple, vertical as v_grid from icon4py.model.common.settings import xp from icon4py.model.common.test_utils import datatest_utils as dt_utils, helpers from icon4py.model.common.test_utils.parallel_helpers import ( # noqa: F401 # import fixtures from test_utils package @@ -31,6 +30,13 @@ ) +UGRID_FILE = dt_utils.GRIDS_PATH.joinpath(dt_utils.R02B04_GLOBAL).joinpath( + "icon_grid_0013_R02B04_R_ugrid.nc" +) +GRID_FILE = dt_utils.GRIDS_PATH.joinpath(dt_utils.R02B04_GLOBAL).joinpath( + "icon_grid_0013_R02B04_R.nc" +) + simple_distribution = xp.asarray( [ 0, # 0c @@ -147,7 +153,7 @@ def test_element_ownership_is_unique(dim, processor_props): # fixture run_properties=processor_props, rank_mapping=simple_distribution, ) - + decomposition_info = halo_generator.construct_decomposition_info() owned = decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED) print(f"\nrank {processor_props.rank} owns {dim} : {owned} ") @@ -180,12 +186,13 @@ def test_element_ownership_is_unique(dim, processor_props): # fixture # check the buffer has all global indices assert xp.all(xp.sort(values) == global_indices(dim)) + @pytest.mark.with_mpi(min_size=4) @pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) def test_halo_constructor_decomposition_info(processor_props, dim): # fixture if processor_props.comm_size != 4: pytest.skip("This test requires exactly 4 MPI ranks.") - + grid = simple.SimpleGrid() halo_generator = HaloGenerator( ugrid=grid, @@ -204,17 +211,14 @@ def test_halo_constructor_decomposition_info(processor_props, dim): # fixture assert xp.setdiff1d(my_owned, owned[dim][processor_props.rank], assume_unique=True).size == 0 -# TODO V2E2V (from grid file vertices_of_vertex) do we use that at all? @pytest.mark.parametrize( - "field_offset", [dims.C2V, dims.E2V, dims.V2C, dims.E2C, dims.C2E, dims.V2E, dims.C2E2C] + "field_offset", + [dims.C2V, dims.E2V, dims.V2C, dims.E2C, dims.C2E, dims.V2E, dims.C2E2C, dims.V2E2V], ) def test_local_connectivities(processor_props, caplog, field_offset): # fixture caplog.set_level(logging.INFO) - grid = grid_file_manager(GRID_FILE).grid - partitioner = SimpleMetisDecomposer() - processor_props.comm_size = 4 - processor_props.rank = 3 - labels = partitioner(grid.connectivities[dims.C2E2CDim], processor_props.comm_size) + grid = grid_file_manager(GRID_FILE).grid + labels = decompose(grid, processor_props) halo_generator = HaloGenerator( ugrid=grid, run_properties=processor_props, @@ -224,25 +228,19 @@ def test_local_connectivities(processor_props, caplog, field_offset): # fixture decomposition_info = halo_generator.construct_decomposition_info() connectivity = halo_generator.construct_local_connectivity(field_offset, decomposition_info) - # TODO (@halungge): think of more valuable assertions... + # there is an neighbor list for each index of the target dimension on the node assert ( connectivity.shape[0] == decomposition_info.global_index( field_offset.target[0], defs.DecompositionInfo.EntryType.ALL ).size ) + # all neighbor indices are valid local indices assert xp.max(connectivity) == xp.max( decomposition_info.local_index(field_offset.source, defs.DecompositionInfo.EntryType.ALL) ) - - - -UGRID_FILE = dt_utils.GRIDS_PATH.joinpath(dt_utils.R02B04_GLOBAL).joinpath( - "icon_grid_0013_R02B04_R_ugrid.nc" -) -GRID_FILE = dt_utils.GRIDS_PATH.joinpath(dt_utils.R02B04_GLOBAL).joinpath( - "icon_grid_0013_R02B04_R.nc" -) + # TODO what else? + # outer halo entries have SKIP_VALUE neighbors (depends on offsets) def grid_file_manager(file: pathlib.Path) -> gm.GridManager: @@ -253,8 +251,6 @@ def grid_file_manager(file: pathlib.Path) -> gm.GridManager: return manager - - def global_indices(dim: dims.Dimension) -> int: mesh = simple.SimpleGrid() return xp.arange(mesh.size[dim], dtype=xp.int32) @@ -280,10 +276,10 @@ def icon_distribution( return distribution -def gather_field(field: xp.ndarray, comm: mpi4py.MPI.Comm, dtype=int) -> tuple: +def gather_field(field: xp.ndarray, comm: mpi4py.MPI.Comm) -> tuple: local_sizes = xp.array(comm.gather(field.size, root=0)) if comm.rank == 0: - recv_buffer = xp.empty(sum(local_sizes), dtype=dtype) + recv_buffer = xp.empty(sum(local_sizes), dtype=field.dtype) else: recv_buffer = None comm.Gatherv(sendbuf=field, recvbuf=(recv_buffer, local_sizes), root=0) @@ -293,11 +289,9 @@ def gather_field(field: xp.ndarray, comm: mpi4py.MPI.Comm, dtype=int) -> tuple: @pytest.mark.xfail(reason="This test is not yet implemented") def test_local_grid(processor_props, caplog): # fixture caplog.set_level(logging.INFO) - + grid = grid_file_manager(GRID_FILE).grid - partitioner = SimpleMetisDecomposer() - labels = partitioner(grid.connectivities[dims.C2E2CDim], - n_part=processor_props.comm_size) + labels = decompose(grid, processor_props) halo_generator = HaloGenerator( ugrid=grid, run_properties=processor_props, @@ -316,11 +310,11 @@ def test_local_grid(processor_props, caplog): # fixture def test_distributed_fields(processor_props): # fixture grid_manager = grid_file_manager(GRID_FILE) - partitioner = SimpleMetisDecomposer() - labels = partitioner(grid_manager.grid.connectivities[dims.C2E2CDim], n_part=processor_props.comm_size) - + global_grid = grid_manager.grid + labels = decompose(global_grid, processor_props) + halo_generator = HaloGenerator( - ugrid=grid_manager.grid, + ugrid=global_grid, run_properties=processor_props, rank_mapping=labels, ) @@ -336,13 +330,13 @@ def test_distributed_fields(processor_props): # fixture f"rank = {processor_props.rank} has size(cell_area): {local_cell_area.shape}, has size(edge_length): {local_edge_length.shape}" ) # the local number of cells must be at most the global number of cells (analytically computed) - assert local_cell_area.size <= grid_manager.grid.global_properties.num_cells + assert local_cell_area.size <= global_grid.global_properties.num_cells # global read: read the same (global fields) global_geometry_fields = grid_manager.read_geometry() - global_vlon = grid_manager.read_coordinates()[gm.GridFile.CoordinateName.VERTEX_LONGITUDE] - global_cell_area = global_geometry_fields[gm.GridFile.GeometryName.CELL_AREA] global_edge_length = global_geometry_fields[gm.GridFile.GeometryName.EDGE_LENGTH] + global_vlon = grid_manager.read_coordinates()[gm.GridFile.CoordinateName.VERTEX_LONGITUDE] + assert_gathered_field_against_global( decomposition_info, processor_props, dims.CellDim, global_cell_area, local_cell_area ) @@ -355,6 +349,12 @@ def test_distributed_fields(processor_props): # fixture ) +def decompose(grid: base_grid.BaseGrid, processor_props): + partitioner = SimpleMetisDecomposer() + labels = partitioner(grid.connectivities[dims.C2E2CDim], n_part=processor_props.comm_size) + return labels + + def assert_gathered_field_against_global( decomposition_info: defs.DecompositionInfo, processor_props: defs.ProcessProperties, @@ -369,16 +369,17 @@ def assert_gathered_field_against_global( owned_entries = local_field[ decomposition_info.local_index(dim, defs.DecompositionInfo.EntryType.OWNED) ] - gathered_sizes, gathered_field = gather_field( - owned_entries, processor_props.comm, dtype=xp.float64 - ) + gathered_sizes, gathered_field = gather_field(owned_entries, processor_props.comm) global_index_sizes, gathered_global_indices = gather_field( decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED), processor_props.comm, - dtype=int, ) if processor_props.rank == 0: assert xp.all(gathered_sizes == global_index_sizes) sorted = xp.zeros(global_reference_field.shape, dtype=xp.float64) sorted[gathered_global_indices] = gathered_field assert helpers.dallclose(sorted, global_reference_field) + + +# TODO add test including halo access: +# Will uses geofac_div and geofac_n2s From 2a0158320be5e0de176f67fddce39179b5bfe973 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 26 Jul 2024 12:51:16 +0200 Subject: [PATCH 012/240] rename IndexTransformation implementation --- .../model/common/decomposition/halo.py | 98 +++++++++++-------- .../icon4py/model/common/grid/grid_manager.py | 2 +- .../model/common/test_utils/grid_utils.py | 2 +- .../tests/decomposition_tests/test_halo.py | 7 +- .../tests/grid_tests/test_grid_manager.py | 6 +- 5 files changed, 65 insertions(+), 50 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 3c329c9155..6edb643d54 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -231,9 +231,8 @@ def _update_owner_mask_by_max_rank_convention( decomp_info.with_dimension(dims.VertexDim, all_vertices, v_owner_mask) return decomp_info - def construct_local_connectivity( - self, field_offset: gtx.FieldOffset, decomposition_info: defs.DecompositionInfo - ) -> xp.ndarray: + def construct_local_connectivity(self, + field_offset: gtx.FieldOffset, decomposition_info: defs.DecompositionInfo) -> xp.ndarray: """ Construct a connectivity table for use on a given rank: it maps from source to target dimension in _local_ indices. @@ -280,44 +279,61 @@ def construct_local_connectivity( return local_connectivity -# should be done in grid manager!sor -def local_grid( - props: defs.ProcessProperties, - decomp_info: defs.DecompositionInfo, - global_params: icon_grid.GlobalGridParams, - num_lev: int, - limited_area: bool = False, - on_gpu: bool = False, -) -> base_grid.BaseGrid: - """ - Constructs a local grid for this rank based on the decomposition info. - TODO (@halungge): for now only returning BaseGrid as we have not start/end indices implementation yet - TODO (@halungge): make sure the INVALID_INDEX is set correctly: - when set in the original (global index) connectivity it should remain - TODO (@halungge): how to handle the (source) indices of last halo line: their (global) neighbors are not all present on the local grid, set INVALID_INDEX (that is what xugrid does) - check what ICON does, (they probably duplicate the valid indices...) - Args: - decomp_info: the decomposition info for this rank - Returns: - local_grid: the local grid - """ - num_vertices = decomp_info.global_index( - dims.VertexDim, defs.DecompositionInfo.EntryType.ALL - ).size - num_edges = decomp_info.global_index(dims.EdgeDim, defs.DecompositionInfo.EntryType.ALL).size - num_cells = decomp_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.ALL).size - grid_size = base_grid.HorizontalGridSize( - num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells - ) - config = base_grid.GridConfig( - horizontal_config=grid_size, vertical_size=num_lev, on_gpu=on_gpu, limited_area=limited_area - ) - - local_grid = ( - icon_grid.IconGrid(uuid.uuid4()).with_config(config).with_global_params(global_params) - ) - # add connectivities - - return local_grid +# should be done in grid manager! + def local_grid(self, + props: defs.ProcessProperties, + decomposition_info: defs.DecompositionInfo, + global_params: icon_grid.GlobalGridParams, + num_lev: int, + limited_area: bool = False, + on_gpu: bool = False, + ) -> base_grid.BaseGrid: + + """ + Constructs a local grid for this rank based on the decomposition info. + TODO (@halungge): for now only returning BaseGrid as we have not start/end indices implementation yet + TODO (@halungge): make sure the INVALID_INDEX is set correctly: - when set in the original (global index) connectivity it should remain + TODO (@halungge): how to handle the (source) indices of last halo line: their (global) neighbors are not all present on the local grid, set INVALID_INDEX (that is what xugrid does) + check what ICON does, (they probably duplicate the valid indices...) + Args: + decomposition_info: the decomposition info for this rank + Returns: + local_grid: the local grid + """ + num_vertices = decomposition_info.global_index( + dims.VertexDim, defs.DecompositionInfo.EntryType.ALL + ).size + num_edges = decomposition_info.global_index(dims.EdgeDim, + defs.DecompositionInfo.EntryType.ALL).size + num_cells = decomposition_info.global_index(dims.CellDim, + defs.DecompositionInfo.EntryType.ALL).size + grid_size = base_grid.HorizontalGridSize( + num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells + ) + config = base_grid.GridConfig( + horizontal_config=grid_size, vertical_size=num_lev, on_gpu=on_gpu, limited_area=limited_area + ) + + local_grid = ( + icon_grid.IconGrid(uuid.uuid4()).with_config(config).with_global_params(global_params).with_connectivities({ + dims.C2EDim: self.construct_local_connectivity(dims.C2EDim, decomposition_info), + dims.E2CDim: self.construct_local_connectivity(dims.E2CDim, decomposition_info), + dims.E2VDim: self.construct_local_connectivity(dims.E2VDim, decomposition_info), + dims.V2EDim: self.construct_local_connectivity(dims.V2EDim, decomposition_info), + dims.C2VDim: self.construct_local_connectivity(dims.C2VDim, decomposition_info), + dims.V2CDim: self.construct_local_connectivity(dims.V2CDim, decomposition_info), + dims.C2E2CDim: self.construct_local_connectivity(dims.C2E2CDim, decomposition_info), + # dims.C2E2CODim + + + + + }) + ) + + + + return local_grid # TODO (@halungge): refine type hints: adjacency_matrix should be a connectivity matrix of C2E2C and diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index ec03eca278..513c570846 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -252,7 +252,7 @@ def get_offset_for_index_field( return np.zeros(array.shape, dtype=gtx.int32) -class ToGt4PyTransformation(IndexTransformation): +class ToZeroBasedIndexTransformation(IndexTransformation): def get_offset_for_index_field(self, array: np.ndarray): """ Calculate the index offset needed for usage with python. diff --git a/model/common/src/icon4py/model/common/test_utils/grid_utils.py b/model/common/src/icon4py/model/common/test_utils/grid_utils.py index ee98c32378..90c3caf11d 100644 --- a/model/common/src/icon4py/model/common/test_utils/grid_utils.py +++ b/model/common/src/icon4py/model/common/test_utils/grid_utils.py @@ -65,7 +65,7 @@ def load_grid_from_file( grid_file: str, num_levels: int, on_gpu: bool, limited_area: bool ) -> icon_grid.IconGrid: manager = gm.GridManager( - gm.ToGt4PyTransformation(), + gm.ToZeroBasedIndexTransformation(), str(grid_file), v_grid.VerticalGridConfig(num_levels=num_levels), ) diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index 8cf659defc..e94d13f62e 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -239,13 +239,13 @@ def test_local_connectivities(processor_props, caplog, field_offset): # fixture assert xp.max(connectivity) == xp.max( decomposition_info.local_index(field_offset.source, defs.DecompositionInfo.EntryType.ALL) ) - # TODO what else? + # TODO what else to assert? # outer halo entries have SKIP_VALUE neighbors (depends on offsets) def grid_file_manager(file: pathlib.Path) -> gm.GridManager: manager = gm.GridManager( - gm.ToGt4PyTransformation(), str(file), v_grid.VerticalGridConfig(num_levels=1) + gm.ToZeroBasedIndexTransformation(), str(file), v_grid.VerticalGridConfig(num_levels=1) ) manager() return manager @@ -255,7 +255,7 @@ def global_indices(dim: dims.Dimension) -> int: mesh = simple.SimpleGrid() return xp.arange(mesh.size[dim], dtype=xp.int32) - +# TODO unused - remove or fix and use? def icon_distribution( props: defs.ProcessProperties, decomposition_info: defs.DecompositionInfo ) -> xp.ndarray: @@ -286,7 +286,6 @@ def gather_field(field: xp.ndarray, comm: mpi4py.MPI.Comm) -> tuple: return local_sizes, recv_buffer -@pytest.mark.xfail(reason="This test is not yet implemented") def test_local_grid(processor_props, caplog): # fixture caplog.set_level(logging.INFO) diff --git a/model/common/tests/grid_tests/test_grid_manager.py b/model/common/tests/grid_tests/test_grid_manager.py index b47df17f55..9533dfd8e6 100644 --- a/model/common/tests/grid_tests/test_grid_manager.py +++ b/model/common/tests/grid_tests/test_grid_manager.py @@ -54,7 +54,7 @@ GridFileName, GridManager, IndexTransformation, - ToGt4PyTransformation, + ToZeroBasedIndexTransformation, ) from icon4py.model.common.grid.horizontal import HorizontalMarkerIndex from icon4py.model.common.grid.simple import SimpleGrid @@ -593,7 +593,7 @@ def test_gridmanager_eval_c2v(caplog, grid_savepoint, grid_file): @functools.cache def init_grid_manager(fname, num_levels=65, transformation=None): if transformation is None: - transformation = ToGt4PyTransformation() + transformation = ToZeroBasedIndexTransformation() grid_manager = GridManager(transformation, fname, VerticalGridConfig(num_levels)) grid_manager() return grid_manager @@ -643,7 +643,7 @@ def test_gridmanager_given_file_not_found_then_abort(): @pytest.mark.parametrize("size", [100, 1500, 20000]) @pytest.mark.with_netcdf def test_gt4py_transform_offset_by_1_where_valid(size): - trafo = ToGt4PyTransformation() + trafo = ToZeroBasedIndexTransformation() rng = np.random.default_rng() input_field = rng.integers(-1, size, size) offset = trafo.get_offset_for_index_field(input_field) From 63ecbef19f9032ade83fdcec45551716f211cd08 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 26 Jul 2024 13:02:36 +0200 Subject: [PATCH 013/240] grid_manger clean up --- .../icon4py/model/common/grid/grid_manager.py | 81 +++++++------------ 1 file changed, 31 insertions(+), 50 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 513c570846..d17bfdaaa9 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -13,7 +13,8 @@ import dataclasses import enum import logging -from typing import Optional, Sequence +import pathlib +from typing import Optional, Sequence, Union import gt4py.next as gtx import numpy as np @@ -274,21 +275,21 @@ class GridManager: def __init__( self, transformation: IndexTransformation, - grid_file: str, # TODO use pathlib.Path? + grid_file: Union[pathlib.Path,str], config: v_grid.VerticalGridConfig, ): self._log = logging.getLogger(__name__) self._transformation = transformation - self._config = config + self._config = config # TODO (@halungge) remove self._grid: Optional[icon_grid.IconGrid] = None - self._file_name = grid_file + self._file_name = str(grid_file) self._dataset = None self._reader = None def __call__(self, on_gpu: bool = False, limited_area=True): self._read_gridfile(self._file_name) - grid = self._construct_grid(self._dataset, on_gpu=on_gpu, limited_area=limited_area) + grid = self._construct_grid(on_gpu=on_gpu, limited_area=limited_area) self._grid = grid def _read_gridfile(self, fname: str) -> None: @@ -331,9 +332,7 @@ def _read_grid_refinement_information(self, dataset): GridFile.GridRefinementName.START_INDEX_VERTICES, ] start_indices = { - dim: self._get_index_field( - reader, start_index_dims[dim_i], transpose=False, dtype=gtx.int32 - )[_CHILD_DOM] + dim: self._get_index_field(start_index_dims[dim_i], transpose=False, dtype=gtx.int32)[_CHILD_DOM] for dim_i, dim in enumerate(global_dimensions.values()) } @@ -343,9 +342,8 @@ def _read_grid_refinement_information(self, dataset): GridFile.GridRefinementName.END_INDEX_VERTICES, ] end_indices = { - dim: self._get_index_field( - reader, end_index_dims[dim_i], transpose=False, apply_offset=False, dtype=gtx.int32 - )[_CHILD_DOM] + dim: self._get_index_field(end_index_dims[dim_i], transpose=False, apply_offset=False, + dtype=gtx.int32)[_CHILD_DOM] for dim_i, dim in enumerate(global_dimensions.values()) } @@ -426,10 +424,8 @@ def _get_index(self, dim: gtx.Dimension, start_marker: int, index_dict): self._log.error(msg) raise IconGridError(msg) from err - def _construct_grid( - self, dataset: Dataset, on_gpu: bool, limited_area: bool - ) -> icon_grid.IconGrid: - return self._from_grid_dataset(dataset, on_gpu=on_gpu, limited_area=limited_area) + def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGrid: + return self._from_grid_dataset(on_gpu=on_gpu, limited_area=limited_area) def get_size(self, dim: gtx.Dimension): if dim == VertexDim: @@ -442,57 +438,42 @@ def get_size(self, dim: gtx.Dimension): self._log.warning(f"cannot determine size of unknown dimension {dim}") raise IconGridError(f"Unknown dimension {dim}") - def _get_index_field( - self, - reader, - field: GridFileName, - transpose=True, - apply_offset=True, - dtype=gtx.int32, - ): - field = reader.int_field(field, transpose=transpose, dtype=dtype) + def _get_index_field(self, field: GridFileName, transpose=True, apply_offset=True, + dtype=gtx.int32): + field = self._reader.int_field(field, transpose=transpose, dtype=dtype) if apply_offset: field = field + self._transformation.get_offset_for_index_field(field) return field - def _from_grid_dataset( - self, dataset: Dataset, on_gpu: bool, limited_area=True - ) -> icon_grid.IconGrid: - reader = GridFile(dataset) - num_cells = reader.dimension(GridFile.DimensionName.CELL_NAME) - num_edges = reader.dimension(GridFile.DimensionName.EDGE_NAME) - num_vertices = reader.dimension(GridFile.DimensionName.VERTEX_NAME) - uuid = dataset.getncattr(GridFile.PropertyName.GRID_ID) - grid_level = dataset.getncattr(GridFile.PropertyName.LEVEL) - grid_root = dataset.getncattr(GridFile.PropertyName.ROOT) + def _from_grid_dataset(self, on_gpu: bool, limited_area=True) -> icon_grid.IconGrid: + + num_cells = self._reader.dimension(GridFile.DimensionName.CELL_NAME) + num_edges = self._reader.dimension(GridFile.DimensionName.EDGE_NAME) + num_vertices = self._reader.dimension(GridFile.DimensionName.VERTEX_NAME) + uuid = self._dataset.getncattr(GridFile.PropertyName.GRID_ID) + grid_level = self._dataset.getncattr(GridFile.PropertyName.LEVEL) + grid_root = self._dataset.getncattr(GridFile.PropertyName.ROOT) global_params = icon_grid.GlobalGridParams(level=grid_level, root=grid_root) grid_size = grid_def.HorizontalGridSize( num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells ) - c2e = self._get_index_field(reader, GridFile.ConnectivityName.C2E) + c2e = self._get_index_field(GridFile.ConnectivityName.C2E) e2c = self._get_index_field( - reader, GridFile.ConnectivityName.E2C - ) # edge_face_connectivity (optional) + GridFile.ConnectivityName.E2C) # edge_face_connectivity (optional) c2v = self._get_index_field( - reader, GridFile.ConnectivityName.C2V - ) # face_node_connectivity (required) + GridFile.ConnectivityName.C2V) # face_node_connectivity (required) v2c = self._get_index_field( - reader, GridFile.ConnectivityName.V2C - ) # node_face_connectivity -- (pentagon/hexagon) + GridFile.ConnectivityName.V2C) # node_face_connectivity -- (pentagon/hexagon) e2v = self._get_index_field( - reader, GridFile.ConnectivityName.E2V - ) # edge_node_connectivity (optionally required) + GridFile.ConnectivityName.E2V) # edge_node_connectivity (optionally required) v2e = self._get_index_field( - reader, GridFile.ConnectivityName.V2E - ) # node_edge_connectivity -- (pentagon/hexagon) + GridFile.ConnectivityName.V2E) # node_edge_connectivity -- (pentagon/hexagon) v2e2v = self._get_index_field( - reader, GridFile.ConnectivityName.V2E2V - ) # node_node_connectivity -- ((pentagon/hexagon)) + GridFile.ConnectivityName.V2E2V) # node_node_connectivity -- ((pentagon/hexagon)) c2e2c = self._get_index_field( - reader, GridFile.ConnectivityName.C2E2C - ) # face_face_connectivity (optional) + GridFile.ConnectivityName.C2E2C) # face_face_connectivity (optional) e2c2v = self._construct_diamond_vertices(e2v, c2v, e2c) e2c2e = self._construct_diamond_edges(e2c, c2e) @@ -505,7 +486,7 @@ def _from_grid_dataset( end_indices, refine_ctrl, refine_ctrl_max, - ) = self._read_grid_refinement_information(dataset) + ) = self._read_grid_refinement_information(self._dataset) config = grid_def.GridConfig( horizontal_config=grid_size, From d70a0341832508f44025fc8d2f58edec0eb2a42c Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 26 Jul 2024 13:16:49 +0200 Subject: [PATCH 014/240] rename global dimension mapping fix imports in grid_manager --- .../common/decomposition/mpi_decomposition.py | 8 +- .../src/icon4py/model/common/dimension.py | 2 +- .../icon4py/model/common/grid/grid_manager.py | 92 ++++++++----------- .../icon4pygen/bindings/locations.py | 8 +- tools/src/icon4pytools/icon4pygen/metadata.py | 10 +- 5 files changed, 50 insertions(+), 70 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py index 80bd82e442..3e30c77300 100644 --- a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py +++ b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py @@ -44,7 +44,7 @@ unstructured = None from icon4py.model.common.decomposition import definitions -from icon4py.model.common.dimension import DimensionKind, global_dimensions +from icon4py.model.common.dimension import HORIZONTAL_DIMENSIONS, DimensionKind if TYPE_CHECKING: @@ -135,14 +135,14 @@ def __init__( dim: self._create_domain_descriptor( dim, ) - for dim in global_dimensions.values() + for dim in HORIZONTAL_DIMENSIONS.values() } log.info(f"domain descriptors for dimensions {self._domain_descriptors.keys()} initialized") self._patterns = { dim: self._create_pattern( dim, ) - for dim in global_dimensions.values() + for dim in HORIZONTAL_DIMENSIONS.values() } log.info(f"patterns for dimensions {self._patterns.keys()} initialized ") self._comm = make_communication_object(self._context) @@ -194,7 +194,7 @@ def _create_pattern(self, horizontal_dim: Dimension): return pattern def exchange(self, dim: definitions.Dimension, *fields: Sequence[Field]): - assert dim in global_dimensions.values() + assert dim in HORIZONTAL_DIMENSIONS.values() pattern = self._patterns[dim] assert pattern is not None, f"pattern for {dim.value} not found" domain_descriptor = self._domain_descriptors[dim] diff --git a/model/common/src/icon4py/model/common/dimension.py b/model/common/src/icon4py/model/common/dimension.py index db1caef440..f508ece920 100644 --- a/model/common/src/icon4py/model/common/dimension.py +++ b/model/common/src/icon4py/model/common/dimension.py @@ -20,7 +20,7 @@ EdgeDim = Dimension("Edge") CellDim = Dimension("Cell") VertexDim = Dimension("Vertex") -global_dimensions = {"CellDim": CellDim, "EdgeDim": EdgeDim, "VertexDim": VertexDim} +HORIZONTAL_DIMENSIONS = {"CellDim": CellDim, "EdgeDim": EdgeDim, "VertexDim": VertexDim} CEDim = Dimension("CE") CECDim = Dimension("CEC") ECDim = Dimension("EC") diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index d17bfdaaa9..d2f9f7e01a 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -34,30 +34,8 @@ def __init__(self, *args, **kwargs): import icon4py.model.common.dimension as dims -from icon4py.model.common.dimension import ( - C2E2C2EDim, - C2E2CDim, - C2E2CODim, - C2EDim, - C2VDim, - CEDim, - CellDim, - E2C2EDim, - E2C2EODim, - E2C2VDim, - E2CDim, - E2VDim, - ECDim, - ECVDim, - EdgeDim, - V2CDim, - V2E2VDim, - V2EDim, - VertexDim, - global_dimensions, -) from icon4py.model.common.grid import ( - base as grid_def, + base as base_grid, icon as icon_grid, vertical as v_grid, ) @@ -313,7 +291,7 @@ def _read_grid_refinement_information(self, dataset): ] refin_ctrl = { dim: reader.int_field(control_dims[dim_i]) - for dim_i, dim in enumerate(global_dimensions.values()) + for dim_i, dim in enumerate(dims.HORIZONTAL_DIMENSIONS.values()) } grf_dims = [ @@ -323,7 +301,7 @@ def _read_grid_refinement_information(self, dataset): ] refin_ctrl_max = { dim: reader.dimension(grf_dims[dim_i]) - for dim_i, dim in enumerate(global_dimensions.values()) + for dim_i, dim in enumerate(dims.HORIZONTAL_DIMENSIONS.values()) } start_index_dims = [ @@ -333,7 +311,7 @@ def _read_grid_refinement_information(self, dataset): ] start_indices = { dim: self._get_index_field(start_index_dims[dim_i], transpose=False, dtype=gtx.int32)[_CHILD_DOM] - for dim_i, dim in enumerate(global_dimensions.values()) + for dim_i, dim in enumerate(dims.HORIZONTAL_DIMENSIONS.values()) } end_index_dims = [ @@ -344,7 +322,7 @@ def _read_grid_refinement_information(self, dataset): end_indices = { dim: self._get_index_field(end_index_dims[dim_i], transpose=False, apply_offset=False, dtype=gtx.int32)[_CHILD_DOM] - for dim_i, dim in enumerate(global_dimensions.values()) + for dim_i, dim in enumerate(dims.HORIZONTAL_DIMENSIONS.values()) } return start_indices, end_indices, refin_ctrl, refin_ctrl_max @@ -384,8 +362,8 @@ def read_geometry(self, decomposition_info: Optional[defs.DecompositionInfo] = N return self._read( decomposition_info, { - CellDim: [GridFile.GeometryName.CELL_AREA], - EdgeDim: [GridFile.GeometryName.EDGE_LENGTH], + dims.CellDim: [GridFile.GeometryName.CELL_AREA], + dims.EdgeDim: [GridFile.GeometryName.EDGE_LENGTH], }, ) @@ -393,15 +371,15 @@ def read_coordinates(self, decomposition_info: Optional[defs.DecompositionInfo] return self._read( decomposition_info, { - CellDim: [ + dims.CellDim: [ GridFile.CoordinateName.CELL_LONGITUDE, GridFile.CoordinateName.CELL_LATITUDE, ], - EdgeDim: [ + dims.EdgeDim: [ GridFile.CoordinateName.EDGE_LONGITUDE, GridFile.CoordinateName.EDGE_LATITUDE, ], - VertexDim: [ + dims.VertexDim: [ GridFile.CoordinateName.VERTEX_LONGITUDE, GridFile.CoordinateName.VERTEX_LATITUDE, ], @@ -428,11 +406,11 @@ def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGri return self._from_grid_dataset(on_gpu=on_gpu, limited_area=limited_area) def get_size(self, dim: gtx.Dimension): - if dim == VertexDim: + if dim == dims.VertexDim: return self._grid.config.num_vertices - elif dim == CellDim: + elif dim == dims.CellDim: return self._grid.config.num_cells - elif dim == EdgeDim: + elif dim == dims.EdgeDim: return self._grid.config.num_edges else: self._log.warning(f"cannot determine size of unknown dimension {dim}") @@ -445,6 +423,8 @@ def _get_index_field(self, field: GridFileName, transpose=True, apply_offset=Tru field = field + self._transformation.get_offset_for_index_field(field) return field + def _from_decomposition_info(self, decomposition_info: defs.DecompositionInfo, on_gpu: bool = False, limited_area:bool = False)-> base_grid.BaseGrid: + pass def _from_grid_dataset(self, on_gpu: bool, limited_area=True) -> icon_grid.IconGrid: num_cells = self._reader.dimension(GridFile.DimensionName.CELL_NAME) @@ -455,7 +435,7 @@ def _from_grid_dataset(self, on_gpu: bool, limited_area=True) -> icon_grid.IconG grid_root = self._dataset.getncattr(GridFile.PropertyName.ROOT) global_params = icon_grid.GlobalGridParams(level=grid_level, root=grid_root) - grid_size = grid_def.HorizontalGridSize( + grid_size = base_grid.HorizontalGridSize( num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells ) c2e = self._get_index_field(GridFile.ConnectivityName.C2E) @@ -488,7 +468,7 @@ def _from_grid_dataset(self, on_gpu: bool, limited_area=True) -> icon_grid.IconG refine_ctrl_max, ) = self._read_grid_refinement_information(self._dataset) - config = grid_def.GridConfig( + config = base_grid.GridConfig( horizontal_config=grid_size, vertical_size=self._config.num_levels, on_gpu=on_gpu, @@ -500,30 +480,30 @@ def _from_grid_dataset(self, on_gpu: bool, limited_area=True) -> icon_grid.IconG .with_global_params(global_params) .with_connectivities( { - C2EDim: c2e, - E2CDim: e2c, - E2VDim: e2v, - V2EDim: v2e, - V2CDim: v2c, - C2VDim: c2v, - C2E2CDim: c2e2c, - C2E2CODim: c2e2c0, - C2E2C2EDim: c2e2c2e, - E2C2VDim: e2c2v, - V2E2VDim: v2e2v, - E2C2EDim: e2c2e, - E2C2EODim: e2c2e0, + dims.C2EDim: c2e, + dims.E2CDim: e2c, + dims.E2VDim: e2v, + dims.V2EDim: v2e, + dims.V2CDim: v2c, + dims.C2VDim: c2v, + dims.C2E2CDim: c2e2c, + dims.C2E2CODim: c2e2c0, + dims.C2E2C2EDim: c2e2c2e, + dims.E2C2VDim: e2c2v, + dims.V2E2VDim: v2e2v, + dims.E2C2EDim: e2c2e, + dims.E2C2EODim: e2c2e0, } ) - .with_start_end_indices(CellDim, start_indices[CellDim], end_indices[CellDim]) - .with_start_end_indices(EdgeDim, start_indices[EdgeDim], end_indices[EdgeDim]) - .with_start_end_indices(VertexDim, start_indices[VertexDim], end_indices[VertexDim]) + .with_start_end_indices(dims.CellDim, start_indices[dims.CellDim], end_indices[dims.CellDim]) + .with_start_end_indices(dims.EdgeDim, start_indices[dims.EdgeDim], end_indices[dims.EdgeDim]) + .with_start_end_indices(dims.VertexDim, start_indices[dims.VertexDim], end_indices[dims.VertexDim]) ) grid.update_size_connectivities( { - ECVDim: grid.size[EdgeDim] * grid.size[E2C2VDim], - CEDim: grid.size[CellDim] * grid.size[C2EDim], - ECDim: grid.size[EdgeDim] * grid.size[E2CDim], + dims.ECVDim: grid.size[dims.EdgeDim] * grid.size[dims.E2C2VDim], + dims.CEDim: grid.size[dims.CellDim] * grid.size[dims.C2EDim], + dims.ECDim: grid.size[dims.EdgeDim] * grid.size[dims.E2CDim], } ) diff --git a/tools/src/icon4pytools/icon4pygen/bindings/locations.py b/tools/src/icon4pytools/icon4pygen/bindings/locations.py index daafadc9d7..201022020f 100644 --- a/tools/src/icon4pytools/icon4pygen/bindings/locations.py +++ b/tools/src/icon4pytools/icon4pygen/bindings/locations.py @@ -15,10 +15,10 @@ from typing import Iterator from gt4py.next.ffront.fbuiltins import Dimension -from icon4py.model.common.dimension import global_dimensions - from icon4pytools.icon4pygen.bindings.codegen.render.location import LocationRenderer +from icon4py.model.common.dimension import HORIZONTAL_DIMENSIONS + class BasicLocation: renderer = LocationRenderer @@ -73,9 +73,9 @@ def __getitem__(self, item: int) -> BasicLocation: return self.chain[item] def to_dim_list(self) -> list[Dimension]: - dims_initials = [key[0] for key in global_dimensions.keys()] + dims_initials = [key[0] for key in HORIZONTAL_DIMENSIONS.keys()] map_to_dim = { - d: list(global_dimensions.values())[d_i] for d_i, d in enumerate(dims_initials) + d: list(HORIZONTAL_DIMENSIONS.values())[d_i] for d_i, d in enumerate(dims_initials) } return [map_to_dim[str(c)] for c in self.chain] diff --git a/tools/src/icon4pytools/icon4pygen/metadata.py b/tools/src/icon4pytools/icon4pygen/metadata.py index cc812bdb75..00368b9645 100644 --- a/tools/src/icon4pytools/icon4pygen/metadata.py +++ b/tools/src/icon4pytools/icon4pygen/metadata.py @@ -17,7 +17,6 @@ from dataclasses import dataclass from typing import Any, Optional, TypeGuard -import icon4py.model.common.dimension from gt4py import eve from gt4py.next.common import Connectivity, Dimension, DimensionKind from gt4py.next.ffront import program_ast as past @@ -26,10 +25,11 @@ from gt4py.next.iterator import ir as itir from gt4py.next.iterator.runtime import FendefDispatcher from gt4py.next.type_system import type_specifications as ts -from icon4py.model.common.dimension import Koff, global_dimensions - from icon4pytools.icon4pygen.bindings.utils import calc_num_neighbors +import icon4py.model.common.dimension +from icon4py.model.common.dimension import HORIZONTAL_DIMENSIONS, Koff + H_START = "horizontal_start" H_END = "horizontal_end" @@ -187,8 +187,8 @@ def provide_neighbor_table(chain: str, is_global: bool) -> DummyConnectivity: skip_values = True include_center = True if chain.count("O") > 0 else False - dims_initials = [key[0] for key in global_dimensions.keys()] - map_to_dim = {d: list(global_dimensions.values())[d_i] for d_i, d in enumerate(dims_initials)} + dims_initials = [key[0] for key in HORIZONTAL_DIMENSIONS.keys()] + map_to_dim = {d: list(HORIZONTAL_DIMENSIONS.values())[d_i] for d_i, d in enumerate(dims_initials)} location_chain: list[Dimension] = [map_to_dim.get(c) for c in chain if c not in ("2", "O")] # type: ignore[misc] # type specified return DummyConnectivity( From dc32f6228e5a6d26787e86730099557d24de0f0b Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 26 Jul 2024 13:30:30 +0200 Subject: [PATCH 015/240] fix imports in test_grid_manager.py --- .../tests/grid_tests/test_grid_manager.py | 343 +++++++++--------- 1 file changed, 162 insertions(+), 181 deletions(-) diff --git a/model/common/tests/grid_tests/test_grid_manager.py b/model/common/tests/grid_tests/test_grid_manager.py index 9533dfd8e6..5b32d4c190 100644 --- a/model/common/tests/grid_tests/test_grid_manager.py +++ b/model/common/tests/grid_tests/test_grid_manager.py @@ -15,17 +15,12 @@ import functools import logging import typing -from uuid import uuid4 +import uuid import numpy as np import pytest -from icon4py.model.common.grid import simple -from icon4py.model.common.test_utils.datatest_utils import ( - GLOBAL_EXPERIMENT, - JABW_EXPERIMENT, - REGIONAL_EXPERIMENT, -) +import icon4py.model.common.test_utils.datatest_utils as dt_utils if typing.TYPE_CHECKING: @@ -36,19 +31,8 @@ except ImportError: pytest.skip("optional netcdf dependency not installed", allow_module_level=True) -from icon4py.model.common.dimension import ( - C2E2CDim, - C2EDim, - C2VDim, - CellDim, - E2C2EDim, - E2CDim, - E2VDim, - EdgeDim, - V2CDim, - V2EDim, - VertexDim, -) +import icon4py.model.common.dimension as dims +from icon4py.model.common.grid import horizontal as h_grid, simple, vertical as v_grid from icon4py.model.common.grid.grid_manager import ( GridFile, GridFileName, @@ -56,11 +40,8 @@ IndexTransformation, ToZeroBasedIndexTransformation, ) -from icon4py.model.common.grid.horizontal import HorizontalMarkerIndex -from icon4py.model.common.grid.simple import SimpleGrid -from icon4py.model.common.grid.vertical import VerticalGridConfig -from .utils import R02B04_GLOBAL, resolve_file_from_gridfile_name +from . import utils SIMPLE_GRID_NC = "simple_grid.nc" @@ -115,18 +96,18 @@ @pytest.fixture def simple_grid_gridfile(tmp_path): path = tmp_path.joinpath(SIMPLE_GRID_NC).absolute() - grid = SimpleGrid() + grid = simple.SimpleGrid() dataset = netCDF4.Dataset(path, "w", format="NETCDF4") - dataset.setncattr(GridFile.PropertyName.GRID_ID, str(uuid4())) + dataset.setncattr(GridFile.PropertyName.GRID_ID, str(uuid.uuid4())) dataset.setncattr(GridFile.PropertyName.LEVEL, 0) dataset.setncattr(GridFile.PropertyName.ROOT, 0) dataset.createDimension(GridFile.DimensionName.VERTEX_NAME, size=grid.num_vertices) dataset.createDimension(GridFile.DimensionName.EDGE_NAME, size=grid.num_edges) dataset.createDimension(GridFile.DimensionName.CELL_NAME, size=grid.num_cells) - dataset.createDimension(GridFile.DimensionName.NEIGHBORS_TO_EDGE_SIZE, size=grid.size[E2VDim]) - dataset.createDimension(GridFile.DimensionName.DIAMOND_EDGE_SIZE, size=grid.size[E2C2EDim]) + dataset.createDimension(GridFile.DimensionName.NEIGHBORS_TO_EDGE_SIZE, size=grid.size[dims.E2VDim]) + dataset.createDimension(GridFile.DimensionName.DIAMOND_EDGE_SIZE, size=grid.size[dims.E2C2EDim]) dataset.createDimension(GridFile.DimensionName.MAX_CHILD_DOMAINS, size=1) # add dummy values for the grf dimensions dataset.createDimension(GridFile.DimensionName.CELL_GRF, size=14) @@ -152,12 +133,12 @@ def simple_grid_gridfile(tmp_path): (GridFile.DimensionName.VERTEX_NAME,), ) - dataset.createDimension(GridFile.DimensionName.NEIGHBORS_TO_CELL_SIZE, size=grid.size[C2EDim]) - dataset.createDimension(GridFile.DimensionName.NEIGHBORS_TO_VERTEX_SIZE, size=grid.size[V2CDim]) + dataset.createDimension(GridFile.DimensionName.NEIGHBORS_TO_CELL_SIZE, size=grid.size[dims.C2EDim]) + dataset.createDimension(GridFile.DimensionName.NEIGHBORS_TO_VERTEX_SIZE, size=grid.size[dims.V2CDim]) _add_to_dataset( dataset, - grid.connectivities[C2EDim], + grid.connectivities[dims.C2EDim], GridFile.ConnectivityName.C2E, ( GridFile.DimensionName.NEIGHBORS_TO_CELL_SIZE, @@ -167,7 +148,7 @@ def simple_grid_gridfile(tmp_path): _add_to_dataset( dataset, - grid.connectivities[E2CDim], + grid.connectivities[dims.E2CDim], GridFile.ConnectivityName.E2C, ( GridFile.DimensionName.NEIGHBORS_TO_EDGE_SIZE, @@ -176,7 +157,7 @@ def simple_grid_gridfile(tmp_path): ) _add_to_dataset( dataset, - grid.connectivities[E2VDim], + grid.connectivities[dims.E2VDim], GridFile.ConnectivityName.E2V, ( GridFile.DimensionName.NEIGHBORS_TO_EDGE_SIZE, @@ -186,7 +167,7 @@ def simple_grid_gridfile(tmp_path): _add_to_dataset( dataset, - grid.connectivities[V2CDim], + grid.connectivities[dims.V2CDim], GridFile.ConnectivityName.V2C, ( GridFile.DimensionName.NEIGHBORS_TO_VERTEX_SIZE, @@ -196,7 +177,7 @@ def simple_grid_gridfile(tmp_path): _add_to_dataset( dataset, - grid.connectivities[C2VDim], + grid.connectivities[dims.C2VDim], GridFile.ConnectivityName.C2V, ( GridFile.DimensionName.NEIGHBORS_TO_CELL_SIZE, @@ -211,7 +192,7 @@ def simple_grid_gridfile(tmp_path): ) _add_to_dataset( dataset, - grid.connectivities[V2EDim], + grid.connectivities[dims.V2EDim], GridFile.ConnectivityName.V2E, ( GridFile.DimensionName.NEIGHBORS_TO_VERTEX_SIZE, @@ -220,7 +201,7 @@ def simple_grid_gridfile(tmp_path): ) _add_to_dataset( dataset, - grid.connectivities[C2E2CDim], + grid.connectivities[dims.C2E2CDim], GridFile.ConnectivityName.C2E2C, ( GridFile.DimensionName.NEIGHBORS_TO_CELL_SIZE, @@ -283,7 +264,7 @@ def _add_to_dataset( def test_gridparser_dimension(simple_grid_gridfile): data = netCDF4.Dataset(simple_grid_gridfile, "r") grid_parser = GridFile(data) - grid = SimpleGrid() + grid = simple.SimpleGrid() assert grid_parser.dimension(GridFile.DimensionName.CELL_NAME) == grid.num_cells assert grid_parser.dimension(GridFile.DimensionName.VERTEX_NAME) == grid.num_vertices assert grid_parser.dimension(GridFile.DimensionName.EDGE_NAME) == grid.num_edges @@ -293,36 +274,36 @@ def test_gridparser_dimension(simple_grid_gridfile): @pytest.mark.with_netcdf @pytest.mark.parametrize( "grid_file, experiment", - [(REGIONAL_EXPERIMENT, REGIONAL_EXPERIMENT), (R02B04_GLOBAL, GLOBAL_EXPERIMENT)], + [(dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), (utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT)], ) def test_gridfile_vertex_cell_edge_dimensions(grid_savepoint, grid_file): - file = resolve_file_from_gridfile_name(grid_file) + file = utils.resolve_file_from_gridfile_name(grid_file) dataset = netCDF4.Dataset(file, "r") grid_file = GridFile(dataset) - assert grid_file.dimension(GridFile.DimensionName.CELL_NAME) == grid_savepoint.num(CellDim) - assert grid_file.dimension(GridFile.DimensionName.EDGE_NAME) == grid_savepoint.num(EdgeDim) - assert grid_file.dimension(GridFile.DimensionName.VERTEX_NAME) == grid_savepoint.num(VertexDim) + assert grid_file.dimension(GridFile.DimensionName.CELL_NAME) == grid_savepoint.num(dims.CellDim) + assert grid_file.dimension(GridFile.DimensionName.EDGE_NAME) == grid_savepoint.num(dims.EdgeDim) + assert grid_file.dimension(GridFile.DimensionName.VERTEX_NAME) == grid_savepoint.num(dims.VertexDim) @pytest.mark.with_netcdf def test_grid_parser_index_fields(simple_grid_gridfile, caplog): caplog.set_level(logging.DEBUG) data = netCDF4.Dataset(simple_grid_gridfile, "r") - grid = SimpleGrid() + grid = simple.SimpleGrid() grid_parser = GridFile(data) assert np.allclose( - grid_parser.int_field(GridFile.ConnectivityName.C2E), grid.connectivities[C2EDim] + grid_parser.int_field(GridFile.ConnectivityName.C2E), grid.connectivities[dims.C2EDim] ) assert np.allclose( - grid_parser.int_field(GridFile.ConnectivityName.E2C), grid.connectivities[E2CDim] + grid_parser.int_field(GridFile.ConnectivityName.E2C), grid.connectivities[dims.E2CDim] ) assert np.allclose( - grid_parser.int_field(GridFile.ConnectivityName.V2E), grid.connectivities[V2EDim] + grid_parser.int_field(GridFile.ConnectivityName.V2E), grid.connectivities[dims.V2EDim] ) assert np.allclose( - grid_parser.int_field(GridFile.ConnectivityName.V2C), grid.connectivities[V2CDim] + grid_parser.int_field(GridFile.ConnectivityName.V2C), grid.connectivities[dims.V2CDim] ) @@ -335,11 +316,11 @@ def test_grid_parser_index_fields(simple_grid_gridfile, caplog): @pytest.mark.with_netcdf @pytest.mark.parametrize( "grid_file, experiment", - [(REGIONAL_EXPERIMENT, REGIONAL_EXPERIMENT), (R02B04_GLOBAL, GLOBAL_EXPERIMENT)], + [(dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), (utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT)], ) def test_gridmanager_eval_v2e(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) - file = resolve_file_from_gridfile_name(grid_file) + file = utils.resolve_file_from_gridfile_name(grid_file) grid = init_grid_manager(file).grid seralized_v2e = grid_savepoint.v2e()[0 : grid.num_vertices, :] # there are vertices at the boundary of a local domain or at a pentagon point that have less than @@ -357,11 +338,11 @@ def test_gridmanager_eval_v2e(caplog, grid_savepoint, grid_file): @pytest.mark.with_netcdf @pytest.mark.parametrize( "grid_file, experiment", - [(REGIONAL_EXPERIMENT, REGIONAL_EXPERIMENT), (R02B04_GLOBAL, GLOBAL_EXPERIMENT)], + [(dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), (utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT)], ) def test_gridmanager_eval_v2c(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) - file = resolve_file_from_gridfile_name(grid_file) + file = utils.resolve_file_from_gridfile_name(grid_file) grid = init_grid_manager(file).grid serialized_v2c = grid_savepoint.v2c()[0 : grid.num_vertices, :] # there are vertices that have less than 6 neighboring cells: either pentagon points or @@ -407,11 +388,11 @@ def reset_invalid_index(index_array: np.ndarray): @pytest.mark.with_netcdf @pytest.mark.parametrize( "grid_file, experiment", - [(REGIONAL_EXPERIMENT, REGIONAL_EXPERIMENT), (R02B04_GLOBAL, GLOBAL_EXPERIMENT)], + [(dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), (utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT)], ) def test_gridmanager_eval_e2v(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) - file = resolve_file_from_gridfile_name(grid_file) + file = utils.resolve_file_from_gridfile_name(grid_file) grid = init_grid_manager(file).grid serialized_e2v = grid_savepoint.e2v()[0 : grid.num_edges, :] @@ -431,7 +412,7 @@ def invalid_index(ar): def _is_local(grid_file: str): - return grid_file == REGIONAL_EXPERIMENT + return grid_file == dt_utils.REGIONAL_EXPERIMENT def assert_invalid_indices(e2c_table: np.ndarray, grid_file: str): @@ -460,11 +441,11 @@ def assert_invalid_indices(e2c_table: np.ndarray, grid_file: str): @pytest.mark.with_netcdf @pytest.mark.parametrize( "grid_file, experiment", - [(REGIONAL_EXPERIMENT, REGIONAL_EXPERIMENT), (R02B04_GLOBAL, GLOBAL_EXPERIMENT)], + [(dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), (utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT)], ) def test_gridmanager_eval_e2c(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) - file = resolve_file_from_gridfile_name(grid_file) + file = utils.resolve_file_from_gridfile_name(grid_file) grid = init_grid_manager(file).grid serialized_e2c = grid_savepoint.e2c()[0 : grid.num_edges, :] e2c_table = grid.get_offset_provider("E2C").table @@ -478,11 +459,11 @@ def test_gridmanager_eval_e2c(caplog, grid_savepoint, grid_file): @pytest.mark.with_netcdf @pytest.mark.parametrize( "grid_file, experiment", - [(REGIONAL_EXPERIMENT, REGIONAL_EXPERIMENT), (R02B04_GLOBAL, GLOBAL_EXPERIMENT)], + [(dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), (utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT)], ) def test_gridmanager_eval_c2e(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) - file = resolve_file_from_gridfile_name(grid_file) + file = utils.resolve_file_from_gridfile_name(grid_file) grid = init_grid_manager(file).grid serialized_c2e = grid_savepoint.c2e()[0 : grid.num_cells, :] @@ -499,11 +480,11 @@ def test_gridmanager_eval_c2e(caplog, grid_savepoint, grid_file): @pytest.mark.with_netcdf @pytest.mark.parametrize( "grid_file, experiment", - [(REGIONAL_EXPERIMENT, REGIONAL_EXPERIMENT), (R02B04_GLOBAL, GLOBAL_EXPERIMENT)], + [(dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), (dt_utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT)], ) def test_gridmanager_eval_c2e2c(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) - file = resolve_file_from_gridfile_name(grid_file) + file = utils.resolve_file_from_gridfile_name(grid_file) grid = init_grid_manager(file).grid assert np.allclose( grid.get_offset_provider("C2E2C").table, @@ -515,11 +496,11 @@ def test_gridmanager_eval_c2e2c(caplog, grid_savepoint, grid_file): @pytest.mark.with_netcdf @pytest.mark.parametrize( "grid_file, experiment", - [(REGIONAL_EXPERIMENT, REGIONAL_EXPERIMENT), (R02B04_GLOBAL, GLOBAL_EXPERIMENT)], + [(dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), (dt_utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT)], ) def test_gridmanager_eval_c2e2cO(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) - file = resolve_file_from_gridfile_name(grid_file) + file = utils.resolve_file_from_gridfile_name(grid_file) grid = init_grid_manager(file).grid serialized_grid = grid_savepoint.construct_icon_grid(on_gpu=False) assert np.allclose( @@ -533,11 +514,11 @@ def test_gridmanager_eval_c2e2cO(caplog, grid_savepoint, grid_file): @pytest.mark.with_netcdf @pytest.mark.parametrize( "grid_file, experiment", - [(REGIONAL_EXPERIMENT, REGIONAL_EXPERIMENT), (R02B04_GLOBAL, GLOBAL_EXPERIMENT)], + [(dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), (utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT)], ) def test_gridmanager_eval_e2c2e(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) - file = resolve_file_from_gridfile_name(grid_file) + file = utils.resolve_file_from_gridfile_name(grid_file) grid = init_grid_manager(file).grid serialized_grid = grid_savepoint.construct_icon_grid(on_gpu=False) serialized_e2c2e = serialized_grid.get_offset_provider("E2C2E").table @@ -548,7 +529,7 @@ def test_gridmanager_eval_e2c2e(caplog, grid_savepoint, grid_file): e2c2e0_table = grid.get_offset_provider("E2C2EO").table assert_invalid_indices(e2c2e_table, grid_file) - # ICON calculates diamond edges only from rl_start = 2 (lateral_boundary(EdgeDim) + 1 for + # ICON calculates diamond edges only from rl_start = 2 (lateral_boundary(dims.EdgeDim) + 1 for # boundaries all values are INVALID even though the half diamond exists (see mo_model_domimp_setup.f90 ll 163ff.) assert_unless_invalid(e2c2e_table, serialized_e2c2e) assert_unless_invalid(e2c2e0_table, serialized_e2c2eO) @@ -563,14 +544,14 @@ def assert_unless_invalid(table, serialized_ref): @pytest.mark.with_netcdf @pytest.mark.parametrize( "grid_file, experiment", - [(REGIONAL_EXPERIMENT, REGIONAL_EXPERIMENT), (R02B04_GLOBAL, GLOBAL_EXPERIMENT)], + [(dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), (utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT)], ) def test_gridmanager_eval_e2c2v(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) - file = resolve_file_from_gridfile_name(grid_file) + file = utils.resolve_file_from_gridfile_name(grid_file) grid = init_grid_manager(file).grid # the "far" (adjacent to edge normal ) is not always there, because ICON only calculates those starting from - # (lateral_boundary(EdgeDim) + 1) to end(EdgeDim) (see mo_intp_coeffs.f90) and only for owned cells + # (lateral_boundary(dims.EdgeDim) + 1) to end(dims.EdgeDim) (see mo_intp_coeffs.f90) and only for owned cells serialized_ref = grid_savepoint.e2c2v()[: grid.num_edges, :] table = grid.get_offset_provider("E2C2V").table assert_unless_invalid(table, serialized_ref) @@ -580,11 +561,11 @@ def test_gridmanager_eval_e2c2v(caplog, grid_savepoint, grid_file): @pytest.mark.with_netcdf @pytest.mark.parametrize( "grid_file, experiment", - [(REGIONAL_EXPERIMENT, REGIONAL_EXPERIMENT), (R02B04_GLOBAL, GLOBAL_EXPERIMENT)], + [(dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), (utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT)], ) def test_gridmanager_eval_c2v(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) - file = resolve_file_from_gridfile_name(grid_file) + file = utils.resolve_file_from_gridfile_name(grid_file) grid = init_grid_manager(file).grid c2v = grid.get_offset_provider("C2V").table assert np.allclose(c2v, grid_savepoint.c2v()[0 : grid.num_cells, :]) @@ -594,12 +575,12 @@ def test_gridmanager_eval_c2v(caplog, grid_savepoint, grid_file): def init_grid_manager(fname, num_levels=65, transformation=None): if transformation is None: transformation = ToZeroBasedIndexTransformation() - grid_manager = GridManager(transformation, fname, VerticalGridConfig(num_levels)) + grid_manager = GridManager(transformation, fname, v_grid.VerticalGridConfig(num_levels)) grid_manager() return grid_manager -@pytest.mark.parametrize("dim, size", [(CellDim, 18), (EdgeDim, 27), (VertexDim, 9)]) +@pytest.mark.parametrize("dim, size", [(dims.CellDim, 18), (dims.EdgeDim, 27), (dims.VertexDim, 9)]) @pytest.mark.with_netcdf def test_grid_manager_getsize(simple_grid_gridfile, dim, size, caplog): caplog.set_level(logging.DEBUG) @@ -618,7 +599,7 @@ def assert_up_to_order(table, diamond_table): @pytest.mark.with_netcdf def test_grid_manager_diamond_offset(simple_grid_gridfile): - simple_grid = SimpleGrid() + simple_grid = simple.SimpleGrid() gm = init_grid_manager( simple_grid_gridfile, num_levels=simple_grid.num_levels, @@ -634,7 +615,7 @@ def test_grid_manager_diamond_offset(simple_grid_gridfile): def test_gridmanager_given_file_not_found_then_abort(): fname = "./unknown_grid.nc" with pytest.raises(SystemExit) as error: - gm = GridManager(IndexTransformation(), fname, VerticalGridConfig(num_levels=80)) + gm = GridManager(IndexTransformation(), fname, v_grid.VerticalGridConfig(num_levels=80)) gm() assert error.type == SystemExit assert error.value == 1 @@ -656,228 +637,228 @@ def test_gt4py_transform_offset_by_1_where_valid(size): "dim, marker, start_index, end_index", [ ( - CellDim, - HorizontalMarkerIndex.interior(CellDim), + dims.CellDim, + h_grid.HorizontalMarkerIndex.interior(dims.CellDim), MCH_CH_R04B09_CELL_DOMAINS["INTERIOR"], MCH_CH_RO4B09_LOCAL_NUM_CELLS, ), ( - CellDim, - HorizontalMarkerIndex.interior(CellDim) + 1, + dims.CellDim, + h_grid.HorizontalMarkerIndex.interior(dims.CellDim) + 1, 0, MCH_CH_R04B09_CELL_DOMAINS["2ND_BOUNDARY_LINE"], ), ( - CellDim, - HorizontalMarkerIndex.local(CellDim) - 2, + dims.CellDim, + h_grid.HorizontalMarkerIndex.local(dims.CellDim) - 2, MCH_CH_RO4B09_LOCAL_NUM_CELLS, MCH_CH_RO4B09_LOCAL_NUM_CELLS, ), ( - CellDim, - HorizontalMarkerIndex.local(CellDim) - 1, + dims.CellDim, + h_grid.HorizontalMarkerIndex.local(dims.CellDim) - 1, MCH_CH_RO4B09_LOCAL_NUM_CELLS, MCH_CH_RO4B09_LOCAL_NUM_CELLS, ), ( - CellDim, - HorizontalMarkerIndex.local(CellDim), + dims.CellDim, + h_grid.HorizontalMarkerIndex.local(dims.CellDim), MCH_CH_RO4B09_LOCAL_NUM_CELLS, MCH_CH_RO4B09_LOCAL_NUM_CELLS, ), ( - CellDim, - HorizontalMarkerIndex.nudging(CellDim), + dims.CellDim, + h_grid.HorizontalMarkerIndex.nudging(dims.CellDim), MCH_CH_R04B09_CELL_DOMAINS["NUDGING"], MCH_CH_R04B09_CELL_DOMAINS["INTERIOR"], ), ( - CellDim, - HorizontalMarkerIndex.lateral_boundary(CellDim) + 3, + dims.CellDim, + h_grid.HorizontalMarkerIndex.lateral_boundary(dims.CellDim) + 3, MCH_CH_R04B09_CELL_DOMAINS["4TH_BOUNDARY_LINE"], MCH_CH_R04B09_CELL_DOMAINS["NUDGING"], ), ( - CellDim, - HorizontalMarkerIndex.lateral_boundary(CellDim) + 2, + dims.CellDim, + h_grid.HorizontalMarkerIndex.lateral_boundary(dims.CellDim) + 2, MCH_CH_R04B09_CELL_DOMAINS["3D_BOUNDARY_LINE"], MCH_CH_R04B09_CELL_DOMAINS["4TH_BOUNDARY_LINE"], ), ( - CellDim, - HorizontalMarkerIndex.lateral_boundary(CellDim) + 1, + dims.CellDim, + h_grid.HorizontalMarkerIndex.lateral_boundary(dims.CellDim) + 1, MCH_CH_R04B09_CELL_DOMAINS["2ND_BOUNDARY_LINE"], MCH_CH_R04B09_CELL_DOMAINS["3D_BOUNDARY_LINE"], ), ( - CellDim, - HorizontalMarkerIndex.lateral_boundary(CellDim) + 0, + dims.CellDim, + h_grid.HorizontalMarkerIndex.lateral_boundary(dims.CellDim) + 0, 0, MCH_CH_R04B09_CELL_DOMAINS["2ND_BOUNDARY_LINE"], ), ( - EdgeDim, - HorizontalMarkerIndex.interior(EdgeDim), + dims.EdgeDim, + h_grid.HorizontalMarkerIndex.interior(dims.EdgeDim), MCH_CH_R04B09_EDGE_DOMAINS["INTERIOR"], MCH_CH_R04B09_LOCAL_NUM_EDGES, ), ( - EdgeDim, - HorizontalMarkerIndex.local(EdgeDim) - 2, + dims.EdgeDim, + h_grid.HorizontalMarkerIndex.local(dims.EdgeDim) - 2, MCH_CH_R04B09_LOCAL_NUM_EDGES, MCH_CH_R04B09_LOCAL_NUM_EDGES, ), ( - EdgeDim, - HorizontalMarkerIndex.local(EdgeDim) - 1, + dims.EdgeDim, + h_grid.HorizontalMarkerIndex.local(dims.EdgeDim) - 1, MCH_CH_R04B09_LOCAL_NUM_EDGES, MCH_CH_R04B09_LOCAL_NUM_EDGES, ), ( - EdgeDim, - HorizontalMarkerIndex.local(EdgeDim), + dims.EdgeDim, + h_grid.HorizontalMarkerIndex.local(dims.EdgeDim), MCH_CH_R04B09_LOCAL_NUM_EDGES, MCH_CH_R04B09_LOCAL_NUM_EDGES, ), ( - EdgeDim, - HorizontalMarkerIndex.nudging(EdgeDim), + dims.EdgeDim, + h_grid.HorizontalMarkerIndex.nudging(dims.EdgeDim), MCH_CH_R04B09_EDGE_DOMAINS["NUDGING"], MCH_CH_R04B09_EDGE_DOMAINS["2ND_NUDGING"], ), ( - EdgeDim, - HorizontalMarkerIndex.nudging(EdgeDim) + 1, + dims.EdgeDim, + h_grid.HorizontalMarkerIndex.nudging(dims.EdgeDim) + 1, MCH_CH_R04B09_EDGE_DOMAINS["2ND_NUDGING"], MCH_CH_R04B09_EDGE_DOMAINS["INTERIOR"], ), ( - EdgeDim, - HorizontalMarkerIndex.lateral_boundary(EdgeDim) + 7, + dims.EdgeDim, + h_grid.HorizontalMarkerIndex.lateral_boundary(dims.EdgeDim) + 7, MCH_CH_R04B09_EDGE_DOMAINS["8TH_BOUNDARY_LINE"], MCH_CH_R04B09_EDGE_DOMAINS["NUDGING"], ), ( - EdgeDim, - HorizontalMarkerIndex.lateral_boundary(EdgeDim) + 6, + dims.EdgeDim, + h_grid.HorizontalMarkerIndex.lateral_boundary(dims.EdgeDim) + 6, MCH_CH_R04B09_EDGE_DOMAINS["7TH_BOUNDARY_LINE"], MCH_CH_R04B09_EDGE_DOMAINS["8TH_BOUNDARY_LINE"], ), ( - EdgeDim, - HorizontalMarkerIndex.lateral_boundary(EdgeDim) + 5, + dims.EdgeDim, + h_grid.HorizontalMarkerIndex.lateral_boundary(dims.EdgeDim) + 5, MCH_CH_R04B09_EDGE_DOMAINS["6TH_BOUNDARY_LINE"], MCH_CH_R04B09_EDGE_DOMAINS["7TH_BOUNDARY_LINE"], ), ( - EdgeDim, - HorizontalMarkerIndex.lateral_boundary(EdgeDim) + 4, + dims.EdgeDim, + h_grid.HorizontalMarkerIndex.lateral_boundary(dims.EdgeDim) + 4, MCH_CH_R04B09_EDGE_DOMAINS["5TH_BOUNDARY_LINE"], MCH_CH_R04B09_EDGE_DOMAINS["6TH_BOUNDARY_LINE"], ), ( - EdgeDim, - HorizontalMarkerIndex.lateral_boundary(EdgeDim) + 3, + dims.EdgeDim, + h_grid.HorizontalMarkerIndex.lateral_boundary(dims.EdgeDim) + 3, MCH_CH_R04B09_EDGE_DOMAINS["4TH_BOUNDARY_LINE"], MCH_CH_R04B09_EDGE_DOMAINS["5TH_BOUNDARY_LINE"], ), ( - EdgeDim, - HorizontalMarkerIndex.lateral_boundary(EdgeDim) + 2, + dims.EdgeDim, + h_grid.HorizontalMarkerIndex.lateral_boundary(dims.EdgeDim) + 2, MCH_CH_R04B09_EDGE_DOMAINS["3D_BOUNDARY_LINE"], MCH_CH_R04B09_EDGE_DOMAINS["4TH_BOUNDARY_LINE"], ), ( - EdgeDim, - HorizontalMarkerIndex.lateral_boundary(EdgeDim) + 1, + dims.EdgeDim, + h_grid.HorizontalMarkerIndex.lateral_boundary(dims.EdgeDim) + 1, MCH_CH_R04B09_EDGE_DOMAINS["2ND_BOUNDARY_LINE"], MCH_CH_R04B09_EDGE_DOMAINS["3D_BOUNDARY_LINE"], ), ( - EdgeDim, - HorizontalMarkerIndex.lateral_boundary(EdgeDim) + 0, + dims.EdgeDim, + h_grid.HorizontalMarkerIndex.lateral_boundary(dims.EdgeDim) + 0, 0, MCH_CH_R04B09_EDGE_DOMAINS["2ND_BOUNDARY_LINE"], ), ( - VertexDim, - HorizontalMarkerIndex.interior(VertexDim), + dims.VertexDim, + h_grid.HorizontalMarkerIndex.interior(dims.VertexDim), MCH_CH_R04B09_VERTEX_DOMAINS["INTERIOR"], MCH_CH_04B09_NUM_VERTICES, ), ( - VertexDim, - HorizontalMarkerIndex.local(VertexDim) - 2, + dims.VertexDim, + h_grid.HorizontalMarkerIndex.local(dims.VertexDim) - 2, MCH_CH_04B09_NUM_VERTICES, MCH_CH_04B09_NUM_VERTICES, ), ( - VertexDim, - HorizontalMarkerIndex.local(VertexDim) - 1, + dims.VertexDim, + h_grid.HorizontalMarkerIndex.local(dims.VertexDim) - 1, MCH_CH_04B09_NUM_VERTICES, MCH_CH_04B09_NUM_VERTICES, ), ( - VertexDim, - HorizontalMarkerIndex.local(VertexDim), + dims.VertexDim, + h_grid.HorizontalMarkerIndex.local(dims.VertexDim), MCH_CH_04B09_NUM_VERTICES, MCH_CH_04B09_NUM_VERTICES, ), ( - VertexDim, - HorizontalMarkerIndex.nudging(VertexDim) + 1, + dims.VertexDim, + h_grid.HorizontalMarkerIndex.nudging(dims.VertexDim) + 1, MCH_CH_04B09_NUM_VERTICES, MCH_CH_04B09_NUM_VERTICES, ), ( - VertexDim, - HorizontalMarkerIndex.nudging(VertexDim), + dims.VertexDim, + h_grid.HorizontalMarkerIndex.nudging(dims.VertexDim), MCH_CH_04B09_NUM_VERTICES, MCH_CH_04B09_NUM_VERTICES, ), ( - VertexDim, - HorizontalMarkerIndex.end(VertexDim), + dims.VertexDim, + h_grid.HorizontalMarkerIndex.end(dims.VertexDim), MCH_CH_04B09_NUM_VERTICES, MCH_CH_04B09_NUM_VERTICES, ), ( - VertexDim, - HorizontalMarkerIndex.lateral_boundary(VertexDim) + 4, + dims.VertexDim, + h_grid.HorizontalMarkerIndex.lateral_boundary(dims.VertexDim) + 4, MCH_CH_R04B09_VERTEX_DOMAINS["5TH_BOUNDARY_LINE"], MCH_CH_R04B09_VERTEX_DOMAINS["INTERIOR"], ), ( - VertexDim, - HorizontalMarkerIndex.lateral_boundary(VertexDim) + 3, + dims.VertexDim, + h_grid.HorizontalMarkerIndex.lateral_boundary(dims.VertexDim) + 3, MCH_CH_R04B09_VERTEX_DOMAINS["4TH_BOUNDARY_LINE"], MCH_CH_R04B09_VERTEX_DOMAINS["5TH_BOUNDARY_LINE"], ), ( - VertexDim, - HorizontalMarkerIndex.lateral_boundary(VertexDim) + 2, + dims.VertexDim, + h_grid.HorizontalMarkerIndex.lateral_boundary(dims.VertexDim) + 2, MCH_CH_R04B09_VERTEX_DOMAINS["3D_BOUNDARY_LINE"], MCH_CH_R04B09_VERTEX_DOMAINS["4TH_BOUNDARY_LINE"], ), ( - VertexDim, - HorizontalMarkerIndex.lateral_boundary(VertexDim) + 1, + dims.VertexDim, + h_grid.HorizontalMarkerIndex.lateral_boundary(dims.VertexDim) + 1, MCH_CH_R04B09_VERTEX_DOMAINS["2ND_BOUNDARY_LINE"], MCH_CH_R04B09_VERTEX_DOMAINS["3D_BOUNDARY_LINE"], ), ( - VertexDim, - HorizontalMarkerIndex.lateral_boundary(VertexDim) + 0, + dims.VertexDim, + h_grid.HorizontalMarkerIndex.lateral_boundary(dims.VertexDim) + 0, 0, MCH_CH_R04B09_VERTEX_DOMAINS["2ND_BOUNDARY_LINE"], ), ], ) -@pytest.mark.parametrize("grid_file, num_levels", [(REGIONAL_EXPERIMENT, 65)]) +@pytest.mark.parametrize("grid_file, num_levels", [(dt_utils.REGIONAL_EXPERIMENT, 65)]) def test_get_start_end_index_for_local_grid( grid_file, num_levels, dim, marker, start_index, end_index ): - file = resolve_file_from_gridfile_name(grid_file) + file = utils.resolve_file_from_gridfile_name(grid_file) from_grid_file = init_grid_manager(file, num_levels=num_levels).grid assert from_grid_file.get_start_index(dim, marker) == start_index assert from_grid_file.get_end_index(dim, marker) == end_index @@ -887,65 +868,65 @@ def test_get_start_end_index_for_local_grid( @pytest.mark.parametrize( "dim, marker,start_index,end_index", [ - (CellDim, HorizontalMarkerIndex.interior(CellDim), 0, 0), - (CellDim, HorizontalMarkerIndex.local(CellDim), 0, R02B04_GLOBAL_NUM_CELLS), - (CellDim, HorizontalMarkerIndex.nudging(CellDim), 0, 0), - (CellDim, HorizontalMarkerIndex.lateral_boundary(CellDim), 0, 0), + (dims.CellDim, h_grid.HorizontalMarkerIndex.interior(dims.CellDim), 0, 0), + (dims.CellDim, h_grid.HorizontalMarkerIndex.local(dims.CellDim), 0, R02B04_GLOBAL_NUM_CELLS), + (dims.CellDim, h_grid.HorizontalMarkerIndex.nudging(dims.CellDim), 0, 0), + (dims.CellDim, h_grid.HorizontalMarkerIndex.lateral_boundary(dims.CellDim), 0, 0), ( - CellDim, - HorizontalMarkerIndex.end(CellDim), + dims.CellDim, + h_grid.HorizontalMarkerIndex.end(dims.CellDim), R02B04_GLOBAL_NUM_CELLS, R02B04_GLOBAL_NUM_CELLS, ), ( - CellDim, - HorizontalMarkerIndex.halo(CellDim), + dims.CellDim, + h_grid.HorizontalMarkerIndex.halo(dims.CellDim), R02B04_GLOBAL_NUM_CELLS, R02B04_GLOBAL_NUM_CELLS, ), - (EdgeDim, HorizontalMarkerIndex.interior(EdgeDim), 0, 0), - (EdgeDim, HorizontalMarkerIndex.local(EdgeDim), 0, R02B04_GLOBAL_NUM_EDGES), - (EdgeDim, HorizontalMarkerIndex.nudging(EdgeDim), 0, 0), - (EdgeDim, HorizontalMarkerIndex.lateral_boundary(EdgeDim), 0, 0), + (dims.EdgeDim, h_grid.HorizontalMarkerIndex.interior(dims.EdgeDim), 0, 0), + (dims.EdgeDim, h_grid.HorizontalMarkerIndex.local(dims.EdgeDim), 0, R02B04_GLOBAL_NUM_EDGES), + (dims.EdgeDim, h_grid.HorizontalMarkerIndex.nudging(dims.EdgeDim), 0, 0), + (dims.EdgeDim, h_grid.HorizontalMarkerIndex.lateral_boundary(dims.EdgeDim), 0, 0), ( - EdgeDim, - HorizontalMarkerIndex.end(EdgeDim), + dims.EdgeDim, + h_grid.HorizontalMarkerIndex.end(dims.EdgeDim), R02B04_GLOBAL_NUM_EDGES, R02B04_GLOBAL_NUM_EDGES, ), ( - EdgeDim, - HorizontalMarkerIndex.halo(EdgeDim), + dims.EdgeDim, + h_grid.HorizontalMarkerIndex.halo(dims.EdgeDim), R02B04_GLOBAL_NUM_EDGES, R02B04_GLOBAL_NUM_EDGES, ), - (VertexDim, HorizontalMarkerIndex.interior(VertexDim), 0, 0), + (dims.VertexDim, h_grid.HorizontalMarkerIndex.interior(dims.VertexDim), 0, 0), ( - VertexDim, - HorizontalMarkerIndex.local(VertexDim), + dims.VertexDim, + h_grid.HorizontalMarkerIndex.local(dims.VertexDim), 0, R02B04_GLOBAL_NUM_VERTICES, ), - (VertexDim, HorizontalMarkerIndex.lateral_boundary(VertexDim), 0, 0), + (dims.VertexDim, h_grid.HorizontalMarkerIndex.lateral_boundary(dims.VertexDim), 0, 0), ( - VertexDim, - HorizontalMarkerIndex.end(VertexDim), + dims.VertexDim, + h_grid.HorizontalMarkerIndex.end(dims.VertexDim), R02B04_GLOBAL_NUM_VERTICES, R02B04_GLOBAL_NUM_VERTICES, ), ( - VertexDim, - HorizontalMarkerIndex.halo(VertexDim), + dims.VertexDim, + h_grid.HorizontalMarkerIndex.halo(dims.VertexDim), R02B04_GLOBAL_NUM_VERTICES, R02B04_GLOBAL_NUM_VERTICES, ), ], ) -@pytest.mark.parametrize("grid_file, num_levels", [(R02B04_GLOBAL, 80)]) +@pytest.mark.parametrize("grid_file, num_levels", [(utils.R02B04_GLOBAL, 80)]) def test_get_start_end_index_for_global_grid( grid_file, num_levels, dim, marker, start_index, end_index ): - file = resolve_file_from_gridfile_name(grid_file) + file = utils.resolve_file_from_gridfile_name(grid_file) from_grid_file = init_grid_manager(file, num_levels=num_levels).grid assert from_grid_file.get_start_index(dim, marker) == start_index assert from_grid_file.get_end_index(dim, marker) == end_index @@ -954,18 +935,18 @@ def test_get_start_end_index_for_global_grid( @pytest.mark.parametrize( "grid_file, global_num_cells", [ - (R02B04_GLOBAL, R02B04_GLOBAL_NUM_CELLS), - (REGIONAL_EXPERIMENT, MCH_CH_RO4B09_GLOBAL_NUM_CELLS), + (utils.R02B04_GLOBAL, R02B04_GLOBAL_NUM_CELLS), + (dt_utils.REGIONAL_EXPERIMENT, MCH_CH_RO4B09_GLOBAL_NUM_CELLS), ], ) def test_grid_level_and_root(grid_file, global_num_cells): - file = resolve_file_from_gridfile_name(grid_file) + file = utils.resolve_file_from_gridfile_name(grid_file) grid = init_grid_manager(file, num_levels=10).grid assert global_num_cells == grid.global_num_cells def test_c2e2c2e(simple_grid_gridfile): - simple_grid = SimpleGrid() + simple_grid = simple.SimpleGrid() gm = init_grid_manager( simple_grid_gridfile, num_levels=simple_grid.num_levels, @@ -980,11 +961,11 @@ def test_c2e2c2e(simple_grid_gridfile): @pytest.mark.with_netcdf @pytest.mark.parametrize( "grid_file, experiment", - [(R02B04_GLOBAL, JABW_EXPERIMENT)], + [(utils.R02B04_GLOBAL, dt_utils.JABW_EXPERIMENT)], ) def test_gridmanager_eval_c2e2c2e(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) - file = resolve_file_from_gridfile_name(grid_file) + file = utils.resolve_file_from_gridfile_name(grid_file) grid = init_grid_manager(file).grid serialized_grid = grid_savepoint.construct_icon_grid(on_gpu=False) assert np.allclose( From 639f21dbc44aa2edc477921647b66d6cbe8a59cf Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 26 Jul 2024 14:29:10 +0200 Subject: [PATCH 016/240] change halo constructor interface: pass connectivity list instead of grid --- .../model/common/decomposition/definitions.py | 2 + .../model/common/decomposition/halo.py | 68 ++++++++++++------- .../icon4py/model/common/grid/grid_manager.py | 44 ++++++++---- .../tests/decomposition_tests/test_halo.py | 20 ++++-- 4 files changed, 90 insertions(+), 44 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 54eeaf79c3..c932f6e4b4 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -34,6 +34,8 @@ class ProcessProperties(Protocol): rank: int comm_name: str comm_size: int + def single_node(self) -> bool: + return self.comm_size == 1 @dataclass(frozen=True, init=False) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 6edb643d54..64637463dc 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -20,7 +20,7 @@ import icon4py.model.common.decomposition.definitions as defs from icon4py.model.common import dimension as dims -from icon4py.model.common.grid import base as base_grid, grid_manager as gm, icon as icon_grid +from icon4py.model.common.grid import base as base_grid, icon as icon_grid from icon4py.model.common.settings import xp @@ -37,7 +37,7 @@ class DecompositionFlag(enum.IntEnum): SECOND_HALO_LINE = 2 -SKIP_VALUE = gm.GridFile.INVALID_INDEX +SKIP_VALUE = -1 class HaloGenerator: @@ -47,32 +47,47 @@ def __init__( self, run_properties: defs.ProcessProperties, rank_mapping: xp.ndarray, - ugrid: base_grid.BaseGrid, + connectivities:dict, + num_levels:int, ): """ Args: run_properties: contains information on the communicator and local compute node. rank_mapping: array with shape (global_num_cells,): mapping of global cell indices to their rank in the distribution - ugrid: the global grid + connectivities: connectivity arrays needed to construct the halos + num_levels: # TODO (@halungge): should not be needed here """ self._props = run_properties self._mapping = rank_mapping - self._global_grid = ugrid - self._connectivities = self._global_grid.connectivities + self._connectivities = connectivities + self._num_levels = num_levels + @property + def face_face_connectivity(self): + return self._connectivity(dims.C2E2CDim) + @property def edge_face_connectivity(self): - return self.connectivity(dims.E2CDim) - + return self._connectivity(dims.E2CDim) + @property - def face_face_connectivity(self): - return self.connectivity(dims.C2E2CDim) + def face_edge_connectivity(self): + return self._connectivity(dims.C2EDim) + @property def node_edge_connectivity(self): - return self.connectivity(dims.V2EDim) + return self._connectivity(dims.V2EDim) + @property + def node_face_connectivity(self): + return self._connectivity(dims.V2CDim) + + @property + def face_node_connectivity(self): + return self._connectivity(dims.C2VDim) + def _validate(self): assert self._mapping.ndim == 1 # the decomposition should match the communicator size @@ -81,7 +96,7 @@ def _validate(self): def _post_init(self): self._validate() - def connectivity(self, dim) -> xp.ndarray: + def _connectivity(self, dim) -> xp.ndarray: try: conn_table = self._connectivities[dim] return conn_table @@ -108,7 +123,7 @@ def next_halo_line(self, cell_line: xp.ndarray, depot=None): return next_halo_cells def _cell_neighbors(self, cells: xp.ndarray): - return xp.unique(self.connectivity(dims.C2E2CDim)[cells, :]) + return xp.unique(self.face_face_connectivity[cells, :]) def _find_neighbors(self, cell_line: xp.ndarray, connectivity: xp.ndarray) -> xp.ndarray: """Get a flattened list of all (unique) neighbors to a given global index list""" @@ -118,10 +133,12 @@ def _find_neighbors(self, cell_line: xp.ndarray, connectivity: xp.ndarray) -> xp return unique_neighbors def find_edge_neighbors_for_cells(self, cell_line: xp.ndarray) -> xp.ndarray: - return self._find_neighbors(cell_line, connectivity=self.connectivity(dims.C2EDim)) + return self._find_neighbors(cell_line, connectivity=self.face_edge_connectivity) + + def find_vertex_neighbors_for_cells(self, cell_line: xp.ndarray) -> xp.ndarray: - return self._find_neighbors(cell_line, connectivity=self.connectivity(dims.C2VDim)) + return self._find_neighbors(cell_line, connectivity=self.face_node_connectivity) def owned_cells(self) -> xp.ndarray: """Returns the global indices of the cells owned by this rank""" @@ -147,7 +164,7 @@ def construct_decomposition_info(self) -> defs.DecompositionInfo: c_owner_mask = xp.isin(all_cells, owned_cells) - decomp_info = defs.DecompositionInfo(klevels=self._global_grid.num_levels).with_dimension( + decomp_info = defs.DecompositionInfo(klevels=self._num_levels).with_dimension( dims.CellDim, all_cells, c_owner_mask ) @@ -226,10 +243,12 @@ def _update_owner_mask_by_max_rank_convention( all_vertices = xp.unique(xp.hstack((vertices_on_owned_cells, vertices_on_first_halo_line))) v_owner_mask = xp.isin(all_vertices, vertices_on_owned_cells) v_owner_mask = _update_owner_mask_by_max_rank_convention( - v_owner_mask, all_vertices, intersect_owned_first_line, self.connectivity(dims.V2CDim) + v_owner_mask, all_vertices, intersect_owned_first_line, self.node_face_connectivity ) decomp_info.with_dimension(dims.VertexDim, all_vertices, v_owner_mask) return decomp_info + + def construct_local_connectivity(self, field_offset: gtx.FieldOffset, decomposition_info: defs.DecompositionInfo) -> xp.ndarray: @@ -247,11 +266,13 @@ def construct_local_connectivity(self, Returns: connectivity are for the same FieldOffset but mapping from local target dimension indices to local source dimension indices. + + # TODO (@halungge): this should be done in the grid manager """ source_dim = field_offset.source target_dim = field_offset.target[0] local_dim = field_offset.target[1] - sliced_connectivity = self.connectivity(local_dim)[ + sliced_connectivity = self._connectivity(local_dim)[ decomposition_info.global_index(target_dim, defs.DecompositionInfo.EntryType.ALL) ] log.debug(f"rank {self._props.rank} has local connectivity f: {sliced_connectivity}") @@ -283,8 +304,7 @@ def construct_local_connectivity(self, def local_grid(self, props: defs.ProcessProperties, decomposition_info: defs.DecompositionInfo, - global_params: icon_grid.GlobalGridParams, - num_lev: int, + limited_area: bool = False, on_gpu: bool = False, ) -> base_grid.BaseGrid: @@ -311,11 +331,11 @@ def local_grid(self, num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells ) config = base_grid.GridConfig( - horizontal_config=grid_size, vertical_size=num_lev, on_gpu=on_gpu, limited_area=limited_area + horizontal_config=grid_size, vertical_size=self._num_levels, on_gpu=on_gpu, limited_area=limited_area ) local_grid = ( - icon_grid.IconGrid(uuid.uuid4()).with_config(config).with_global_params(global_params).with_connectivities({ + icon_grid.IconGrid(uuid.uuid4()).with_config(config).with_connectivities({ dims.C2EDim: self.construct_local_connectivity(dims.C2EDim, decomposition_info), dims.E2CDim: self.construct_local_connectivity(dims.E2CDim, decomposition_info), dims.E2VDim: self.construct_local_connectivity(dims.E2VDim, decomposition_info), @@ -324,8 +344,8 @@ def local_grid(self, dims.V2CDim: self.construct_local_connectivity(dims.V2CDim, decomposition_info), dims.C2E2CDim: self.construct_local_connectivity(dims.C2E2CDim, decomposition_info), # dims.C2E2CODim - - + # etc... # TODO (@halungge): this should all go to grid_manager + }) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index d2f9f7e01a..09e7084c91 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -19,7 +19,7 @@ import gt4py.next as gtx import numpy as np -from icon4py.model.common.decomposition import definitions as defs +from icon4py.model.common.decomposition import definitions as decomposition, halo try: @@ -252,15 +252,18 @@ class GridManager: def __init__( self, + transformation: IndexTransformation, grid_file: Union[pathlib.Path,str], - config: v_grid.VerticalGridConfig, + config: v_grid.VerticalGridConfig, # TODO (@halungge) remove + run_properties: decomposition.ProcessProperties = decomposition.SingleNodeProcessProperties(), ): self._log = logging.getLogger(__name__) + self._run_properties = run_properties self._transformation = transformation - self._config = config # TODO (@halungge) remove - self._grid: Optional[icon_grid.IconGrid] = None + self._config = config self._file_name = str(grid_file) + self._grid: Optional[icon_grid.IconGrid] = None self._dataset = None self._reader = None @@ -329,15 +332,15 @@ def _read_grid_refinement_information(self, dataset): def _read( self, - decomposition_info: defs.DecompositionInfo, + decomposition_info: decomposition.DecompositionInfo, fields: dict[dims.Dimension, Sequence[GridFileName]], ): (cells_on_node, edges_on_node, vertices_on_node) = ( ( - decomposition_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.ALL), - decomposition_info.global_index(dims.EdgeDim, defs.DecompositionInfo.EntryType.ALL), + decomposition_info.global_index(dims.CellDim, decomposition.DecompositionInfo.EntryType.ALL), + decomposition_info.global_index(dims.EdgeDim, decomposition.DecompositionInfo.EntryType.ALL), decomposition_info.global_index( - dims.VertexDim, defs.DecompositionInfo.EntryType.ALL + dims.VertexDim, decomposition.DecompositionInfo.EntryType.ALL ), ) if decomposition_info is not None @@ -358,7 +361,7 @@ def _read_local(fields: dict[dims.Dimension, Sequence[GridFileName]]): return _read_local(fields) - def read_geometry(self, decomposition_info: Optional[defs.DecompositionInfo] = None): + def read_geometry(self, decomposition_info: Optional[decomposition.DecompositionInfo] = None): return self._read( decomposition_info, { @@ -367,7 +370,7 @@ def read_geometry(self, decomposition_info: Optional[defs.DecompositionInfo] = N }, ) - def read_coordinates(self, decomposition_info: Optional[defs.DecompositionInfo] = None): + def read_coordinates(self, decomposition_info: Optional[decomposition.DecompositionInfo] = None): return self._read( decomposition_info, { @@ -403,8 +406,23 @@ def _get_index(self, dim: gtx.Dimension, start_marker: int, index_dict): raise IconGridError(msg) from err def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGrid: - return self._from_grid_dataset(on_gpu=on_gpu, limited_area=limited_area) - + if self._run_properties.single_node(): + return self._from_grid_dataset(on_gpu=on_gpu, limited_area=limited_area) + else: + # TODO decompose! + decompose = halo.SimpleMetisDecomposer(self._run_properties) + cell_neighbors = self._reader.int_field(GridFile.ConnectivityName.C2E2C) + cells_to_rank_mapping = decompose(cell_neighbors, self._run_properties.comm_size) + global_connectivities = {dims.C2E2CDim: cell_neighbors, + dims.C2EDim: self._reader.int_field(GridFile.ConnectivityName.C2E), + dims.E2CDim: self._reader.int_field(GridFile.ConnectivityName.E2C), + dims.V2EDim: self._reader.int_field(GridFile.ConnectivityName.V2E), + dims.V2CDim: self._reader.int_field(GridFile.ConnectivityName.V2C), + dims.C2VDim: self._reader.int_field(GridFile.ConnectivityName.C2V), + } + + global_grid = icon_grid.IconGrid().with_connectivities(global_connectivities) + halo_constructor = halo.HaloGenerator(self._run_properties, cells_to_rank_mapping, global_connectivities, self._config.num_levels) def get_size(self, dim: gtx.Dimension): if dim == dims.VertexDim: return self._grid.config.num_vertices @@ -423,7 +441,7 @@ def _get_index_field(self, field: GridFileName, transpose=True, apply_offset=Tru field = field + self._transformation.get_offset_for_index_field(field) return field - def _from_decomposition_info(self, decomposition_info: defs.DecompositionInfo, on_gpu: bool = False, limited_area:bool = False)-> base_grid.BaseGrid: + def _from_decomposition_info(self, decomposition_info: decomposition.DecompositionInfo, on_gpu: bool = False, limited_area:bool = False)-> base_grid.BaseGrid: pass def _from_grid_dataset(self, on_gpu: bool, limited_area=True) -> icon_grid.IconGrid: diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index e94d13f62e..e0f551d172 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -131,9 +131,10 @@ def test_halo_constructor_owned_cells(processor_props): # fixture grid = simple.SimpleGrid() halo_generator = HaloGenerator( - ugrid=grid, + connectivities=grid.connectivities, run_properties=processor_props, rank_mapping=simple_distribution, + num_levels=1, ) my_owned_cells = halo_generator.owned_cells() @@ -149,9 +150,10 @@ def test_element_ownership_is_unique(dim, processor_props): # fixture pytest.skip("This test requires exactly 4 MPI ranks.") grid = simple.SimpleGrid() halo_generator = HaloGenerator( - ugrid=grid, + connectivities=grid.connectivities, run_properties=processor_props, rank_mapping=simple_distribution, + num_levels=1, ) decomposition_info = halo_generator.construct_decomposition_info() @@ -195,9 +197,10 @@ def test_halo_constructor_decomposition_info(processor_props, dim): # fixture grid = simple.SimpleGrid() halo_generator = HaloGenerator( - ugrid=grid, + connectivities=grid.connectivities, run_properties=processor_props, rank_mapping=simple_distribution, + num_levels=1, ) decomp_info = halo_generator.construct_decomposition_info() @@ -220,9 +223,10 @@ def test_local_connectivities(processor_props, caplog, field_offset): # fixture grid = grid_file_manager(GRID_FILE).grid labels = decompose(grid, processor_props) halo_generator = HaloGenerator( - ugrid=grid, + connectivities=grid.connectivities, run_properties=processor_props, rank_mapping=labels, + num_levels=1, ) decomposition_info = halo_generator.construct_decomposition_info() @@ -285,16 +289,17 @@ def gather_field(field: xp.ndarray, comm: mpi4py.MPI.Comm) -> tuple: comm.Gatherv(sendbuf=field, recvbuf=(recv_buffer, local_sizes), root=0) return local_sizes, recv_buffer - +@pytest.mark.xfail(reason="Not implemented yet") def test_local_grid(processor_props, caplog): # fixture caplog.set_level(logging.INFO) grid = grid_file_manager(GRID_FILE).grid labels = decompose(grid, processor_props) halo_generator = HaloGenerator( - ugrid=grid, + connectivities=grid.connectivities, run_properties=processor_props, rank_mapping=labels, + num_levels=1, ) decomposition_info = halo_generator.construct_decomposition_info() local_grid = halo_generator.local_grid(decomposition_info) @@ -313,9 +318,10 @@ def test_distributed_fields(processor_props): # fixture labels = decompose(global_grid, processor_props) halo_generator = HaloGenerator( - ugrid=global_grid, + connectivities=global_grid.connectivities, run_properties=processor_props, rank_mapping=labels, + num_levels=1, ) decomposition_info = halo_generator.construct_decomposition_info() # distributed read: read one field per dimension From 6a1cd8c8e771bfebb9efeb55e91a7b3bdeb88722 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 26 Jul 2024 15:03:05 +0200 Subject: [PATCH 017/240] move construction of local connectivity to grid_manager.py --- .../model/common/decomposition/halo.py | 114 --------- .../icon4py/model/common/grid/grid_manager.py | 217 ++++++++++++++---- .../tests/decomposition_tests/test_halo.py | 38 +-- .../tests/grid_tests/test_grid_manager.py | 41 +++- 4 files changed, 213 insertions(+), 197 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 64637463dc..befa568449 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -13,14 +13,10 @@ import enum import logging -import uuid from typing import Protocol -import gt4py.next as gtx - import icon4py.model.common.decomposition.definitions as defs from icon4py.model.common import dimension as dims -from icon4py.model.common.grid import base as base_grid, icon as icon_grid from icon4py.model.common.settings import xp @@ -37,9 +33,6 @@ class DecompositionFlag(enum.IntEnum): SECOND_HALO_LINE = 2 -SKIP_VALUE = -1 - - class HaloGenerator: """Creates necessary halo information for a given rank.""" @@ -247,113 +240,6 @@ def _update_owner_mask_by_max_rank_convention( ) decomp_info.with_dimension(dims.VertexDim, all_vertices, v_owner_mask) return decomp_info - - - - def construct_local_connectivity(self, - field_offset: gtx.FieldOffset, decomposition_info: defs.DecompositionInfo) -> xp.ndarray: - """ - Construct a connectivity table for use on a given rank: it maps from source to target dimension in _local_ indices. - - Starting from the connectivity table on the global grid - - we reduce it to the lines for the locally present entries of the the target dimension - - the reduced connectivity then still maps to global source dimension indices: - we replace those source dimension indices not present on the node to SKIP_VALUE and replace the rest with the local indices - - Args: - field_offset: FieldOffset for which we want to construct the local connectivity table - decomposition_info: DecompositionInfo for the current rank. - - Returns: - connectivity are for the same FieldOffset but mapping from local target dimension indices to local source dimension indices. - - # TODO (@halungge): this should be done in the grid manager - """ - source_dim = field_offset.source - target_dim = field_offset.target[0] - local_dim = field_offset.target[1] - sliced_connectivity = self._connectivity(local_dim)[ - decomposition_info.global_index(target_dim, defs.DecompositionInfo.EntryType.ALL) - ] - log.debug(f"rank {self._props.rank} has local connectivity f: {sliced_connectivity}") - - global_idx = decomposition_info.global_index( - source_dim, defs.DecompositionInfo.EntryType.ALL - ) - - # replace indices in the connectivity that do not exist on the local node by the SKIP_VALUE (those are for example neighbors of the outermost halo points) - local_connectivity = xp.where( - xp.isin(sliced_connectivity, global_idx), sliced_connectivity, SKIP_VALUE - ) - - # map to local source indices - sorted_index_of_global_idx = xp.argsort(global_idx) - global_idx_sorted = global_idx[sorted_index_of_global_idx] - for i in xp.arange(local_connectivity.shape[0]): - valid_neighbor_mask = local_connectivity[i, :] != SKIP_VALUE - positions = xp.searchsorted( - global_idx_sorted, local_connectivity[i, valid_neighbor_mask] - ) - indices = sorted_index_of_global_idx[positions] - local_connectivity[i, valid_neighbor_mask] = indices - log.debug(f"rank {self._props.rank} has local connectivity f: {local_connectivity}") - return local_connectivity - - -# should be done in grid manager! - def local_grid(self, - props: defs.ProcessProperties, - decomposition_info: defs.DecompositionInfo, - - limited_area: bool = False, - on_gpu: bool = False, - ) -> base_grid.BaseGrid: - - """ - Constructs a local grid for this rank based on the decomposition info. - TODO (@halungge): for now only returning BaseGrid as we have not start/end indices implementation yet - TODO (@halungge): make sure the INVALID_INDEX is set correctly: - when set in the original (global index) connectivity it should remain - TODO (@halungge): how to handle the (source) indices of last halo line: their (global) neighbors are not all present on the local grid, set INVALID_INDEX (that is what xugrid does) - check what ICON does, (they probably duplicate the valid indices...) - Args: - decomposition_info: the decomposition info for this rank - Returns: - local_grid: the local grid - """ - num_vertices = decomposition_info.global_index( - dims.VertexDim, defs.DecompositionInfo.EntryType.ALL - ).size - num_edges = decomposition_info.global_index(dims.EdgeDim, - defs.DecompositionInfo.EntryType.ALL).size - num_cells = decomposition_info.global_index(dims.CellDim, - defs.DecompositionInfo.EntryType.ALL).size - grid_size = base_grid.HorizontalGridSize( - num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells - ) - config = base_grid.GridConfig( - horizontal_config=grid_size, vertical_size=self._num_levels, on_gpu=on_gpu, limited_area=limited_area - ) - - local_grid = ( - icon_grid.IconGrid(uuid.uuid4()).with_config(config).with_connectivities({ - dims.C2EDim: self.construct_local_connectivity(dims.C2EDim, decomposition_info), - dims.E2CDim: self.construct_local_connectivity(dims.E2CDim, decomposition_info), - dims.E2VDim: self.construct_local_connectivity(dims.E2VDim, decomposition_info), - dims.V2EDim: self.construct_local_connectivity(dims.V2EDim, decomposition_info), - dims.C2VDim: self.construct_local_connectivity(dims.C2VDim, decomposition_info), - dims.V2CDim: self.construct_local_connectivity(dims.V2CDim, decomposition_info), - dims.C2E2CDim: self.construct_local_connectivity(dims.C2E2CDim, decomposition_info), - # dims.C2E2CODim - # etc... # TODO (@halungge): this should all go to grid_manager - - - - }) - ) - - - - return local_grid # TODO (@halungge): refine type hints: adjacency_matrix should be a connectivity matrix of C2E2C and diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 09e7084c91..030e633fd2 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -14,12 +14,19 @@ import enum import logging import pathlib +import uuid from typing import Optional, Sequence, Union import gt4py.next as gtx import numpy as np -from icon4py.model.common.decomposition import definitions as decomposition, halo +from icon4py.model.common import dimension as dims +from icon4py.model.common.decomposition import ( + definitions as decomposition, + definitions as defs, + halo, +) +from icon4py.model.common.settings import xp try: @@ -33,7 +40,6 @@ def __init__(self, *args, **kwargs): raise ModuleNotFoundError("NetCDF4 is not installed.") -import icon4py.model.common.dimension as dims from icon4py.model.common.grid import ( base as base_grid, icon as icon_grid, @@ -409,20 +415,26 @@ def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGri if self._run_properties.single_node(): return self._from_grid_dataset(on_gpu=on_gpu, limited_area=limited_area) else: - # TODO decompose! - decompose = halo.SimpleMetisDecomposer(self._run_properties) - cell_neighbors = self._reader.int_field(GridFile.ConnectivityName.C2E2C) - cells_to_rank_mapping = decompose(cell_neighbors, self._run_properties.comm_size) - global_connectivities = {dims.C2E2CDim: cell_neighbors, - dims.C2EDim: self._reader.int_field(GridFile.ConnectivityName.C2E), - dims.E2CDim: self._reader.int_field(GridFile.ConnectivityName.E2C), - dims.V2EDim: self._reader.int_field(GridFile.ConnectivityName.V2E), - dims.V2CDim: self._reader.int_field(GridFile.ConnectivityName.V2C), - dims.C2VDim: self._reader.int_field(GridFile.ConnectivityName.C2V), - } + decompose = halo.SimpleMetisDecomposer() + + global_connectivities = { + dims.C2E2CDim: self._reader.int_field(GridFile.ConnectivityName.C2E2C), + dims.C2EDim: self._reader.int_field(GridFile.ConnectivityName.C2E), + dims.E2CDim: self._reader.int_field(GridFile.ConnectivityName.E2C), + dims.V2EDim: self._reader.int_field(GridFile.ConnectivityName.V2E), + dims.E2VDim: self._reader.int_field(GridFile.ConnectivityName.E2V), + dims.V2CDim: self._reader.int_field(GridFile.ConnectivityName.V2C), + dims.C2VDim: self._reader.int_field(GridFile.ConnectivityName.C2V), + dims.V2E2VDim: self._reader.int_field(GridFile.ConnectivityName.V2E2V), + } + cells_to_rank_mapping = decompose( + global_connectivities[dims.C2EDim], self._run_properties.comm_size) + halo_constructor = halo.HaloGenerator(self._run_properties, cells_to_rank_mapping, + global_connectivities, self._config.num_levels) + decomposition_info = halo_constructor.construct_decomposition_info() + self._from_decomposition_info(decomposition_info, on_gpu=on_gpu, limited_area=limited_area) - global_grid = icon_grid.IconGrid().with_connectivities(global_connectivities) - halo_constructor = halo.HaloGenerator(self._run_properties, cells_to_rank_mapping, global_connectivities, self._config.num_levels) + def get_size(self, dim: gtx.Dimension): if dim == dims.VertexDim: return self._grid.config.num_vertices @@ -442,22 +454,15 @@ def _get_index_field(self, field: GridFileName, transpose=True, apply_offset=Tru return field def _from_decomposition_info(self, decomposition_info: decomposition.DecompositionInfo, on_gpu: bool = False, limited_area:bool = False)-> base_grid.BaseGrid: - pass + grid = self._initialize_global(limited_area, on_gpu) + + + def _from_grid_dataset(self, on_gpu: bool, limited_area=True) -> icon_grid.IconGrid: - num_cells = self._reader.dimension(GridFile.DimensionName.CELL_NAME) - num_edges = self._reader.dimension(GridFile.DimensionName.EDGE_NAME) - num_vertices = self._reader.dimension(GridFile.DimensionName.VERTEX_NAME) - uuid = self._dataset.getncattr(GridFile.PropertyName.GRID_ID) - grid_level = self._dataset.getncattr(GridFile.PropertyName.LEVEL) - grid_root = self._dataset.getncattr(GridFile.PropertyName.ROOT) - global_params = icon_grid.GlobalGridParams(level=grid_level, root=grid_root) + grid = self._initialize_global(limited_area, on_gpu) - grid_size = base_grid.HorizontalGridSize( - num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells - ) c2e = self._get_index_field(GridFile.ConnectivityName.C2E) - e2c = self._get_index_field( GridFile.ConnectivityName.E2C) # edge_face_connectivity (optional) c2v = self._get_index_field( @@ -479,24 +484,9 @@ def _from_grid_dataset(self, on_gpu: bool, limited_area=True) -> icon_grid.IconG c2e2c2e = self._construct_triangle_edges(c2e2c, c2e) c2e2c0 = np.column_stack((np.asarray(range(c2e2c.shape[0])), c2e2c)) - ( - start_indices, - end_indices, - refine_ctrl, - refine_ctrl_max, - ) = self._read_grid_refinement_information(self._dataset) + - config = base_grid.GridConfig( - horizontal_config=grid_size, - vertical_size=self._config.num_levels, - on_gpu=on_gpu, - limited_area=limited_area, - ) - grid = ( - icon_grid.IconGrid(uuid) - .with_config(config) - .with_global_params(global_params) - .with_connectivities( + grid.with_connectivities( { dims.C2EDim: c2e, dims.E2CDim: e2c, @@ -513,10 +503,14 @@ def _from_grid_dataset(self, on_gpu: bool, limited_area=True) -> icon_grid.IconG dims.E2C2EODim: e2c2e0, } ) - .with_start_end_indices(dims.CellDim, start_indices[dims.CellDim], end_indices[dims.CellDim]) - .with_start_end_indices(dims.EdgeDim, start_indices[dims.EdgeDim], end_indices[dims.EdgeDim]) - .with_start_end_indices(dims.VertexDim, start_indices[dims.VertexDim], end_indices[dims.VertexDim]) - ) + ( + start_indices, + end_indices, + refine_ctrl, + refine_ctrl_max, + ) = self._read_grid_refinement_information(self._dataset) + grid.with_start_end_indices(dims.CellDim, start_indices[dims.CellDim], end_indices[dims.CellDim]).with_start_end_indices(dims.EdgeDim, start_indices[dims.EdgeDim], end_indices[dims.EdgeDim]).with_start_end_indices(dims.VertexDim, start_indices[dims.VertexDim], end_indices[dims.VertexDim]) + grid.update_size_connectivities( { dims.ECVDim: grid.size[dims.EdgeDim] * grid.size[dims.E2C2VDim], @@ -527,6 +521,26 @@ def _from_grid_dataset(self, on_gpu: bool, limited_area=True) -> icon_grid.IconG return grid + def _initialize_global(self, limited_area, on_gpu): + num_cells = self._reader.dimension(GridFile.DimensionName.CELL_NAME) + num_edges = self._reader.dimension(GridFile.DimensionName.EDGE_NAME) + num_vertices = self._reader.dimension(GridFile.DimensionName.VERTEX_NAME) + uuid = self._dataset.getncattr(GridFile.PropertyName.GRID_ID) + grid_level = self._dataset.getncattr(GridFile.PropertyName.LEVEL) + grid_root = self._dataset.getncattr(GridFile.PropertyName.ROOT) + global_params = icon_grid.GlobalGridParams(level=grid_level, root=grid_root) + grid_size = base_grid.HorizontalGridSize( + num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells + ) + config = base_grid.GridConfig( + horizontal_config=grid_size, + vertical_size=self._config.num_levels, + on_gpu=on_gpu, + limited_area=limited_area, + ) + grid = icon_grid.IconGrid(uuid).with_config(config).with_global_params(global_params) + return grid + @staticmethod def _construct_diamond_vertices( e2v: np.ndarray, c2v: np.ndarray, e2c: np.ndarray @@ -653,3 +667,110 @@ def _patch_with_dummy_lastline(ar): axis=0, ) return patched_ar + + +SKIP_VALUE = -1 + + +def construct_local_connectivity( + field_offset: gtx.FieldOffset, decomposition_info: defs.DecompositionInfo, connectivities:dict) -> xp.ndarray: + """ + Construct a connectivity table for use on a given rank: it maps from source to target dimension in _local_ indices. + + Starting from the connectivity table on the global grid + - we reduce it to the lines for the locally present entries of the the target dimension + - the reduced connectivity then still maps to global source dimension indices: + we replace those source dimension indices not present on the node to SKIP_VALUE and replace the rest with the local indices + + Args: + field_offset: FieldOffset for which we want to construct the local connectivity table + decomposition_info: DecompositionInfo for the current rank. + + Returns: + connectivity are for the same FieldOffset but mapping from local target dimension indices to local source dimension indices. + """ + source_dim = field_offset.source + target_dim = field_offset.target[0] + local_dim = field_offset.target[1] + sliced_connectivity = connectivities[local_dim][ + decomposition_info.global_index(target_dim, defs.DecompositionInfo.EntryType.ALL) + ] + #log.debug(f"rank {self._props.rank} has local connectivity f: {sliced_connectivity}") + + global_idx = decomposition_info.global_index( + source_dim, defs.DecompositionInfo.EntryType.ALL + ) + + # replace indices in the connectivity that do not exist on the local node by the SKIP_VALUE (those are for example neighbors of the outermost halo points) + local_connectivity = xp.where( + xp.isin(sliced_connectivity, global_idx), sliced_connectivity, SKIP_VALUE + ) + + # map to local source indices + sorted_index_of_global_idx = xp.argsort(global_idx) + global_idx_sorted = global_idx[sorted_index_of_global_idx] + for i in xp.arange(local_connectivity.shape[0]): + valid_neighbor_mask = local_connectivity[i, :] != SKIP_VALUE + positions = xp.searchsorted( + global_idx_sorted, local_connectivity[i, valid_neighbor_mask] + ) + indices = sorted_index_of_global_idx[positions] + local_connectivity[i, valid_neighbor_mask] = indices + #log.debug(f"rank {self._props.rank} has local connectivity f: {local_connectivity}") + return local_connectivity + + + # should be done in grid manager! + def local_grid(self, + props: defs.ProcessProperties, + decomposition_info: defs.DecompositionInfo, + + limited_area: bool = False, + on_gpu: bool = False, + ) -> base_grid.BaseGrid: + + """ + Constructs a local grid for this rank based on the decomposition info. + TODO (@halungge): for now only returning BaseGrid as we have not start/end indices implementation yet + TODO (@halungge): make sure the INVALID_INDEX is set correctly: - when set in the original (global index) connectivity it should remain + TODO (@halungge): how to handle the (source) indices of last halo line: their (global) neighbors are not all present on the local grid, set INVALID_INDEX (that is what xugrid does) + check what ICON does, (they probably duplicate the valid indices...) + Args: + decomposition_info: the decomposition info for this rank + Returns: + local_grid: the local grid + """ + num_vertices = decomposition_info.global_index( + dims.VertexDim, defs.DecompositionInfo.EntryType.ALL + ).size + num_edges = decomposition_info.global_index(dims.EdgeDim, + defs.DecompositionInfo.EntryType.ALL).size + num_cells = decomposition_info.global_index(dims.CellDim, + defs.DecompositionInfo.EntryType.ALL).size + grid_size = base_grid.HorizontalGridSize( + num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells + ) + config = base_grid.GridConfig( + horizontal_config=grid_size, vertical_size=self._num_levels, on_gpu=on_gpu, limited_area=limited_area + ) + + local_grid = ( + icon_grid.IconGrid(uuid.uuid4()).with_config(config).with_connectivities({ + dims.C2EDim: self.construct_local_connectivity(dims.C2EDim, decomposition_info), + dims.E2CDim: self.construct_local_connectivity(dims.E2CDim, decomposition_info), + dims.E2VDim: self.construct_local_connectivity(dims.E2VDim, decomposition_info), + dims.V2EDim: self.construct_local_connectivity(dims.V2EDim, decomposition_info), + dims.C2VDim: self.construct_local_connectivity(dims.C2VDim, decomposition_info), + dims.V2CDim: self.construct_local_connectivity(dims.V2CDim, decomposition_info), + dims.C2E2CDim: self.construct_local_connectivity(dims.C2E2CDim, decomposition_info), + # dims.C2E2CODim + # etc... # TODO (@halungge): this should all go to grid_manager + + + + }) + ) + + + + return local_grid diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index e0f551d172..4a881aecec 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -20,7 +20,10 @@ import icon4py.model.common.dimension as dims import icon4py.model.common.grid.grid_manager as gm from icon4py.model.common.decomposition import definitions as defs -from icon4py.model.common.decomposition.halo import HaloGenerator, SimpleMetisDecomposer +from icon4py.model.common.decomposition.halo import ( + HaloGenerator, + SimpleMetisDecomposer, +) from icon4py.model.common.grid import base as base_grid, simple, vertical as v_grid from icon4py.model.common.settings import xp from icon4py.model.common.test_utils import datatest_utils as dt_utils, helpers @@ -214,39 +217,6 @@ def test_halo_constructor_decomposition_info(processor_props, dim): # fixture assert xp.setdiff1d(my_owned, owned[dim][processor_props.rank], assume_unique=True).size == 0 -@pytest.mark.parametrize( - "field_offset", - [dims.C2V, dims.E2V, dims.V2C, dims.E2C, dims.C2E, dims.V2E, dims.C2E2C, dims.V2E2V], -) -def test_local_connectivities(processor_props, caplog, field_offset): # fixture - caplog.set_level(logging.INFO) - grid = grid_file_manager(GRID_FILE).grid - labels = decompose(grid, processor_props) - halo_generator = HaloGenerator( - connectivities=grid.connectivities, - run_properties=processor_props, - rank_mapping=labels, - num_levels=1, - ) - - decomposition_info = halo_generator.construct_decomposition_info() - - connectivity = halo_generator.construct_local_connectivity(field_offset, decomposition_info) - # there is an neighbor list for each index of the target dimension on the node - assert ( - connectivity.shape[0] - == decomposition_info.global_index( - field_offset.target[0], defs.DecompositionInfo.EntryType.ALL - ).size - ) - # all neighbor indices are valid local indices - assert xp.max(connectivity) == xp.max( - decomposition_info.local_index(field_offset.source, defs.DecompositionInfo.EntryType.ALL) - ) - # TODO what else to assert? - # outer halo entries have SKIP_VALUE neighbors (depends on offsets) - - def grid_file_manager(file: pathlib.Path) -> gm.GridManager: manager = gm.GridManager( gm.ToZeroBasedIndexTransformation(), str(file), v_grid.VerticalGridConfig(num_levels=1) diff --git a/model/common/tests/grid_tests/test_grid_manager.py b/model/common/tests/grid_tests/test_grid_manager.py index 5b32d4c190..3901b37af7 100644 --- a/model/common/tests/grid_tests/test_grid_manager.py +++ b/model/common/tests/grid_tests/test_grid_manager.py @@ -21,6 +21,10 @@ import pytest import icon4py.model.common.test_utils.datatest_utils as dt_utils +from icon4py.model.common import dimension as dims +from icon4py.model.common.decomposition import definitions as defs, halo +from icon4py.model.common.decomposition.halo import HaloGenerator +from icon4py.model.common.settings import xp if typing.TYPE_CHECKING: @@ -31,7 +35,6 @@ except ImportError: pytest.skip("optional netcdf dependency not installed", allow_module_level=True) -import icon4py.model.common.dimension as dims from icon4py.model.common.grid import horizontal as h_grid, simple, vertical as v_grid from icon4py.model.common.grid.grid_manager import ( GridFile, @@ -39,6 +42,7 @@ GridManager, IndexTransformation, ToZeroBasedIndexTransformation, + construct_local_connectivity, ) from . import utils @@ -973,3 +977,38 @@ def test_gridmanager_eval_c2e2c2e(caplog, grid_savepoint, grid_file): serialized_grid.get_offset_provider("C2E2C2E").table, ) assert grid.get_offset_provider("C2E2C2E").table.shape == (grid.num_cells, 9) + +@pytest.mark.with_mpi +@pytest.mark.parametrize( + "field_offset", + [dims.C2V, dims.E2V, dims.V2C, dims.E2C, dims.C2E, dims.V2E, dims.C2E2C, dims.V2E2V], +) +def test_local_connectivities(processor_props, caplog, field_offset): # fixture + caplog.set_level(logging.INFO) + file = utils.resolve_file_from_gridfile_name(utils.R02B04_GLOBAL) + grid = init_grid_manager(file).grid + partitioner = halo.SimpleMetisDecomposer() + labels = partitioner(grid.connectivities[dims.C2E2CDim], n_part=processor_props.comm_size) + halo_generator = HaloGenerator( + connectivities=grid.connectivities, + run_properties=processor_props, + rank_mapping=labels, + num_levels=1, + ) + + decomposition_info = halo_generator.construct_decomposition_info() + + connectivity = construct_local_connectivity(field_offset, decomposition_info, connectivities=grid.connectivities) + # there is an neighbor list for each index of the target dimension on the node + assert ( + connectivity.shape[0] + == decomposition_info.global_index( + field_offset.target[0], defs.DecompositionInfo.EntryType.ALL + ).size + ) + # all neighbor indices are valid local indices + assert xp.max(connectivity) == xp.max( + decomposition_info.local_index(field_offset.source, defs.DecompositionInfo.EntryType.ALL) + ) + # TODO what else to assert? + # outer halo entries have SKIP_VALUE neighbors (depends on offsets) From a6c94b73bd23304562005683f73fecc2d5081fdc Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 26 Jul 2024 17:34:58 +0200 Subject: [PATCH 018/240] WIP --- .../model/common/decomposition/halo.py | 11 +- .../src/icon4py/model/common/exceptions.py | 4 + .../icon4py/model/common/grid/grid_manager.py | 204 +++++++++++------- .../tests/grid_tests/test_grid_manager.py | 2 +- 4 files changed, 134 insertions(+), 87 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index befa568449..0ef9f5811e 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -15,14 +15,17 @@ import logging from typing import Protocol +import gt4py.next as gtx + import icon4py.model.common.decomposition.definitions as defs -from icon4py.model.common import dimension as dims +from icon4py.model.common import dimension as dims, exceptions from icon4py.model.common.settings import xp log = logging.getLogger(__name__) + # TODO (@halungge) do we need three of those: one for each dimension? class DecompositionFlag(enum.IntEnum): #: cell is owned by this rank @@ -40,7 +43,7 @@ def __init__( self, run_properties: defs.ProcessProperties, rank_mapping: xp.ndarray, - connectivities:dict, + connectivities:dict[gtx.Dimension, xp.ndarray], num_levels:int, ): """ @@ -89,12 +92,12 @@ def _validate(self): def _post_init(self): self._validate() - def _connectivity(self, dim) -> xp.ndarray: + def _connectivity(self, dim: gtx.Dimension) -> xp.ndarray: try: conn_table = self._connectivities[dim] return conn_table except KeyError as err: - raise (f"Connectivity for dimension {dim} is not available") from err + raise exceptions.MissingConnectivity(f"Connectivity for offset {dim} is not available") from err def next_halo_line(self, cell_line: xp.ndarray, depot=None): """Returns the global indices of the next halo line. diff --git a/model/common/src/icon4py/model/common/exceptions.py b/model/common/src/icon4py/model/common/exceptions.py index 5b8ab21635..ef604db2cd 100644 --- a/model/common/src/icon4py/model/common/exceptions.py +++ b/model/common/src/icon4py/model/common/exceptions.py @@ -19,3 +19,7 @@ class InvalidConfigError(Exception): class IncompleteStateError(Exception): def __init__(self, field_name): super().__init__(f"Field '{field_name}' is missing in state.") + + +class MissingConnectivity(ValueError): + pass \ No newline at end of file diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 030e633fd2..691b6081cf 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -14,7 +14,6 @@ import enum import logging import pathlib -import uuid from typing import Optional, Sequence, Union import gt4py.next as gtx @@ -411,30 +410,130 @@ def _get_index(self, dim: gtx.Dimension, start_marker: int, index_dict): self._log.error(msg) raise IconGridError(msg) from err + def _from_grid_dataset(self, grid, on_gpu: bool, limited_area=True) -> icon_grid.IconGrid: + + e2c2v = self._construct_diamond_vertices(grid.connectivities[dims.E2VDim], + grid.connectivities[dims.C2VDim], + grid.connectivities[dims.E2CDim]) + e2c2e = self._construct_diamond_edges(grid.connectivities[dims.E2CDim], + grid.connectivities[dims.C2EDim]) + e2c2e0 = np.column_stack((np.asarray(range(e2c2e.shape[0])), e2c2e)) + + c2e2c2e = self._construct_triangle_edges(grid.connectivities[dims.C2E2CDim], + grid.connectivities[dims.C2EDim]) + c2e2c0 = np.column_stack((np.asarray(range(grid.connectivities[dims.C2E2CDim].shape[0])), + (grid.connectivities[dims.C2E2CDim]))) + + grid.with_connectivities( + { + dims.C2E2CODim: c2e2c0, + dims.C2E2C2EDim: c2e2c2e, + dims.E2C2VDim: e2c2v, + dims.E2C2EDim: e2c2e, + dims.E2C2EODim: e2c2e0, + } + ) + grid.update_size_connectivities( + { + dims.ECVDim: grid.size[dims.EdgeDim] * grid.size[dims.E2C2VDim], + dims.CEDim: grid.size[dims.CellDim] * grid.size[dims.C2EDim], + dims.ECDim: grid.size[dims.EdgeDim] * grid.size[dims.E2CDim], + } + ) + self._add_start_end_indices(grid) + + return grid + + def _add_start_end_indices(self, grid): + ( + start_indices, + end_indices, + refine_ctrl, + refine_ctrl_max, + ) = self._read_grid_refinement_information(self._dataset) + grid.with_start_end_indices(dims.CellDim, start_indices[dims.CellDim], + end_indices[dims.CellDim]).with_start_end_indices(dims.EdgeDim, + start_indices[ + dims.EdgeDim], + end_indices[ + dims.EdgeDim]).with_start_end_indices( + dims.VertexDim, start_indices[dims.VertexDim], end_indices[dims.VertexDim]) + def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGrid: + grid = self._initialize_global(limited_area, on_gpu) + + global_connectivities = { + dims.C2E2C: self._reader.int_field(GridFile.ConnectivityName.C2E2C), + dims.C2E: self._reader.int_field(GridFile.ConnectivityName.C2E), + dims.E2C: self._reader.int_field(GridFile.ConnectivityName.E2C), + dims.V2E: self._reader.int_field(GridFile.ConnectivityName.V2E), + dims.E2V: self._reader.int_field(GridFile.ConnectivityName.E2V), + dims.V2C: self._reader.int_field(GridFile.ConnectivityName.V2C), + dims.C2V: self._reader.int_field(GridFile.ConnectivityName.C2V), + dims.V2E2V: self._reader.int_field(GridFile.ConnectivityName.V2E2V), + } + conn = {o.target[1]: global_connectivities[o] for o in global_connectivities.keys()} + grid.with_connectivities(conn) + return self._compute_derived_connectivities(grid, on_gpu, limited_area) if self._run_properties.single_node(): - return self._from_grid_dataset(on_gpu=on_gpu, limited_area=limited_area) + grid.with_connectivities({o.target[1]:global_connectivities[o] for o in global_connectivities.keys()}) else: decompose = halo.SimpleMetisDecomposer() - - global_connectivities = { - dims.C2E2CDim: self._reader.int_field(GridFile.ConnectivityName.C2E2C), - dims.C2EDim: self._reader.int_field(GridFile.ConnectivityName.C2E), - dims.E2CDim: self._reader.int_field(GridFile.ConnectivityName.E2C), - dims.V2EDim: self._reader.int_field(GridFile.ConnectivityName.V2E), - dims.E2VDim: self._reader.int_field(GridFile.ConnectivityName.E2V), - dims.V2CDim: self._reader.int_field(GridFile.ConnectivityName.V2C), - dims.C2VDim: self._reader.int_field(GridFile.ConnectivityName.C2V), - dims.V2E2VDim: self._reader.int_field(GridFile.ConnectivityName.V2E2V), - } cells_to_rank_mapping = decompose( - global_connectivities[dims.C2EDim], self._run_properties.comm_size) + global_connectivities[dims.C2E2C], self._run_properties.comm_size) halo_constructor = halo.HaloGenerator(self._run_properties, cells_to_rank_mapping, global_connectivities, self._config.num_levels) decomposition_info = halo_constructor.construct_decomposition_info() - self._from_decomposition_info(decomposition_info, on_gpu=on_gpu, limited_area=limited_area) - + + local_connectivities = { + offset.target[1]: construct_local_connectivity(offset, decomposition_info, global_connectivities[offset]) + for offset, conn in global_connectivities.keys() + } + grid.with_connectivities(local_connectivities) + e2c2v = self._construct_diamond_vertices(grid.connectivities[dims.E2VDim], + grid.connectivities[dims.C2VDim], + grid.connectivities[dims.E2CDim]) + e2c2e = self._construct_diamond_edges(grid.connectivities[dims.E2CDim], + grid.connectivities[dims.C2EDim]) + e2c2e0 = np.column_stack((np.asarray(range(e2c2e.shape[0])), e2c2e)) + + c2e2c2e = self._construct_triangle_edges(grid.connectivities[dims.C2E2CDim], + grid.connectivities[dims.C2EDim]) + c2e2c0 = np.column_stack((np.asarray(range(grid.connectivities[dims.C2E2CDim].shape[0])), + (grid.connectivities[dims.C2E2CDim]))) + grid.with_connectivities( + { + dims.C2E2CODim: c2e2c0, + dims.C2E2C2EDim: c2e2c2e, + dims.E2C2VDim: e2c2v, + dims.E2C2EDim: e2c2e, + dims.E2C2EODim: e2c2e0, + } + ) + ( + start_indices, + end_indices, + refine_ctrl, + refine_ctrl_max, + ) = self._read_grid_refinement_information(self._dataset) + grid.with_start_end_indices(dims.CellDim, start_indices[dims.CellDim], + end_indices[dims.CellDim]).with_start_end_indices(dims.EdgeDim, + start_indices[ + dims.EdgeDim], + end_indices[ + dims.EdgeDim]).with_start_end_indices( + dims.VertexDim, start_indices[dims.VertexDim], end_indices[dims.VertexDim]) + + grid.update_size_connectivities( + { + dims.ECVDim: grid.size[dims.EdgeDim] * grid.size[dims.E2C2VDim], + dims.CEDim: grid.size[dims.CellDim] * grid.size[dims.C2EDim], + dims.ECDim: grid.size[dims.EdgeDim] * grid.size[dims.E2CDim], + } + ) + return grid + def get_size(self, dim: gtx.Dimension): if dim == dims.VertexDim: return self._grid.config.num_vertices @@ -452,16 +551,12 @@ def _get_index_field(self, field: GridFileName, transpose=True, apply_offset=Tru if apply_offset: field = field + self._transformation.get_offset_for_index_field(field) return field - - def _from_decomposition_info(self, decomposition_info: decomposition.DecompositionInfo, on_gpu: bool = False, limited_area:bool = False)-> base_grid.BaseGrid: - grid = self._initialize_global(limited_area, on_gpu) - def _from_grid_dataset(self, on_gpu: bool, limited_area=True) -> icon_grid.IconGrid: - - grid = self._initialize_global(limited_area, on_gpu) + def _compute_derived_connectivities(self, grid, on_gpu: bool, limited_area=True) -> icon_grid.IconGrid: + c2e = self._get_index_field(GridFile.ConnectivityName.C2E) e2c = self._get_index_field( GridFile.ConnectivityName.E2C) # edge_face_connectivity (optional) @@ -669,11 +764,10 @@ def _patch_with_dummy_lastline(ar): return patched_ar -SKIP_VALUE = -1 def construct_local_connectivity( - field_offset: gtx.FieldOffset, decomposition_info: defs.DecompositionInfo, connectivities:dict) -> xp.ndarray: + field_offset: gtx.FieldOffset, decomposition_info: defs.DecompositionInfo, connectivity:xp.ndarray) -> xp.ndarray: """ Construct a connectivity table for use on a given rank: it maps from source to target dimension in _local_ indices. @@ -691,8 +785,7 @@ def construct_local_connectivity( """ source_dim = field_offset.source target_dim = field_offset.target[0] - local_dim = field_offset.target[1] - sliced_connectivity = connectivities[local_dim][ + sliced_connectivity = connectivity[ decomposition_info.global_index(target_dim, defs.DecompositionInfo.EntryType.ALL) ] #log.debug(f"rank {self._props.rank} has local connectivity f: {sliced_connectivity}") @@ -703,14 +796,14 @@ def construct_local_connectivity( # replace indices in the connectivity that do not exist on the local node by the SKIP_VALUE (those are for example neighbors of the outermost halo points) local_connectivity = xp.where( - xp.isin(sliced_connectivity, global_idx), sliced_connectivity, SKIP_VALUE + xp.isin(sliced_connectivity, global_idx), sliced_connectivity, GridFile.INVALID_INDEX ) # map to local source indices sorted_index_of_global_idx = xp.argsort(global_idx) global_idx_sorted = global_idx[sorted_index_of_global_idx] for i in xp.arange(local_connectivity.shape[0]): - valid_neighbor_mask = local_connectivity[i, :] != SKIP_VALUE + valid_neighbor_mask = local_connectivity[i, :] != GridFile.INVALID_INDEX positions = xp.searchsorted( global_idx_sorted, local_connectivity[i, valid_neighbor_mask] ) @@ -720,57 +813,4 @@ def construct_local_connectivity( return local_connectivity - # should be done in grid manager! - def local_grid(self, - props: defs.ProcessProperties, - decomposition_info: defs.DecompositionInfo, - - limited_area: bool = False, - on_gpu: bool = False, - ) -> base_grid.BaseGrid: - - """ - Constructs a local grid for this rank based on the decomposition info. - TODO (@halungge): for now only returning BaseGrid as we have not start/end indices implementation yet - TODO (@halungge): make sure the INVALID_INDEX is set correctly: - when set in the original (global index) connectivity it should remain - TODO (@halungge): how to handle the (source) indices of last halo line: their (global) neighbors are not all present on the local grid, set INVALID_INDEX (that is what xugrid does) - check what ICON does, (they probably duplicate the valid indices...) - Args: - decomposition_info: the decomposition info for this rank - Returns: - local_grid: the local grid - """ - num_vertices = decomposition_info.global_index( - dims.VertexDim, defs.DecompositionInfo.EntryType.ALL - ).size - num_edges = decomposition_info.global_index(dims.EdgeDim, - defs.DecompositionInfo.EntryType.ALL).size - num_cells = decomposition_info.global_index(dims.CellDim, - defs.DecompositionInfo.EntryType.ALL).size - grid_size = base_grid.HorizontalGridSize( - num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells - ) - config = base_grid.GridConfig( - horizontal_config=grid_size, vertical_size=self._num_levels, on_gpu=on_gpu, limited_area=limited_area - ) - - local_grid = ( - icon_grid.IconGrid(uuid.uuid4()).with_config(config).with_connectivities({ - dims.C2EDim: self.construct_local_connectivity(dims.C2EDim, decomposition_info), - dims.E2CDim: self.construct_local_connectivity(dims.E2CDim, decomposition_info), - dims.E2VDim: self.construct_local_connectivity(dims.E2VDim, decomposition_info), - dims.V2EDim: self.construct_local_connectivity(dims.V2EDim, decomposition_info), - dims.C2VDim: self.construct_local_connectivity(dims.C2VDim, decomposition_info), - dims.V2CDim: self.construct_local_connectivity(dims.V2CDim, decomposition_info), - dims.C2E2CDim: self.construct_local_connectivity(dims.C2E2CDim, decomposition_info), - # dims.C2E2CODim - # etc... # TODO (@halungge): this should all go to grid_manager - - - - }) - ) - - - - return local_grid + \ No newline at end of file diff --git a/model/common/tests/grid_tests/test_grid_manager.py b/model/common/tests/grid_tests/test_grid_manager.py index 3901b37af7..4fd0d75ab7 100644 --- a/model/common/tests/grid_tests/test_grid_manager.py +++ b/model/common/tests/grid_tests/test_grid_manager.py @@ -998,7 +998,7 @@ def test_local_connectivities(processor_props, caplog, field_offset): # fixture decomposition_info = halo_generator.construct_decomposition_info() - connectivity = construct_local_connectivity(field_offset, decomposition_info, connectivities=grid.connectivities) + connectivity = construct_local_connectivity(field_offset, decomposition_info, connectivity=grid.connectivities[field_offset.target[1]]) # there is an neighbor list for each index of the target dimension on the node assert ( connectivity.shape[0] From 81e35b30f21cc1f2e904f88afbffc7b83e002d89 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 30 Jul 2024 08:59:14 +0200 Subject: [PATCH 019/240] refactor grid_manger to construct local grids WIP(1) --- .../icon4py/model/common/grid/grid_manager.py | 43 ++++++++----------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 691b6081cf..95dd2c050c 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -405,11 +405,9 @@ def _get_index(self, dim: gtx.Dimension, start_marker: int, index_dict): raise IconGridError(msg) try: return index_dict[dim][start_marker] - except KeyError as err: + except KeyError: msg = f"start, end indices for dimension {dim} not present" self._log.error(msg) - raise IconGridError(msg) from err - def _from_grid_dataset(self, grid, on_gpu: bool, limited_area=True) -> icon_grid.IconGrid: e2c2v = self._construct_diamond_vertices(grid.connectivities[dims.E2VDim], @@ -440,7 +438,7 @@ def _from_grid_dataset(self, grid, on_gpu: bool, limited_area=True) -> icon_grid dims.ECDim: grid.size[dims.EdgeDim] * grid.size[dims.E2CDim], } ) - self._add_start_end_indices(grid) + return grid @@ -463,24 +461,26 @@ def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGri grid = self._initialize_global(limited_area, on_gpu) global_connectivities = { - dims.C2E2C: self._reader.int_field(GridFile.ConnectivityName.C2E2C), - dims.C2E: self._reader.int_field(GridFile.ConnectivityName.C2E), - dims.E2C: self._reader.int_field(GridFile.ConnectivityName.E2C), - dims.V2E: self._reader.int_field(GridFile.ConnectivityName.V2E), - dims.E2V: self._reader.int_field(GridFile.ConnectivityName.E2V), - dims.V2C: self._reader.int_field(GridFile.ConnectivityName.V2C), - dims.C2V: self._reader.int_field(GridFile.ConnectivityName.C2V), - dims.V2E2V: self._reader.int_field(GridFile.ConnectivityName.V2E2V), + dims.C2E2CDim: self._reader.int_field(GridFile.ConnectivityName.C2E2C), + dims.C2EDim: self._reader.int_field(GridFile.ConnectivityName.C2E), + dims.E2CDim: self._reader.int_field(GridFile.ConnectivityName.E2C), + dims.V2EDim: self._reader.int_field(GridFile.ConnectivityName.V2E), + dims.E2VDim: self._reader.int_field(GridFile.ConnectivityName.E2V), + dims.V2CDim: self._reader.int_field(GridFile.ConnectivityName.V2C), + dims.C2VDim: self._reader.int_field(GridFile.ConnectivityName.C2V), + dims.V2E2VDim: self._reader.int_field(GridFile.ConnectivityName.V2E2V), } - conn = {o.target[1]: global_connectivities[o] for o in global_connectivities.keys()} - grid.with_connectivities(conn) - return self._compute_derived_connectivities(grid, on_gpu, limited_area) + + grid.with_connectivities(global_connectivities) + self._add_start_end_indices(grid) + return self._compute_derived_connectivities(grid) + #return self._add_derived_connectivities(grid) if self._run_properties.single_node(): - grid.with_connectivities({o.target[1]:global_connectivities[o] for o in global_connectivities.keys()}) + grid.with_connectivities(global_connectivities) else: decompose = halo.SimpleMetisDecomposer() cells_to_rank_mapping = decompose( - global_connectivities[dims.C2E2C], self._run_properties.comm_size) + global_connectivities[dims.C2E2CDim], self._run_properties.comm_size) halo_constructor = halo.HaloGenerator(self._run_properties, cells_to_rank_mapping, global_connectivities, self._config.num_levels) decomposition_info = halo_constructor.construct_decomposition_info() @@ -554,7 +554,7 @@ def _get_index_field(self, field: GridFileName, transpose=True, apply_offset=Tru - def _compute_derived_connectivities(self, grid, on_gpu: bool, limited_area=True) -> icon_grid.IconGrid: + def _compute_derived_connectivities(self, grid) -> icon_grid.IconGrid: c2e = self._get_index_field(GridFile.ConnectivityName.C2E) @@ -598,13 +598,6 @@ def _compute_derived_connectivities(self, grid, on_gpu: bool, limited_area=True) dims.E2C2EODim: e2c2e0, } ) - ( - start_indices, - end_indices, - refine_ctrl, - refine_ctrl_max, - ) = self._read_grid_refinement_information(self._dataset) - grid.with_start_end_indices(dims.CellDim, start_indices[dims.CellDim], end_indices[dims.CellDim]).with_start_end_indices(dims.EdgeDim, start_indices[dims.EdgeDim], end_indices[dims.EdgeDim]).with_start_end_indices(dims.VertexDim, start_indices[dims.VertexDim], end_indices[dims.VertexDim]) grid.update_size_connectivities( { From 696203fab2c8b1d9145227b0a38d3022e424c25e Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 6 Aug 2024 14:33:39 +0200 Subject: [PATCH 020/240] refactor grid manager move test to for local grid to test_grid_manager.py --- .../model/common/decomposition/definitions.py | 13 +- .../model/common/decomposition/halo.py | 108 ++++++-- .../icon4py/model/common/grid/grid_manager.py | 165 ++++-------- .../tests/decomposition_tests/test_halo.py | 235 +++++++++++------- .../tests/grid_tests/test_grid_manager.py | 7 +- 5 files changed, 303 insertions(+), 225 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 833beb92d3..a8792a636e 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -75,12 +75,20 @@ class EntryType(IntEnum): HALO = 2 @builder.builder - def with_dimension(self, dim: Dimension, global_index: np.ndarray, owner_mask: np.ndarray): + def with_dimension( + self, + dim: Dimension, + global_index: np.ndarray, + owner_mask: np.ndarray, + halo_levels: np.ndarray, + ): masked_global_index = ma.array(global_index, mask=owner_mask) self._global_index[dim] = masked_global_index + self._halo_levels[dim] = halo_levels def __init__(self, klevels: int): self._global_index = {} + self._halo_levels = {} self._klevels = klevels @property @@ -121,6 +129,9 @@ def global_index(self, dim: Dimension, entry_type: EntryType = EntryType.ALL): case _: raise NotImplementedError() + def halo_levels(self, dim: Dimension): + return self._halo_levels[dim] + class ExchangeResult(Protocol): def wait(self): diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index a17d724d5d..617ad06ba3 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -27,12 +27,34 @@ # TODO (@halungge) do we need three of those: one for each dimension? class DecompositionFlag(enum.IntEnum): - #: cell is owned by this rank - OWNED = (0,) - #: cell is in the first halo line: that is cells that share and edge with an owned cell - FIRST_HALO_LINE = (1,) - #: cell is in the second halo line: that is cells that share only a vertex with an owned cell (and at least an edge with a FIRST_HALO_LINE cell) + UNDEFINED = -1 + OWNED = 0 + """used for locally owned cells, vertices, edges""" + + FIRST_HALO_LINE = 1 + """ + used for: + - cells that share 1 edge with an OWNED cell + - vertices that are on OWNED cell + - edges that are on OWNED cell + """ + SECOND_HALO_LINE = 2 + """ + used for: + - cells that share a vertex with an OWNED cell + - vertices that are on a cell(FIRST_HALO_LINE) but not on an owned cell + - edges that have _exactly_ one vertex shared with and OWNED Cell + """ + + THIRD_HALO_LINE = 3 + """ + This type does not exist in ICON. It denotes the "closing/far" edges of the SECOND_HALO_LINE cells + used for: + - cells (NOT USED) + - vertices (NOT USED) + - edges that are only on the cell(SECOND_HALO_LINE) + """ class HaloGenerator: @@ -140,7 +162,7 @@ def owned_cells(self) -> xp.ndarray: return xp.asarray(owned_cells).nonzero()[0] # TODO (@halungge): move out of halo generator? - def construct_decomposition_info(self) -> defs.DecompositionInfo: + def __call__(self) -> defs.DecompositionInfo: """ Constructs the DecompositionInfo for the current rank. @@ -156,10 +178,13 @@ def construct_decomposition_info(self) -> defs.DecompositionInfo: total_halo_cells = xp.hstack((first_halo_cells, second_halo_cells)) all_cells = xp.hstack((owned_cells, total_halo_cells)) - c_owner_mask = xp.isin(all_cells, owned_cells) - + cell_owner_mask = xp.isin(all_cells, owned_cells) + cell_halo_levels = DecompositionFlag.UNDEFINED * xp.ones(all_cells.size, dtype=int) + cell_halo_levels[cell_owner_mask] = DecompositionFlag.OWNED + cell_halo_levels[xp.isin(all_cells, first_halo_cells)] = DecompositionFlag.FIRST_HALO_LINE + cell_halo_levels[xp.isin(all_cells, second_halo_cells)] = DecompositionFlag.SECOND_HALO_LINE decomp_info = defs.DecompositionInfo(klevels=self._num_levels).with_dimension( - dims.CellDim, all_cells, c_owner_mask + dims.CellDim, all_cells, cell_owner_mask, cell_halo_levels ) #: edges @@ -167,17 +192,20 @@ def construct_decomposition_info(self) -> defs.DecompositionInfo: edges_on_first_halo_line = self.find_edge_neighbors_for_cells(first_halo_cells) edges_on_second_halo_line = self.find_edge_neighbors_for_cells(second_halo_cells) + level_two_edges = xp.setdiff1d(edges_on_first_halo_line, edges_on_owned_cells) all_edges = xp.hstack( ( edges_on_owned_cells, - xp.setdiff1d(edges_on_first_halo_line, edges_on_owned_cells), + level_two_edges, xp.setdiff1d(edges_on_second_halo_line, edges_on_first_halo_line), ) ) all_edges = xp.unique(all_edges) # We need to reduce the overlap: # `edges_on_owned_cells` and `edges_on_first_halo_line` both contain the edges on the cutting line. - intersect_owned_first_line = xp.intersect1d(edges_on_owned_cells, edges_on_first_halo_line) + edge_intersect_owned_first_line = xp.intersect1d( + edges_on_owned_cells, edges_on_first_halo_line + ) def _update_owner_mask_by_max_rank_convention( owner_mask, all_indices, indices_on_cutting_line, target_connectivity @@ -218,28 +246,53 @@ def _update_owner_mask_by_max_rank_convention( edge_owner_mask = _update_owner_mask_by_max_rank_convention( edge_owner_mask, all_edges, - intersect_owned_first_line, + edge_intersect_owned_first_line, self.edge_face_connectivity, ) - decomp_info.with_dimension(dims.EdgeDim, all_edges, edge_owner_mask) + edge_halo_levels = DecompositionFlag.UNDEFINED * xp.ones(all_edges.shape, dtype=int) + edge_halo_levels[edge_owner_mask] = DecompositionFlag.OWNED + edge_halo_levels[ + xp.logical_and( + xp.logical_not(edge_owner_mask), xp.isin(all_edges, edge_intersect_owned_first_line) + ) + ] = DecompositionFlag.FIRST_HALO_LINE + edge_halo_levels[xp.isin(all_edges, level_two_edges)] = DecompositionFlag.SECOND_HALO_LINE + decomp_info.with_dimension(dims.EdgeDim, all_edges, edge_owner_mask, edge_halo_levels) # vertices - vertices_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) - vertices_on_first_halo_line = self.find_vertex_neighbors_for_cells(first_halo_cells) - vertices_on_second_halo_line = self.find_vertex_neighbors_for_cells( + vertex_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) + vertex_on_first_halo_line = self.find_vertex_neighbors_for_cells(first_halo_cells) + vertex_on_second_halo_line = self.find_vertex_neighbors_for_cells( second_halo_cells - ) # TODO (@halungge): do we need that? - intersect_owned_first_line = xp.intersect1d( - vertices_on_owned_cells, vertices_on_first_halo_line + ) # TODO (@halungge): do we need that at all? + vertex_intersect_owned_first_line = xp.intersect1d( + vertex_on_owned_cells, vertex_on_first_halo_line ) # create decomposition_info for vertices - all_vertices = xp.unique(xp.hstack((vertices_on_owned_cells, vertices_on_first_halo_line))) - v_owner_mask = xp.isin(all_vertices, vertices_on_owned_cells) - v_owner_mask = _update_owner_mask_by_max_rank_convention( - v_owner_mask, all_vertices, intersect_owned_first_line, self.node_face_connectivity + all_vertices = xp.unique(xp.hstack((vertex_on_owned_cells, vertex_on_first_halo_line))) + vertex_owner_mask = xp.isin(all_vertices, vertex_on_owned_cells) + vertex_owner_mask = _update_owner_mask_by_max_rank_convention( + vertex_owner_mask, + all_vertices, + vertex_intersect_owned_first_line, + self.node_face_connectivity, + ) + vertex_second_level = xp.setdiff1d(vertex_on_first_halo_line, vertex_on_owned_cells) + vertex_halo_levels = DecompositionFlag.UNDEFINED * xp.ones(all_vertices.size, dtype=int) + vertex_halo_levels[vertex_owner_mask] = DecompositionFlag.OWNED + vertex_halo_levels[ + xp.logical_and( + xp.logical_not(vertex_owner_mask), + xp.isin(all_vertices, vertex_intersect_owned_first_line), + ) + ] = DecompositionFlag.FIRST_HALO_LINE + vertex_halo_levels[ + xp.isin(all_vertices, vertex_second_level) + ] = DecompositionFlag.SECOND_HALO_LINE + decomp_info.with_dimension( + dims.VertexDim, all_vertices, vertex_owner_mask, vertex_halo_levels ) - decomp_info.with_dimension(dims.VertexDim, all_vertices, v_owner_mask) return decomp_info @@ -261,7 +314,7 @@ class SimpleMetisDecomposer(Decomposer): https://documen.tician.de/pymetis/functionality.html """ - def __call__(self, adjacency_matrix, n_part: int) -> xp.ndarray: + def __call__(self, adjacency_matrix: xp.ndarray, n_part: int) -> xp.ndarray: """ Generate partition labesl for this grid topology using METIS: https://github.com/KarypisLab/METIS @@ -278,3 +331,8 @@ def __call__(self, adjacency_matrix, n_part: int) -> xp.ndarray: cut_count, partition_index = pymetis.part_graph(nparts=n_part, adjacency=adjacency_matrix) return xp.array(partition_index) + + +class SingleNodeDecomposer(Decomposer): + def __call__(self, adjacency_matrix: xp.ndarray, n_part: int) -> xp.ndarray: + return xp.zeros(adjacency_matrix.shape[0], dtype=xp.int32) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 17868d4a6d..cdd7d6fcab 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -446,13 +446,7 @@ def _from_grid_dataset(self, grid, on_gpu: bool, limited_area=True) -> icon_grid dims.E2C2EODim: e2c2e0, } ) - grid.update_size_connectivities( - { - dims.ECVDim: grid.size[dims.EdgeDim] * grid.size[dims.E2C2VDim], - dims.CEDim: grid.size[dims.CellDim] * grid.size[dims.C2EDim], - dims.ECDim: grid.size[dims.EdgeDim] * grid.size[dims.E2CDim], - } - ) + self._update_size_for_1d_sparse_dims(grid) return grid @@ -471,26 +465,35 @@ def _add_start_end_indices(self, grid): dims.VertexDim, start_indices[dims.VertexDim], end_indices[dims.VertexDim] ) + # TODO (@halungge) + # - remove duplication, + # - only read fields globally that are used for halo construction + # - make halo constructor transparent + def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGrid: grid = self._initialize_global(limited_area, on_gpu) global_connectivities = { - dims.C2E2CDim: self._reader.int_field(GridFile.ConnectivityName.C2E2C), - dims.C2EDim: self._reader.int_field(GridFile.ConnectivityName.C2E), - dims.E2CDim: self._reader.int_field(GridFile.ConnectivityName.E2C), - dims.V2EDim: self._reader.int_field(GridFile.ConnectivityName.V2E), - dims.E2VDim: self._reader.int_field(GridFile.ConnectivityName.E2V), - dims.V2CDim: self._reader.int_field(GridFile.ConnectivityName.V2C), - dims.C2VDim: self._reader.int_field(GridFile.ConnectivityName.C2V), - dims.V2E2VDim: self._reader.int_field(GridFile.ConnectivityName.V2E2V), + dims.C2E2C: self._get_index_field(GridFile.ConnectivityName.C2E2C), + dims.C2E: self._get_index_field(GridFile.ConnectivityName.C2E), + dims.E2C: self._get_index_field(GridFile.ConnectivityName.E2C), + dims.V2E: self._get_index_field(GridFile.ConnectivityName.V2E), + dims.E2V: self._get_index_field(GridFile.ConnectivityName.E2V), + dims.V2C: self._get_index_field(GridFile.ConnectivityName.V2C), + dims.C2V: self._get_index_field(GridFile.ConnectivityName.C2V), + dims.V2E2V: self._get_index_field(GridFile.ConnectivityName.V2E2V), } - - grid.with_connectivities(global_connectivities) - self._add_start_end_indices(grid) - return self._compute_derived_connectivities(grid) - # return self._add_derived_connectivities(grid) if self._run_properties.single_node(): - grid.with_connectivities(global_connectivities) + global_connectivities.update( + { + dims.E2V: self._get_index_field(GridFile.ConnectivityName.E2V), + dims.C2V: self._get_index_field(GridFile.ConnectivityName.C2V), + } + ) + grid.with_connectivities({o.target[1]: c for o, c in global_connectivities.items()}) + self._add_derived_connectivities(grid) + self._add_start_end_indices(grid) + return grid else: decompose = halo.SimpleMetisDecomposer() cells_to_rank_mapping = decompose( @@ -502,8 +505,13 @@ def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGri global_connectivities, self._config.num_levels, ) - decomposition_info = halo_constructor.construct_decomposition_info() - + decomposition_info = halo_constructor() + global_connectivities.update( + { + dims.E2V: self._get_index_field(GridFile.ConnectivityName.E2V), + dims.C2V: self._get_index_field(GridFile.ConnectivityName.C2V), + } + ) local_connectivities = { offset.target[1]: construct_local_connectivity( offset, decomposition_info, global_connectivities[offset] @@ -511,56 +519,10 @@ def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGri for offset, conn in global_connectivities.keys() } grid.with_connectivities(local_connectivities) - e2c2v = self._construct_diamond_vertices( - grid.connectivities[dims.E2VDim], - grid.connectivities[dims.C2VDim], - grid.connectivities[dims.E2CDim], - ) - e2c2e = self._construct_diamond_edges( - grid.connectivities[dims.E2CDim], grid.connectivities[dims.C2EDim] - ) - e2c2e0 = np.column_stack((np.asarray(range(e2c2e.shape[0])), e2c2e)) - - c2e2c2e = self._construct_triangle_edges( - grid.connectivities[dims.C2E2CDim], grid.connectivities[dims.C2EDim] - ) - c2e2c0 = np.column_stack( - ( - np.asarray(range(grid.connectivities[dims.C2E2CDim].shape[0])), - (grid.connectivities[dims.C2E2CDim]), - ) - ) - grid.with_connectivities( - { - dims.C2E2CODim: c2e2c0, - dims.C2E2C2EDim: c2e2c2e, - dims.E2C2VDim: e2c2v, - dims.E2C2EDim: e2c2e, - dims.E2C2EODim: e2c2e0, - } - ) - - ( - start_indices, - end_indices, - refine_ctrl, - refine_ctrl_max, - ) = self._read_grid_refinement_information(self._dataset) - grid.with_start_end_indices( - dims.CellDim, start_indices[dims.CellDim], end_indices[dims.CellDim] - ).with_start_end_indices( - dims.EdgeDim, start_indices[dims.EdgeDim], end_indices[dims.EdgeDim] - ).with_start_end_indices( - dims.VertexDim, start_indices[dims.VertexDim], end_indices[dims.VertexDim] - ) + self._add_derived_connectivities(grid) + # self._add_start_end_indices(grid) - grid.update_size_connectivities( - { - dims.ECVDim: grid.size[dims.EdgeDim] * grid.size[dims.E2C2VDim], - dims.CEDim: grid.size[dims.CellDim] * grid.size[dims.C2EDim], - dims.ECDim: grid.size[dims.EdgeDim] * grid.size[dims.E2CDim], - } - ) + self._update_size_for_1d_sparse_dims(grid) return grid def get_size(self, dim: gtx.Dimension): @@ -582,55 +544,40 @@ def _get_index_field( field = field + self._transformation.get_offset_for_index_field(field) return field - def _compute_derived_connectivities(self, grid) -> icon_grid.IconGrid: - c2e = self._get_index_field(GridFile.ConnectivityName.C2E) - e2c = self._get_index_field( - GridFile.ConnectivityName.E2C - ) # edge_face_connectivity (optional) - c2v = self._get_index_field( - GridFile.ConnectivityName.C2V - ) # face_node_connectivity (required) - v2c = self._get_index_field( - GridFile.ConnectivityName.V2C - ) # node_face_connectivity -- (pentagon/hexagon) - e2v = self._get_index_field( - GridFile.ConnectivityName.E2V - ) # edge_node_connectivity (optionally required) - v2e = self._get_index_field( - GridFile.ConnectivityName.V2E - ) # node_edge_connectivity -- (pentagon/hexagon) - v2e2v = self._get_index_field( - GridFile.ConnectivityName.V2E2V - ) # node_node_connectivity -- ((pentagon/hexagon)) - c2e2c = self._get_index_field( - GridFile.ConnectivityName.C2E2C - ) # face_face_connectivity (optional) - - e2c2v = self._construct_diamond_vertices(e2v, c2v, e2c) - e2c2e = self._construct_diamond_edges(e2c, c2e) + def _add_derived_connectivities(self, grid) -> icon_grid.IconGrid: + e2c2v = self._construct_diamond_vertices( + grid.connectivities[dims.E2VDim], + grid.connectivities[dims.C2VDim], + grid.connectivities[dims.E2CDim], + ) + e2c2e = self._construct_diamond_edges( + grid.connectivities[dims.E2CDim], grid.connectivities[dims.C2EDim] + ) e2c2e0 = np.column_stack((np.asarray(range(e2c2e.shape[0])), e2c2e)) - c2e2c2e = self._construct_triangle_edges(c2e2c, c2e) - c2e2c0 = np.column_stack((np.asarray(range(c2e2c.shape[0])), c2e2c)) + c2e2c2e = self._construct_triangle_edges( + grid.connectivities[dims.C2E2CDim], grid.connectivities[dims.C2EDim] + ) + c2e2c0 = np.column_stack( + ( + np.asarray(range(grid.connectivities[dims.C2E2CDim].shape[0])), + (grid.connectivities[dims.C2E2CDim]), + ) + ) grid.with_connectivities( { - dims.C2EDim: c2e, - dims.E2CDim: e2c, - dims.E2VDim: e2v, - dims.V2EDim: v2e, - dims.V2CDim: v2c, - dims.C2VDim: c2v, - dims.C2E2CDim: c2e2c, dims.C2E2CODim: c2e2c0, dims.C2E2C2EDim: c2e2c2e, dims.E2C2VDim: e2c2v, - dims.V2E2VDim: v2e2v, dims.E2C2EDim: e2c2e, dims.E2C2EODim: e2c2e0, } ) + return grid + + def _update_size_for_1d_sparse_dims(self, grid): grid.update_size_connectivities( { dims.ECVDim: grid.size[dims.EdgeDim] * grid.size[dims.E2C2VDim], @@ -639,8 +586,6 @@ def _compute_derived_connectivities(self, grid) -> icon_grid.IconGrid: } ) - return grid - def _initialize_global(self, limited_area, on_gpu): num_cells = self._reader.dimension(GridFile.DimensionName.CELL_NAME) num_edges = self._reader.dimension(GridFile.DimensionName.EDGE_NAME) diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index 7e94f71c66..c67f6801ae 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -10,17 +10,19 @@ # distribution for a copy of the license or check . # # SPDX-License-Identifier: GPL-3.0-or-later -import logging import pathlib +import gt4py.next as gtx import mpi4py import mpi4py.MPI +import numpy as np import pytest import icon4py.model.common.dimension as dims import icon4py.model.common.grid.grid_manager as gm from icon4py.model.common.decomposition import definitions as defs from icon4py.model.common.decomposition.halo import ( + DecompositionFlag, HaloGenerator, SimpleMetisDecomposer, ) @@ -40,7 +42,7 @@ "icon_grid_0013_R02B04_R.nc" ) -simple_distribution = xp.asarray( +SIMPLE_DISTRIBUTION = xp.asarray( [ 0, # 0c 1, # 1c @@ -62,52 +64,66 @@ 1, # 17c ] ) -cell_own = {0: [0, 3, 4, 6, 7, 10], 1: [1, 2, 5, 14, 17], 2: [8, 9, 11], 3: [12, 13, 15, 16]} -cell_halos = { - 0: [2, 15, 1, 11, 13, 9, 17, 5, 12, 14, 8, 16], - 1: [4, 16, 3, 8, 15, 11, 13, 0, 7, 6, 9, 10, 12], - 2: [5, 6, 12, 14, 7, 2, 1, 4, 3, 10, 15, 16, 17], - 3: [9, 10, 17, 14, 0, 1, 6, 7, 8, 2, 3, 4, 5, 11], +_CELL_OWN = {0: [0, 3, 4, 6, 7, 10], 1: [1, 2, 5, 14, 17], 2: [8, 9, 11], 3: [12, 13, 15, 16]} +_CELL_FIRST_HALO_LINE = { + 0: [1, 11, 13, 9, 2, 15], + 1: [3, 8, 4, 11, 16, 13, 15], + 2: [5, 7, 6, 12, 14], + 3: [9, 10, 17, 14, 0, 1], +} +_CELL_SECOND_HALO_LINE = { + 0: [17, 5, 12, 14, 8, 16], + 1: [0, 7, 6, 9, 10, 12], + 2: [2, 1, 4, 3, 10, 15, 16, 17], + 3: [6, 7, 8, 2, 3, 4, 5, 11], +} + +_CELL_HALO = { + 0: _CELL_FIRST_HALO_LINE[0] + _CELL_SECOND_HALO_LINE[0], + 1: _CELL_FIRST_HALO_LINE[1] + _CELL_SECOND_HALO_LINE[1], + 2: _CELL_FIRST_HALO_LINE[2] + _CELL_SECOND_HALO_LINE[2], + 3: _CELL_FIRST_HALO_LINE[3] + _CELL_SECOND_HALO_LINE[3], } -edge_own = { +_EDGE_OWN = { 0: [1, 5, 12, 13, 14, 9], 1: [8, 7, 6, 25, 4, 2], 2: [16, 11, 15, 17, 10, 24], 3: [19, 23, 22, 26, 0, 3, 20, 18, 21], } - -edge_halos = { - 0: [0, 4, 21, 10, 2, 3, 8, 6, 7, 19, 20, 17, 16, 11, 18, 26, 25, 15, 23, 24, 22], - 1: [ - 5, - 12, - 22, - 23, - 3, - 1, - 9, - 15, +_EDGE_FIRST_HALO_LINE = {0: [0, 4, 17, 21, 10, 2], 1: [3, 15, 20, 26, 24], 2: [18], 3: []} +_EDGE_SECOND_HALO_LINE = { + 0: [ + 8, 16, - 11, - 19, - 20, - 0, - 17, 24, - 21, + 25, 26, - 13, - 10, - 14, + 22, + 23, 18, + 11, + 6, ], - 2: [7, 6, 9, 8, 14, 18, 19, 23, 25, 20, 12, 13, 2, 3, 4, 5, 1, 21, 22, 0, 26], + 1: [0, 1, 9, 5, 12, 17, 16, 22, 3, 21, 13, 17, 11, 18, 23], + 2: [6, 9, 14, 18, 19, 23, 25, 20, 13, 3, 4, 5, 1, 21, 22, 0, 26], + 3: [], +} +_EDGE_THIRD_HALO_LINE = { + 0: [19, 7, 20], + 1: [10, 14], + 2: [], + 3: [], +} +_EDGE_HALO = { + 0: _EDGE_FIRST_HALO_LINE[0] + _EDGE_SECOND_HALO_LINE[0] + _EDGE_THIRD_HALO_LINE[0], + 1: _EDGE_FIRST_HALO_LINE[1] + _EDGE_SECOND_HALO_LINE[1] + [10, 14], + 2: _EDGE_FIRST_HALO_LINE[2] + _EDGE_SECOND_HALO_LINE[2] + _EDGE_THIRD_HALO_LINE[2], 3: [10, 11, 13, 14, 25, 6, 24, 1, 5, 4, 8, 9, 17, 12, 15, 16, 2, 7], } -vertex_own = { +_VERTEX_OWN = { 0: [4], 1: [], 2: [3, 5], @@ -120,46 +136,72 @@ 8, ], } -vertex_halo = { - 0: [0, 1, 2, 3, 5, 6, 7, 8], - 1: [1, 2, 0, 5, 3, 8, 6, 7, 4], - 2: [8, 6, 7, 4, 0, 2, 1], +_VERTEX_FIRST_HALO_LINE = { + 0: [0, 1, 5, 8, 7, 3], + 1: [1, 2, 0, 5, 3, 8, 6], + 2: [ + 6, + 8, + 7, + ], + 3: [], +} +_VERTEX_SECOND_HALO_LINE = { + 0: [2, 6], + 1: [7, 4], + 2: [4, 0, 2, 1], 3: [3, 4, 5], } +_VERTEX_HALO = { + 0: _VERTEX_FIRST_HALO_LINE[0] + _VERTEX_SECOND_HALO_LINE[0], + 1: _VERTEX_FIRST_HALO_LINE[1] + _VERTEX_SECOND_HALO_LINE[1], + 2: _VERTEX_FIRST_HALO_LINE[2] + _VERTEX_SECOND_HALO_LINE[2], + 3: _VERTEX_FIRST_HALO_LINE[3] + _VERTEX_SECOND_HALO_LINE[3], +} -owned = {dims.CellDim: cell_own, dims.EdgeDim: edge_own, dims.VertexDim: vertex_own} -halos = {dims.CellDim: cell_halos, dims.EdgeDim: edge_halos, dims.VertexDim: vertex_halo} +OWNED = {dims.CellDim: _CELL_OWN, dims.EdgeDim: _EDGE_OWN, dims.VertexDim: _VERTEX_OWN} +HALO = {dims.CellDim: _CELL_HALO, dims.EdgeDim: _EDGE_HALO, dims.VertexDim: _VERTEX_HALO} +FIRST_HALO_LINE = { + dims.CellDim: _CELL_FIRST_HALO_LINE, + dims.VertexDim: _VERTEX_FIRST_HALO_LINE, + dims.EdgeDim: _EDGE_FIRST_HALO_LINE, +} +SECOND_HALO_LINE = { + dims.CellDim: _CELL_SECOND_HALO_LINE, + dims.VertexDim: _VERTEX_SECOND_HALO_LINE, + dims.EdgeDim: _EDGE_SECOND_HALO_LINE, +} -def test_halo_constructor_owned_cells(processor_props): # fixture +def test_halo_constructor_owned_cells(processor_props): # noqa F811 # fixture grid = simple.SimpleGrid() halo_generator = HaloGenerator( connectivities=grid.connectivities, run_properties=processor_props, - rank_mapping=simple_distribution, + rank_mapping=SIMPLE_DISTRIBUTION, num_levels=1, ) my_owned_cells = halo_generator.owned_cells() print(f"rank {processor_props.rank} owns {my_owned_cells} ") - assert my_owned_cells.size == len(cell_own[processor_props.rank]) - assert xp.setdiff1d(my_owned_cells, cell_own[processor_props.rank]).size == 0 + assert my_owned_cells.size == len(_CELL_OWN[processor_props.rank]) + assert xp.setdiff1d(my_owned_cells, _CELL_OWN[processor_props.rank]).size == 0 @pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) @pytest.mark.mpi(min_size=4) -def test_element_ownership_is_unique(dim, processor_props): # fixture +def test_element_ownership_is_unique(dim, processor_props): # noqa F811 # fixture if processor_props.comm_size != 4: pytest.skip("This test requires exactly 4 MPI ranks.") grid = simple.SimpleGrid() halo_generator = HaloGenerator( connectivities=grid.connectivities, run_properties=processor_props, - rank_mapping=simple_distribution, + rank_mapping=SIMPLE_DISTRIBUTION, num_levels=1, ) - decomposition_info = halo_generator.construct_decomposition_info() + decomposition_info = halo_generator() owned = decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED) print(f"\nrank {processor_props.rank} owns {dim} : {owned} ") if not mpi4py.MPI.Is_initialized(): @@ -175,13 +217,11 @@ def test_element_ownership_is_unique(dim, processor_props): # fixture print(f"rank {processor_props.rank} send_buf: {send_buf}") if processor_props.rank == 0: print(f"local_sizes: {local_sizes}") - # recv_buffer = xp.empty(sum(local_sizes), dtype=int) recv_buffer = -1 * xp.ones((4, buffer_size), dtype=int) print(f"{recv_buffer.shape}") else: recv_buffer = None - # TODO (@halungge) Gatherv does not work if one of the buffers has size-0 (VertexDim) - # comm.Gatherv(sendbuf=owned, recvbuf=(recv_buffer, local_sizes), root=0) + # Gatherv does not work if one of the buffers has size-0 (VertexDim) comm.Gather(sendbuf=send_buf, recvbuf=recv_buffer, root=0) if processor_props.rank == 0: print(f"global indices: {recv_buffer}") @@ -192,9 +232,9 @@ def test_element_ownership_is_unique(dim, processor_props): # fixture assert xp.all(xp.sort(values) == global_indices(dim)) -@pytest.mark.with_mpi(min_size=4) -@pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) -def test_halo_constructor_decomposition_info(processor_props, dim): # fixture +@pytest.mark.mpi(min_size=4) +@pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim]) # TODO dims.EdgeDim, +def test_halo_constructor_decomposition_info_global_indices(processor_props, dim): # noqa F811 # fixture if processor_props.comm_size != 4: pytest.skip("This test requires exactly 4 MPI ranks.") @@ -202,19 +242,63 @@ def test_halo_constructor_decomposition_info(processor_props, dim): # fixture halo_generator = HaloGenerator( connectivities=grid.connectivities, run_properties=processor_props, - rank_mapping=simple_distribution, + rank_mapping=SIMPLE_DISTRIBUTION, num_levels=1, ) - decomp_info = halo_generator.construct_decomposition_info() + decomp_info = halo_generator() my_halo = decomp_info.global_index(dim, defs.DecompositionInfo.EntryType.HALO) print(f"rank {processor_props.rank} has halo {dim} : {my_halo}") - assert my_halo.size == len(halos[dim][processor_props.rank]) - assert xp.setdiff1d(my_halo, halos[dim][processor_props.rank], assume_unique=True).size == 0 + assert my_halo.size == len(HALO[dim][processor_props.rank]) + assert xp.setdiff1d(my_halo, HALO[dim][processor_props.rank], assume_unique=True).size == 0 my_owned = decomp_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED) print(f"rank {processor_props.rank} owns {dim} : {my_owned} ") - assert my_owned.size == len(owned[dim][processor_props.rank]) - assert xp.setdiff1d(my_owned, owned[dim][processor_props.rank], assume_unique=True).size == 0 + assert_same_entries(dim, my_owned, OWNED, processor_props.rank) + + +def assert_same_entries( + dim: gtx.Dimension, my_owned: np.ndarray, reference: dict[int, list], rank: int +): + assert my_owned.size == len(reference[dim][rank]) + assert xp.setdiff1d(my_owned, reference[dim][rank], assume_unique=True).size == 0 + + +@pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim]) # TODO: dims.EdgeDim, +def test_halo_constructor_decomposition_info_halo_levels(processor_props, dim): # noqa F811 # fixture + grid = simple.SimpleGrid() + halo_generator = HaloGenerator( + connectivities=grid.connectivities, + run_properties=processor_props, + rank_mapping=SIMPLE_DISTRIBUTION, + num_levels=1, + ) + processor_props.rank = 1 + decomp_info = halo_generator() + my_halo_levels = decomp_info.halo_levels(dim) + print(f"{dim.value}: rank {processor_props.rank} has halo levels {my_halo_levels} ") + if dim != dims.EdgeDim: + assert xp.all( + my_halo_levels != DecompositionFlag.UNDEFINED + ), ( + "All indices should have a defined DecompositionFlag" + ) # THIS WILL CURRENTLY FAIL FOR EDGES + assert xp.where(my_halo_levels == DecompositionFlag.OWNED)[0].size == len( + OWNED[dim][processor_props.rank] + ) + owned_local_indices = decomp_info.local_index(dim, defs.DecompositionInfo.EntryType.OWNED) + assert xp.all( + my_halo_levels[owned_local_indices] == DecompositionFlag.OWNED + ), "owned local indices should have DecompositionFlag.OWNED" + first_halo_line_local_index = xp.where(my_halo_levels == DecompositionFlag.FIRST_HALO_LINE)[0] + first_halo_line_global_index = decomp_info.global_index( + dim, defs.DecompositionInfo.EntryType.ALL + )[first_halo_line_local_index] + assert_same_entries(dim, first_halo_line_global_index, FIRST_HALO_LINE, processor_props.rank) + second_halo_line_local_index = xp.where(my_halo_levels == DecompositionFlag.SECOND_HALO_LINE)[0] + second_halo_line_global_index = decomp_info.global_index( + dim, defs.DecompositionInfo.EntryType.ALL + )[second_halo_line_local_index] + assert_same_entries(dim, second_halo_line_global_index, SECOND_HALO_LINE, processor_props.rank) def grid_file_manager(file: pathlib.Path) -> gm.GridManager: @@ -261,29 +345,8 @@ def gather_field(field: xp.ndarray, comm: mpi4py.MPI.Comm) -> tuple: return local_sizes, recv_buffer -@pytest.mark.xfail(reason="Not implemented yet") -def test_local_grid(processor_props, caplog): # fixture - caplog.set_level(logging.INFO) - - grid = grid_file_manager(GRID_FILE).grid - labels = decompose(grid, processor_props) - halo_generator = HaloGenerator( - connectivities=grid.connectivities, - run_properties=processor_props, - rank_mapping=labels, - num_levels=1, - ) - decomposition_info = halo_generator.construct_decomposition_info() - local_grid = halo_generator.local_grid(decomposition_info) - - assert ( - local_grid.num_cells - == decomposition_info.global_index(dims.CellDim, defs.DecompositionInfo.EntryType.All).size - ) - - -@pytest.mark.with_mpi -def test_distributed_fields(processor_props): # fixture +@pytest.mark.mpi +def test_distributed_fields(processor_props): # noqa F811 # fixture grid_manager = grid_file_manager(GRID_FILE) global_grid = grid_manager.grid @@ -295,7 +358,7 @@ def test_distributed_fields(processor_props): # fixture rank_mapping=labels, num_levels=1, ) - decomposition_info = halo_generator.construct_decomposition_info() + decomposition_info = halo_generator() # distributed read: read one field per dimension local_geometry_fields = grid_manager.read_geometry(decomposition_info) local_cell_area = local_geometry_fields[gm.GridFile.GeometryName.CELL_AREA] @@ -326,7 +389,7 @@ def test_distributed_fields(processor_props): # fixture ) -def decompose(grid: base_grid.BaseGrid, processor_props): +def decompose(grid: base_grid.BaseGrid, processor_props): # noqa F811 # fixture partitioner = SimpleMetisDecomposer() labels = partitioner(grid.connectivities[dims.C2E2CDim], n_part=processor_props.comm_size) return labels @@ -334,7 +397,7 @@ def decompose(grid: base_grid.BaseGrid, processor_props): def assert_gathered_field_against_global( decomposition_info: defs.DecompositionInfo, - processor_props: defs.ProcessProperties, + processor_props: defs.ProcessProperties, # noqa F811 # fixture dim: dims.Dimension, global_reference_field: xp.ndarray, local_field: xp.ndarray, @@ -353,9 +416,9 @@ def assert_gathered_field_against_global( ) if processor_props.rank == 0: assert xp.all(gathered_sizes == global_index_sizes) - sorted = xp.zeros(global_reference_field.shape, dtype=xp.float64) - sorted[gathered_global_indices] = gathered_field - assert helpers.dallclose(sorted, global_reference_field) + sorted_ = xp.zeros(global_reference_field.shape, dtype=xp.float64) + sorted_[gathered_global_indices] = gathered_field + assert helpers.dallclose(sorted_, global_reference_field) # TODO add test including halo access: diff --git a/model/common/tests/grid_tests/test_grid_manager.py b/model/common/tests/grid_tests/test_grid_manager.py index 95392de42f..ef52e07bef 100644 --- a/model/common/tests/grid_tests/test_grid_manager.py +++ b/model/common/tests/grid_tests/test_grid_manager.py @@ -1040,7 +1040,8 @@ def test_local_connectivities(processor_props, caplog, field_offset): # fixture file = utils.resolve_file_from_gridfile_name(utils.R02B04_GLOBAL) grid = init_grid_manager(file).grid partitioner = halo.SimpleMetisDecomposer() - labels = partitioner(grid.connectivities[dims.C2E2CDim], n_part=processor_props.comm_size) + face_face_connectivity = grid.connectivities[dims.C2E2CDim] + labels = partitioner(face_face_connectivity, n_part=processor_props.comm_size) halo_generator = HaloGenerator( connectivities=grid.connectivities, run_properties=processor_props, @@ -1048,7 +1049,7 @@ def test_local_connectivities(processor_props, caplog, field_offset): # fixture num_levels=1, ) - decomposition_info = halo_generator.construct_decomposition_info() + decomposition_info = halo_generator() connectivity = construct_local_connectivity( field_offset, decomposition_info, connectivity=grid.connectivities[field_offset.target[1]] @@ -1065,4 +1066,4 @@ def test_local_connectivities(processor_props, caplog, field_offset): # fixture decomposition_info.local_index(field_offset.source, defs.DecompositionInfo.EntryType.ALL) ) # TODO what else to assert? - # outer halo entries have SKIP_VALUE neighbors (depends on offsets) + # - outer halo entries have SKIP_VALUE neighbors (depends on offsets) From 276db9dd36a3e072c78880ab7d5525aec6fa1d67 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 7 Aug 2024 10:08:05 +0200 Subject: [PATCH 021/240] halo levels for edges in test data (simple distribution) --- .../tests/decomposition_tests/test_halo.py | 37 +++++++------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index c67f6801ae..dc25561c58 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -94,33 +94,22 @@ } _EDGE_FIRST_HALO_LINE = {0: [0, 4, 17, 21, 10, 2], 1: [3, 15, 20, 26, 24], 2: [18], 3: []} _EDGE_SECOND_HALO_LINE = { - 0: [ - 8, - 16, - 24, - 25, - 26, - 22, - 23, - 18, - 11, - 6, - ], - 1: [0, 1, 9, 5, 12, 17, 16, 22, 3, 21, 13, 17, 11, 18, 23], - 2: [6, 9, 14, 18, 19, 23, 25, 20, 13, 3, 4, 5, 1, 21, 22, 0, 26], - 3: [], + 0: [3,6,7,8,15,24,25,26,16,22,23,18,19,20,11], + 1: [0,1,2,5,9,12,11,10,13,16,17,18,19,21,22,23], + 2: [2,9,12,4,8,7,14,21,13,20,19,23,22,26,25], + 3: [11,10,14,13,17,24,25,6,2,1,5,4,8,7], } _EDGE_THIRD_HALO_LINE = { - 0: [19, 7, 20], - 1: [10, 14], - 2: [], - 3: [], + 0: [], + 1: [14], + 2: [0,3,6,1], + 3: [9,12,15,16], } _EDGE_HALO = { 0: _EDGE_FIRST_HALO_LINE[0] + _EDGE_SECOND_HALO_LINE[0] + _EDGE_THIRD_HALO_LINE[0], - 1: _EDGE_FIRST_HALO_LINE[1] + _EDGE_SECOND_HALO_LINE[1] + [10, 14], + 1: _EDGE_FIRST_HALO_LINE[1] + _EDGE_SECOND_HALO_LINE[1] + _EDGE_THIRD_HALO_LINE[1], 2: _EDGE_FIRST_HALO_LINE[2] + _EDGE_SECOND_HALO_LINE[2] + _EDGE_THIRD_HALO_LINE[2], - 3: [10, 11, 13, 14, 25, 6, 24, 1, 5, 4, 8, 9, 17, 12, 15, 16, 2, 7], + 3: _EDGE_FIRST_HALO_LINE[3] + _EDGE_SECOND_HALO_LINE[3] + _EDGE_THIRD_HALO_LINE[3] } _VERTEX_OWN = { @@ -233,7 +222,7 @@ def test_element_ownership_is_unique(dim, processor_props): # noqa F811 # fixtu @pytest.mark.mpi(min_size=4) -@pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim]) # TODO dims.EdgeDim, +@pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) def test_halo_constructor_decomposition_info_global_indices(processor_props, dim): # noqa F811 # fixture if processor_props.comm_size != 4: pytest.skip("This test requires exactly 4 MPI ranks.") @@ -263,7 +252,7 @@ def assert_same_entries( assert xp.setdiff1d(my_owned, reference[dim][rank], assume_unique=True).size == 0 -@pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim]) # TODO: dims.EdgeDim, +@pytest.mark.parametrize("dim", [dims.EdgeDim]) #dims.CellDim, dims.VertexDim, def test_halo_constructor_decomposition_info_halo_levels(processor_props, dim): # noqa F811 # fixture grid = simple.SimpleGrid() halo_generator = HaloGenerator( @@ -272,7 +261,7 @@ def test_halo_constructor_decomposition_info_halo_levels(processor_props, dim): rank_mapping=SIMPLE_DISTRIBUTION, num_levels=1, ) - processor_props.rank = 1 + processor_props.rank = 0 decomp_info = halo_generator() my_halo_levels = decomp_info.halo_levels(dim) print(f"{dim.value}: rank {processor_props.rank} has halo levels {my_halo_levels} ") From b8c6cc0d000a88e94f5c9715ee04c08eee4cada4 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 7 Aug 2024 12:46:30 +0200 Subject: [PATCH 022/240] inject decomposer --- .../src/icon4py/model/common/grid/grid_manager.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index cdd7d6fcab..03ceac830f 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -14,7 +14,7 @@ import enum import logging import pathlib -from typing import Optional, Sequence, Union +from typing import Callable, Optional, Sequence, Union import gt4py.next as gtx import numpy as np @@ -245,7 +245,7 @@ def get_offset_for_index_field(self, array: np.ndarray): """ return np.asarray(np.where(array == GridFile.INVALID_INDEX, 0, -1), dtype=gtx.int32) - +# TODO make this a context manager class GridManager: """ Read ICON grid file and set up IconGrid. @@ -259,13 +259,20 @@ def __init__( transformation: IndexTransformation, grid_file: Union[pathlib.Path, str], config: v_grid.VerticalGridConfig, # TODO (@halungge) remove + decomposer: Callable[[np.ndarray, int], np.ndarray] = None, run_properties: decomposition.ProcessProperties = decomposition.SingleNodeProcessProperties(), + ): self._log = logging.getLogger(__name__) self._run_properties = run_properties self._transformation = transformation - self._config = config self._file_name = str(grid_file) + + if decomposer is None: + self._decompose = halo.SingleNodeDecomposer() if run_properties.single_node() else halo.SimpleMetisDecomposer() + else: + self._decompose = decomposer + self._config = config self._grid: Optional[icon_grid.IconGrid] = None self._dataset = None self._reader = None From 31b660f3fd718a947e15d495c775736a004fc74f Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 7 Aug 2024 13:17:47 +0200 Subject: [PATCH 023/240] compute vertex halos before edges --- .../mpi_tests/test_parallel_diffusion.py | 2 +- .../model/common/decomposition/halo.py | 139 +++++++++--------- 2 files changed, 72 insertions(+), 69 deletions(-) diff --git a/model/atmosphere/diffusion/tests/diffusion_tests/mpi_tests/test_parallel_diffusion.py b/model/atmosphere/diffusion/tests/diffusion_tests/mpi_tests/test_parallel_diffusion.py index 340f65b579..bca7a01c38 100644 --- a/model/atmosphere/diffusion/tests/diffusion_tests/mpi_tests/test_parallel_diffusion.py +++ b/model/atmosphere/diffusion/tests/diffusion_tests/mpi_tests/test_parallel_diffusion.py @@ -123,7 +123,7 @@ def test_parallel_diffusion( print(f"rank={processor_props.rank}/{processor_props.comm_size}: diffusion run ") verify_diffusion_fields( - config=config, + config=config, diagnostic_state=diagnostic_state, prognostic_state=prognostic_state, diffusion_savepoint=diffusion_savepoint_exit, diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 617ad06ba3..5fd4f6e9c1 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -161,6 +161,40 @@ def owned_cells(self) -> xp.ndarray: owned_cells = self._mapping == self._props.rank return xp.asarray(owned_cells).nonzero()[0] + def _update_owner_mask_by_max_rank_convention(self, + owner_mask, all_indices, indices_on_cutting_line, target_connectivity + ): + """ + In order to have unique ownership of edges (and vertices) among nodes there needs to be + a convention as to where those elements on the cutting line go: + according to a remark in `mo_decomposition_tools.f90` ICON puts them to the node + with the higher rank. + + # TODO (@halungge): can we add an assert for the target dimension of the connectivity being cells. + Args: + owner_mask: owner mask for the dimension + all_indices: (global) indices of the dimension + indices_on_cutting_line: global indices of the elements on the cutting line + target_connectivity: connectivity matrix mapping the dimension d to faces + Returns: + updated owner mask + """ + for index in indices_on_cutting_line: + local_index = xp.nonzero(all_indices == index)[0][0] + owning_ranks = self._mapping[target_connectivity[index]] + assert ( + xp.unique(owning_ranks).size > 1 + ), f"rank {self._props.rank}: all neighboring cells are owned by the same rank" + assert ( + self._props.rank in owning_ranks + ), f"rank {self._props.rank}: neither of the neighboring cells: {owning_ranks} is owned by me" + # assign the index to the rank with the higher rank + if max(owning_ranks) > self._props.rank: + owner_mask[local_index] = False + else: + owner_mask[local_index] = True + return owner_mask + # TODO (@halungge): move out of halo generator? def __call__(self) -> defs.DecompositionInfo: """ @@ -187,6 +221,41 @@ def __call__(self) -> defs.DecompositionInfo: dims.CellDim, all_cells, cell_owner_mask, cell_halo_levels ) + #: vertices + vertex_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) + vertex_on_first_halo_line = self.find_vertex_neighbors_for_cells(first_halo_cells) + vertex_on_second_halo_line = self.find_vertex_neighbors_for_cells( + second_halo_cells + ) # TODO (@halungge): do we need that at all? + vertex_intersect_owned_first_line = xp.intersect1d( + vertex_on_owned_cells, vertex_on_first_halo_line + ) + + # create decomposition_info for vertices + all_vertices = xp.unique(xp.hstack((vertex_on_owned_cells, vertex_on_first_halo_line))) + vertex_owner_mask = xp.isin(all_vertices, vertex_on_owned_cells) + vertex_owner_mask = self._update_owner_mask_by_max_rank_convention( + vertex_owner_mask, + all_vertices, + vertex_intersect_owned_first_line, + self.node_face_connectivity, + ) + vertex_second_level = xp.setdiff1d(vertex_on_first_halo_line, vertex_on_owned_cells) + vertex_halo_levels = DecompositionFlag.UNDEFINED * xp.ones(all_vertices.size, dtype=int) + vertex_halo_levels[vertex_owner_mask] = DecompositionFlag.OWNED + vertex_halo_levels[ + xp.logical_and( + xp.logical_not(vertex_owner_mask), + xp.isin(all_vertices, vertex_intersect_owned_first_line), + ) + ] = DecompositionFlag.FIRST_HALO_LINE + vertex_halo_levels[ + xp.isin(all_vertices, vertex_second_level) + ] = DecompositionFlag.SECOND_HALO_LINE + decomp_info.with_dimension( + dims.VertexDim, all_vertices, vertex_owner_mask, vertex_halo_levels + ) + #: edges edges_on_owned_cells = self.find_edge_neighbors_for_cells(owned_cells) edges_on_first_halo_line = self.find_edge_neighbors_for_cells(first_halo_cells) @@ -207,43 +276,11 @@ def __call__(self) -> defs.DecompositionInfo: edges_on_owned_cells, edges_on_first_halo_line ) - def _update_owner_mask_by_max_rank_convention( - owner_mask, all_indices, indices_on_cutting_line, target_connectivity - ): - """ - In order to have unique ownership of edges (and vertices) among nodes there needs to be - a convention as to where those elements on the cutting line go: - according to a remark in `mo_decomposition_tools.f90` ICON puts them to the node - with the higher rank. - - # TODO (@halungge): can we add an assert for the target dimension of the connectivity being cells. - Args: - owner_mask: owner mask for the dimension - all_indices: (global) indices of the dimension - indices_on_cutting_line: global indices of the elements on the cutting line - target_connectivity: connectivity matrix mapping the dimension d to faces - Returns: - updated owner mask - """ - for index in indices_on_cutting_line: - local_index = xp.nonzero(all_indices == index)[0][0] - owning_ranks = self._mapping[target_connectivity[index]] - assert ( - xp.unique(owning_ranks).size > 1 - ), f"rank {self._props.rank}: all neighboring cells are owned by the same rank" - assert ( - self._props.rank in owning_ranks - ), f"rank {self._props.rank}: neither of the neighboring cells: {owning_ranks} is owned by me" - # assign the index to the rank with the higher rank - if max(owning_ranks) > self._props.rank: - owner_mask[local_index] = False - else: - owner_mask[local_index] = True - return owner_mask + # construct the owner mask edge_owner_mask = xp.isin(all_edges, edges_on_owned_cells) - edge_owner_mask = _update_owner_mask_by_max_rank_convention( + edge_owner_mask = self._update_owner_mask_by_max_rank_convention( edge_owner_mask, all_edges, edge_intersect_owned_first_line, @@ -259,40 +296,6 @@ def _update_owner_mask_by_max_rank_convention( edge_halo_levels[xp.isin(all_edges, level_two_edges)] = DecompositionFlag.SECOND_HALO_LINE decomp_info.with_dimension(dims.EdgeDim, all_edges, edge_owner_mask, edge_halo_levels) - # vertices - vertex_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) - vertex_on_first_halo_line = self.find_vertex_neighbors_for_cells(first_halo_cells) - vertex_on_second_halo_line = self.find_vertex_neighbors_for_cells( - second_halo_cells - ) # TODO (@halungge): do we need that at all? - vertex_intersect_owned_first_line = xp.intersect1d( - vertex_on_owned_cells, vertex_on_first_halo_line - ) - - # create decomposition_info for vertices - all_vertices = xp.unique(xp.hstack((vertex_on_owned_cells, vertex_on_first_halo_line))) - vertex_owner_mask = xp.isin(all_vertices, vertex_on_owned_cells) - vertex_owner_mask = _update_owner_mask_by_max_rank_convention( - vertex_owner_mask, - all_vertices, - vertex_intersect_owned_first_line, - self.node_face_connectivity, - ) - vertex_second_level = xp.setdiff1d(vertex_on_first_halo_line, vertex_on_owned_cells) - vertex_halo_levels = DecompositionFlag.UNDEFINED * xp.ones(all_vertices.size, dtype=int) - vertex_halo_levels[vertex_owner_mask] = DecompositionFlag.OWNED - vertex_halo_levels[ - xp.logical_and( - xp.logical_not(vertex_owner_mask), - xp.isin(all_vertices, vertex_intersect_owned_first_line), - ) - ] = DecompositionFlag.FIRST_HALO_LINE - vertex_halo_levels[ - xp.isin(all_vertices, vertex_second_level) - ] = DecompositionFlag.SECOND_HALO_LINE - decomp_info.with_dimension( - dims.VertexDim, all_vertices, vertex_owner_mask, vertex_halo_levels - ) return decomp_info From b24613cbad7014209ca2b65aa44590350984385a Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 7 Aug 2024 13:57:48 +0200 Subject: [PATCH 024/240] add halo level tests for edges --- .../model/common/decomposition/halo.py | 22 +++++--- .../tests/decomposition_tests/test_halo.py | 52 ++++++++++--------- 2 files changed, 41 insertions(+), 33 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 5fd4f6e9c1..b498264627 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -143,16 +143,19 @@ def next_halo_line(self, cell_line: xp.ndarray, depot=None): def _cell_neighbors(self, cells: xp.ndarray): return xp.unique(self.face_face_connectivity[cells, :]) - def _find_neighbors(self, cell_line: xp.ndarray, connectivity: xp.ndarray) -> xp.ndarray: + def _find_neighbors(self, source_indices: xp.ndarray, connectivity: xp.ndarray) -> xp.ndarray: """Get a flattened list of all (unique) neighbors to a given global index list""" - neighbors = connectivity[cell_line, :] + neighbors = connectivity[source_indices, :] shp = neighbors.shape unique_neighbors = xp.unique(neighbors.reshape(shp[0] * shp[1])) return unique_neighbors def find_edge_neighbors_for_cells(self, cell_line: xp.ndarray) -> xp.ndarray: return self._find_neighbors(cell_line, connectivity=self.face_edge_connectivity) - + + def find_edge_neighbors_for_vertices(self, vertex_line: xp.ndarray) -> xp.ndarray: + return self._find_neighbors(vertex_line, connectivity=self.node_edge_connectivity) + def find_vertex_neighbors_for_cells(self, cell_line: xp.ndarray) -> xp.ndarray: return self._find_neighbors(cell_line, connectivity=self.face_node_connectivity) @@ -227,7 +230,8 @@ def __call__(self) -> defs.DecompositionInfo: vertex_on_second_halo_line = self.find_vertex_neighbors_for_cells( second_halo_cells ) # TODO (@halungge): do we need that at all? - vertex_intersect_owned_first_line = xp.intersect1d( + + vertex_on_cutting_line = xp.intersect1d( vertex_on_owned_cells, vertex_on_first_halo_line ) @@ -237,7 +241,7 @@ def __call__(self) -> defs.DecompositionInfo: vertex_owner_mask = self._update_owner_mask_by_max_rank_convention( vertex_owner_mask, all_vertices, - vertex_intersect_owned_first_line, + vertex_on_cutting_line, self.node_face_connectivity, ) vertex_second_level = xp.setdiff1d(vertex_on_first_halo_line, vertex_on_owned_cells) @@ -246,7 +250,7 @@ def __call__(self) -> defs.DecompositionInfo: vertex_halo_levels[ xp.logical_and( xp.logical_not(vertex_owner_mask), - xp.isin(all_vertices, vertex_intersect_owned_first_line), + xp.isin(all_vertices, vertex_on_cutting_line), ) ] = DecompositionFlag.FIRST_HALO_LINE vertex_halo_levels[ @@ -260,8 +264,10 @@ def __call__(self) -> defs.DecompositionInfo: edges_on_owned_cells = self.find_edge_neighbors_for_cells(owned_cells) edges_on_first_halo_line = self.find_edge_neighbors_for_cells(first_halo_cells) edges_on_second_halo_line = self.find_edge_neighbors_for_cells(second_halo_cells) - - level_two_edges = xp.setdiff1d(edges_on_first_halo_line, edges_on_owned_cells) + + level_two_edges = xp.setdiff1d(self.find_edge_neighbors_for_vertices(vertex_on_cutting_line), edges_on_owned_cells) + + #level_two_edges = xp.setdiff1d(edges_on_first_halo_line, edges_on_owned_cells) all_edges = xp.hstack( ( edges_on_owned_cells, diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index dc25561c58..d1f52f8096 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -64,13 +64,20 @@ 1, # 17c ] ) -_CELL_OWN = {0: [0, 3, 4, 6, 7, 10], 1: [1, 2, 5, 14, 17], 2: [8, 9, 11], 3: [12, 13, 15, 16]} +_CELL_OWN = { + 0: [0, 3, 4, 6, 7, 10], + 1: [1, 2, 5, 14, 17], + 2: [8, 9, 11], + 3: [12, 13, 15, 16] +} + _CELL_FIRST_HALO_LINE = { 0: [1, 11, 13, 9, 2, 15], 1: [3, 8, 4, 11, 16, 13, 15], 2: [5, 7, 6, 12, 14], 3: [9, 10, 17, 14, 0, 1], } + _CELL_SECOND_HALO_LINE = { 0: [17, 5, 12, 14, 8, 16], 1: [0, 7, 6, 9, 10, 12], @@ -85,25 +92,31 @@ 3: _CELL_FIRST_HALO_LINE[3] + _CELL_SECOND_HALO_LINE[3], } - _EDGE_OWN = { 0: [1, 5, 12, 13, 14, 9], 1: [8, 7, 6, 25, 4, 2], 2: [16, 11, 15, 17, 10, 24], 3: [19, 23, 22, 26, 0, 3, 20, 18, 21], } -_EDGE_FIRST_HALO_LINE = {0: [0, 4, 17, 21, 10, 2], 1: [3, 15, 20, 26, 24], 2: [18], 3: []} + +_EDGE_FIRST_HALO_LINE = { + 0: [0, 4, 17, 21, 10, 2], + 1: [3, 15, 20, 26, 24], + 2: [18], + 3: []} + _EDGE_SECOND_HALO_LINE = { - 0: [3,6,7,8,15,24,25,26,16,22,23,18,19,20,11], - 1: [0,1,2,5,9,12,11,10,13,16,17,18,19,21,22,23], - 2: [2,9,12,4,8,7,14,21,13,20,19,23,22,26,25], - 3: [11,10,14,13,17,24,25,6,2,1,5,4,8,7], + 0: [3, 6, 7, 8, 15, 24, 25, 26, 16, 22, 23, 18, 19, 20, 11], + 1: [0, 1, 5, 9, 12, 11, 10, 13, 16, 17, 18, 19, 21, 22, 23], + 2: [2, 9, 12, 4, 8, 7, 14, 21, 13, 19, 20, 22, 23, 25, 26], + 3: [11, 10, 14, 13, 16, 17, 24, 25, 6, 2, 1, 5, 4, 8, 7], } + _EDGE_THIRD_HALO_LINE = { 0: [], 1: [14], - 2: [0,3,6,1], - 3: [9,12,15,16], + 2: [0,1, 3, 5, 6], + 3: [9, 12, 15], } _EDGE_HALO = { 0: _EDGE_FIRST_HALO_LINE[0] + _EDGE_SECOND_HALO_LINE[0] + _EDGE_THIRD_HALO_LINE[0], @@ -116,25 +129,16 @@ 0: [4], 1: [], 2: [3, 5], - 3: [ - 0, - 1, - 2, - 6, - 7, - 8, - ], + 3: [0, 1, 2, 6, 7, 8,], } + _VERTEX_FIRST_HALO_LINE = { 0: [0, 1, 5, 8, 7, 3], 1: [1, 2, 0, 5, 3, 8, 6], - 2: [ - 6, - 8, - 7, - ], + 2: [6, 8, 7,], 3: [], } + _VERTEX_SECOND_HALO_LINE = { 0: [2, 6], 1: [7, 4], @@ -226,7 +230,6 @@ def test_element_ownership_is_unique(dim, processor_props): # noqa F811 # fixtu def test_halo_constructor_decomposition_info_global_indices(processor_props, dim): # noqa F811 # fixture if processor_props.comm_size != 4: pytest.skip("This test requires exactly 4 MPI ranks.") - grid = simple.SimpleGrid() halo_generator = HaloGenerator( connectivities=grid.connectivities, @@ -252,7 +255,7 @@ def assert_same_entries( assert xp.setdiff1d(my_owned, reference[dim][rank], assume_unique=True).size == 0 -@pytest.mark.parametrize("dim", [dims.EdgeDim]) #dims.CellDim, dims.VertexDim, +@pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) def test_halo_constructor_decomposition_info_halo_levels(processor_props, dim): # noqa F811 # fixture grid = simple.SimpleGrid() halo_generator = HaloGenerator( @@ -261,7 +264,6 @@ def test_halo_constructor_decomposition_info_halo_levels(processor_props, dim): rank_mapping=SIMPLE_DISTRIBUTION, num_levels=1, ) - processor_props.rank = 0 decomp_info = halo_generator() my_halo_levels = decomp_info.halo_levels(dim) print(f"{dim.value}: rank {processor_props.rank} has halo levels {my_halo_levels} ") From 9c134ed3bfae423d6a04ba46e578ebf97d6b8b54 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 13 Aug 2024 16:48:55 +0200 Subject: [PATCH 025/240] add decomposition info to grid manager --- model/common/src/icon4py/model/common/grid/grid_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 03ceac830f..366de65d1e 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -274,6 +274,7 @@ def __init__( self._decompose = decomposer self._config = config self._grid: Optional[icon_grid.IconGrid] = None + self._decomposition_info: Optional[decomposition.DecompositionInfo] = None self._dataset = None self._reader = None From cedf7170f073ee2c0b0ee9f5f0d17f92bd56cdb1 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 13 Aug 2024 16:49:13 +0200 Subject: [PATCH 026/240] add decomposition info to grid manager --- model/common/src/icon4py/model/common/grid/grid_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 366de65d1e..a1038262f9 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -503,8 +503,7 @@ def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGri self._add_start_end_indices(grid) return grid else: - decompose = halo.SimpleMetisDecomposer() - cells_to_rank_mapping = decompose( + cells_to_rank_mapping = self._decompose( global_connectivities[dims.C2E2CDim], self._run_properties.comm_size ) halo_constructor = halo.HaloGenerator( From 4a2effd2c9fd056ee99c271746728c07f21e7ef3 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 29 Aug 2024 14:31:24 +0200 Subject: [PATCH 027/240] rename global_dims to horizontal_dims --- .../common/decomposition/mpi_decomposition.py | 6 +- .../src/icon4py/model/common/dimension.py | 2 +- .../icon4py/model/common/grid/grid_manager.py | 324 ++++++++++-------- .../icon4pygen/bindings/locations.py | 4 +- tools/src/icon4pytools/icon4pygen/metadata.py | 4 +- 5 files changed, 181 insertions(+), 159 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py index 94ae8b8e02..9d1b6c5747 100644 --- a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py +++ b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py @@ -130,14 +130,14 @@ def __init__( dim: self._create_domain_descriptor( dim, ) - for dim in dims.global_dimensions.values() + for dim in dims.horizontal_dims.values() } log.info(f"domain descriptors for dimensions {self._domain_descriptors.keys()} initialized") self._patterns = { dim: self._create_pattern( dim, ) - for dim in dims.global_dimensions.values() + for dim in dims.horizontal_dims.values() } log.info(f"patterns for dimensions {self._patterns.keys()} initialized ") self._comm = make_communication_object(self._context) @@ -189,7 +189,7 @@ def _create_pattern(self, horizontal_dim: Dimension): return pattern def exchange(self, dim: definitions.Dimension, *fields: Sequence[Field]): - assert dim in dims.global_dimensions.values() + assert dim in dims.horizontal_dims.values() pattern = self._patterns[dim] assert pattern is not None, f"pattern for {dim.value} not found" domain_descriptor = self._domain_descriptors[dim] diff --git a/model/common/src/icon4py/model/common/dimension.py b/model/common/src/icon4py/model/common/dimension.py index 6d202dda69..c4e168152c 100644 --- a/model/common/src/icon4py/model/common/dimension.py +++ b/model/common/src/icon4py/model/common/dimension.py @@ -15,7 +15,7 @@ EdgeDim = Dimension("Edge") CellDim = Dimension("Cell") VertexDim = Dimension("Vertex") -global_dimensions = {"CellDim": CellDim, "EdgeDim": EdgeDim, "VertexDim": VertexDim} +horizontal_dims = {"CellDim": CellDim, "EdgeDim": EdgeDim, "VertexDim": VertexDim} CEDim = Dimension("CE") CECDim = Dimension("CEC") ECDim = Dimension("EC") diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 73cd52f4d5..bbc7421e64 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -133,6 +133,7 @@ class DimensionName(GridFileName): CELL_GRF = "cell_grf" EDGE_GRF = "edge_grf" VERTEX_GRF = "vert_grf" + CHILD_DOMAINS = "max_chdom" class GridRefinementName(GridFileName): """Names of arrays in grid file defining the grid control, definition of boundaries layers, start and end indices of horizontal zones.""" @@ -301,7 +302,7 @@ def _read_grid_refinement_information(self, dataset): ] refin_ctrl = { dim: reader.int_field(control_dims[i]) - for i, dim in enumerate(dims.global_dimensions.values()) + for i, dim in enumerate(dims.horizontal_dims.values()) } grf_dims = [ @@ -311,7 +312,7 @@ def _read_grid_refinement_information(self, dataset): ] refin_ctrl_max = { dim: reader.dimension(grf_dims[i]) - for i, dim in enumerate(dims.global_dimensions.values()) + for i, dim in enumerate(dims.horizontal_dims.values()) } start_index_dims = [ @@ -323,7 +324,7 @@ def _read_grid_refinement_information(self, dataset): dim: self._get_index_field( start_index_dims[i], transpose=False, dtype=gtx.int32 )[_CHILD_DOM] - for i, dim in enumerate(dims.global_dimensions.values()) + for i, dim in enumerate(dims.horizontal_dims.values()) } end_index_dims = [ @@ -335,7 +336,7 @@ def _read_grid_refinement_information(self, dataset): dim: self._get_index_field( end_index_dims[i], transpose=False, apply_offset=False, dtype=gtx.int32 )[_CHILD_DOM] - for i, dim in enumerate(dims.global_dimensions.values()) + for i, dim in enumerate(dims.horizontal_dims.values()) } return start_indices, end_indices, refin_ctrl, refin_ctrl_max @@ -502,13 +503,13 @@ def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGri cells_to_rank_mapping = self._decompose( global_connectivities[dims.C2E2CDim], self._run_properties.comm_size ) - halo_constructor = halo.HaloGenerator( + construct_decomposition_info = halo.HaloGenerator( self._run_properties, cells_to_rank_mapping, global_connectivities, self._config.num_levels, ) - decomposition_info = halo_constructor() + decomposition_info = construct_decomposition_info() global_connectivities.update( { dims.E2V: self._get_index_field(GridFile.ConnectivityName.E2V), @@ -522,12 +523,15 @@ def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGri for offset, conn in global_connectivities.keys() } grid.with_connectivities(local_connectivities) - self._add_derived_connectivities(grid) + _add_derived_connectivities(grid) + # it has a fixed size that is a dimension in the grid file + # self._add_start_end_indices(grid) - self._update_size_for_1d_sparse_dims(grid) + _update_size_for_1d_sparse_dims(grid) return grid - + + #TODO (@halungge) is this used? def get_size(self, dim: gtx.Dimension): if dim == dims.VertexDim: return self._grid.config.num_vertices @@ -546,48 +550,63 @@ def _get_index_field( if apply_offset: field = field + self._transformation.get_offset_for_index_field(field) return field + + def _construct_start_indices(self, decomposition_info: decomposition.DecompositionInfo): + num_child_domins = self._reader.dimension(GridFile.DimensionName.MAX_CHILD_DOMAINS) + grf_cell_dim = self._reader.dimension(GridFile.DimensionName.CELL_GRF) + # single node case: just read? + #TODO switch from numpy to xp. ... + start_index = np.zeros((num_child_domins, grf_cell_dim), dtype=gtx.int32) + + + + + + + +########################### + +def _add_derived_connectivities(grid) -> icon_grid.IconGrid: + e2c2v = _construct_diamond_vertices( + grid.connectivities[dims.E2VDim], + grid.connectivities[dims.C2VDim], + grid.connectivities[dims.E2CDim], + ) + e2c2e = _construct_diamond_edges( + grid.connectivities[dims.E2CDim], grid.connectivities[dims.C2EDim] + ) + e2c2e0 = np.column_stack((np.asarray(range(e2c2e.shape[0])), e2c2e)) - def _add_derived_connectivities(self, grid) -> icon_grid.IconGrid: - e2c2v = self._construct_diamond_vertices( - grid.connectivities[dims.E2VDim], - grid.connectivities[dims.C2VDim], - grid.connectivities[dims.E2CDim], - ) - e2c2e = self._construct_diamond_edges( - grid.connectivities[dims.E2CDim], grid.connectivities[dims.C2EDim] - ) - e2c2e0 = np.column_stack((np.asarray(range(e2c2e.shape[0])), e2c2e)) - - c2e2c2e = self._construct_triangle_edges( - grid.connectivities[dims.C2E2CDim], grid.connectivities[dims.C2EDim] - ) - c2e2c0 = np.column_stack( - ( - np.asarray(range(grid.connectivities[dims.C2E2CDim].shape[0])), - (grid.connectivities[dims.C2E2CDim]), - ) + c2e2c2e = _construct_triangle_edges( + grid.connectivities[dims.C2E2CDim], grid.connectivities[dims.C2EDim] + ) + c2e2c0 = np.column_stack( + ( + np.asarray(range(grid.connectivities[dims.C2E2CDim].shape[0])), + (grid.connectivities[dims.C2E2CDim]), ) + ) - grid.with_connectivities( - { - dims.C2E2CODim: c2e2c0, - dims.C2E2C2EDim: c2e2c2e, - dims.E2C2VDim: e2c2v, - dims.E2C2EDim: e2c2e, - dims.E2C2EODim: e2c2e0, - } - ) + grid.with_connectivities( + { + dims.C2E2CODim: c2e2c0, + dims.C2E2C2EDim: c2e2c2e, + dims.E2C2VDim: e2c2v, + dims.E2C2EDim: e2c2e, + dims.E2C2EODim: e2c2e0, + } + ) - return grid + return grid - def _update_size_for_1d_sparse_dims(self, grid): - grid.update_size_connectivities( - { - dims.ECVDim: grid.size[dims.EdgeDim] * grid.size[dims.E2C2VDim], - dims.CEDim: grid.size[dims.CellDim] * grid.size[dims.C2EDim], - dims.ECDim: grid.size[dims.EdgeDim] * grid.size[dims.E2CDim], - } - ) +def _update_size_for_1d_sparse_dims(grid): + grid.update_size_connectivities( + { + dims.ECVDim: grid.size[dims.EdgeDim] * grid.size[dims.E2C2VDim], + dims.CEDim: grid.size[dims.CellDim] * grid.size[dims.C2EDim], + dims.ECDim: grid.size[dims.EdgeDim] * grid.size[dims.E2CDim], + } + ) def _initialize_global(self, limited_area, on_gpu): num_cells = self._reader.dimension(GridFile.DimensionName.CELL_NAME) @@ -609,111 +628,111 @@ def _initialize_global(self, limited_area, on_gpu): grid = icon_grid.IconGrid(uuid).with_config(config).with_global_params(global_params) return grid - @staticmethod - def _construct_diamond_vertices( - e2v: np.ndarray, c2v: np.ndarray, e2c: np.ndarray - ) -> np.ndarray: - r""" - Construct the connectivity table for the vertices of a diamond in the ICON triangular grid. - - Starting from the e2v and c2v connectivity the connectivity table for e2c2v is built up. - - v0 - / \ - / \ - / \ - / \ - v1---e0---v3 - \ / - \ / - \ / - \ / - v2 - For example for this diamond: e0 -> (v0, v1, v2, v3) - Ordering is the same as ICON uses. - - Args: - e2v: np.ndarray containing the connectivity table for edge-to-vertex - c2v: np.ndarray containing the connectivity table for cell-to-vertex - e2c: np.ndarray containing the connectivity table for edge-to-cell - - Returns: np.ndarray containing the connectivity table for edge-to-vertex on the diamond - """ - dummy_c2v = _patch_with_dummy_lastline(c2v) - expanded = dummy_c2v[e2c[:, :], :] - sh = expanded.shape - flat = expanded.reshape(sh[0], sh[1] * sh[2]) - far_indices = np.zeros_like(e2v) - # TODO (magdalena) vectorize speed this up? - for i in range(sh[0]): - far_indices[i, :] = flat[i, ~np.isin(flat[i, :], e2v[i, :])][:2] - return np.hstack((e2v, far_indices)) - - @staticmethod - def _construct_diamond_edges(e2c: np.ndarray, c2e: np.ndarray) -> np.ndarray: - r""" - Construct the connectivity table for the edges of a diamond in the ICON triangular grid. - - Starting from the e2c and c2e connectivity the connectivity table for e2c2e is built up. - - / \ - / \ - e2 e1 - / c0 \ - ----e0---- - \ c1 / - e3 e4 - \ / - \ / - - For example, for this diamond for e0 -> (e1, e2, e3, e4) - - - Args: - e2c: np.ndarray containing the connectivity table for edge-to-cell - c2e: np.ndarray containing the connectivity table for cell-to-edge - - Returns: np.ndarray containing the connectivity table for central edge-to- boundary edges - on the diamond - """ - dummy_c2e = _patch_with_dummy_lastline(c2e) - expanded = dummy_c2e[e2c[:, :], :] - sh = expanded.shape - flattened = expanded.reshape(sh[0], sh[1] * sh[2]) - - diamond_sides = 4 - e2c2e = GridFile.INVALID_INDEX * np.ones((sh[0], diamond_sides), dtype=gtx.int32) - for i in range(sh[0]): - var = flattened[i, (~np.isin(flattened[i, :], np.asarray([i, GridFile.INVALID_INDEX])))] - e2c2e[i, : var.shape[0]] = var - return e2c2e - - def _construct_triangle_edges(self, c2e2c, c2e): - """Compute the connectivity from a central cell to all neighboring edges of its cell neighbors. - - ____e3________e7____ - \ c1 / \ c3 / - \ / \ / - e4 e2 e1 e8 - \ / c0 \ / - ----e0---- - \ c2 / - e5 e6 - \ / - \ / - - For example, for the triangular shape above, c0 -> (e3, e4, e2, e0, e5, e6, e7, e1, e8). - - Args: - c2e2c: shape (n_cell, 3) connectivity table from a central cell to its cell neighbors - c2e: shape (n_cell, 3), connectivity table from a cell to its neighboring edges - Returns: - np.ndarray: shape(n_cells, 9) connectivity table from a central cell to all neighboring - edges of its cell neighbors - """ - dummy_c2e = _patch_with_dummy_lastline(c2e) - table = np.reshape(dummy_c2e[c2e2c[:, :], :], (c2e2c.shape[0], 9)) - return table + +def _construct_diamond_vertices( + e2v: np.ndarray, c2v: np.ndarray, e2c: np.ndarray +) -> np.ndarray: + r""" + Construct the connectivity table for the vertices of a diamond in the ICON triangular grid. + + Starting from the e2v and c2v connectivity the connectivity table for e2c2v is built up. + + v0 + / \ + / \ + / \ + / \ + v1---e0---v3 + \ / + \ / + \ / + \ / + v2 + For example for this diamond: e0 -> (v0, v1, v2, v3) + Ordering is the same as ICON uses. + + Args: + e2v: np.ndarray containing the connectivity table for edge-to-vertex + c2v: np.ndarray containing the connectivity table for cell-to-vertex + e2c: np.ndarray containing the connectivity table for edge-to-cell + + Returns: np.ndarray containing the connectivity table for edge-to-vertex on the diamond + """ + dummy_c2v = _patch_with_dummy_lastline(c2v) + expanded = dummy_c2v[e2c[:, :], :] + sh = expanded.shape + flat = expanded.reshape(sh[0], sh[1] * sh[2]) + far_indices = np.zeros_like(e2v) + # TODO (magdalena) vectorize speed this up? + for i in range(sh[0]): + far_indices[i, :] = flat[i, ~np.isin(flat[i, :], e2v[i, :])][:2] + return np.hstack((e2v, far_indices)) + + +def _construct_diamond_edges(e2c: np.ndarray, c2e: np.ndarray) -> np.ndarray: + r""" + Construct the connectivity table for the edges of a diamond in the ICON triangular grid. + + Starting from the e2c and c2e connectivity the connectivity table for e2c2e is built up. + + / \ + / \ + e2 e1 + / c0 \ + ----e0---- + \ c1 / + e3 e4 + \ / + \ / + + For example, for this diamond for e0 -> (e1, e2, e3, e4) + + + Args: + e2c: np.ndarray containing the connectivity table for edge-to-cell + c2e: np.ndarray containing the connectivity table for cell-to-edge + + Returns: np.ndarray containing the connectivity table for central edge-to- boundary edges + on the diamond + """ + dummy_c2e = _patch_with_dummy_lastline(c2e) + expanded = dummy_c2e[e2c[:, :], :] + sh = expanded.shape + flattened = expanded.reshape(sh[0], sh[1] * sh[2]) + + diamond_sides = 4 + e2c2e = GridFile.INVALID_INDEX * np.ones((sh[0], diamond_sides), dtype=gtx.int32) + for i in range(sh[0]): + var = flattened[i, (~np.isin(flattened[i, :], np.asarray([i, GridFile.INVALID_INDEX])))] + e2c2e[i, : var.shape[0]] = var + return e2c2e + +def _construct_triangle_edges(c2e2c, c2e): + r"""Compute the connectivity from a central cell to all neighboring edges of its cell neighbors. + + ____e3________e7____ + \ c1 / \ c3 / + \ / \ / + e4 e2 e1 e8 + \ / c0 \ / + ----e0---- + \ c2 / + e5 e6 + \ / + \ / + + For example, for the triangular shape above, c0 -> (e3, e4, e2, e0, e5, e6, e7, e1, e8). + + Args: + c2e2c: shape (n_cell, 3) connectivity table from a central cell to its cell neighbors + c2e: shape (n_cell, 3), connectivity table from a cell to its neighboring edges + Returns: + np.ndarray: shape(n_cells, 9) connectivity table from a central cell to all neighboring + edges of its cell neighbors + """ + dummy_c2e = _patch_with_dummy_lastline(c2e) + table = np.reshape(dummy_c2e[c2e2c[:, :], :], (c2e2c.shape[0], 9)) + return table def _patch_with_dummy_lastline(ar): @@ -753,6 +772,7 @@ def construct_local_connectivity( Args: field_offset: FieldOffset for which we want to construct the local connectivity table decomposition_info: DecompositionInfo for the current rank. + connectivity: Returns: connectivity are for the same FieldOffset but mapping from local target dimension indices to local source dimension indices. @@ -781,3 +801,5 @@ def construct_local_connectivity( local_connectivity[i, valid_neighbor_mask] = indices # log.debug(f"rank {self._props.rank} has local connectivity f: {local_connectivity}") return local_connectivity + + diff --git a/tools/src/icon4pytools/icon4pygen/bindings/locations.py b/tools/src/icon4pytools/icon4pygen/bindings/locations.py index a45189b1e2..b7d32812a6 100644 --- a/tools/src/icon4pytools/icon4pygen/bindings/locations.py +++ b/tools/src/icon4pytools/icon4pygen/bindings/locations.py @@ -68,9 +68,9 @@ def __getitem__(self, item: int) -> BasicLocation: return self.chain[item] def to_dim_list(self) -> list[Dimension]: - dims_initials = [key[0] for key in dims.global_dimensions.keys()] + dims_initials = [key[0] for key in dims.horizontal_dims.keys()] map_to_dim = { - d: list(dims.global_dimensions.values())[d_i] for d_i, d in enumerate(dims_initials) + d: list(dims.horizontal_dims.values())[d_i] for d_i, d in enumerate(dims_initials) } return [map_to_dim[str(c)] for c in self.chain] diff --git a/tools/src/icon4pytools/icon4pygen/metadata.py b/tools/src/icon4pytools/icon4pygen/metadata.py index 3d2fd37969..5e25509ed9 100644 --- a/tools/src/icon4pytools/icon4pygen/metadata.py +++ b/tools/src/icon4pytools/icon4pygen/metadata.py @@ -183,9 +183,9 @@ def provide_neighbor_table(chain: str, is_global: bool) -> DummyConnectivity: skip_values = True include_center = True if chain.count("O") > 0 else False - dims_initials = [key[0] for key in dims.global_dimensions.keys()] + dims_initials = [key[0] for key in dims.horizontal_dims.keys()] map_to_dim = { - d: list(dims.global_dimensions.values())[d_i] for d_i, d in enumerate(dims_initials) + d: list(dims.horizontal_dims.values())[d_i] for d_i, d in enumerate(dims_initials) } location_chain: list[Dimension] = [map_to_dim.get(c) for c in chain if c not in ("2", "O")] # type: ignore[misc] # type specified From 59d813a4f927531f68b91c66ac7332af3f33ef26 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 30 Aug 2024 09:24:45 +0200 Subject: [PATCH 028/240] start/end index (WIP) --- .../icon4py/model/common/grid/grid_manager.py | 120 +++++++++++------- .../tests/grid_tests/test_grid_manager.py | 26 +++- 2 files changed, 101 insertions(+), 45 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index bbc7421e64..69c3807398 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -18,9 +18,9 @@ from icon4py.model.common import dimension as dims from icon4py.model.common.decomposition import ( definitions as decomposition, - definitions as defs, halo, ) +from icon4py.model.common.grid import horizontal as h_grid from icon4py.model.common.settings import xp @@ -42,10 +42,15 @@ def __init__(self, *args, **kwargs): ) +class ReadType(enum.IntEnum): + FLOAT = 0 + INT = 1 + class GridFileName(str, enum.Enum): pass + @dataclasses.dataclass class GridFileField: name: GridFileName @@ -147,6 +152,7 @@ class GridRefinementName(GridFileName): #: refine control value of vertex indices CONTROL_VERTICES = "refin_v_ctrl" + #: start indices of horizontal grid zones for cell fields START_INDEX_CELLS = "start_idx_c" @@ -188,7 +194,7 @@ def int_field(self, name: GridFileName, transpose=True, dtype=gtx.int32) -> np.n try: nc_variable = self._dataset.variables[name] - self._log.debug(f"reading {name}: {nc_variable}") + self._log.debug(f"reading {name}: {nc_variable}: transposing = {transpose}") data = nc_variable[:] data = np.array(data, dtype=dtype) @@ -198,13 +204,16 @@ def int_field(self, name: GridFileName, transpose=True, dtype=gtx.int32) -> np.n self._log.warning(msg) raise IconGridError(msg) from err - def float_field( + def array_1d( self, name: GridFileName, indices: np.ndarray = None, dtype=gtx.float64 ) -> np.ndarray: - """Read a float field from the grid file. + """Read a field from the grid file. If a index array is given it only reads the values at those positions. - TODO (@halungge): currently only supports 1D fields + Args: + name: name of the field to read + indices: indices to read + dtype: datatype of the field """ try: # use python slice? 2D fields @@ -346,6 +355,7 @@ def _read( decomposition_info: decomposition.DecompositionInfo, fields: dict[dims.Dimension, Sequence[GridFileName]], ): + reader = self._reader.array_1d (cells_on_node, edges_on_node, vertices_on_node) = ( ( decomposition_info.global_index( @@ -367,9 +377,9 @@ def _read_local(fields: dict[dims.Dimension, Sequence[GridFileName]]): edge_fields = fields.get(dims.EdgeDim, []) vertex_fields = fields.get(dims.VertexDim, []) vals = ( - {name: self._reader.float_field(name, cells_on_node) for name in cell_fields} - | {name: self._reader.float_field(name, edges_on_node) for name in edge_fields} - | {name: self._reader.float_field(name, vertices_on_node) for name in vertex_fields} + {name: reader(name, cells_on_node) for name in cell_fields} + | {name:reader(name, edges_on_node) for name in edge_fields} + | {name: reader(name, vertices_on_node) for name in vertex_fields} ) return vals @@ -422,17 +432,17 @@ def _get_index(self, dim: gtx.Dimension, start_marker: int, index_dict): self._log.error(msg) def _from_grid_dataset(self, grid, on_gpu: bool, limited_area=True) -> icon_grid.IconGrid: - e2c2v = self._construct_diamond_vertices( + e2c2v = _construct_diamond_vertices( grid.connectivities[dims.E2VDim], grid.connectivities[dims.C2VDim], grid.connectivities[dims.E2CDim], ) - e2c2e = self._construct_diamond_edges( + e2c2e = _construct_diamond_edges( grid.connectivities[dims.E2CDim], grid.connectivities[dims.C2EDim] ) e2c2e0 = np.column_stack((np.asarray(range(e2c2e.shape[0])), e2c2e)) - c2e2c2e = self._construct_triangle_edges( + c2e2c2e = _construct_triangle_edges( grid.connectivities[dims.C2E2CDim], grid.connectivities[dims.C2EDim] ) c2e2c0 = np.column_stack( @@ -451,7 +461,7 @@ def _from_grid_dataset(self, grid, on_gpu: bool, limited_area=True) -> icon_grid dims.E2C2EODim: e2c2e0, } ) - self._update_size_for_1d_sparse_dims(grid) + _update_size_for_1d_sparse_dims(grid) return grid @@ -469,6 +479,7 @@ def _add_start_end_indices(self, grid): ).with_start_end_indices( dims.VertexDim, start_indices[dims.VertexDim], end_indices[dims.VertexDim] ) + # TODO (@halungge) # - remove duplication, @@ -496,7 +507,7 @@ def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGri } ) grid.with_connectivities({o.target[1]: c for o, c in global_connectivities.items()}) - self._add_derived_connectivities(grid) + _add_derived_connectivities(grid) self._add_start_end_indices(grid) return grid else: @@ -526,7 +537,7 @@ def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGri _add_derived_connectivities(grid) # it has a fixed size that is a dimension in the grid file - # self._add_start_end_indices(grid) + _update_size_for_1d_sparse_dims(grid) return grid @@ -546,20 +557,62 @@ def get_size(self, dim: gtx.Dimension): def _get_index_field( self, field: GridFileName, transpose=True, apply_offset=True, dtype=gtx.int32 ): - field = self._reader.int_field(field, transpose=transpose, dtype=dtype) + field = self._reader.int_field(field, dtype=dtype, transpose=transpose) if apply_offset: field = field + self._transformation.get_offset_for_index_field(field) return field def _construct_start_indices(self, decomposition_info: decomposition.DecompositionInfo): - num_child_domins = self._reader.dimension(GridFile.DimensionName.MAX_CHILD_DOMAINS) - grf_cell_dim = self._reader.dimension(GridFile.DimensionName.CELL_GRF) - # single node case: just read? - #TODO switch from numpy to xp. ... - start_index = np.zeros((num_child_domins, grf_cell_dim), dtype=gtx.int32) + num_child_domains = self._reader.dimension(GridFile.DimensionName.MAX_CHILD_DOMAINS) + grid_refinement_dimensions = {dim: self._reader.dimension(name) for dim, name in {dims.CellDim: GridFile.DimensionName.CELL_GRF, + dims.EdgeDim: GridFile.DimensionName.EDGE_GRF, + dims.VertexDim: GridFile.DimensionName.VERTEX_GRF}.items()} + start_index = {dim: np.zeros((num_child_domains, size), dtype=gtx.int32) for dim, size in grid_refinement_dimensions.items()} + end_index = {dim: np.zeros((num_child_domains, size), dtype=gtx.int32) for dim, size in + grid_refinement_dimensions.items()} + field_names = {dims.CellDim: [GridFile.GridRefinementName.CONTROL_CELLS], + dims.EdgeDim: [GridFile.GridRefinementName.CONTROL_EDGES], + dims.VertexDim: [GridFile.GridRefinementName.CONTROL_EDGES]} + + refine_control_fields = self._read(decomposition_info, fields=field_names) + # do we need to modify the refinement control for "halo" + dim = dims.CellDim + local_size = decomposition_info.global_index(dim, decomposition.DecompositionInfo.EntryType.ALL).shape[0] + start_index[dim][h_grid.HorizontalMarkerIndex.end(dim)] = local_size + start_index[dim][h_grid.HorizontalMarkerIndex.local(dim)] = 0 + # find first local halo index which is not on lateral boundary + local_halo_index = decomposition_info.local_index(dim, decomposition.DecompositionInfo.EntryType.HALO) + ref_ctrl = refine_control_fields[dim] + minimial_local_halo_index = np.min(np.where(ref_ctrl == local_halo_index) + start_index[dim][h_grid.HorizontalMarkerIndex.halo(dim)] = minimial_local_halo_index + # for global + + + + def _initialize_global(self, limited_area, on_gpu): + num_cells = self._reader.dimension(GridFile.DimensionName.CELL_NAME) + num_edges = self._reader.dimension(GridFile.DimensionName.EDGE_NAME) + num_vertices = self._reader.dimension(GridFile.DimensionName.VERTEX_NAME) + uuid = self._dataset.getncattr(GridFile.PropertyName.GRID_ID) + grid_level = self._dataset.getncattr(GridFile.PropertyName.LEVEL) + grid_root = self._dataset.getncattr(GridFile.PropertyName.ROOT) + global_params = icon_grid.GlobalGridParams(level=grid_level, root=grid_root) + grid_size = base_grid.HorizontalGridSize( + num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells + ) + config = base_grid.GridConfig( + horizontal_config=grid_size, + vertical_size=self._config.num_levels, + on_gpu=on_gpu, + limited_area=limited_area, + ) + grid = icon_grid.IconGrid(uuid).with_config(config).with_global_params(global_params) + return grid + + @@ -608,26 +661,7 @@ def _update_size_for_1d_sparse_dims(grid): } ) - def _initialize_global(self, limited_area, on_gpu): - num_cells = self._reader.dimension(GridFile.DimensionName.CELL_NAME) - num_edges = self._reader.dimension(GridFile.DimensionName.EDGE_NAME) - num_vertices = self._reader.dimension(GridFile.DimensionName.VERTEX_NAME) - uuid = self._dataset.getncattr(GridFile.PropertyName.GRID_ID) - grid_level = self._dataset.getncattr(GridFile.PropertyName.LEVEL) - grid_root = self._dataset.getncattr(GridFile.PropertyName.ROOT) - global_params = icon_grid.GlobalGridParams(level=grid_level, root=grid_root) - grid_size = base_grid.HorizontalGridSize( - num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells - ) - config = base_grid.GridConfig( - horizontal_config=grid_size, - vertical_size=self._config.num_levels, - on_gpu=on_gpu, - limited_area=limited_area, - ) - grid = icon_grid.IconGrid(uuid).with_config(config).with_global_params(global_params) - return grid - + def _construct_diamond_vertices( e2v: np.ndarray, c2v: np.ndarray, e2c: np.ndarray @@ -758,7 +792,7 @@ def _patch_with_dummy_lastline(ar): def construct_local_connectivity( field_offset: gtx.FieldOffset, - decomposition_info: defs.DecompositionInfo, + decomposition_info: decomposition.DecompositionInfo, connectivity: xp.ndarray, ) -> xp.ndarray: """ @@ -780,11 +814,11 @@ def construct_local_connectivity( source_dim = field_offset.source target_dim = field_offset.target[0] sliced_connectivity = connectivity[ - decomposition_info.global_index(target_dim, defs.DecompositionInfo.EntryType.ALL) + decomposition_info.global_index(target_dim, decomposition.DecompositionInfo.EntryType.ALL) ] # log.debug(f"rank {self._props.rank} has local connectivity f: {sliced_connectivity}") - global_idx = decomposition_info.global_index(source_dim, defs.DecompositionInfo.EntryType.ALL) + global_idx = decomposition_info.global_index(source_dim, decomposition.DecompositionInfo.EntryType.ALL) # replace indices in the connectivity that do not exist on the local node by the SKIP_VALUE (those are for example neighbors of the outermost halo points) local_connectivity = xp.where( diff --git a/model/common/tests/grid_tests/test_grid_manager.py b/model/common/tests/grid_tests/test_grid_manager.py index a54b03b2c8..d6a3a2d64d 100644 --- a/model/common/tests/grid_tests/test_grid_manager.py +++ b/model/common/tests/grid_tests/test_grid_manager.py @@ -327,7 +327,7 @@ def test_grid_parser_index_fields(simple_grid_gridfile, caplog): @pytest.mark.parametrize( "grid_file, experiment", [ - (dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), + #(dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), (utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT), ], ) @@ -346,6 +346,28 @@ def test_gridmanager_eval_v2e(caplog, grid_savepoint, grid_file): assert np.allclose(v2e_table, seralized_v2e) +pytest.mark.datatest +@pytest.mark.with_netcdf +@pytest.mark.parametrize( + "grid_file, experiment", + [ + (dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), + #(utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT), + ], +) +def test_refin_ctrl(grid_savepoint, grid_file, experiment): + file = utils.resolve_file_from_gridfile_name(grid_file) + gm = init_grid_manager(file) + start_index, end_index, refin_ctrl, refin_ctrl_max = gm._read_grid_refinement_information(gm._dataset) + refin_ctrl_c = grid_savepoint.refin_ctrl(dims.CellDim) + refin_ctrl_e = grid_savepoint.refin_ctrl(dims.EdgeDim) + + + refin_ctrl_v = grid_savepoint.refin_ctrl(dims.VertexDim) + assert np.allclose(refin_ctrl_c.ndarray, refin_ctrl[dims.CellDim]) + assert np.allclose(refin_ctrl_e.ndarray, refin_ctrl[dims.EdgeDim]) + assert np.allclose(refin_ctrl_v.ndarray, refin_ctrl[dims.VertexDim]) + # v2c: exists in serial, simple, grid @pytest.mark.datatest @pytest.mark.with_netcdf @@ -1025,7 +1047,7 @@ def test_gridmanager_eval_c2e2c2e(caplog, grid_savepoint, grid_file): assert grid.get_offset_provider("C2E2C2E").table.shape == (grid.num_cells, 9) -@pytest.mark.with_mpi +@pytest.mark.mpi @pytest.mark.parametrize( "field_offset", [dims.C2V, dims.E2V, dims.V2C, dims.E2C, dims.C2E, dims.V2E, dims.C2E2C, dims.V2E2V], From a1291381681e101c426e49073d074aebfaca8e6a Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 3 Sep 2024 10:37:40 +0200 Subject: [PATCH 029/240] start / end indices (WIP) add refinement.py --- .../icon4py/model/common/grid/grid_manager.py | 2 +- .../icon4py/model/common/grid/refinement.py | 43 +++++++++++++++++++ .../tests/grid_tests/test_refinement.py | 37 ++++++++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 model/common/src/icon4py/model/common/grid/refinement.py create mode 100644 model/common/tests/grid_tests/test_refinement.py diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 69c3807398..a26efabd70 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -584,7 +584,7 @@ def _construct_start_indices(self, decomposition_info: decomposition.Decompositi # find first local halo index which is not on lateral boundary local_halo_index = decomposition_info.local_index(dim, decomposition.DecompositionInfo.EntryType.HALO) ref_ctrl = refine_control_fields[dim] - minimial_local_halo_index = np.min(np.where(ref_ctrl == local_halo_index) + minimial_local_halo_index = np.min(np.where(ref_ctrl == local_halo_index)) start_index[dim][h_grid.HorizontalMarkerIndex.halo(dim)] = minimial_local_halo_index diff --git a/model/common/src/icon4py/model/common/grid/refinement.py b/model/common/src/icon4py/model/common/grid/refinement.py new file mode 100644 index 0000000000..7b953293ce --- /dev/null +++ b/model/common/src/icon4py/model/common/grid/refinement.py @@ -0,0 +1,43 @@ +import dataclasses +from typing import Final + + +""" +Refinement control for ICON grid. + +Grid refinement is used in the context of +- local area grids to determine the type of a grid point, +- nested horizontal grids to determine the nested overlap regions for feedback +- domain decomposition to order grid points + +See Zaengl et al. Grid Refinement in ICON v2.6.4 (Geosci. Model Dev., 15, 7153–7176, 202) + + +""" + + +_MAX_ORDERED: Final[int] = 14 +"""Lateral boundary points are ordered and have an index indicating the (cell) s distance to the boundary, +generally the number of ordered rows can be defined in the grid generator, but it will never exceed 14. +""" + + +_UNORDERED: Final[tuple[int, int]] = (0, -4) +"""Value indicating a point is int the unordered interior (fully prognostic) region: this is encoded by 0 or -4 in coarser parent grid.""" + +_MIN_ORDERED: Final[int] = -3 +"""For coarse parent grids the overlapping boundary regions are counted with negative values, from -1 to max -3, (as -4 is used to mark interior points)""" + +@dataclasses.dataclass(frozen=True) +class RefinementValue(): + value: int + + def __post_init__(self): + assert _UNORDERED[1] <= self.value <= _MAX_ORDERED, f"Invalid refinement control constant {self.value}" + + + def is_nested(self) -> bool: + return self.value < 0 + + def is_ordered(self) -> bool: + return self.value not in _UNORDERED \ No newline at end of file diff --git a/model/common/tests/grid_tests/test_refinement.py b/model/common/tests/grid_tests/test_refinement.py new file mode 100644 index 0000000000..27d1fed2b0 --- /dev/null +++ b/model/common/tests/grid_tests/test_refinement.py @@ -0,0 +1,37 @@ +import pytest + +import icon4py.model.common.grid.refinement as refin + + +def out_of_range(): + for v in range(-15, -4): + yield v + for v in range(15, 20): + yield v + + +def refinement_value(): + for v in range(-4, 14): + yield v + + +@pytest.mark.parametrize("value", refinement_value()) +def test_ordered(value): + ordered = refin.RefinementValue(value) + if ordered.value == 0 or ordered.value == -4: + assert not ordered.is_ordered() + else: + assert ordered.is_ordered() + +@pytest.mark.parametrize("value", refinement_value()) +def test_nested(value): + nested = refin.RefinementValue(value) + if nested.value < 0: + assert nested.is_nested() + else: + assert not nested.is_nested() + +@pytest.mark.parametrize("value", out_of_range()) +def test_valid_refinement_values(value): + with pytest.raises(AssertionError): + refin.RefinementValue(value) \ No newline at end of file From 82c2cf53ec1c8efa8909e18e396ad00641759f9b Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 4 Sep 2024 09:33:35 +0200 Subject: [PATCH 030/240] compute start end indices for Zones: local, halos --- .../model/common/decomposition/definitions.py | 36 ++ .../model/common/decomposition/halo.py | 62 +-- .../icon4py/model/common/grid/grid_manager.py | 363 ++++++++++-------- .../icon4py/model/common/grid/horizontal.py | 2 +- .../icon4py/model/common/grid/refinement.py | 36 +- .../tests/decomposition_tests/test_halo.py | 2 +- .../tests/grid_tests/test_grid_manager.py | 340 ++++++++-------- model/common/tests/grid_tests/test_icon.py | 17 +- model/common/tests/grid_tests/utils.py | 6 + 9 files changed, 496 insertions(+), 368 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 6b58028535..636673ce17 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -8,6 +8,7 @@ from __future__ import annotations +import enum import functools import logging from dataclasses import dataclass @@ -18,6 +19,7 @@ import numpy.ma as ma from gt4py.next import Dimension +from icon4py.model.common.settings import xp from icon4py.model.common.utils import builder @@ -127,6 +129,9 @@ def global_index(self, dim: Dimension, entry_type: EntryType = EntryType.ALL): def halo_levels(self, dim: Dimension): return self._halo_levels[dim] + def halo_level_mask(self, dim: Dimension, level: DecompositionFlag): + return xp.where(self._halo_levels[dim] == level, True, False) + class ExchangeResult(Protocol): def wait(self): @@ -239,3 +244,34 @@ def create_single_node_exchange( props: SingleNodeProcessProperties, decomp_info: DecompositionInfo ) -> ExchangeRuntime: return SingleNodeExchange() + + +class DecompositionFlag(enum.IntEnum): + UNDEFINED = -1 + OWNED = 0 + """used for locally owned cells, vertices, edges""" + + FIRST_HALO_LINE = 1 + """ + used for: + - cells that share 1 edge with an OWNED cell + - vertices that are on OWNED cell + - edges that are on OWNED cell + """ + + SECOND_HALO_LINE = 2 + """ + used for: + - cells that share a vertex with an OWNED cell + - vertices that are on a cell(FIRST_HALO_LINE) but not on an owned cell + - edges that have _exactly_ one vertex shared with and OWNED Cell + """ + + THIRD_HALO_LINE = 3 + """ + This type does not exist in ICON. It denotes the "closing/far" edges of the SECOND_HALO_LINE cells + used for: + - cells (NOT USED) + - vertices (NOT USED) + - edges that are only on the cell(SECOND_HALO_LINE) + """ diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index d9057c513c..3ea2f5ed52 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -19,7 +19,6 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -import enum import logging from typing import Protocol @@ -34,35 +33,6 @@ # TODO (@halungge) do we need three of those: one for each dimension? -class DecompositionFlag(enum.IntEnum): - UNDEFINED = -1 - OWNED = 0 - """used for locally owned cells, vertices, edges""" - - FIRST_HALO_LINE = 1 - """ - used for: - - cells that share 1 edge with an OWNED cell - - vertices that are on OWNED cell - - edges that are on OWNED cell - """ - - SECOND_HALO_LINE = 2 - """ - used for: - - cells that share a vertex with an OWNED cell - - vertices that are on a cell(FIRST_HALO_LINE) but not on an owned cell - - edges that have _exactly_ one vertex shared with and OWNED Cell - """ - - THIRD_HALO_LINE = 3 - """ - This type does not exist in ICON. It denotes the "closing/far" edges of the SECOND_HALO_LINE cells - used for: - - cells (NOT USED) - - vertices (NOT USED) - - edges that are only on the cell(SECOND_HALO_LINE) - """ class HaloGenerator: @@ -224,10 +194,14 @@ def __call__(self) -> defs.DecompositionInfo: all_cells = xp.hstack((owned_cells, total_halo_cells)) cell_owner_mask = xp.isin(all_cells, owned_cells) - cell_halo_levels = DecompositionFlag.UNDEFINED * xp.ones(all_cells.size, dtype=int) - cell_halo_levels[cell_owner_mask] = DecompositionFlag.OWNED - cell_halo_levels[xp.isin(all_cells, first_halo_cells)] = DecompositionFlag.FIRST_HALO_LINE - cell_halo_levels[xp.isin(all_cells, second_halo_cells)] = DecompositionFlag.SECOND_HALO_LINE + cell_halo_levels = defs.DecompositionFlag.UNDEFINED * xp.ones(all_cells.size, dtype=int) + cell_halo_levels[cell_owner_mask] = defs.DecompositionFlag.OWNED + cell_halo_levels[ + xp.isin(all_cells, first_halo_cells) + ] = defs.DecompositionFlag.FIRST_HALO_LINE + cell_halo_levels[ + xp.isin(all_cells, second_halo_cells) + ] = defs.DecompositionFlag.SECOND_HALO_LINE decomp_info = defs.DecompositionInfo(klevels=self._num_levels).with_dimension( dims.CellDim, all_cells, cell_owner_mask, cell_halo_levels ) @@ -251,17 +225,19 @@ def __call__(self) -> defs.DecompositionInfo: self.node_face_connectivity, ) vertex_second_level = xp.setdiff1d(vertex_on_first_halo_line, vertex_on_owned_cells) - vertex_halo_levels = DecompositionFlag.UNDEFINED * xp.ones(all_vertices.size, dtype=int) - vertex_halo_levels[vertex_owner_mask] = DecompositionFlag.OWNED + vertex_halo_levels = defs.DecompositionFlag.UNDEFINED * xp.ones( + all_vertices.size, dtype=int + ) + vertex_halo_levels[vertex_owner_mask] = defs.DecompositionFlag.OWNED vertex_halo_levels[ xp.logical_and( xp.logical_not(vertex_owner_mask), xp.isin(all_vertices, vertex_on_cutting_line), ) - ] = DecompositionFlag.FIRST_HALO_LINE + ] = defs.DecompositionFlag.FIRST_HALO_LINE vertex_halo_levels[ xp.isin(all_vertices, vertex_second_level) - ] = DecompositionFlag.SECOND_HALO_LINE + ] = defs.DecompositionFlag.SECOND_HALO_LINE decomp_info.with_dimension( dims.VertexDim, all_vertices, vertex_owner_mask, vertex_halo_levels ) @@ -298,14 +274,16 @@ def __call__(self) -> defs.DecompositionInfo: edge_intersect_owned_first_line, self.edge_face_connectivity, ) - edge_halo_levels = DecompositionFlag.UNDEFINED * xp.ones(all_edges.shape, dtype=int) - edge_halo_levels[edge_owner_mask] = DecompositionFlag.OWNED + edge_halo_levels = defs.DecompositionFlag.UNDEFINED * xp.ones(all_edges.shape, dtype=int) + edge_halo_levels[edge_owner_mask] = defs.DecompositionFlag.OWNED edge_halo_levels[ xp.logical_and( xp.logical_not(edge_owner_mask), xp.isin(all_edges, edge_intersect_owned_first_line) ) - ] = DecompositionFlag.FIRST_HALO_LINE - edge_halo_levels[xp.isin(all_edges, level_two_edges)] = DecompositionFlag.SECOND_HALO_LINE + ] = defs.DecompositionFlag.FIRST_HALO_LINE + edge_halo_levels[ + xp.isin(all_edges, level_two_edges) + ] = defs.DecompositionFlag.SECOND_HALO_LINE decomp_info.with_dimension(dims.EdgeDim, all_edges, edge_owner_mask, edge_halo_levels) return decomp_info diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index d677b9bcf9..8682bf0ae0 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -20,7 +20,7 @@ definitions as decomposition, halo, ) -from icon4py.model.common.grid import horizontal as h_grid +from icon4py.model.common.grid import horizontal as h_grid, refinement as refin from icon4py.model.common.settings import xp @@ -64,123 +64,129 @@ def _validate_shape(data: np.array, field_definition: GridFileField): ) -class GridFile: - """Represent and ICON netcdf grid file.""" +class PropertyName(GridFileName): + GRID_ID = "uuidOfHGrid" + PARENT_GRID_ID = "uuidOfParHGrid" + LEVEL = "grid_level" + ROOT = "grid_root" - INVALID_INDEX = -1 - class PropertyName(GridFileName): - GRID_ID = "uuidOfHGrid" - PARENT_GRID_ID = "uuidOfParHGrid" - LEVEL = "grid_level" - ROOT = "grid_root" +class ConnectivityName(GridFileName): + """Names for connectivities used in the grid file.""" + + # e2c2e/e2c2eO: diamond edges (including origin) not present in grid file-> construct + # from e2c and c2e + # e2c2v: diamond vertices: not present in grid file -> constructed from e2c and c2v + + #: name of C2E2C connectivity in grid file: dims(nv=3, cell) + C2E2C = "neighbor_cell_index" + + #: name of V2E2V connectivity in gridfile: dims(ne=6, vertex), + #: all vertices of a pentagon/hexagon, same as V2C2V + V2E2V = "vertices_of_vertex" # does not exist in simple.py - class ConnectivityName(GridFileName): - """Names for connectivities used in the grid file.""" + #: name of V2E dimension in grid file: dims(ne=6, vertex) + V2E = "edges_of_vertex" - # e2c2e/e2c2eO: diamond edges (including origin) not present in grid file-> construct - # from e2c and c2e - # e2c2v: diamond vertices: not present in grid file -> constructed from e2c and c2v + #: name fo V2C connectivity in grid file: dims(ne=6, vertex) + V2C = "cells_of_vertex" - #: name of C2E2C connectivity in grid file: dims(nv=3, cell) - C2E2C = "neighbor_cell_index" + #: name of E2V connectivity in grid file: dims(nc=2, edge) + E2V = "edge_vertices" - #: name of V2E2V connectivity in gridfile: dims(ne=6, vertex), - #: all vertices of a pentagon/hexagon, same as V2C2V - V2E2V = "vertices_of_vertex" # does not exist in simple.py + #: name of C2V connectivity in grid file: dims(nv=3, cell) + C2V = "vertex_of_cell" # does not exist in grid.simple.py - #: name of V2E dimension in grid file: dims(ne=6, vertex) - V2E = "edges_of_vertex" + #: name of E2C connectivity in grid file: dims(nc=2, edge) + E2C = "adjacent_cell_of_edge" - #: name fo V2C connectivity in grid file: dims(ne=6, vertex) - V2C = "cells_of_vertex" + #: name of C2E connectivity in grid file: dims(nv=3, cell) + C2E = "edge_of_cell" - #: name of E2V connectivity in grid file: dims(nc=2, edge) - E2V = "edge_vertices" - #: name of C2V connectivity in grid file: dims(nv=3, cell) - C2V = "vertex_of_cell" # does not exist in grid.simple.py +class DimensionName(GridFileName): + """Dimension values (sizes) used in grid file.""" - #: name of E2C connectivity in grid file: dims(nc=2, edge) - E2C = "adjacent_cell_of_edge" + #: number of vertices + VERTEX_NAME = "vertex" - #: name of C2E connectivity in grid file: dims(nv=3, cell) - C2E = "edge_of_cell" + #: number of edges + EDGE_NAME = "edge" - class DimensionName(GridFileName): - """Dimension values (sizes) used in grid file.""" + #: number of cells + CELL_NAME = "cell" - #: number of vertices - VERTEX_NAME = "vertex" + #: number of edges in a diamond: 4 + DIAMOND_EDGE_SIZE = "no" - #: number of edges - EDGE_NAME = "edge" + #: number of edges/cells neighboring one vertex: 6 (for regular, non pentagons) + NEIGHBORS_TO_VERTEX_SIZE = "ne" - #: number of cells - CELL_NAME = "cell" + #: number of cells edges, vertices and cells neighboring a cell: 3 + NEIGHBORS_TO_CELL_SIZE = "nv" - #: number of edges in a diamond: 4 - DIAMOND_EDGE_SIZE = "no" + #: number of vertices/cells neighboring an edge: 2 + NEIGHBORS_TO_EDGE_SIZE = "nc" - #: number of edges/cells neighboring one vertex: 6 (for regular, non pentagons) - NEIGHBORS_TO_VERTEX_SIZE = "ne" + #: number of child domains (for nesting) + MAX_CHILD_DOMAINS = "max_chdom" - #: number of cells edges, vertices and cells neighboring a cell: 3 - NEIGHBORS_TO_CELL_SIZE = "nv" + #: Grid refinement: maximal number in grid-refinement (refin_ctl) array for each dimension + CELL_GRF = "cell_grf" + EDGE_GRF = "edge_grf" + VERTEX_GRF = "vert_grf" + CHILD_DOMAINS = "max_chdom" - #: number of vertices/cells neighboring an edge: 2 - NEIGHBORS_TO_EDGE_SIZE = "nc" - #: number of child domains (for nesting) - MAX_CHILD_DOMAINS = "max_chdom" +class GeometryName(GridFileName): + CELL_AREA = "cell_area" + EDGE_LENGTH = "edge_length" - #: Grid refinement: maximal number in grid-refinement (refin_ctl) array for each dimension - CELL_GRF = "cell_grf" - EDGE_GRF = "edge_grf" - VERTEX_GRF = "vert_grf" - CHILD_DOMAINS = "max_chdom" - class GridRefinementName(GridFileName): - """Names of arrays in grid file defining the grid control, definition of boundaries layers, start and end indices of horizontal zones.""" +class CoordinateName(GridFileName): + CELL_LONGITUDE = "clon" + CELL_LATITUDE = "clat" + EDGE_LONGITUDE = "elon" + EDGE_LATITUDE = "elat" + VERTEX_LONGITUDE = "vlon" + VERTEX_LATITUDE = "vlat" - #: refine control value of cell indices - CONTROL_CELLS = "refin_c_ctrl" - #: refine control value of edge indices - CONTROL_EDGES = "refin_e_ctrl" +class GridRefinementName(GridFileName): + """Names of arrays in grid file defining the grid control, definition of boundaries layers, start and end indices of horizontal zones.""" - #: refine control value of vertex indices - CONTROL_VERTICES = "refin_v_ctrl" + #: refine control value of cell indices + CONTROL_CELLS = "refin_c_ctrl" - #: start indices of horizontal grid zones for cell fields - START_INDEX_CELLS = "start_idx_c" + #: refine control value of edge indices + CONTROL_EDGES = "refin_e_ctrl" - #: start indices of horizontal grid zones for edge fields - START_INDEX_EDGES = "start_idx_e" + #: refine control value of vertex indices + CONTROL_VERTICES = "refin_v_ctrl" - #: start indices of horizontal grid zones for vertex fields - START_INDEX_VERTICES = "start_idx_v" + #: start indices of horizontal grid zones for cell fields + START_INDEX_CELLS = "start_idx_c" - #: end indices of horizontal grid zones for cell fields - END_INDEX_CELLS = "end_idx_c" + #: start indices of horizontal grid zones for edge fields + START_INDEX_EDGES = "start_idx_e" - #: end indices of horizontal grid zones for edge fields - END_INDEX_EDGES = "end_idx_e" + #: start indices of horizontal grid zones for vertex fields + START_INDEX_VERTICES = "start_idx_v" - #: end indices of horizontal grid zones for vertex fields - END_INDEX_VERTICES = "end_idx_v" + #: end indices of horizontal grid zones for cell fields + END_INDEX_CELLS = "end_idx_c" - class GeometryName(GridFileName): - CELL_AREA = "cell_area" - EDGE_LENGTH = "edge_length" + #: end indices of horizontal grid zones for edge fields + END_INDEX_EDGES = "end_idx_e" - class CoordinateName(GridFileName): - CELL_LONGITUDE = "clon" - CELL_LATITUDE = "clat" - EDGE_LONGITUDE = "elon" - EDGE_LATITUDE = "elat" - VERTEX_LONGITUDE = "vlon" - VERTEX_LATITUDE = "vlat" + #: end indices of horizontal grid zones for vertex fields + END_INDEX_VERTICES = "end_idx_v" + + +class GridFile: + """Represent and ICON netcdf grid file.""" + + INVALID_INDEX = -1 def __init__(self, dataset: Dataset): self._dataset = dataset @@ -215,8 +221,7 @@ def array_1d( dtype: datatype of the field """ try: - # use python slice? 2D fields - # (sparse, horizontal) + # use python slice? 2D fields (sparse, horizontal) variable = self._dataset.variables[name] self._log.debug(f"reading {name}: {variable}") data = variable[:] if indices is None else variable[indices] @@ -308,9 +313,9 @@ def _read_grid_refinement_information(self, dataset): reader = GridFile(dataset) control_dims = [ - GridFile.GridRefinementName.CONTROL_CELLS, - GridFile.GridRefinementName.CONTROL_EDGES, - GridFile.GridRefinementName.CONTROL_VERTICES, + GridRefinementName.CONTROL_CELLS, + GridRefinementName.CONTROL_EDGES, + GridRefinementName.CONTROL_VERTICES, ] refin_ctrl = { dim: reader.int_field(control_dims[i]) @@ -318,9 +323,9 @@ def _read_grid_refinement_information(self, dataset): } grf_dims = [ - GridFile.DimensionName.CELL_GRF, - GridFile.DimensionName.EDGE_GRF, - GridFile.DimensionName.VERTEX_GRF, + DimensionName.CELL_GRF, + DimensionName.EDGE_GRF, + DimensionName.VERTEX_GRF, ] refin_ctrl_max = { dim: reader.dimension(grf_dims[i]) @@ -328,9 +333,9 @@ def _read_grid_refinement_information(self, dataset): } start_index_dims = [ - GridFile.GridRefinementName.START_INDEX_CELLS, - GridFile.GridRefinementName.START_INDEX_EDGES, - GridFile.GridRefinementName.START_INDEX_VERTICES, + GridRefinementName.START_INDEX_CELLS, + GridRefinementName.START_INDEX_EDGES, + GridRefinementName.START_INDEX_VERTICES, ] start_indices = { dim: self._get_index_field(start_index_dims[i], transpose=False, dtype=gtx.int32)[ @@ -340,9 +345,9 @@ def _read_grid_refinement_information(self, dataset): } end_index_dims = [ - GridFile.GridRefinementName.END_INDEX_CELLS, - GridFile.GridRefinementName.END_INDEX_EDGES, - GridFile.GridRefinementName.END_INDEX_VERTICES, + GridRefinementName.END_INDEX_CELLS, + GridRefinementName.END_INDEX_EDGES, + GridRefinementName.END_INDEX_VERTICES, ] end_indices = { dim: self._get_index_field( @@ -380,9 +385,9 @@ def _read_local(fields: dict[dims.Dimension, Sequence[GridFileName]]): edge_fields = fields.get(dims.EdgeDim, []) vertex_fields = fields.get(dims.VertexDim, []) vals = ( - {name: reader(name, cells_on_node) for name in cell_fields} - | {name: reader(name, edges_on_node) for name in edge_fields} - | {name: reader(name, vertices_on_node) for name in vertex_fields} + {name: reader(name, cells_on_node, dtype=gtx.int32) for name in cell_fields} + | {name: reader(name, edges_on_node, dtype=gtx.int32) for name in edge_fields} + | {name: reader(name, vertices_on_node, dtype=gtx.int32) for name in vertex_fields} ) return vals @@ -393,8 +398,8 @@ def read_geometry(self, decomposition_info: Optional[decomposition.Decomposition return self._read( decomposition_info, { - dims.CellDim: [GridFile.GeometryName.CELL_AREA], - dims.EdgeDim: [GridFile.GeometryName.EDGE_LENGTH], + dims.CellDim: [GeometryName.CELL_AREA], + dims.EdgeDim: [GeometryName.EDGE_LENGTH], }, ) @@ -405,16 +410,16 @@ def read_coordinates( decomposition_info, { dims.CellDim: [ - GridFile.CoordinateName.CELL_LONGITUDE, - GridFile.CoordinateName.CELL_LATITUDE, + CoordinateName.CELL_LONGITUDE, + CoordinateName.CELL_LATITUDE, ], dims.EdgeDim: [ - GridFile.CoordinateName.EDGE_LONGITUDE, - GridFile.CoordinateName.EDGE_LATITUDE, + CoordinateName.EDGE_LONGITUDE, + CoordinateName.EDGE_LATITUDE, ], dims.VertexDim: [ - GridFile.CoordinateName.VERTEX_LONGITUDE, - GridFile.CoordinateName.VERTEX_LATITUDE, + CoordinateName.VERTEX_LONGITUDE, + CoordinateName.VERTEX_LATITUDE, ], }, ) @@ -468,7 +473,7 @@ def _from_grid_dataset(self, grid, on_gpu: bool, limited_area=True) -> icon_grid return grid - def _add_start_end_indices(self, grid): + def _read_start_end_indices(self, grid): ( start_indices, end_indices, @@ -492,25 +497,25 @@ def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGri grid = self._initialize_global(limited_area, on_gpu) global_connectivities = { - dims.C2E2C: self._get_index_field(GridFile.ConnectivityName.C2E2C), - dims.C2E: self._get_index_field(GridFile.ConnectivityName.C2E), - dims.E2C: self._get_index_field(GridFile.ConnectivityName.E2C), - dims.V2E: self._get_index_field(GridFile.ConnectivityName.V2E), - dims.E2V: self._get_index_field(GridFile.ConnectivityName.E2V), - dims.V2C: self._get_index_field(GridFile.ConnectivityName.V2C), - dims.C2V: self._get_index_field(GridFile.ConnectivityName.C2V), - dims.V2E2V: self._get_index_field(GridFile.ConnectivityName.V2E2V), + dims.C2E2C: self._get_index_field(ConnectivityName.C2E2C), + dims.C2E: self._get_index_field(ConnectivityName.C2E), + dims.E2C: self._get_index_field(ConnectivityName.E2C), + dims.V2E: self._get_index_field(ConnectivityName.V2E), + dims.E2V: self._get_index_field(ConnectivityName.E2V), + dims.V2C: self._get_index_field(ConnectivityName.V2C), + dims.C2V: self._get_index_field(ConnectivityName.C2V), + dims.V2E2V: self._get_index_field(ConnectivityName.V2E2V), } if self._run_properties.single_node(): global_connectivities.update( { - dims.E2V: self._get_index_field(GridFile.ConnectivityName.E2V), - dims.C2V: self._get_index_field(GridFile.ConnectivityName.C2V), + dims.E2V: self._get_index_field(ConnectivityName.E2V), + dims.C2V: self._get_index_field(ConnectivityName.C2V), } ) grid.with_connectivities({o.target[1]: c for o, c in global_connectivities.items()}) _add_derived_connectivities(grid) - self._add_start_end_indices(grid) + self._read_start_end_indices(grid) return grid else: cells_to_rank_mapping = self._decompose( @@ -525,8 +530,8 @@ def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGri decomposition_info = construct_decomposition_info() global_connectivities.update( { - dims.E2V: self._get_index_field(GridFile.ConnectivityName.E2V), - dims.C2V: self._get_index_field(GridFile.ConnectivityName.C2V), + dims.E2V: self._get_index_field(ConnectivityName.E2V), + dims.C2V: self._get_index_field(ConnectivityName.C2V), } ) local_connectivities = { @@ -563,55 +568,107 @@ def _get_index_field( return field def _construct_start_indices(self, decomposition_info: decomposition.DecompositionInfo): - num_child_domains = self._reader.dimension(GridFile.DimensionName.MAX_CHILD_DOMAINS) grid_refinement_dimensions = { dim: self._reader.dimension(name) for dim, name in { - dims.CellDim: GridFile.DimensionName.CELL_GRF, - dims.EdgeDim: GridFile.DimensionName.EDGE_GRF, - dims.VertexDim: GridFile.DimensionName.VERTEX_GRF, + dims.CellDim: DimensionName.CELL_GRF, + dims.EdgeDim: DimensionName.EDGE_GRF, + dims.VertexDim: DimensionName.VERTEX_GRF, }.items() } start_index = { - dim: np.zeros((num_child_domains, size), dtype=gtx.int32) + dim: np.zeros((size,), dtype=gtx.int32) for dim, size in grid_refinement_dimensions.items() } end_index = { - dim: np.zeros((num_child_domains, size), dtype=gtx.int32) + dim: np.zeros((size,), dtype=gtx.int32) for dim, size in grid_refinement_dimensions.items() } field_names = { - dims.CellDim: [GridFile.GridRefinementName.CONTROL_CELLS], - dims.EdgeDim: [GridFile.GridRefinementName.CONTROL_EDGES], - dims.VertexDim: [GridFile.GridRefinementName.CONTROL_EDGES], + dims.CellDim: (GridRefinementName.CONTROL_CELLS,), + dims.EdgeDim: (GridRefinementName.CONTROL_EDGES,), + dims.VertexDim: (GridRefinementName.CONTROL_VERTICES,), } - refine_control_fields = self._read(decomposition_info, fields=field_names) + field_list = self._read(decomposition_info, fields=field_names) + refine_control_fields = {dim: field_list[name[0]] for dim, name in field_names.items()} # do we need to modify the refinement control for "halo" - dim = dims.CellDim - local_size = decomposition_info.global_index( - dim, decomposition.DecompositionInfo.EntryType.ALL - ).shape[0] - start_index[dim][h_grid.HorizontalMarkerIndex.end(dim)] = local_size - start_index[dim][h_grid.HorizontalMarkerIndex.local(dim)] = 0 - # find first local halo index which is not on lateral boundary - local_halo_index = decomposition_info.local_index( - dim, decomposition.DecompositionInfo.EntryType.HALO - ) - ref_ctrl = refine_control_fields[dim] - minimial_local_halo_index = np.min(np.where(ref_ctrl == local_halo_index)) - start_index[dim][h_grid.HorizontalMarkerIndex.halo(dim)] = minimial_local_halo_index + for dim in dims.horizontal_dims.values(): + domain = h_grid.domain(dim) + local_size = decomposition_info.global_index( + dim, decomposition.DecompositionInfo.EntryType.ALL + ).shape[0] + # END: size of the local grid for both start and end index + start_index[dim][domain(h_grid.Zone.END)()] = local_size + end_index[dim][domain(h_grid.Zone.END)()] = local_size + + local_owned_index = decomposition_info.local_index( + dim, decomposition.DecompositionInfo.EntryType.OWNED + ) + unordered = refin.is_unordered(refine_control_fields[dim], dim) + + # LOCAL: all values up to the halos + start_index[dim][domain(h_grid.Zone.LOCAL)()] = 0 + end_index[dim][domain(h_grid.Zone.LOCAL)()] = np.max(local_owned_index) + gtx.int32(1) + + # INTERIOR: unordered prognostic region + start_index[dim][domain(h_grid.Zone.INTERIOR)()] = np.min(local_owned_index[unordered]) + max_local_interior = np.max(local_owned_index[unordered]) + gtx.int32(1) + end_index[dim][domain(h_grid.Zone.INTERIOR)()] = max_local_interior + + # HALO lines + # find first local halo index which is in the unordered region (0 or -4) so not on boundary. + # TODO nudging: where does ICON group nudging halo points grouped? + local_halo_index = decomposition_info.local_index( + dim, decomposition.DecompositionInfo.EntryType.HALO + ) + + first_halo_mask = decomposition_info.halo_level_mask( + dim, decomposition.DecompositionFlag.FIRST_HALO_LINE + ) + second_halo_mask = decomposition_info.halo_level_mask( + dim, decomposition.DecompositionFlag.SECOND_HALO_LINE + ) + + minimal_halo_index = ( + local_size + if local_halo_index.size == 0 + else np.min(np.nonzero(np.logical_and(unordered, first_halo_mask))) + ) + assert ( + minimal_halo_index >= 0 and minimal_halo_index >= max_local_interior + ), "first halo line should start after last interior cell" + + minimal_second_halo_index = ( + local_size + if local_halo_index.size == 0 + else np.min(np.nonzero(np.logical_and(unordered, second_halo_mask))) + ) + assert ( + minimal_second_halo_index >= 0 and minimal_second_halo_index >= minimal_halo_index + ), "second halo line should start after first halo line" + + start_index[dim][domain(h_grid.Zone.HALO)()] = minimal_halo_index + end_index[dim][domain(h_grid.Zone.HALO)()] = minimal_second_halo_index + + start_index[dim][domain(h_grid.Zone.HALO_LEVEL_2)()] = minimal_second_halo_index + end_index[dim][domain(h_grid.Zone.HALO_LEVEL_2)()] = local_size + self.grid.with_start_end_indices(dim, start_index[dim], end_index[dim]) + + # TODO global grid so no lateral boundary / nudging levels: need to make sure they are empty + # for now (global grid) we set them to be empty - # for global + # TODO (@halungge) do the same for edges and vertices + # def _initialize_global(self, limited_area, on_gpu): - num_cells = self._reader.dimension(GridFile.DimensionName.CELL_NAME) - num_edges = self._reader.dimension(GridFile.DimensionName.EDGE_NAME) - num_vertices = self._reader.dimension(GridFile.DimensionName.VERTEX_NAME) - uuid = self._dataset.getncattr(GridFile.PropertyName.GRID_ID) - grid_level = self._dataset.getncattr(GridFile.PropertyName.LEVEL) - grid_root = self._dataset.getncattr(GridFile.PropertyName.ROOT) + num_cells = self._reader.dimension(DimensionName.CELL_NAME) + num_edges = self._reader.dimension(DimensionName.EDGE_NAME) + num_vertices = self._reader.dimension(DimensionName.VERTEX_NAME) + uuid = self._dataset.getncattr(PropertyName.GRID_ID) + grid_level = self._dataset.getncattr(PropertyName.LEVEL) + grid_root = self._dataset.getncattr(PropertyName.ROOT) global_params = icon_grid.GlobalGridParams(level=grid_level, root=grid_root) grid_size = base_grid.HorizontalGridSize( num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells diff --git a/model/common/src/icon4py/model/common/grid/horizontal.py b/model/common/src/icon4py/model/common/grid/horizontal.py index 88614f3cc1..8e600943ac 100644 --- a/model/common/src/icon4py/model/common/grid/horizontal.py +++ b/model/common/src/icon4py/model/common/grid/horizontal.py @@ -346,7 +346,7 @@ class Domain(Protocol): _index: int def __str__(self): - return f"{self.dim}: {self._marker} /[ {self._index}]" + return f"Domain (dim = {self.dim}: zone = {self._marker} /[ {self._index}])" @abstractmethod def _valid(self, marker: Zone) -> bool: diff --git a/model/common/src/icon4py/model/common/grid/refinement.py b/model/common/src/icon4py/model/common/grid/refinement.py index fc54f45c0d..918cbb6cce 100644 --- a/model/common/src/icon4py/model/common/grid/refinement.py +++ b/model/common/src/icon4py/model/common/grid/refinement.py @@ -9,6 +9,9 @@ import dataclasses from typing import Final +from icon4py.model.common import dimension as dims +from icon4py.model.common.settings import xp + """ Refinement control for ICON grid. @@ -25,30 +28,53 @@ """ -_MAX_ORDERED: Final[int] = 14 +_MAX_ORDERED: Final[dict[dims.Dimension, int]] = { + dims.CellDim: 14, + dims.EdgeDim: 15, + dims.VertexDim: 16, +} """Lateral boundary points are ordered and have an index indicating the (cell) s distance to the boundary, generally the number of ordered rows can be defined in the grid generator, but it will never exceed 14. """ -_UNORDERED: Final[tuple[int, int]] = (0, -4) +_UNORDERED: Final[dict[dims.Dimension : tuple[int, int]]] = { + dims.CellDim: (0, -4), + dims.EdgeDim: (0, -8), + dims.VertexDim: (0, -4), +} """Value indicating a point is int the unordered interior (fully prognostic) region: this is encoded by 0 or -4 in coarser parent grid.""" -_MIN_ORDERED: Final[int] = -3 +_MIN_ORDERED: Final[dict[dims.Dimension, int]] = { + dim: value[1] + 1 for dim, value in _UNORDERED.items() +} """For coarse parent grids the overlapping boundary regions are counted with negative values, from -1 to max -3, (as -4 is used to mark interior points)""" @dataclasses.dataclass(frozen=True) class RefinementValue: value: int + dim: dims.Dimension def __post_init__(self): assert ( - _UNORDERED[1] <= self.value <= _MAX_ORDERED + _UNORDERED[self.dim][1] <= self.value <= _MAX_ORDERED[self.dim] ), f"Invalid refinement control constant {self.value}" def is_nested(self) -> bool: return self.value < 0 def is_ordered(self) -> bool: - return self.value not in _UNORDERED + return self.value not in _UNORDERED[self.dim] + + +def is_unordered(field: xp.ndarray, dim: dims.Dimension) -> xp.ndarray: + assert field.dtype == xp.int32 or field.dtype == xp.int64, f"not an integer type {field.dtype}" + return xp.where( + field == _UNORDERED[dim][0], True, xp.where(field == _UNORDERED[dim][1], True, False) + ) + + +def to_unnested(field: xp.ndarray, dim: dims.Dimension) -> xp.ndarray: + assert field.dtype == xp.int32 or field.dtype == xp.int64, f"not an integer type {field.dtype}" + return xp.where(field == _UNORDERED[dim][1], 0, xp.where(field < 0, -field, field)) diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index 204622899f..48308b66f0 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -29,8 +29,8 @@ import icon4py.model.common.dimension as dims import icon4py.model.common.grid.grid_manager as gm from icon4py.model.common.decomposition import definitions as defs +from icon4py.model.common.decomposition.definitions import DecompositionFlag from icon4py.model.common.decomposition.halo import ( - DecompositionFlag, HaloGenerator, SimpleMetisDecomposer, ) diff --git a/model/common/tests/grid_tests/test_grid_manager.py b/model/common/tests/grid_tests/test_grid_manager.py index 9114bef213..1e478163aa 100644 --- a/model/common/tests/grid_tests/test_grid_manager.py +++ b/model/common/tests/grid_tests/test_grid_manager.py @@ -19,7 +19,13 @@ import icon4py.model.common.test_utils.datatest_utils as dt_utils from icon4py.model.common import dimension as dims from icon4py.model.common.decomposition import definitions as defs, halo -from icon4py.model.common.grid import simple, vertical as v_grid +from icon4py.model.common.grid import ( + grid_manager as gm, + horizontal as h_grid, + refinement as refin, + simple, + vertical as v_grid, +) from icon4py.model.common.settings import xp @@ -31,14 +37,6 @@ except ImportError: pytest.skip("optional netcdf dependency not installed", allow_module_level=True) -from icon4py.model.common.grid.grid_manager import ( - GridFile, - GridFileName, - GridManager, - IndexTransformation, - ToZeroBasedIndexTransformation, - construct_local_connectivity, -) from . import utils @@ -98,157 +96,151 @@ def simple_grid_gridfile(tmp_path): grid = simple.SimpleGrid() dataset = netCDF4.Dataset(path, "w", format="NETCDF4") - dataset.setncattr(GridFile.PropertyName.GRID_ID, str(uuid.uuid4())) - dataset.setncattr(GridFile.PropertyName.LEVEL, 0) - dataset.setncattr(GridFile.PropertyName.ROOT, 0) - dataset.createDimension(GridFile.DimensionName.VERTEX_NAME, size=grid.num_vertices) - - dataset.createDimension(GridFile.DimensionName.EDGE_NAME, size=grid.num_edges) - dataset.createDimension(GridFile.DimensionName.CELL_NAME, size=grid.num_cells) - dataset.createDimension( - GridFile.DimensionName.NEIGHBORS_TO_EDGE_SIZE, size=grid.size[dims.E2VDim] - ) - dataset.createDimension(GridFile.DimensionName.DIAMOND_EDGE_SIZE, size=grid.size[dims.E2C2EDim]) - dataset.createDimension(GridFile.DimensionName.MAX_CHILD_DOMAINS, size=1) + dataset.setncattr(gm.PropertyName.GRID_ID, str(uuid.uuid4())) + dataset.setncattr(gm.PropertyName.LEVEL, 0) + dataset.setncattr(gm.PropertyName.ROOT, 0) + dataset.createDimension(gm.DimensionName.VERTEX_NAME, size=grid.num_vertices) + + dataset.createDimension(gm.DimensionName.EDGE_NAME, size=grid.num_edges) + dataset.createDimension(gm.DimensionName.CELL_NAME, size=grid.num_cells) + dataset.createDimension(gm.DimensionName.NEIGHBORS_TO_EDGE_SIZE, size=grid.size[dims.E2VDim]) + dataset.createDimension(gm.DimensionName.DIAMOND_EDGE_SIZE, size=grid.size[dims.E2C2EDim]) + dataset.createDimension(gm.DimensionName.MAX_CHILD_DOMAINS, size=1) # add dummy values for the grf dimensions - dataset.createDimension(GridFile.DimensionName.CELL_GRF, size=14) - dataset.createDimension(GridFile.DimensionName.EDGE_GRF, size=24) - dataset.createDimension(GridFile.DimensionName.VERTEX_GRF, size=13) + dataset.createDimension(gm.DimensionName.CELL_GRF, size=14) + dataset.createDimension(gm.DimensionName.EDGE_GRF, size=24) + dataset.createDimension(gm.DimensionName.VERTEX_GRF, size=13) _add_to_dataset( dataset, np.zeros(grid.num_edges), - GridFile.GridRefinementName.CONTROL_EDGES, - (GridFile.DimensionName.EDGE_NAME,), + gm.GridRefinementName.CONTROL_EDGES, + (gm.DimensionName.EDGE_NAME,), ) _add_to_dataset( dataset, np.zeros(grid.num_cells), - GridFile.GridRefinementName.CONTROL_CELLS, - (GridFile.DimensionName.CELL_NAME,), + gm.GridRefinementName.CONTROL_CELLS, + (gm.DimensionName.CELL_NAME,), ) _add_to_dataset( dataset, np.zeros(grid.num_vertices), - GridFile.GridRefinementName.CONTROL_VERTICES, - (GridFile.DimensionName.VERTEX_NAME,), + gm.GridRefinementName.CONTROL_VERTICES, + (gm.DimensionName.VERTEX_NAME,), ) - dataset.createDimension( - GridFile.DimensionName.NEIGHBORS_TO_CELL_SIZE, size=grid.size[dims.C2EDim] - ) - dataset.createDimension( - GridFile.DimensionName.NEIGHBORS_TO_VERTEX_SIZE, size=grid.size[dims.V2CDim] - ) + dataset.createDimension(gm.DimensionName.NEIGHBORS_TO_CELL_SIZE, size=grid.size[dims.C2EDim]) + dataset.createDimension(gm.DimensionName.NEIGHBORS_TO_VERTEX_SIZE, size=grid.size[dims.V2CDim]) _add_to_dataset( dataset, grid.connectivities[dims.C2EDim], - GridFile.ConnectivityName.C2E, + gm.ConnectivityName.C2E, ( - GridFile.DimensionName.NEIGHBORS_TO_CELL_SIZE, - GridFile.DimensionName.CELL_NAME, + gm.DimensionName.NEIGHBORS_TO_CELL_SIZE, + gm.DimensionName.CELL_NAME, ), ) _add_to_dataset( dataset, grid.connectivities[dims.E2CDim], - GridFile.ConnectivityName.E2C, + gm.ConnectivityName.E2C, ( - GridFile.DimensionName.NEIGHBORS_TO_EDGE_SIZE, - GridFile.DimensionName.EDGE_NAME, + gm.DimensionName.NEIGHBORS_TO_EDGE_SIZE, + gm.DimensionName.EDGE_NAME, ), ) _add_to_dataset( dataset, grid.connectivities[dims.E2VDim], - GridFile.ConnectivityName.E2V, + gm.ConnectivityName.E2V, ( - GridFile.DimensionName.NEIGHBORS_TO_EDGE_SIZE, - GridFile.DimensionName.EDGE_NAME, + gm.DimensionName.NEIGHBORS_TO_EDGE_SIZE, + gm.DimensionName.EDGE_NAME, ), ) _add_to_dataset( dataset, grid.connectivities[dims.V2CDim], - GridFile.ConnectivityName.V2C, + gm.ConnectivityName.V2C, ( - GridFile.DimensionName.NEIGHBORS_TO_VERTEX_SIZE, - GridFile.DimensionName.VERTEX_NAME, + gm.DimensionName.NEIGHBORS_TO_VERTEX_SIZE, + gm.DimensionName.VERTEX_NAME, ), ) _add_to_dataset( dataset, grid.connectivities[dims.C2VDim], - GridFile.ConnectivityName.C2V, + gm.ConnectivityName.C2V, ( - GridFile.DimensionName.NEIGHBORS_TO_CELL_SIZE, - GridFile.DimensionName.CELL_NAME, + gm.DimensionName.NEIGHBORS_TO_CELL_SIZE, + gm.DimensionName.CELL_NAME, ), ) _add_to_dataset( dataset, np.zeros((grid.num_vertices, 4), dtype=np.int32), - GridFile.ConnectivityName.V2E2V, - (GridFile.DimensionName.DIAMOND_EDGE_SIZE, GridFile.DimensionName.VERTEX_NAME), + gm.ConnectivityName.V2E2V, + (gm.DimensionName.DIAMOND_EDGE_SIZE, gm.DimensionName.VERTEX_NAME), ) _add_to_dataset( dataset, grid.connectivities[dims.V2EDim], - GridFile.ConnectivityName.V2E, + gm.ConnectivityName.V2E, ( - GridFile.DimensionName.NEIGHBORS_TO_VERTEX_SIZE, - GridFile.DimensionName.VERTEX_NAME, + gm.DimensionName.NEIGHBORS_TO_VERTEX_SIZE, + gm.DimensionName.VERTEX_NAME, ), ) _add_to_dataset( dataset, grid.connectivities[dims.C2E2CDim], - GridFile.ConnectivityName.C2E2C, + gm.ConnectivityName.C2E2C, ( - GridFile.DimensionName.NEIGHBORS_TO_CELL_SIZE, - GridFile.DimensionName.CELL_NAME, + gm.DimensionName.NEIGHBORS_TO_CELL_SIZE, + gm.DimensionName.CELL_NAME, ), ) _add_to_dataset( dataset, np.ones((1, 24), dtype=np.int32), - GridFile.GridRefinementName.START_INDEX_EDGES, - (GridFile.DimensionName.MAX_CHILD_DOMAINS, GridFile.DimensionName.EDGE_GRF), + gm.GridRefinementName.START_INDEX_EDGES, + (gm.DimensionName.MAX_CHILD_DOMAINS, gm.DimensionName.EDGE_GRF), ) _add_to_dataset( dataset, np.ones((1, 14), dtype=np.int32), - GridFile.GridRefinementName.START_INDEX_CELLS, - (GridFile.DimensionName.MAX_CHILD_DOMAINS, GridFile.DimensionName.CELL_GRF), + gm.GridRefinementName.START_INDEX_CELLS, + (gm.DimensionName.MAX_CHILD_DOMAINS, gm.DimensionName.CELL_GRF), ) _add_to_dataset( dataset, np.ones((1, 13), dtype=np.int32), - GridFile.GridRefinementName.START_INDEX_VERTICES, - (GridFile.DimensionName.MAX_CHILD_DOMAINS, GridFile.DimensionName.VERTEX_GRF), + gm.GridRefinementName.START_INDEX_VERTICES, + (gm.DimensionName.MAX_CHILD_DOMAINS, gm.DimensionName.VERTEX_GRF), ) _add_to_dataset( dataset, np.ones((1, 24), dtype=np.int32), - GridFile.GridRefinementName.END_INDEX_EDGES, - (GridFile.DimensionName.MAX_CHILD_DOMAINS, GridFile.DimensionName.EDGE_GRF), + gm.GridRefinementName.END_INDEX_EDGES, + (gm.DimensionName.MAX_CHILD_DOMAINS, gm.DimensionName.EDGE_GRF), ) _add_to_dataset( dataset, np.ones((1, 14), dtype=np.int32), - GridFile.GridRefinementName.END_INDEX_CELLS, - (GridFile.DimensionName.MAX_CHILD_DOMAINS, GridFile.DimensionName.CELL_GRF), + gm.GridRefinementName.END_INDEX_CELLS, + (gm.DimensionName.MAX_CHILD_DOMAINS, gm.DimensionName.CELL_GRF), ) _add_to_dataset( dataset, np.ones((1, 13), dtype=np.int32), - GridFile.GridRefinementName.END_INDEX_VERTICES, - (GridFile.DimensionName.MAX_CHILD_DOMAINS, GridFile.DimensionName.VERTEX_GRF), + gm.GridRefinementName.END_INDEX_VERTICES, + (gm.DimensionName.MAX_CHILD_DOMAINS, gm.DimensionName.VERTEX_GRF), ) dataset.close() yield path @@ -259,61 +251,67 @@ def _add_to_dataset( dataset: netCDF4.Dataset, data: np.ndarray, var_name: str, - dims: tuple[GridFileName, GridFileName], + dims: tuple[gm.GridFileName, gm.GridFileName], ): var = dataset.createVariable(var_name, np.int32, dims) var[:] = np.transpose(data)[:] +@functools.cache +def grid_manager(fname, num_levels=65, transformation=None) -> gm.GridManager: + if transformation is None: + transformation = gm.ToZeroBasedIndexTransformation() + grid_manager = gm.GridManager(transformation, fname, v_grid.VerticalGridConfig(num_levels)) + grid_manager() + return grid_manager + + @pytest.mark.with_netcdf def test_gridparser_dimension(simple_grid_gridfile): data = netCDF4.Dataset(simple_grid_gridfile, "r") - grid_parser = GridFile(data) + parser = gm.GridFile(data) grid = simple.SimpleGrid() - assert grid_parser.dimension(GridFile.DimensionName.CELL_NAME) == grid.num_cells - assert grid_parser.dimension(GridFile.DimensionName.VERTEX_NAME) == grid.num_vertices - assert grid_parser.dimension(GridFile.DimensionName.EDGE_NAME) == grid.num_edges + assert parser.dimension(gm.DimensionName.CELL_NAME) == grid.num_cells + assert parser.dimension(gm.DimensionName.VERTEX_NAME) == grid.num_vertices + assert parser.dimension(gm.DimensionName.EDGE_NAME) == grid.num_edges @pytest.mark.datatest @pytest.mark.with_netcdf @pytest.mark.parametrize( - "grid_file, experiment", + "parser, experiment", [ (dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), (utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT), ], ) -def test_gridfile_vertex_cell_edge_dimensions(grid_savepoint, grid_file): - file = utils.resolve_file_from_gridfile_name(grid_file) - dataset = netCDF4.Dataset(file, "r") - grid_file = GridFile(dataset) +def test_gridfile_vertex_cell_edge_dimensions(grid_savepoint, parser): + file = utils.resolve_file_from_gridfile_name(parser) + parser = gm.GridFile(netCDF4.Dataset(file, "r")) - assert grid_file.dimension(GridFile.DimensionName.CELL_NAME) == grid_savepoint.num(dims.CellDim) - assert grid_file.dimension(GridFile.DimensionName.EDGE_NAME) == grid_savepoint.num(dims.EdgeDim) - assert grid_file.dimension(GridFile.DimensionName.VERTEX_NAME) == grid_savepoint.num( - dims.VertexDim - ) + assert parser.dimension(gm.DimensionName.CELL_NAME) == grid_savepoint.num(dims.CellDim) + assert parser.dimension(gm.DimensionName.EDGE_NAME) == grid_savepoint.num(dims.EdgeDim) + assert parser.dimension(gm.DimensionName.VERTEX_NAME) == grid_savepoint.num(dims.VertexDim) @pytest.mark.with_netcdf def test_grid_parser_index_fields(simple_grid_gridfile, caplog): caplog.set_level(logging.DEBUG) data = netCDF4.Dataset(simple_grid_gridfile, "r") - grid = simple.SimpleGrid() - grid_parser = GridFile(data) + simple_grid = simple.SimpleGrid() + parser = gm.GridFile(data) assert np.allclose( - grid_parser.int_field(GridFile.ConnectivityName.C2E), grid.connectivities[dims.C2EDim] + parser.int_field(gm.ConnectivityName.C2E), simple_grid.connectivities[dims.C2EDim] ) assert np.allclose( - grid_parser.int_field(GridFile.ConnectivityName.E2C), grid.connectivities[dims.E2CDim] + parser.int_field(gm.ConnectivityName.E2C), simple_grid.connectivities[dims.E2CDim] ) assert np.allclose( - grid_parser.int_field(GridFile.ConnectivityName.V2E), grid.connectivities[dims.V2EDim] + parser.int_field(gm.ConnectivityName.V2E), simple_grid.connectivities[dims.V2EDim] ) assert np.allclose( - grid_parser.int_field(GridFile.ConnectivityName.V2C), grid.connectivities[dims.V2CDim] + parser.int_field(gm.ConnectivityName.V2C), simple_grid.connectivities[dims.V2CDim] ) @@ -327,15 +325,15 @@ def test_grid_parser_index_fields(simple_grid_gridfile, caplog): @pytest.mark.parametrize( "grid_file, experiment", [ - # (dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), + (dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), (utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT), ], ) def test_gridmanager_eval_v2e(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) file = utils.resolve_file_from_gridfile_name(grid_file) - grid = init_grid_manager(file).grid - seralized_v2e = grid_savepoint.v2e()[0 : grid.num_vertices, :] + grid = grid_manager(file).grid + seralized_v2e = grid_savepoint.v2e() # there are vertices at the boundary of a local domain or at a pentagon point that have less than # 6 neighbors hence there are "Missing values" in the grid file # they get substituted by the "last valid index" in preprocessing step in icon. @@ -346,30 +344,23 @@ def test_gridmanager_eval_v2e(caplog, grid_savepoint, grid_file): assert np.allclose(v2e_table, seralized_v2e) -pytest.mark.datatest - - @pytest.mark.with_netcdf @pytest.mark.parametrize( "grid_file, experiment", [ (dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), - # (utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT), + (utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT), ], ) -def test_refin_ctrl(grid_savepoint, grid_file, experiment): +@pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) +def test_refin_ctrl(grid_savepoint, grid_file, experiment, dim): file = utils.resolve_file_from_gridfile_name(grid_file) - gm = init_grid_manager(file) + gm = grid_manager(file) start_index, end_index, refin_ctrl, refin_ctrl_max = gm._read_grid_refinement_information( gm._dataset ) - refin_ctrl_c = grid_savepoint.refin_ctrl(dims.CellDim) - refin_ctrl_e = grid_savepoint.refin_ctrl(dims.EdgeDim) - - refin_ctrl_v = grid_savepoint.refin_ctrl(dims.VertexDim) - assert np.allclose(refin_ctrl_c.ndarray, refin_ctrl[dims.CellDim]) - assert np.allclose(refin_ctrl_e.ndarray, refin_ctrl[dims.EdgeDim]) - assert np.allclose(refin_ctrl_v.ndarray, refin_ctrl[dims.VertexDim]) + refin_ctrl_serialized = grid_savepoint.refin_ctrl(dim) + assert np.all(refin_ctrl_serialized.ndarray == refin.to_unnested(refin_ctrl[dim], dim)) # v2c: exists in serial, simple, grid @@ -385,8 +376,8 @@ def test_refin_ctrl(grid_savepoint, grid_file, experiment): def test_gridmanager_eval_v2c(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) file = utils.resolve_file_from_gridfile_name(grid_file) - grid = init_grid_manager(file).grid - serialized_v2c = grid_savepoint.v2c()[0 : grid.num_vertices, :] + grid = grid_manager(file).grid + serialized_v2c = grid_savepoint.v2c() # there are vertices that have less than 6 neighboring cells: either pentagon points or # vertices at the boundary of the domain for a limited area mode # hence in the grid file there are "missing values" @@ -422,7 +413,7 @@ def reset_invalid_index(index_array: np.ndarray): """ for i in range(0, index_array.shape[0]): uq, index = np.unique(index_array[i, :], return_index=True) - index_array[i, max(index) + 1 :] = GridFile.INVALID_INDEX + index_array[i, max(index) + 1 :] = gm.GridFile.INVALID_INDEX # e2v: exists in serial, simple, grid @@ -438,7 +429,7 @@ def reset_invalid_index(index_array: np.ndarray): def test_gridmanager_eval_e2v(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) file = utils.resolve_file_from_gridfile_name(grid_file) - grid = init_grid_manager(file).grid + grid = grid_manager(file).grid serialized_e2v = grid_savepoint.e2v()[0 : grid.num_edges, :] # all vertices in the system have to neighboring edges, there no edges that point nowhere @@ -453,7 +444,7 @@ def has_invalid_index(ar: np.ndarray): def invalid_index(ar): - return np.where(ar == GridFile.INVALID_INDEX) + return np.where(ar == gm.GridFile.INVALID_INDEX) def _is_local(grid_file: str): @@ -494,8 +485,8 @@ def assert_invalid_indices(e2c_table: np.ndarray, grid_file: str): def test_gridmanager_eval_e2c(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) file = utils.resolve_file_from_gridfile_name(grid_file) - grid = init_grid_manager(file).grid - serialized_e2c = grid_savepoint.e2c()[0 : grid.num_edges, :] + grid = grid_manager(file).grid + serialized_e2c = grid_savepoint.e2c() e2c_table = grid.get_offset_provider("E2C").table assert_invalid_indices(serialized_e2c, grid_file) assert_invalid_indices(e2c_table, grid_file) @@ -515,9 +506,9 @@ def test_gridmanager_eval_e2c(caplog, grid_savepoint, grid_file): def test_gridmanager_eval_c2e(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) file = utils.resolve_file_from_gridfile_name(grid_file) - grid = init_grid_manager(file).grid + grid = grid_manager(file).grid - serialized_c2e = grid_savepoint.c2e()[0 : grid.num_cells, :] + serialized_c2e = grid_savepoint.c2e() # no cells with less than 3 neighboring edges exist, otherwise the cell is not there in the # first place # hence there are no "missing values" in the grid file @@ -539,10 +530,10 @@ def test_gridmanager_eval_c2e(caplog, grid_savepoint, grid_file): def test_gridmanager_eval_c2e2c(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) file = utils.resolve_file_from_gridfile_name(grid_file) - grid = init_grid_manager(file).grid + grid = grid_manager(file).grid assert np.allclose( grid.get_offset_provider("C2E2C").table, - grid_savepoint.c2e2c()[0 : grid.num_cells, :], + grid_savepoint.c2e2c(), ) @@ -558,7 +549,7 @@ def test_gridmanager_eval_c2e2c(caplog, grid_savepoint, grid_file): def test_gridmanager_eval_c2e2cO(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) file = utils.resolve_file_from_gridfile_name(grid_file) - grid = init_grid_manager(file).grid + grid = grid_manager(file).grid serialized_grid = grid_savepoint.construct_icon_grid(on_gpu=False) assert np.allclose( grid.get_offset_provider("C2E2CO").table, @@ -579,7 +570,7 @@ def test_gridmanager_eval_c2e2cO(caplog, grid_savepoint, grid_file): def test_gridmanager_eval_e2c2e(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) file = utils.resolve_file_from_gridfile_name(grid_file) - grid = init_grid_manager(file).grid + grid = grid_manager(file).grid serialized_grid = grid_savepoint.construct_icon_grid(on_gpu=False) serialized_e2c2e = serialized_grid.get_offset_provider("E2C2E").table serialized_e2c2eO = serialized_grid.get_offset_provider("E2C2EO").table @@ -612,10 +603,11 @@ def assert_unless_invalid(table, serialized_ref): def test_gridmanager_eval_e2c2v(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) file = utils.resolve_file_from_gridfile_name(grid_file) - grid = init_grid_manager(file).grid + gm = grid_manager(file) + grid = gm.grid # the "far" (adjacent to edge normal ) is not always there, because ICON only calculates those starting from # (lateral_boundary(dims.EdgeDim) + 1) to end(dims.EdgeDim) (see mo_intp_coeffs.f90) and only for owned cells - serialized_ref = grid_savepoint.e2c2v()[: grid.num_edges, :] + serialized_ref = grid_savepoint.e2c2v() table = grid.get_offset_provider("E2C2V").table assert_unless_invalid(table, serialized_ref) @@ -632,29 +624,20 @@ def test_gridmanager_eval_e2c2v(caplog, grid_savepoint, grid_file): def test_gridmanager_eval_c2v(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) file = utils.resolve_file_from_gridfile_name(grid_file) - grid = init_grid_manager(file).grid + grid = grid_manager(file).grid c2v = grid.get_offset_provider("C2V").table - assert np.allclose(c2v, grid_savepoint.c2v()[0 : grid.num_cells, :]) - - -@functools.cache -def init_grid_manager(fname, num_levels=65, transformation=None): - if transformation is None: - transformation = ToZeroBasedIndexTransformation() - grid_manager = GridManager(transformation, fname, v_grid.VerticalGridConfig(num_levels)) - grid_manager() - return grid_manager + assert np.allclose(c2v, grid_savepoint.c2v()) @pytest.mark.parametrize("dim, size", [(dims.CellDim, 18), (dims.EdgeDim, 27), (dims.VertexDim, 9)]) @pytest.mark.with_netcdf def test_grid_manager_getsize(simple_grid_gridfile, dim, size, caplog): caplog.set_level(logging.DEBUG) - gm = init_grid_manager( - simple_grid_gridfile, num_levels=10, transformation=IndexTransformation() + manager = grid_manager( + simple_grid_gridfile, num_levels=10, transformation=gm.IndexTransformation() ) - gm() - assert size == gm.get_size(dim) + + assert size == manager.get_size(dim) def assert_up_to_order(table, diamond_table): @@ -666,14 +649,13 @@ def assert_up_to_order(table, diamond_table): @pytest.mark.with_netcdf def test_grid_manager_diamond_offset(simple_grid_gridfile): simple_grid = simple.SimpleGrid() - gm = init_grid_manager( + manager = grid_manager( simple_grid_gridfile, num_levels=simple_grid.num_levels, - transformation=IndexTransformation(), + transformation=gm.IndexTransformation(), ) - gm() - icon_grid = gm.grid - table = icon_grid.get_offset_provider("E2C2V").table + + table = manager.grid.get_offset_provider("E2C2V").table assert_up_to_order(table, simple_grid.diamond_table) @@ -681,8 +663,10 @@ def test_grid_manager_diamond_offset(simple_grid_gridfile): def test_gridmanager_given_file_not_found_then_abort(): fname = "./unknown_grid.nc" with pytest.raises(SystemExit) as error: - gm = GridManager(IndexTransformation(), fname, v_grid.VerticalGridConfig(num_levels=80)) - gm() + manager = gm.GridManager( + gm.IndexTransformation(), fname, v_grid.VerticalGridConfig(num_levels=80) + ) + manager() assert error.type == SystemExit assert error.value == 1 @@ -690,7 +674,7 @@ def test_gridmanager_given_file_not_found_then_abort(): @pytest.mark.parametrize("size", [100, 1500, 20000]) @pytest.mark.with_netcdf def test_gt4py_transform_offset_by_1_where_valid(size): - trafo = ToZeroBasedIndexTransformation() + trafo = gm.ToZeroBasedIndexTransformation() rng = np.random.default_rng() input_field = rng.integers(-1, size, size) offset = trafo.get_offset_for_index_field(input_field) @@ -707,19 +691,18 @@ def test_gt4py_transform_offset_by_1_where_valid(size): ) def test_grid_level_and_root(grid_file, global_num_cells): file = utils.resolve_file_from_gridfile_name(grid_file) - grid = init_grid_manager(file, num_levels=10).grid - assert global_num_cells == grid.global_num_cells + assert global_num_cells == grid_manager(file, num_levels=10).grid.global_num_cells def test_c2e2c2e(simple_grid_gridfile): simple_grid = simple.SimpleGrid() - gm = init_grid_manager( + manager = grid_manager( simple_grid_gridfile, num_levels=simple_grid.num_levels, - transformation=IndexTransformation(), + transformation=gm.IndexTransformation(), ) - gm() - table = gm.grid.get_offset_provider("C2E2C2E").table + + table = manager.grid.get_offset_provider("C2E2C2E").table assert_up_to_order(table, simple.SimpleGridData.c2e2c2e_table) @@ -732,7 +715,7 @@ def test_c2e2c2e(simple_grid_gridfile): def test_gridmanager_eval_c2e2c2e(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) file = utils.resolve_file_from_gridfile_name(grid_file) - grid = init_grid_manager(file).grid + grid = grid_manager(file).grid serialized_grid = grid_savepoint.construct_icon_grid(on_gpu=False) assert np.allclose( grid.get_offset_provider("C2E2C2E").table, @@ -749,7 +732,7 @@ def test_gridmanager_eval_c2e2c2e(caplog, grid_savepoint, grid_file): def test_local_connectivities(processor_props, caplog, field_offset): # fixture caplog.set_level(logging.INFO) file = utils.resolve_file_from_gridfile_name(utils.R02B04_GLOBAL) - grid = init_grid_manager(file).grid + grid = grid_manager(file).grid partitioner = halo.SimpleMetisDecomposer() face_face_connectivity = grid.connectivities[dims.C2E2CDim] labels = partitioner(face_face_connectivity, n_part=processor_props.comm_size) @@ -762,7 +745,7 @@ def test_local_connectivities(processor_props, caplog, field_offset): # fixture decomposition_info = halo_generator() - connectivity = construct_local_connectivity( + connectivity = gm.construct_local_connectivity( field_offset, decomposition_info, connectivity=grid.connectivities[field_offset.target[1]] ) # there is an neighbor list for each index of the target dimension on the node @@ -778,3 +761,50 @@ def test_local_connectivities(processor_props, caplog, field_offset): # fixture ) # TODO what else to assert? # - outer halo entries have SKIP_VALUE neighbors (depends on offsets) + + +@pytest.mark.parametrize( + "grid_file, experiment", [(utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT)] +) +@pytest.mark.parametrize("dim", utils.horizontal_dim()) +def test_start_end_index(processor_props, caplog, dim, experiment, grid_file, icon_grid): + caplog.set_level(logging.DEBUG) + file = utils.resolve_file_from_gridfile_name(grid_file) + manager = grid_manager(file) + grid = manager.grid + # if dim is not dims.CellDim: + # pytest.mark.xfail(reason="not yet implemented for other dimensions than CellDim") + + partitioner = halo.SimpleMetisDecomposer() + + face_face_connectivity = grid.connectivities[dims.C2E2CDim] + labels = partitioner(face_face_connectivity, n_part=processor_props.comm_size) + halo_generator = halo.HaloGenerator( + connectivities=grid.connectivities, + run_properties=processor_props, + rank_mapping=labels, + num_levels=1, + ) + + decomposition_info = halo_generator() + manager._construct_start_indices(decomposition_info) + + for domain in domains(dim): + assert grid.start_index(domain) == icon_grid.start_index( + domain + ), f"start index wrong for domain {domain}" + assert grid.end_index(domain) == icon_grid.end_index( + domain + ), f"end index wrong for domain {domain}" + + +def domains(dim: dims.Dimension): + # TODO (@halungge) INTERIOR?? for global single node grid + # zones = [ h_grid.Zone.LOCAL, h_grid.Zone.INTERIOR, h_grid.Zone.HALO, h_grid.Zone.HALO_LEVEL_2] + zones = [ + h_grid.Zone.LOCAL, + h_grid.Zone.HALO, + h_grid.Zone.HALO_LEVEL_2, + ] + for z in zones: + yield h_grid.domain(dim)(z) diff --git a/model/common/tests/grid_tests/test_icon.py b/model/common/tests/grid_tests/test_icon.py index bbef2c6458..20374de19d 100644 --- a/model/common/tests/grid_tests/test_icon.py +++ b/model/common/tests/grid_tests/test_icon.py @@ -44,11 +44,6 @@ def nudging(): yield marker -def horizontal_dim(): - for dim in (dims.VertexDim, dims.EdgeDim, dims.CellDim): - yield dim - - LATERAL_BOUNDARY_IDX = { dims.CellDim: [0, 850, 1688, 2511, 3316, 4104], dims.EdgeDim: [0, 428, 1278, 1700, 2538, 2954, 3777, 4184, 4989, 5387, 6176], @@ -73,7 +68,7 @@ def horizontal_dim(): @pytest.mark.datatest @pytest.mark.parametrize("source", ("serialbox", "file")) -@pytest.mark.parametrize("dim", horizontal_dim()) +@pytest.mark.parametrize("dim", utils.horizontal_dim()) @pytest.mark.parametrize("marker", [h_grid.Zone.HALO, h_grid.Zone.HALO_LEVEL_2]) def test_halo(icon_grid, source, dim, marker): # working around the fact that fixtures cannot be used in parametrized functions @@ -86,7 +81,7 @@ def test_halo(icon_grid, source, dim, marker): @pytest.mark.datatest @pytest.mark.parametrize("source", ("serialbox", "file")) -@pytest.mark.parametrize("dim", horizontal_dim()) +@pytest.mark.parametrize("dim", utils.horizontal_dim()) def test_local(dim, source, icon_grid): # working around the fact that fixtures cannot be used in parametrized functions grid = icon_grid if source == "serialbox" else grid_from_file() @@ -97,7 +92,7 @@ def test_local(dim, source, icon_grid): @pytest.mark.datatest @pytest.mark.parametrize("source", ("serialbox", "file")) -@pytest.mark.parametrize("dim", horizontal_dim()) +@pytest.mark.parametrize("dim", utils.horizontal_dim()) @pytest.mark.parametrize("marker", lateral_boundary()) def test_lateral_boundary(icon_grid, source, dim, marker): # working around the fact that fixtures cannot be used in parametrized functions @@ -117,7 +112,7 @@ def test_lateral_boundary(icon_grid, source, dim, marker): @pytest.mark.datatest @pytest.mark.parametrize("source", ("serialbox", "file")) -@pytest.mark.parametrize("dim", horizontal_dim()) +@pytest.mark.parametrize("dim", utils.horizontal_dim()) def test_end(icon_grid, source, dim): # working around the fact that fixtures cannot be used in parametrized functions grid = icon_grid if source == "serialbox" else grid_from_file() @@ -129,7 +124,7 @@ def test_end(icon_grid, source, dim): @pytest.mark.datatest @pytest.mark.parametrize("source", ("serialbox", "file")) @pytest.mark.parametrize("marker", nudging()) -@pytest.mark.parametrize("dim", horizontal_dim()) +@pytest.mark.parametrize("dim", utils.horizontal_dim()) def test_nudging(icon_grid, source, dim, marker): # working around the fact that fixtures cannot be used in parametrized functions grid = icon_grid if source == "serialbox" else grid_from_file() @@ -148,7 +143,7 @@ def test_nudging(icon_grid, source, dim, marker): @pytest.mark.datatest @pytest.mark.parametrize("source", ("serialbox", "file")) -@pytest.mark.parametrize("dim", horizontal_dim()) +@pytest.mark.parametrize("dim", utils.horizontal_dim()) def test_interior(icon_grid, source, dim): # working around the fact that fixtures cannot be used in parametrized functions grid = icon_grid if source == "serialbox" else grid_from_file() diff --git a/model/common/tests/grid_tests/utils.py b/model/common/tests/grid_tests/utils.py index d4a1ad98ae..07abe879e3 100644 --- a/model/common/tests/grid_tests/utils.py +++ b/model/common/tests/grid_tests/utils.py @@ -8,6 +8,7 @@ from pathlib import Path +from icon4py.model.common import dimension as dims from icon4py.model.common.test_utils.data_handling import download_and_extract from icon4py.model.common.test_utils.datatest_utils import ( GRIDS_PATH, @@ -48,3 +49,8 @@ def resolve_file_from_gridfile_name(name: str) -> Path: return gridfile else: raise ValueError(f"invalid name: use one of {R02B04_GLOBAL, REGIONAL_EXPERIMENT}") + + +def horizontal_dim(): + for dim in (dims.VertexDim, dims.EdgeDim, dims.CellDim): + yield dim From 2ef989dc3152f702204af0cc6e03cc58f1eb1413 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 4 Sep 2024 12:02:59 +0200 Subject: [PATCH 031/240] refactor: grid manager - module level log - add builder for decomposition, remove from constructor - start/end index add workaround for global INTERIOR --- .../icon4py/model/common/grid/grid_manager.py | 78 +++++++++++-------- .../icon4py/model/common/grid/horizontal.py | 4 + .../src/icon4py/model/common/grid/icon.py | 3 + .../icon4py/model/common/grid/refinement.py | 6 +- .../mpi_tests/test_parallel_icon.py | 6 +- .../tests/grid_tests/test_grid_manager.py | 67 ++++++++++------ .../tests/grid_tests/test_horizontal.py | 27 ++++++- model/common/tests/grid_tests/test_icon.py | 2 +- .../tests/grid_tests/test_refinement.py | 70 ++++++++++------- 9 files changed, 167 insertions(+), 96 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 8682bf0ae0..88a1369791 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -22,6 +22,7 @@ ) from icon4py.model.common.grid import horizontal as h_grid, refinement as refin from icon4py.model.common.settings import xp +from icon4py.model.common.utils import builder try: @@ -42,6 +43,10 @@ def __init__(self, *args, **kwargs): ) +_log = logging.getLogger(__name__) +_single_node_properties = decomposition.SingleNodeProcessProperties() + + class ReadType(enum.IntEnum): FLOAT = 0 INT = 1 @@ -190,7 +195,7 @@ class GridFile: def __init__(self, dataset: Dataset): self._dataset = dataset - self._log = logging.getLogger(__name__) + _log = logging.getLogger(__name__) def dimension(self, name: GridFileName) -> int: return self._dataset.dimensions[name].size @@ -199,14 +204,14 @@ def int_field(self, name: GridFileName, transpose=True, dtype=gtx.int32) -> np.n try: nc_variable = self._dataset.variables[name] - self._log.debug(f"reading {name}: {nc_variable}: transposing = {transpose}") + _log.debug(f"reading {name}: {nc_variable}: transposing = {transpose}") data = nc_variable[:] data = np.array(data, dtype=dtype) return np.transpose(data) if transpose else data except KeyError as err: msg = f"{name} does not exist in dataset" - self._log.warning(msg) + _log.warning(msg) raise IconGridError(msg) from err def array_1d( @@ -223,13 +228,13 @@ def array_1d( try: # use python slice? 2D fields (sparse, horizontal) variable = self._dataset.variables[name] - self._log.debug(f"reading {name}: {variable}") + _log.debug(f"reading {name}: {variable}") data = variable[:] if indices is None else variable[indices] data = np.array(data, dtype=dtype) return data except KeyError as err: msg = f"{name} does not exist in dataset" - self._log.warning(msg) + _log.warning(msg) raise IconGridError(msg) from err @@ -258,54 +263,55 @@ def get_offset_for_index_field(self, array: np.ndarray): # TODO make this a context manager class GridManager: - """ - Read ICON grid file and set up IconGrid. - - Reads an ICON grid file and extracts connectivity arrays and start-, end-indices for horizontal - domain boundaries. Provides an IconGrid instance for further usage. - """ - def __init__( self, transformation: IndexTransformation, grid_file: Union[pathlib.Path, str], config: v_grid.VerticalGridConfig, # TODO (@halungge) remove - decomposer: Optional[Callable[[np.ndarray, int], np.ndarray]] = None, - run_properties: decomposition.ProcessProperties = decomposition.SingleNodeProcessProperties(), + decomposer: Optional[Callable[[np.ndarray, int], np.ndarray]] = halo.SingleNodeDecomposer(), + run_properties: decomposition.ProcessProperties = _single_node_properties, # get rid of that? only used info from decomposer? ): - self._log = logging.getLogger(__name__) self._run_properties = run_properties self._transformation = transformation self._file_name = str(grid_file) - - if decomposer is None: - self._decompose = ( - halo.SingleNodeDecomposer() - if run_properties.single_node() - else halo.SimpleMetisDecomposer() - ) - else: - self._decompose = decomposer + self._decompose = decomposer self._config = config self._grid: Optional[icon_grid.IconGrid] = None self._decomposition_info: Optional[decomposition.DecompositionInfo] = None self._dataset = None self._reader = None + """ + Read ICON grid file and set up IconGrid. + + Reads an ICON grid file and extracts connectivity arrays and start-, end-indices for horizontal + domain boundaries. Provides an IconGrid instance for further usage. + """ + + # TODO # add args to __call__? + @builder.builder + def with_decomposer( + self, + decomposer: Callable[[np.ndarray, int], np.ndarray], + run_properties: decomposition.ProcessProperties, + ): + self._run_properties = run_properties + self._decompose = decomposer + def __call__(self, on_gpu: bool = False, limited_area=True): self._read_gridfile(self._file_name) - grid = self._construct_grid(on_gpu=on_gpu, limited_area=limited_area) self._grid = grid + return self def _read_gridfile(self, fname: str) -> None: try: dataset = Dataset(self._file_name, "r", format="NETCDF4") - self._log.debug(dataset) + _log.debug(dataset) self._dataset = dataset self._reader = GridFile(dataset) except FileNotFoundError: - self._log.error(f"gridfile {fname} not found, aborting") + _log.error(f"gridfile {fname} not found, aborting") exit(1) def _read_grid_refinement_information(self, dataset): @@ -431,13 +437,13 @@ def grid(self): def _get_index(self, dim: gtx.Dimension, start_marker: int, index_dict): if dim.kind != gtx.DimensionKind.HORIZONTAL: msg = f"getting start index in horizontal domain with non - horizontal dimension {dim}" - self._log.warning(msg) + _log.warning(msg) raise IconGridError(msg) try: return index_dict[dim][start_marker] except KeyError: msg = f"start, end indices for dimension {dim} not present" - self._log.error(msg) + _log.error(msg) def _from_grid_dataset(self, grid, on_gpu: bool, limited_area=True) -> icon_grid.IconGrid: e2c2v = _construct_diamond_vertices( @@ -542,9 +548,11 @@ def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGri } grid.with_connectivities(local_connectivities) _add_derived_connectivities(grid) - # it has a fixed size that is a dimension in the grid file + _update_size_for_1d_sparse_dims(grid) + start_indices, end_indices = self._construct_start_indices(decomposition_info) + for dim in dims.horizontal_dims.values(): + grid.with_start_end_indices(dim, start_indices[dim], end_indices[dim]) - _update_size_for_1d_sparse_dims(grid) return grid # TODO (@halungge) is this used? @@ -556,7 +564,7 @@ def get_size(self, dim: gtx.Dimension): elif dim == dims.EdgeDim: return self._grid.config.num_edges else: - self._log.warning(f"cannot determine size of unknown dimension {dim}") + _log.warning(f"cannot determine size of unknown dimension {dim}") raise IconGridError(f"Unknown dimension {dim}") def _get_index_field( @@ -567,7 +575,9 @@ def _get_index_field( field = field + self._transformation.get_offset_for_index_field(field) return field - def _construct_start_indices(self, decomposition_info: decomposition.DecompositionInfo): + def _construct_start_indices( + self, decomposition_info: decomposition.DecompositionInfo + ) -> tuple[dict[dims.Dimension, np.ndarray], dict[dims.Dimension, np.ndarray]]: grid_refinement_dimensions = { dim: self._reader.dimension(name) for dim, name in { @@ -654,7 +664,7 @@ def _construct_start_indices(self, decomposition_info: decomposition.Decompositi start_index[dim][domain(h_grid.Zone.HALO_LEVEL_2)()] = minimal_second_halo_index end_index[dim][domain(h_grid.Zone.HALO_LEVEL_2)()] = local_size - self.grid.with_start_end_indices(dim, start_index[dim], end_index[dim]) + return start_index, end_index # TODO global grid so no lateral boundary / nudging levels: need to make sure they are empty # for now (global grid) we set them to be empty diff --git a/model/common/src/icon4py/model/common/grid/horizontal.py b/model/common/src/icon4py/model/common/grid/horizontal.py index 8e600943ac..59c388fdf0 100644 --- a/model/common/src/icon4py/model/common/grid/horizontal.py +++ b/model/common/src/icon4py/model/common/grid/horizontal.py @@ -352,6 +352,10 @@ def __str__(self): def _valid(self, marker: Zone) -> bool: ... + @property + def zone(self) -> Zone: + return self._marker + def marker(self, marker: Zone): assert self._valid(marker), f" Domain `{marker}` not a valid zone for use with '{self.dim}'" self._marker = marker diff --git a/model/common/src/icon4py/model/common/grid/icon.py b/model/common/src/icon4py/model/common/grid/icon.py index d550da3245..9611d8820a 100644 --- a/model/common/src/icon4py/model/common/grid/icon.py +++ b/model/common/src/icon4py/model/common/grid/icon.py @@ -187,4 +187,7 @@ def end_index(self, domain: h_grid.Domain): For a given dimension, returns the end index of the horizontal region in a field given by the marker. """ + if domain.zone == h_grid.Zone.INTERIOR and not self.limited_area: + # special treatment because this value is not set properly in the underlying data, for a global grid + return self.size[domain.dim] return self._end_indices[domain.dim][domain()] diff --git a/model/common/src/icon4py/model/common/grid/refinement.py b/model/common/src/icon4py/model/common/grid/refinement.py index 918cbb6cce..36cb4f3859 100644 --- a/model/common/src/icon4py/model/common/grid/refinement.py +++ b/model/common/src/icon4py/model/common/grid/refinement.py @@ -30,8 +30,8 @@ _MAX_ORDERED: Final[dict[dims.Dimension, int]] = { dims.CellDim: 14, - dims.EdgeDim: 15, - dims.VertexDim: 16, + dims.EdgeDim: 14, + dims.VertexDim: 14, } """Lateral boundary points are ordered and have an index indicating the (cell) s distance to the boundary, generally the number of ordered rows can be defined in the grid generator, but it will never exceed 14. @@ -53,8 +53,8 @@ @dataclasses.dataclass(frozen=True) class RefinementValue: - value: int dim: dims.Dimension + value: int def __post_init__(self): assert ( diff --git a/model/common/tests/grid_tests/mpi_tests/test_parallel_icon.py b/model/common/tests/grid_tests/mpi_tests/test_parallel_icon.py index ad8375082b..65bdff5689 100644 --- a/model/common/tests/grid_tests/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/grid_tests/mpi_tests/test_parallel_icon.py @@ -17,7 +17,7 @@ processor_props, ) -from .. import test_icon +from .. import utils try: @@ -49,7 +49,7 @@ def test_props(processor_props): # noqa: F811 # fixture @pytest.mark.datatest @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -@pytest.mark.parametrize("dim", test_icon.horizontal_dim()) +@pytest.mark.parametrize("dim", utils.horizontal_dim()) def test_distributed_local(processor_props, dim, icon_grid, caplog): # noqa: F811 # fixture caplog.set_level(logging.INFO) check_comm_size(processor_props) @@ -106,7 +106,7 @@ def test_distributed_local(processor_props, dim, icon_grid, caplog): # noqa: F8 @pytest.mark.datatest @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi -@pytest.mark.parametrize("dim", test_icon.horizontal_dim()) +@pytest.mark.parametrize("dim", utils.horizontal_dim()) @pytest.mark.parametrize("marker", [h_grid.Zone.HALO, h_grid.Zone.HALO_LEVEL_2]) def test_distributed_halo(processor_props, dim, marker, icon_grid): # noqa: F811 # fixture check_comm_size(processor_props) diff --git a/model/common/tests/grid_tests/test_grid_manager.py b/model/common/tests/grid_tests/test_grid_manager.py index 1e478163aa..a12df44cef 100644 --- a/model/common/tests/grid_tests/test_grid_manager.py +++ b/model/common/tests/grid_tests/test_grid_manager.py @@ -763,48 +763,67 @@ def test_local_connectivities(processor_props, caplog, field_offset): # fixture # - outer halo entries have SKIP_VALUE neighbors (depends on offsets) +@pytest.mark.with_netcdf @pytest.mark.parametrize( - "grid_file, experiment", [(utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT)] + "grid_file, experiment", + [ + (utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT), + # (dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT) + ], ) @pytest.mark.parametrize("dim", utils.horizontal_dim()) -def test_start_end_index(processor_props, caplog, dim, experiment, grid_file, icon_grid): - caplog.set_level(logging.DEBUG) +def test_start_end_index(processor_props, caplog, dim, experiment, grid_file): + caplog.set_level(logging.INFO) file = utils.resolve_file_from_gridfile_name(grid_file) - manager = grid_manager(file) - grid = manager.grid - # if dim is not dims.CellDim: - # pytest.mark.xfail(reason="not yet implemented for other dimensions than CellDim") + limited_area = experiment == dt_utils.REGIONAL_EXPERIMENT + manager = gm.GridManager( + gm.ToZeroBasedIndexTransformation(), file, v_grid.VerticalGridConfig(1) + ) + manager(limited_area=limited_area) + single_node_grid = manager.grid partitioner = halo.SimpleMetisDecomposer() - face_face_connectivity = grid.connectivities[dims.C2E2CDim] - labels = partitioner(face_face_connectivity, n_part=processor_props.comm_size) - halo_generator = halo.HaloGenerator( - connectivities=grid.connectivities, - run_properties=processor_props, - rank_mapping=labels, - num_levels=1, - ) - - decomposition_info = halo_generator() - manager._construct_start_indices(decomposition_info) + manager.with_decomposer(partitioner, processor_props) # add these args to __call__? + manager(limited_area=limited_area) + grid = manager.grid - for domain in domains(dim): - assert grid.start_index(domain) == icon_grid.start_index( + for domain in global_grid_domains(dim): + assert grid.start_index(domain) == single_node_grid.start_index( domain ), f"start index wrong for domain {domain}" - assert grid.end_index(domain) == icon_grid.end_index( + assert grid.end_index(domain) == single_node_grid.end_index( domain ), f"end index wrong for domain {domain}" -def domains(dim: dims.Dimension): - # TODO (@halungge) INTERIOR?? for global single node grid - # zones = [ h_grid.Zone.LOCAL, h_grid.Zone.INTERIOR, h_grid.Zone.HALO, h_grid.Zone.HALO_LEVEL_2] +def global_grid_domains(dim: dims.Dimension): zones = [ + h_grid.Zone.END, h_grid.Zone.LOCAL, + h_grid.Zone.INTERIOR, h_grid.Zone.HALO, h_grid.Zone.HALO_LEVEL_2, ] + + yield from _domain(dim, zones) + + +def _domain(dim, zones): for z in zones: yield h_grid.domain(dim)(z) + + +def boundary_domains(dim: dims.Dimension): + zones = [ + h_grid.Zone.LATERAL_BOUNDARY, + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2, + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_3, + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_4, + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_5, + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_6, + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_7, + h_grid.Zone.NUDGING, + h_grid.Zone.NUDGING_LEVEL_2, + ] + yield from _domain(dim, zones) diff --git a/model/common/tests/grid_tests/test_horizontal.py b/model/common/tests/grid_tests/test_horizontal.py index 9ea87d9d53..93ef0c4987 100644 --- a/model/common/tests/grid_tests/test_horizontal.py +++ b/model/common/tests/grid_tests/test_horizontal.py @@ -5,12 +5,17 @@ # # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause +import logging + import pytest import icon4py.model.common.dimension as dims import icon4py.model.common.grid.horizontal as h_grid -from . import test_icon +from . import utils + + +log = logging.getLogger(__name__) @pytest.mark.parametrize("dim", [dims.C2EDim, dims.C2E2C2EDim, dims.E2VDim, dims.V2EDim, dims.KDim]) @@ -25,10 +30,11 @@ def zones(): yield zone -@pytest.mark.parametrize("dim", test_icon.horizontal_dim()) +@pytest.mark.parametrize("dim", utils.horizontal_dim()) @pytest.mark.parametrize("zone", zones()) -def test_domain_raises_for_invalid_zones(dim, zone): - print(f"dim={dim}, zone={zone},") +def test_domain_raises_for_invalid_zones(dim, zone, caplog): + caplog.set_level(logging.DEBUG) + log.debug(f"dim={dim}, zone={zone},") if dim == dims.CellDim or dim == dims.VertexDim: if zone in ( h_grid.Zone.LATERAL_BOUNDARY_LEVEL_5, @@ -38,3 +44,16 @@ def test_domain_raises_for_invalid_zones(dim, zone): with pytest.raises(AssertionError) as e: h_grid.domain(dim)(zone) e.match("not a valid zone") + + +@pytest.mark.parametrize("dim", utils.horizontal_dim()) +def test_zone_and_domain_index(dim, caplog): + """test mostly used for documentation purposes""" + caplog.set_level(logging.INFO) + for zone in zones(): + try: + domain = h_grid.domain(dim)(zone) + log.info(f"dim={dim}: zone={zone:16}: index={domain():3}") + assert domain() <= h_grid._BOUNDS[dim][1] + except AssertionError: + log.info(f"dim={dim}: zone={zone:16}: invalid") diff --git a/model/common/tests/grid_tests/test_icon.py b/model/common/tests/grid_tests/test_icon.py index 20374de19d..078a147776 100644 --- a/model/common/tests/grid_tests/test_icon.py +++ b/model/common/tests/grid_tests/test_icon.py @@ -26,7 +26,7 @@ def grid_from_file() -> icon.IconGrid: file_name = utils.resolve_file_from_gridfile_name("mch_ch_r04b09_dsl") manager = gm.GridManager( - gm.ToZeroBasedIndexTransformation(), str(file_name), v_grid.VerticalGridConfig(65) + gm.ToZeroBasedIndexTransformation(), str(file_name), v_grid.VerticalGridConfig(1) ) manager() return manager.grid diff --git a/model/common/tests/grid_tests/test_refinement.py b/model/common/tests/grid_tests/test_refinement.py index 97334a7a79..385c159c7a 100644 --- a/model/common/tests/grid_tests/test_refinement.py +++ b/model/common/tests/grid_tests/test_refinement.py @@ -8,40 +8,56 @@ import pytest +import icon4py.model.common.dimension as dims import icon4py.model.common.grid.refinement as refin - -def out_of_range(): - for v in range(-15, -4): - yield v - for v in range(15, 20): - yield v +from . import utils -def refinement_value(): - for v in range(-4, 14): +def out_of_range(dim: dims.Dimension): + lower = range(-25, -9) if dim == dims.EdgeDim else range(-25, -5) + for v in lower: yield v - -@pytest.mark.parametrize("value", refinement_value()) -def test_ordered(value): - ordered = refin.RefinementValue(value) - if ordered.value == 0 or ordered.value == -4: - assert not ordered.is_ordered() - else: - assert ordered.is_ordered() + for v in range(15, 25): + yield v -@pytest.mark.parametrize("value", refinement_value()) -def test_nested(value): - nested = refin.RefinementValue(value) - if nested.value < 0: - assert nested.is_nested() - else: - assert not nested.is_nested() +def refinement_value(dim: dims.Dimension): + lower = -8 if dim == dims.EdgeDim else -4 + for v in range(lower, 14): + yield v -@pytest.mark.parametrize("value", out_of_range()) -def test_valid_refinement_values(value): - with pytest.raises(AssertionError): - refin.RefinementValue(value) +# TODO (@halungge) fix this test -- too complex +@pytest.mark.parametrize("dim", utils.horizontal_dim()) +def test_ordered(dim): + for value in refinement_value(dim): + ordered = refin.RefinementValue(dim, value) + if dim == dims.EdgeDim: + if ordered.value == 0 or ordered.value == -8: + assert not ordered.is_ordered() + else: + assert ordered.is_ordered() + else: + if ordered.value == 0 or ordered.value == -4: + assert not ordered.is_ordered() + else: + assert ordered.is_ordered() + + +@pytest.mark.parametrize("dim", utils.horizontal_dim()) +def test_nested(dim): + for value in refinement_value(dim): + nested = refin.RefinementValue(dim, value) + if nested.value < 0: + assert nested.is_nested() + else: + assert not nested.is_nested() + + +@pytest.mark.parametrize("dim", utils.horizontal_dim()) +def test_valid_refinement_values(dim): + for value in out_of_range(dim): + with pytest.raises(AssertionError): + refin.RefinementValue(dim, value) From 73091d7ee1b6489784a314cb08c8aee77b151e10 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 4 Sep 2024 12:48:05 +0200 Subject: [PATCH 032/240] add boundary zones to test --- .../tests/grid_tests/test_grid_manager.py | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/model/common/tests/grid_tests/test_grid_manager.py b/model/common/tests/grid_tests/test_grid_manager.py index a12df44cef..900f2bd985 100644 --- a/model/common/tests/grid_tests/test_grid_manager.py +++ b/model/common/tests/grid_tests/test_grid_manager.py @@ -796,6 +796,17 @@ def test_start_end_index(processor_props, caplog, dim, experiment, grid_file): domain ), f"end index wrong for domain {domain}" + for domain in valid_boundary_zones_for_dim(dim): + if limited_area: + assert grid.start_index(domain) == 0 + assert grid.end_index(domain) == 0 + assert grid.start_index(domain) == single_node_grid.start_index( + domain + ), f"start index wrong for domain {domain}" + assert grid.end_index(domain) == single_node_grid.end_index( + domain + ), f"end index wrong for domain {domain}" + def global_grid_domains(dim: dims.Dimension): zones = [ @@ -810,11 +821,15 @@ def global_grid_domains(dim: dims.Dimension): def _domain(dim, zones): - for z in zones: - yield h_grid.domain(dim)(z) + domain = h_grid.domain(dim) + for zone in zones: + try: + yield domain(zone) + except AssertionError: + ... -def boundary_domains(dim: dims.Dimension): +def valid_boundary_zones_for_dim(dim: dims.Dimension): zones = [ h_grid.Zone.LATERAL_BOUNDARY, h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2, @@ -826,4 +841,5 @@ def boundary_domains(dim: dims.Dimension): h_grid.Zone.NUDGING, h_grid.Zone.NUDGING_LEVEL_2, ] + yield from _domain(dim, zones) From 6095199d708b59545380939855e393623414e61f Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 5 Sep 2024 09:20:22 +0200 Subject: [PATCH 033/240] use GridFile as content manager in grid_manager --- .../icon4py/model/common/grid/grid_manager.py | 83 ++++++++++++------- .../tests/grid_tests/test_grid_manager.py | 61 +++++++------- 2 files changed, 83 insertions(+), 61 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 88a1369791..5834efdbe4 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -193,19 +193,36 @@ class GridFile: INVALID_INDEX = -1 - def __init__(self, dataset: Dataset): - self._dataset = dataset - _log = logging.getLogger(__name__) - + def __init__(self, file_name: str): + self._filename = file_name + + def __enter__(self): + self._dataset = Dataset(self._filename, "r", format="NETCDF4") + _log.debug(self._dataset) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + + if exc_type is not None: + _log.debug(f"Exception '{exc_type}: {exc_val}' while reading the grid file {self._filename}") + if exc_type is FileNotFoundError: + raise FileNotFoundError(f"gridfile {self._filename} not found, aborting") + + + _log.info(f"Closing dataset: {self._filename}") + self.close() + def dimension(self, name: GridFileName) -> int: return self._dataset.dimensions[name].size + + def attribute(self, name:PropertyName): + return self._dataset.getncattr(name) def int_field(self, name: GridFileName, transpose=True, dtype=gtx.int32) -> np.ndarray: try: nc_variable = self._dataset.variables[name] _log.debug(f"reading {name}: {nc_variable}: transposing = {transpose}") - data = nc_variable[:] data = np.array(data, dtype=dtype) return np.transpose(data) if transpose else data @@ -236,7 +253,9 @@ def array_1d( msg = f"{name} does not exist in dataset" _log.warning(msg) raise IconGridError(msg) from err - + + def close(self): + self._dataset.close() class IconGridError(RuntimeError): pass @@ -268,8 +287,8 @@ def __init__( transformation: IndexTransformation, grid_file: Union[pathlib.Path, str], config: v_grid.VerticalGridConfig, # TODO (@halungge) remove - decomposer: Optional[Callable[[np.ndarray, int], np.ndarray]] = halo.SingleNodeDecomposer(), - run_properties: decomposition.ProcessProperties = _single_node_properties, # get rid of that? only used info from decomposer? + decomposer: Callable[[np.ndarray, int], np.ndarray] = halo.SingleNodeDecomposer(), + run_properties: decomposition.ProcessProperties = _single_node_properties, ): self._run_properties = run_properties self._transformation = transformation @@ -278,7 +297,6 @@ def __init__( self._config = config self._grid: Optional[icon_grid.IconGrid] = None self._decomposition_info: Optional[decomposition.DecompositionInfo] = None - self._dataset = None self._reader = None """ @@ -299,32 +317,29 @@ def with_decomposer( self._decompose = decomposer def __call__(self, on_gpu: bool = False, limited_area=True): - self._read_gridfile(self._file_name) - grid = self._construct_grid(on_gpu=on_gpu, limited_area=limited_area) - self._grid = grid - return self + with GridFile(self._file_name) as self._reader: + grid = self._construct_grid(on_gpu=on_gpu, limited_area=limited_area) + self._grid = grid + self._start, self._end, self._refinement, self._refinement_max = self._read_grid_refinement_information() + return self - def _read_gridfile(self, fname: str) -> None: + def _read_gridfile(self) -> None: try: - dataset = Dataset(self._file_name, "r", format="NETCDF4") - _log.debug(dataset) - self._dataset = dataset - self._reader = GridFile(dataset) + self._reader = GridFile(self._file_name) except FileNotFoundError: - _log.error(f"gridfile {fname} not found, aborting") + _log.error(f"gridfile {self._file_name} not found, aborting") exit(1) - def _read_grid_refinement_information(self, dataset): + def _read_grid_refinement_information(self): _CHILD_DOM = 0 - reader = GridFile(dataset) - + control_dims = [ GridRefinementName.CONTROL_CELLS, GridRefinementName.CONTROL_EDGES, GridRefinementName.CONTROL_VERTICES, ] refin_ctrl = { - dim: reader.int_field(control_dims[i]) + dim: self._reader.int_field(control_dims[i]) for i, dim in enumerate(dims.horizontal_dims.values()) } @@ -334,7 +349,7 @@ def _read_grid_refinement_information(self, dataset): DimensionName.VERTEX_GRF, ] refin_ctrl_max = { - dim: reader.dimension(grf_dims[i]) + dim: self._reader.dimension(grf_dims[i]) for i, dim in enumerate(dims.horizontal_dims.values()) } @@ -433,6 +448,18 @@ def read_coordinates( @property def grid(self): return self._grid + + @property + def start_indices(self): + return self._start + + @property + def end_indices(self): + return self._end + + @property + def refinement(self): + return self._refinement def _get_index(self, dim: gtx.Dimension, start_marker: int, index_dict): if dim.kind != gtx.DimensionKind.HORIZONTAL: @@ -485,7 +512,7 @@ def _read_start_end_indices(self, grid): end_indices, refine_ctrl, refine_ctrl_max, - ) = self._read_grid_refinement_information(self._dataset) + ) = self._read_grid_refinement_information() grid.with_start_end_indices( dims.CellDim, start_indices[dims.CellDim], end_indices[dims.CellDim] ).with_start_end_indices( @@ -676,9 +703,9 @@ def _initialize_global(self, limited_area, on_gpu): num_cells = self._reader.dimension(DimensionName.CELL_NAME) num_edges = self._reader.dimension(DimensionName.EDGE_NAME) num_vertices = self._reader.dimension(DimensionName.VERTEX_NAME) - uuid = self._dataset.getncattr(PropertyName.GRID_ID) - grid_level = self._dataset.getncattr(PropertyName.LEVEL) - grid_root = self._dataset.getncattr(PropertyName.ROOT) + uuid = self._reader.attribute(PropertyName.GRID_ID) + grid_level = self._reader.attribute(PropertyName.LEVEL) + grid_root = self._reader.attribute(PropertyName.ROOT) global_params = icon_grid.GlobalGridParams(level=grid_level, root=grid_root) grid_size = base_grid.HorizontalGridSize( num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells diff --git a/model/common/tests/grid_tests/test_grid_manager.py b/model/common/tests/grid_tests/test_grid_manager.py index 900f2bd985..edfe6911a5 100644 --- a/model/common/tests/grid_tests/test_grid_manager.py +++ b/model/common/tests/grid_tests/test_grid_manager.py @@ -267,13 +267,13 @@ def grid_manager(fname, num_levels=65, transformation=None) -> gm.GridManager: @pytest.mark.with_netcdf -def test_gridparser_dimension(simple_grid_gridfile): - data = netCDF4.Dataset(simple_grid_gridfile, "r") - parser = gm.GridFile(data) +def test_gridfile_dimension(simple_grid_gridfile): grid = simple.SimpleGrid() - assert parser.dimension(gm.DimensionName.CELL_NAME) == grid.num_cells - assert parser.dimension(gm.DimensionName.VERTEX_NAME) == grid.num_vertices - assert parser.dimension(gm.DimensionName.EDGE_NAME) == grid.num_edges + + with gm.GridFile(str(simple_grid_gridfile)) as parser: + assert parser.dimension(gm.DimensionName.CELL_NAME) == grid.num_cells + assert parser.dimension(gm.DimensionName.VERTEX_NAME) == grid.num_vertices + assert parser.dimension(gm.DimensionName.EDGE_NAME) == grid.num_edges @pytest.mark.datatest @@ -287,33 +287,31 @@ def test_gridparser_dimension(simple_grid_gridfile): ) def test_gridfile_vertex_cell_edge_dimensions(grid_savepoint, parser): file = utils.resolve_file_from_gridfile_name(parser) - parser = gm.GridFile(netCDF4.Dataset(file, "r")) - - assert parser.dimension(gm.DimensionName.CELL_NAME) == grid_savepoint.num(dims.CellDim) - assert parser.dimension(gm.DimensionName.EDGE_NAME) == grid_savepoint.num(dims.EdgeDim) - assert parser.dimension(gm.DimensionName.VERTEX_NAME) == grid_savepoint.num(dims.VertexDim) + with gm.GridFile(str(file)) as parser: + assert parser.dimension(gm.DimensionName.CELL_NAME) == grid_savepoint.num(dims.CellDim) + assert parser.dimension(gm.DimensionName.EDGE_NAME) == grid_savepoint.num(dims.EdgeDim) + assert parser.dimension(gm.DimensionName.VERTEX_NAME) == grid_savepoint.num(dims.VertexDim) @pytest.mark.with_netcdf -def test_grid_parser_index_fields(simple_grid_gridfile, caplog): +def test_gridfile_index_fields(simple_grid_gridfile, caplog): caplog.set_level(logging.DEBUG) - data = netCDF4.Dataset(simple_grid_gridfile, "r") simple_grid = simple.SimpleGrid() - parser = gm.GridFile(data) - - assert np.allclose( - parser.int_field(gm.ConnectivityName.C2E), simple_grid.connectivities[dims.C2EDim] - ) - assert np.allclose( - parser.int_field(gm.ConnectivityName.E2C), simple_grid.connectivities[dims.E2CDim] - ) - assert np.allclose( - parser.int_field(gm.ConnectivityName.V2E), simple_grid.connectivities[dims.V2EDim] - ) - assert np.allclose( - parser.int_field(gm.ConnectivityName.V2C), simple_grid.connectivities[dims.V2CDim] - ) - + with gm.GridFile(str(simple_grid_gridfile)) as parser: + + assert np.allclose( + parser.int_field(gm.ConnectivityName.C2E), simple_grid.connectivities[dims.C2EDim] + ) + assert np.allclose( + parser.int_field(gm.ConnectivityName.E2C), simple_grid.connectivities[dims.E2CDim] + ) + assert np.allclose( + parser.int_field(gm.ConnectivityName.V2E), simple_grid.connectivities[dims.V2EDim] + ) + assert np.allclose( + parser.int_field(gm.ConnectivityName.V2C), simple_grid.connectivities[dims.V2CDim] + ) + # TODO @magdalena add test cases for hexagon vertices v2e2v # v2e2v: grid,??? @@ -356,9 +354,7 @@ def test_gridmanager_eval_v2e(caplog, grid_savepoint, grid_file): def test_refin_ctrl(grid_savepoint, grid_file, experiment, dim): file = utils.resolve_file_from_gridfile_name(grid_file) gm = grid_manager(file) - start_index, end_index, refin_ctrl, refin_ctrl_max = gm._read_grid_refinement_information( - gm._dataset - ) + refin_ctrl = gm.refinement refin_ctrl_serialized = grid_savepoint.refin_ctrl(dim) assert np.all(refin_ctrl_serialized.ndarray == refin.to_unnested(refin_ctrl[dim], dim)) @@ -662,12 +658,11 @@ def test_grid_manager_diamond_offset(simple_grid_gridfile): @pytest.mark.with_netcdf def test_gridmanager_given_file_not_found_then_abort(): fname = "./unknown_grid.nc" - with pytest.raises(SystemExit) as error: + with pytest.raises(FileNotFoundError) as error: manager = gm.GridManager( gm.IndexTransformation(), fname, v_grid.VerticalGridConfig(num_levels=80) ) manager() - assert error.type == SystemExit assert error.value == 1 From a2d3d5e7993f8dccd0b960bf6a81ce7a073a2972 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 5 Sep 2024 11:55:10 +0200 Subject: [PATCH 034/240] make GridManager implement context manager remove dtype arg from int fields as it is always the same --- .../icon4py/model/common/grid/grid_manager.py | 92 ++++++++++++------- .../tests/decomposition_tests/test_halo.py | 15 ++- .../mpi_tests/test_parallel_icon.py | 3 + 3 files changed, 77 insertions(+), 33 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 5834efdbe4..48b56bd32a 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -197,8 +197,7 @@ def __init__(self, file_name: str): self._filename = file_name def __enter__(self): - self._dataset = Dataset(self._filename, "r", format="NETCDF4") - _log.debug(self._dataset) + self.open() return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -208,23 +207,35 @@ def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is FileNotFoundError: raise FileNotFoundError(f"gridfile {self._filename} not found, aborting") - _log.info(f"Closing dataset: {self._filename}") self.close() def dimension(self, name: GridFileName) -> int: + """Read a dimension with name 'name' from the grid file.""" return self._dataset.dimensions[name].size def attribute(self, name:PropertyName): + "Read a global attribute with name 'name' from the grid file." return self._dataset.getncattr(name) - - def int_field(self, name: GridFileName, transpose=True, dtype=gtx.int32) -> np.ndarray: + + # TODO add index list for reading, is it obsolete or should become read2d? + def int_field(self, name: GridFileName, transpose:bool=True) -> np.ndarray: + """Read a integer field from the grid file. + + Reads as int32. + + Args: + name: name of the field to read + transpose: flag to indicate whether the file should be transposed (for 2d fields) + Returns: + np.ndarray: field data + + """ try: nc_variable = self._dataset.variables[name] - _log.debug(f"reading {name}: {nc_variable}: transposing = {transpose}") data = nc_variable[:] - data = np.array(data, dtype=dtype) + data = np.array(data, dtype=gtx.int32) return np.transpose(data) if transpose else data except KeyError as err: msg = f"{name} does not exist in dataset" @@ -232,7 +243,7 @@ def int_field(self, name: GridFileName, transpose=True, dtype=gtx.int32) -> np.n raise IconGridError(msg) from err def array_1d( - self, name: GridFileName, indices: np.ndarray = None, dtype=gtx.float64 + self, name: GridFileName, indices: np.ndarray = None, dtype:np.dtype=gtx.float64 ) -> np.ndarray: """Read a field from the grid file. @@ -256,7 +267,10 @@ def array_1d( def close(self): self._dataset.close() - + def open(self): + self._dataset = Dataset(self._filename, "r", format="NETCDF4") + _log.debug(f"opened data set: {self._dataset}") + class IconGridError(RuntimeError): pass @@ -316,21 +330,32 @@ def with_decomposer( self._run_properties = run_properties self._decompose = decomposer - def __call__(self, on_gpu: bool = False, limited_area=True): - with GridFile(self._file_name) as self._reader: - grid = self._construct_grid(on_gpu=on_gpu, limited_area=limited_area) - self._grid = grid - self._start, self._end, self._refinement, self._refinement_max = self._read_grid_refinement_information() - return self + def __enter__(self): + self._reader = GridFile(self._file_name) + self._reader.open() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self._reader.close() + if exc_type is not None: + _log.debug(f"Exception '{exc_type}: {exc_val}' while reading the grid file {self._file_name}") + if exc_type is FileNotFoundError: + raise FileNotFoundError(f"gridfile {self._file_name} not found, aborting") - def _read_gridfile(self) -> None: - try: - self._reader = GridFile(self._file_name) - except FileNotFoundError: - _log.error(f"gridfile {self._file_name} not found, aborting") - exit(1) + def __call__(self, on_gpu: bool = False, limited_area=True): + self.__enter__() + + grid = self._construct_grid(on_gpu=on_gpu, limited_area=limited_area) + self._grid = grid + self._start, self._end, self._refinement, self._refinement_max = self._read_grid_refinement_information() + return self + def _open_gridfile(self) -> None: + self._reader = GridFile(self._file_name) + self.reader.open() + def _read_grid_refinement_information(self): + assert self._reader is not None, "grid file not opened!" _CHILD_DOM = 0 control_dims = [ @@ -359,7 +384,7 @@ def _read_grid_refinement_information(self): GridRefinementName.START_INDEX_VERTICES, ] start_indices = { - dim: self._get_index_field(start_index_dims[i], transpose=False, dtype=gtx.int32)[ + dim: self._get_index_field(start_index_dims[i], transpose=False)[ _CHILD_DOM ] for i, dim in enumerate(dims.horizontal_dims.values()) @@ -372,7 +397,7 @@ def _read_grid_refinement_information(self): ] end_indices = { dim: self._get_index_field( - end_index_dims[i], transpose=False, apply_offset=False, dtype=gtx.int32 + end_index_dims[i], transpose=False, apply_offset=False )[_CHILD_DOM] for i, dim in enumerate(dims.horizontal_dims.values()) } @@ -381,10 +406,13 @@ def _read_grid_refinement_information(self): def _read( self, + reader_func: Callable[ + [GridFileName, np.ndarray, np.dtype], np.ndarray], decomposition_info: decomposition.DecompositionInfo, fields: dict[dims.Dimension, Sequence[GridFileName]], + ): - reader = self._reader.array_1d + (cells_on_node, edges_on_node, vertices_on_node) = ( ( decomposition_info.global_index( @@ -406,17 +434,18 @@ def _read_local(fields: dict[dims.Dimension, Sequence[GridFileName]]): edge_fields = fields.get(dims.EdgeDim, []) vertex_fields = fields.get(dims.VertexDim, []) vals = ( - {name: reader(name, cells_on_node, dtype=gtx.int32) for name in cell_fields} - | {name: reader(name, edges_on_node, dtype=gtx.int32) for name in edge_fields} - | {name: reader(name, vertices_on_node, dtype=gtx.int32) for name in vertex_fields} + {name: reader_func(name, cells_on_node, dtype=gtx.int32) for name in cell_fields} + | {name: reader_func(name, edges_on_node, dtype=gtx.int32) for name in edge_fields} + | {name: reader_func(name, vertices_on_node, dtype=gtx.int32) for name in vertex_fields} ) return vals return _read_local(fields) - def read_geometry(self, decomposition_info: Optional[decomposition.DecompositionInfo] = None): + def _read_geometry(self, decomposition_info: Optional[decomposition.DecompositionInfo] = None): return self._read( + self._reader.array_1d, decomposition_info, { dims.CellDim: [GeometryName.CELL_AREA], @@ -428,6 +457,7 @@ def read_coordinates( self, decomposition_info: Optional[decomposition.DecompositionInfo] = None ): return self._read( + self._reader.array_1d, decomposition_info, { dims.CellDim: [ @@ -595,9 +625,9 @@ def get_size(self, dim: gtx.Dimension): raise IconGridError(f"Unknown dimension {dim}") def _get_index_field( - self, field: GridFileName, transpose=True, apply_offset=True, dtype=gtx.int32 + self, field: GridFileName, transpose=True, apply_offset=True ): - field = self._reader.int_field(field, dtype=dtype, transpose=transpose) + field = self._reader.int_field(field, transpose=transpose) if apply_offset: field = field + self._transformation.get_offset_for_index_field(field) return field @@ -723,7 +753,7 @@ def _initialize_global(self, limited_area, on_gpu): ########################### -def _add_derived_connectivities(grid) -> icon_grid.IconGrid: +def _add_derived_connectivities(grid:icon_grid.IconGrid) -> icon_grid.IconGrid: e2c2v = _construct_diamond_vertices( grid.connectivities[dims.E2VDim], grid.connectivities[dims.C2VDim], diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index 48308b66f0..386fa2391e 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -361,7 +361,7 @@ def test_distributed_fields(processor_props): # noqa F811 # fixture ) decomposition_info = halo_generator() # distributed read: read one field per dimension - local_geometry_fields = grid_manager.read_geometry(decomposition_info) + local_geometry_fields = grid_manager._read_geometry(decomposition_info) local_cell_area = local_geometry_fields[gm.GridFile.GeometryName.CELL_AREA] local_edge_length = local_geometry_fields[gm.GridFile.GeometryName.EDGE_LENGTH] local_vlon = grid_manager.read_coordinates(decomposition_info)[ @@ -373,7 +373,7 @@ def test_distributed_fields(processor_props): # noqa F811 # fixture # the local number of cells must be at most the global number of cells (analytically computed) assert local_cell_area.size <= global_grid.global_properties.num_cells # global read: read the same (global fields) - global_geometry_fields = grid_manager.read_geometry() + global_geometry_fields = grid_manager._read_geometry() global_cell_area = global_geometry_fields[gm.GridFile.GeometryName.CELL_AREA] global_edge_length = global_geometry_fields[gm.GridFile.GeometryName.EDGE_LENGTH] global_vlon = grid_manager.read_coordinates()[gm.GridFile.CoordinateName.VERTEX_LONGITUDE] @@ -424,3 +424,14 @@ def assert_gathered_field_against_global( # TODO add test including halo access: # Will uses geofac_div and geofac_n2s + +def test_halo_neighbor_access_c2e(): + ... + # geofac_div = primal_edge_length(C2E) * edge_orientation / area + + # 1. read grid and distribue - GridManager + + # 2. get geometry fields (from GridManger) primal_edge_length, edge_orientation, area (local read) + # 3. compute geofac_div = primal_edge_length * edge_orientation / area + #4. gather geofac_div + # 5 compare (possible reorder \ No newline at end of file diff --git a/model/common/tests/grid_tests/mpi_tests/test_parallel_icon.py b/model/common/tests/grid_tests/mpi_tests/test_parallel_icon.py index 65bdff5689..8c03bd010d 100644 --- a/model/common/tests/grid_tests/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/grid_tests/mpi_tests/test_parallel_icon.py @@ -121,3 +121,6 @@ def test_distributed_halo(processor_props, dim, marker, icon_grid): # noqa: F81 assert start_index == HALO_IDX[processor_props.comm_size][dim][rank][num - 1] assert end_index == HALO_IDX[processor_props.comm_size][dim][rank][num] + + + From 51bc2cd8c31b4fc925f6a2006986214408712cfd Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 6 Sep 2024 10:05:35 +0200 Subject: [PATCH 035/240] get dummy parallel test_parallel_grid_manager.py to run --- .../icon4py/model/common/grid/grid_manager.py | 38 ++++---- .../common/test_utils/datatest_fixtures.py | 8 +- .../model/common/test_utils/datatest_utils.py | 5 +- .../mpi_tests/test_parallel_grid_manager.py | 57 ++++++++++++ .../tests/grid_tests/test_grid_manager.py | 88 ++++++------------- model/common/tests/grid_tests/utils.py | 38 ++++++++ 6 files changed, 152 insertions(+), 82 deletions(-) create mode 100644 model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 48b56bd32a..f5b1ced8da 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -329,27 +329,36 @@ def with_decomposer( ): self._run_properties = run_properties self._decompose = decomposer - - def __enter__(self): + + def open(self): self._reader = GridFile(self._file_name) self._reader.open() + + def close(self): + self._reader.close() + + def __enter__(self): + self.open() return self def __exit__(self, exc_type, exc_val, exc_tb): - self._reader.close() + self.close() if exc_type is not None: _log.debug(f"Exception '{exc_type}: {exc_val}' while reading the grid file {self._file_name}") if exc_type is FileNotFoundError: raise FileNotFoundError(f"gridfile {self._file_name} not found, aborting") - def __call__(self, on_gpu: bool = False, limited_area=True): - self.__enter__() - + def read(self, on_gpu: bool = False, limited_area=True): + if not self._reader: + self.open() grid = self._construct_grid(on_gpu=on_gpu, limited_area=limited_area) self._grid = grid self._start, self._end, self._refinement, self._refinement_max = self._read_grid_refinement_information() return self + def __call__(self, on_gpu: bool = False, limited_area=True): + self.read(on_gpu=on_gpu, limited_area=limited_area) + def _open_gridfile(self) -> None: self._reader = GridFile(self._file_name) self.reader.open() @@ -568,21 +577,17 @@ def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGri dims.V2C: self._get_index_field(ConnectivityName.V2C), dims.C2V: self._get_index_field(ConnectivityName.C2V), dims.V2E2V: self._get_index_field(ConnectivityName.V2E2V), + dims.E2V: self._get_index_field(ConnectivityName.E2V), + dims.C2V: self._get_index_field(ConnectivityName.C2V), } if self._run_properties.single_node(): - global_connectivities.update( - { - dims.E2V: self._get_index_field(ConnectivityName.E2V), - dims.C2V: self._get_index_field(ConnectivityName.C2V), - } - ) grid.with_connectivities({o.target[1]: c for o, c in global_connectivities.items()}) _add_derived_connectivities(grid) self._read_start_end_indices(grid) return grid else: cells_to_rank_mapping = self._decompose( - global_connectivities[dims.C2E2CDim], self._run_properties.comm_size + global_connectivities[dims.C2E2C], self._run_properties.comm_size ) construct_decomposition_info = halo.HaloGenerator( self._run_properties, @@ -591,12 +596,7 @@ def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGri self._config.num_levels, ) decomposition_info = construct_decomposition_info() - global_connectivities.update( - { - dims.E2V: self._get_index_field(ConnectivityName.E2V), - dims.C2V: self._get_index_field(ConnectivityName.C2V), - } - ) + local_connectivities = { offset.target[1]: construct_local_connectivity( offset, decomposition_info, global_connectivities[offset] diff --git a/model/common/src/icon4py/model/common/test_utils/datatest_fixtures.py b/model/common/src/icon4py/model/common/test_utils/datatest_fixtures.py index a9d3a0feb8..37c512df76 100644 --- a/model/common/src/icon4py/model/common/test_utils/datatest_fixtures.py +++ b/model/common/src/icon4py/model/common/test_utils/datatest_fixtures.py @@ -25,7 +25,10 @@ def processor_props(request): @pytest.fixture(scope="session") def ranked_data_path(processor_props): - return dt_utils.get_ranked_data_path(dt_utils.SERIALIZED_DATA_PATH, processor_props) + + path = dt_utils.get_ranked_data_path(dt_utils.SERIALIZED_DATA_PATH, processor_props) + print(path) + return path @pytest.fixture @@ -43,6 +46,7 @@ def download_ser_data(request, processor_props, ranked_data_path, experiment, py try: destination_path = dt_utils.get_datapath_for_experiment(ranked_data_path, experiment) + if experiment == dt_utils.GLOBAL_EXPERIMENT: uri = dt_utils.DATA_URIS_APE[processor_props.comm_size] elif experiment == dt_utils.JABW_EXPERIMENT: @@ -63,7 +67,7 @@ def download_ser_data(request, processor_props, ranked_data_path, experiment, py processor_props.comm.barrier() except KeyError as err: raise AssertionError( - f"no data for communicator of size {processor_props.comm_size} exists, use 1, 2 or 4" + f"no data url for experiment {experiment} and comm size {processor_props.comm_size} exists." ) from err diff --git a/model/common/src/icon4py/model/common/test_utils/datatest_utils.py b/model/common/src/icon4py/model/common/test_utils/datatest_utils.py index b922e0f0ee..9c843ce776 100644 --- a/model/common/src/icon4py/model/common/test_utils/datatest_utils.py +++ b/model/common/src/icon4py/model/common/test_utils/datatest_utils.py @@ -62,7 +62,10 @@ def get_test_data_root_path() -> pathlib.Path: 2: "https://polybox.ethz.ch/index.php/s/P6F6ZbzWHI881dZ/download", 4: "https://polybox.ethz.ch/index.php/s/NfES3j9no15A0aX/download", } -DATA_URIS_APE = {1: "https://polybox.ethz.ch/index.php/s/y9WRP1mpPlf2BtM/download"} +DATA_URIS_APE = {1: "https://polybox.ethz.ch/index.php/s/y9WRP1mpPlf2BtM/download", + 2: "https://polybox.ethz.ch/index.php/s/1Z9Z2Z2Z2Z2Z2Z2/download", + 4: "https://polybox.ethz.ch/index.php/s/1Z9Z2Z2Z2Z2Z2Z2/download"} + DATA_URIS_JABW = {1: "https://polybox.ethz.ch/index.php/s/kp9Rab00guECrEd/download"} DATA_URIS_GAUSS3D = {1: "https://polybox.ethz.ch/index.php/s/IiRimdJH2ZBZ1od/download"} DATA_URIS_WK = {1: "https://polybox.ethz.ch/index.php/s/91DEUGmAkBgrXO6/download"} diff --git a/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py new file mode 100644 index 0000000000..ff0368f5c3 --- /dev/null +++ b/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py @@ -0,0 +1,57 @@ +import logging + +import pytest + +from icon4py.model.common.decomposition import halo +from icon4py.model.common.grid import grid_manager as gm, vertical as v_grid +from icon4py.model.common.test_utils import datatest_utils as dt_utils +from icon4py.model.common.test_utils.parallel_helpers import ( # noqa: F401 # import fixtures from test_utils package + check_comm_size, + processor_props, +) + +from .. import utils + + +try: + import mpi4py # noqa F401: import mpi4py to check for optional mpi dependency +except ImportError: + pytest.skip("Skipping parallel on single node installation", allow_module_level=True) + +#mpi marker meses up mpi initialization +#@pytest.mark.mpi(min_size=2) +@pytest.mark.parametrize("processor_props", [True], indirect=True) +def test_props(caplog, processor_props): # noqa: F811 # fixture + caplog.set_level(logging.DEBUG) + """dummy test to check setup""" + assert processor_props.comm_size > 1 + + +@pytest.mark.mpi +@pytest.mark.parametrize("processor_props", [True], indirect=True) +@pytest.mark.parametrize( + "grid_file, experiment", + [ + (utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT), + # (dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT) + ], +) +@pytest.mark.parametrize("dim", utils.horizontal_dim()) +def test_start_end_index(caplog, processor_props, grid_file, experiment, dim, icon_grid): # noqa: F811 # fixture + + caplog.set_level(logging.INFO) + file = utils.resolve_file_from_gridfile_name(grid_file) + limited_area = experiment == dt_utils.REGIONAL_EXPERIMENT + partitioner = halo.SimpleMetisDecomposer() + manager = gm.GridManager(gm.ToZeroBasedIndexTransformation(), file, v_grid.VerticalGridConfig(1)) + with manager.with_decomposer(partitioner, processor_props) as manage: + manage(limited_area=limited_area) + grid = manage.grid + + for domain in utils.global_grid_domains(dim): + assert grid.start_index(domain) == utils.single_node_grid.start_index( + domain + ), f"start index wrong for domain {domain}" + assert grid.end_index(domain) == utils.single_node_grid.end_index( + domain + ), f"end index wrong for domain {domain}" \ No newline at end of file diff --git a/model/common/tests/grid_tests/test_grid_manager.py b/model/common/tests/grid_tests/test_grid_manager.py index edfe6911a5..20235b47d6 100644 --- a/model/common/tests/grid_tests/test_grid_manager.py +++ b/model/common/tests/grid_tests/test_grid_manager.py @@ -21,7 +21,6 @@ from icon4py.model.common.decomposition import definitions as defs, halo from icon4py.model.common.grid import ( grid_manager as gm, - horizontal as h_grid, refinement as refin, simple, vertical as v_grid, @@ -89,6 +88,8 @@ "END": 31558, } +zero_base = gm.ToZeroBasedIndexTransformation() +vertical = v_grid.VerticalGridConfig(num_levels=80) @pytest.fixture def simple_grid_gridfile(tmp_path): @@ -330,16 +331,18 @@ def test_gridfile_index_fields(simple_grid_gridfile, caplog): def test_gridmanager_eval_v2e(caplog, grid_savepoint, grid_file): caplog.set_level(logging.DEBUG) file = utils.resolve_file_from_gridfile_name(grid_file) - grid = grid_manager(file).grid - seralized_v2e = grid_savepoint.v2e() - # there are vertices at the boundary of a local domain or at a pentagon point that have less than - # 6 neighbors hence there are "Missing values" in the grid file - # they get substituted by the "last valid index" in preprocessing step in icon. - assert not has_invalid_index(seralized_v2e) - v2e_table = grid.get_offset_provider("V2E").table - assert has_invalid_index(v2e_table) - reset_invalid_index(seralized_v2e) - assert np.allclose(v2e_table, seralized_v2e) + with gm.GridManager(zero_base, file, vertical) as manager: + manager.read() + grid = manager.grid + seralized_v2e = grid_savepoint.v2e() + # there are vertices at the boundary of a local domain or at a pentagon point that have less than + # 6 neighbors hence there are "Missing values" in the grid file + # they get substituted by the "last valid index" in preprocessing step in icon. + assert not has_invalid_index(seralized_v2e) + v2e_table = grid.get_offset_provider("V2E").table + assert has_invalid_index(v2e_table) + reset_invalid_index(seralized_v2e) + assert np.allclose(v2e_table, seralized_v2e) @pytest.mark.with_netcdf @@ -353,10 +356,11 @@ def test_gridmanager_eval_v2e(caplog, grid_savepoint, grid_file): @pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) def test_refin_ctrl(grid_savepoint, grid_file, experiment, dim): file = utils.resolve_file_from_gridfile_name(grid_file) - gm = grid_manager(file) - refin_ctrl = gm.refinement - refin_ctrl_serialized = grid_savepoint.refin_ctrl(dim) - assert np.all(refin_ctrl_serialized.ndarray == refin.to_unnested(refin_ctrl[dim], dim)) + with gm.GridManager(zero_base, file, vertical) as manager: + manager.read() + refin_ctrl = manager.refinement + refin_ctrl_serialized = grid_savepoint.refin_ctrl(dim) + assert np.all(refin_ctrl_serialized.ndarray == refin.to_unnested(refin_ctrl[dim], dim)) # v2c: exists in serial, simple, grid @@ -771,19 +775,18 @@ def test_start_end_index(processor_props, caplog, dim, experiment, grid_file): caplog.set_level(logging.INFO) file = utils.resolve_file_from_gridfile_name(grid_file) limited_area = experiment == dt_utils.REGIONAL_EXPERIMENT - manager = gm.GridManager( - gm.ToZeroBasedIndexTransformation(), file, v_grid.VerticalGridConfig(1) - ) - manager(limited_area=limited_area) - single_node_grid = manager.grid + manager = gm.GridManager(zero_base, file, vertical) + with manager as single_node: + single_node(limited_area=limited_area) + single_node_grid = single_node.grid partitioner = halo.SimpleMetisDecomposer() - manager.with_decomposer(partitioner, processor_props) # add these args to __call__? - manager(limited_area=limited_area) - grid = manager.grid + with manager.with_decomposer(partitioner, processor_props) as partitioned: # add these args to __call__? + partitioned(limited_area=limited_area) + grid = partitioned.grid - for domain in global_grid_domains(dim): + for domain in utils.global_grid_domains(dim): assert grid.start_index(domain) == single_node_grid.start_index( domain ), f"start index wrong for domain {domain}" @@ -791,7 +794,7 @@ def test_start_end_index(processor_props, caplog, dim, experiment, grid_file): domain ), f"end index wrong for domain {domain}" - for domain in valid_boundary_zones_for_dim(dim): + for domain in utils.valid_boundary_zones_for_dim(dim): if limited_area: assert grid.start_index(domain) == 0 assert grid.end_index(domain) == 0 @@ -803,38 +806,3 @@ def test_start_end_index(processor_props, caplog, dim, experiment, grid_file): ), f"end index wrong for domain {domain}" -def global_grid_domains(dim: dims.Dimension): - zones = [ - h_grid.Zone.END, - h_grid.Zone.LOCAL, - h_grid.Zone.INTERIOR, - h_grid.Zone.HALO, - h_grid.Zone.HALO_LEVEL_2, - ] - - yield from _domain(dim, zones) - - -def _domain(dim, zones): - domain = h_grid.domain(dim) - for zone in zones: - try: - yield domain(zone) - except AssertionError: - ... - - -def valid_boundary_zones_for_dim(dim: dims.Dimension): - zones = [ - h_grid.Zone.LATERAL_BOUNDARY, - h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2, - h_grid.Zone.LATERAL_BOUNDARY_LEVEL_3, - h_grid.Zone.LATERAL_BOUNDARY_LEVEL_4, - h_grid.Zone.LATERAL_BOUNDARY_LEVEL_5, - h_grid.Zone.LATERAL_BOUNDARY_LEVEL_6, - h_grid.Zone.LATERAL_BOUNDARY_LEVEL_7, - h_grid.Zone.NUDGING, - h_grid.Zone.NUDGING_LEVEL_2, - ] - - yield from _domain(dim, zones) diff --git a/model/common/tests/grid_tests/utils.py b/model/common/tests/grid_tests/utils.py index 07abe879e3..a7d8d6c9ef 100644 --- a/model/common/tests/grid_tests/utils.py +++ b/model/common/tests/grid_tests/utils.py @@ -9,6 +9,7 @@ from pathlib import Path from icon4py.model.common import dimension as dims +from icon4py.model.common.grid import horizontal as h_grid from icon4py.model.common.test_utils.data_handling import download_and_extract from icon4py.model.common.test_utils.datatest_utils import ( GRIDS_PATH, @@ -54,3 +55,40 @@ def resolve_file_from_gridfile_name(name: str) -> Path: def horizontal_dim(): for dim in (dims.VertexDim, dims.EdgeDim, dims.CellDim): yield dim + + +def global_grid_domains(dim: dims.Dimension): + zones = [ + h_grid.Zone.END, + h_grid.Zone.LOCAL, + h_grid.Zone.INTERIOR, + h_grid.Zone.HALO, + h_grid.Zone.HALO_LEVEL_2, + ] + + yield from _domain(dim, zones) + + +def _domain(dim, zones): + domain = h_grid.domain(dim) + for zone in zones: + try: + yield domain(zone) + except AssertionError: + ... + + +def valid_boundary_zones_for_dim(dim: dims.Dimension): + zones = [ + h_grid.Zone.LATERAL_BOUNDARY, + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2, + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_3, + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_4, + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_5, + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_6, + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_7, + h_grid.Zone.NUDGING, + h_grid.Zone.NUDGING_LEVEL_2, + ] + + yield from _domain(dim, zones) From 5d9a331a73f4c3cffa0a58dc6750018bdaf2ffd9 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 6 Sep 2024 11:40:51 +0200 Subject: [PATCH 036/240] add data URI for parallel APE R02B04 experiment --- .../icon4py/model/common/grid/grid_manager.py | 98 ++++++++++--------- .../common/test_utils/datatest_fixtures.py | 3 +- .../model/common/test_utils/datatest_utils.py | 8 +- .../tests/decomposition_tests/test_halo.py | 9 +- .../mpi_tests/test_parallel_grid_manager.py | 22 +++-- .../mpi_tests/test_parallel_icon.py | 4 +- .../tests/grid_tests/test_grid_manager.py | 12 +-- 7 files changed, 86 insertions(+), 70 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index f5b1ced8da..dabe8282eb 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -195,41 +195,42 @@ class GridFile: def __init__(self, file_name: str): self._filename = file_name - + def __enter__(self): self.open() return self - + def __exit__(self, exc_type, exc_val, exc_tb): - if exc_type is not None: - _log.debug(f"Exception '{exc_type}: {exc_val}' while reading the grid file {self._filename}") + _log.debug( + f"Exception '{exc_type}: {exc_val}' while reading the grid file {self._filename}" + ) if exc_type is FileNotFoundError: raise FileNotFoundError(f"gridfile {self._filename} not found, aborting") - + _log.info(f"Closing dataset: {self._filename}") self.close() - + def dimension(self, name: GridFileName) -> int: """Read a dimension with name 'name' from the grid file.""" return self._dataset.dimensions[name].size - - def attribute(self, name:PropertyName): + + def attribute(self, name: PropertyName): "Read a global attribute with name 'name' from the grid file." return self._dataset.getncattr(name) - + # TODO add index list for reading, is it obsolete or should become read2d? - def int_field(self, name: GridFileName, transpose:bool=True) -> np.ndarray: + def int_field(self, name: GridFileName, transpose: bool = True) -> np.ndarray: """Read a integer field from the grid file. - + Reads as int32. - + Args: name: name of the field to read transpose: flag to indicate whether the file should be transposed (for 2d fields) Returns: - np.ndarray: field data - + np.ndarray: field data + """ try: nc_variable = self._dataset.variables[name] @@ -243,7 +244,7 @@ def int_field(self, name: GridFileName, transpose:bool=True) -> np.ndarray: raise IconGridError(msg) from err def array_1d( - self, name: GridFileName, indices: np.ndarray = None, dtype:np.dtype=gtx.float64 + self, name: GridFileName, indices: np.ndarray = None, dtype: np.dtype = gtx.float64 ) -> np.ndarray: """Read a field from the grid file. @@ -264,13 +265,15 @@ def array_1d( msg = f"{name} does not exist in dataset" _log.warning(msg) raise IconGridError(msg) from err - + def close(self): self._dataset.close() + def open(self): self._dataset = Dataset(self._filename, "r", format="NETCDF4") _log.debug(f"opened data set: {self._dataset}") - + + class IconGridError(RuntimeError): pass @@ -329,44 +332,51 @@ def with_decomposer( ): self._run_properties = run_properties self._decompose = decomposer - + def open(self): self._reader = GridFile(self._file_name) self._reader.open() - + def close(self): self._reader.close() - + def __enter__(self): self.open() return self - + def __exit__(self, exc_type, exc_val, exc_tb): self.close() if exc_type is not None: - _log.debug(f"Exception '{exc_type}: {exc_val}' while reading the grid file {self._file_name}") + _log.debug( + f"Exception '{exc_type}: {exc_val}' while reading the grid file {self._file_name}" + ) if exc_type is FileNotFoundError: raise FileNotFoundError(f"gridfile {self._file_name} not found, aborting") def read(self, on_gpu: bool = False, limited_area=True): - if not self._reader: + if not self._reader: self.open() grid = self._construct_grid(on_gpu=on_gpu, limited_area=limited_area) self._grid = grid - self._start, self._end, self._refinement, self._refinement_max = self._read_grid_refinement_information() + ( + self._start, + self._end, + self._refinement, + self._refinement_max, + ) = self._read_grid_refinement_information() return self - def __call__(self, on_gpu: bool = False, limited_area=True): + def __call__(self, on_gpu: bool = False, limited_area=True): self.read(on_gpu=on_gpu, limited_area=limited_area) def _open_gridfile(self) -> None: self._reader = GridFile(self._file_name) self.reader.open() - + def _read_grid_refinement_information(self): assert self._reader is not None, "grid file not opened!" _CHILD_DOM = 0 - + control_dims = [ GridRefinementName.CONTROL_CELLS, GridRefinementName.CONTROL_EDGES, @@ -393,9 +403,7 @@ def _read_grid_refinement_information(self): GridRefinementName.START_INDEX_VERTICES, ] start_indices = { - dim: self._get_index_field(start_index_dims[i], transpose=False)[ - _CHILD_DOM - ] + dim: self._get_index_field(start_index_dims[i], transpose=False)[_CHILD_DOM] for i, dim in enumerate(dims.horizontal_dims.values()) } @@ -405,9 +413,9 @@ def _read_grid_refinement_information(self): GridRefinementName.END_INDEX_VERTICES, ] end_indices = { - dim: self._get_index_field( - end_index_dims[i], transpose=False, apply_offset=False - )[_CHILD_DOM] + dim: self._get_index_field(end_index_dims[i], transpose=False, apply_offset=False)[ + _CHILD_DOM + ] for i, dim in enumerate(dims.horizontal_dims.values()) } @@ -415,13 +423,10 @@ def _read_grid_refinement_information(self): def _read( self, - reader_func: Callable[ - [GridFileName, np.ndarray, np.dtype], np.ndarray], + reader_func: Callable[[GridFileName, np.ndarray, np.dtype], np.ndarray], decomposition_info: decomposition.DecompositionInfo, fields: dict[dims.Dimension, Sequence[GridFileName]], - ): - (cells_on_node, edges_on_node, vertices_on_node) = ( ( decomposition_info.global_index( @@ -445,7 +450,10 @@ def _read_local(fields: dict[dims.Dimension, Sequence[GridFileName]]): vals = ( {name: reader_func(name, cells_on_node, dtype=gtx.int32) for name in cell_fields} | {name: reader_func(name, edges_on_node, dtype=gtx.int32) for name in edge_fields} - | {name: reader_func(name, vertices_on_node, dtype=gtx.int32) for name in vertex_fields} + | { + name: reader_func(name, vertices_on_node, dtype=gtx.int32) + for name in vertex_fields + } ) return vals @@ -487,15 +495,15 @@ def read_coordinates( @property def grid(self): return self._grid - + @property def start_indices(self): return self._start - + @property def end_indices(self): return self._end - + @property def refinement(self): return self._refinement @@ -596,7 +604,7 @@ def _construct_grid(self, on_gpu: bool, limited_area: bool) -> icon_grid.IconGri self._config.num_levels, ) decomposition_info = construct_decomposition_info() - + local_connectivities = { offset.target[1]: construct_local_connectivity( offset, decomposition_info, global_connectivities[offset] @@ -624,9 +632,7 @@ def get_size(self, dim: gtx.Dimension): _log.warning(f"cannot determine size of unknown dimension {dim}") raise IconGridError(f"Unknown dimension {dim}") - def _get_index_field( - self, field: GridFileName, transpose=True, apply_offset=True - ): + def _get_index_field(self, field: GridFileName, transpose=True, apply_offset=True): field = self._reader.int_field(field, transpose=transpose) if apply_offset: field = field + self._transformation.get_offset_for_index_field(field) @@ -753,7 +759,7 @@ def _initialize_global(self, limited_area, on_gpu): ########################### -def _add_derived_connectivities(grid:icon_grid.IconGrid) -> icon_grid.IconGrid: +def _add_derived_connectivities(grid: icon_grid.IconGrid) -> icon_grid.IconGrid: e2c2v = _construct_diamond_vertices( grid.connectivities[dims.E2VDim], grid.connectivities[dims.C2VDim], diff --git a/model/common/src/icon4py/model/common/test_utils/datatest_fixtures.py b/model/common/src/icon4py/model/common/test_utils/datatest_fixtures.py index 37c512df76..b4b66136cc 100644 --- a/model/common/src/icon4py/model/common/test_utils/datatest_fixtures.py +++ b/model/common/src/icon4py/model/common/test_utils/datatest_fixtures.py @@ -25,7 +25,6 @@ def processor_props(request): @pytest.fixture(scope="session") def ranked_data_path(processor_props): - path = dt_utils.get_ranked_data_path(dt_utils.SERIALIZED_DATA_PATH, processor_props) print(path) return path @@ -67,7 +66,7 @@ def download_ser_data(request, processor_props, ranked_data_path, experiment, py processor_props.comm.barrier() except KeyError as err: raise AssertionError( - f"no data url for experiment {experiment} and comm size {processor_props.comm_size} exists." + f"No data URL for experiment {experiment} and comm size {processor_props.comm_size} available." ) from err diff --git a/model/common/src/icon4py/model/common/test_utils/datatest_utils.py b/model/common/src/icon4py/model/common/test_utils/datatest_utils.py index 9c843ce776..5cd5515f44 100644 --- a/model/common/src/icon4py/model/common/test_utils/datatest_utils.py +++ b/model/common/src/icon4py/model/common/test_utils/datatest_utils.py @@ -62,9 +62,11 @@ def get_test_data_root_path() -> pathlib.Path: 2: "https://polybox.ethz.ch/index.php/s/P6F6ZbzWHI881dZ/download", 4: "https://polybox.ethz.ch/index.php/s/NfES3j9no15A0aX/download", } -DATA_URIS_APE = {1: "https://polybox.ethz.ch/index.php/s/y9WRP1mpPlf2BtM/download", - 2: "https://polybox.ethz.ch/index.php/s/1Z9Z2Z2Z2Z2Z2Z2/download", - 4: "https://polybox.ethz.ch/index.php/s/1Z9Z2Z2Z2Z2Z2Z2/download"} +DATA_URIS_APE = { + 1: "https://polybox.ethz.ch/index.php/s/y9WRP1mpPlf2BtM/download", + 2: "https://polybox.ethz.ch/index.php/s/VBSETx0dW18cEvG/download", + 4: "https://polybox.ethz.ch/index.php/s/H0MmLV4uUI8DDcM/download", +} DATA_URIS_JABW = {1: "https://polybox.ethz.ch/index.php/s/kp9Rab00guECrEd/download"} DATA_URIS_GAUSS3D = {1: "https://polybox.ethz.ch/index.php/s/IiRimdJH2ZBZ1od/download"} diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/test_halo.py index 386fa2391e..bee554b24a 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/test_halo.py @@ -425,13 +425,14 @@ def assert_gathered_field_against_global( # TODO add test including halo access: # Will uses geofac_div and geofac_n2s + def test_halo_neighbor_access_c2e(): ... # geofac_div = primal_edge_length(C2E) * edge_orientation / area - + # 1. read grid and distribue - GridManager - + # 2. get geometry fields (from GridManger) primal_edge_length, edge_orientation, area (local read) # 3. compute geofac_div = primal_edge_length * edge_orientation / area - #4. gather geofac_div - # 5 compare (possible reorder \ No newline at end of file + # 4. gather geofac_div + # 5 compare (possible reorder diff --git a/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py index ff0368f5c3..e956097d13 100644 --- a/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py @@ -1,3 +1,11 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + import logging import pytest @@ -18,8 +26,9 @@ except ImportError: pytest.skip("Skipping parallel on single node installation", allow_module_level=True) -#mpi marker meses up mpi initialization -#@pytest.mark.mpi(min_size=2) + +# mpi marker meses up mpi initialization +# @pytest.mark.mpi(min_size=2) @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_props(caplog, processor_props): # noqa: F811 # fixture caplog.set_level(logging.DEBUG) @@ -38,20 +47,21 @@ def test_props(caplog, processor_props): # noqa: F811 # fixture ) @pytest.mark.parametrize("dim", utils.horizontal_dim()) def test_start_end_index(caplog, processor_props, grid_file, experiment, dim, icon_grid): # noqa: F811 # fixture - caplog.set_level(logging.INFO) file = utils.resolve_file_from_gridfile_name(grid_file) limited_area = experiment == dt_utils.REGIONAL_EXPERIMENT partitioner = halo.SimpleMetisDecomposer() - manager = gm.GridManager(gm.ToZeroBasedIndexTransformation(), file, v_grid.VerticalGridConfig(1)) + manager = gm.GridManager( + gm.ToZeroBasedIndexTransformation(), file, v_grid.VerticalGridConfig(1) + ) with manager.with_decomposer(partitioner, processor_props) as manage: manage(limited_area=limited_area) grid = manage.grid - + for domain in utils.global_grid_domains(dim): assert grid.start_index(domain) == utils.single_node_grid.start_index( domain ), f"start index wrong for domain {domain}" assert grid.end_index(domain) == utils.single_node_grid.end_index( domain - ), f"end index wrong for domain {domain}" \ No newline at end of file + ), f"end index wrong for domain {domain}" diff --git a/model/common/tests/grid_tests/mpi_tests/test_parallel_icon.py b/model/common/tests/grid_tests/mpi_tests/test_parallel_icon.py index 8c03bd010d..13c0aec9b0 100644 --- a/model/common/tests/grid_tests/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/grid_tests/mpi_tests/test_parallel_icon.py @@ -121,6 +121,4 @@ def test_distributed_halo(processor_props, dim, marker, icon_grid): # noqa: F81 assert start_index == HALO_IDX[processor_props.comm_size][dim][rank][num - 1] assert end_index == HALO_IDX[processor_props.comm_size][dim][rank][num] - - - + \ No newline at end of file diff --git a/model/common/tests/grid_tests/test_grid_manager.py b/model/common/tests/grid_tests/test_grid_manager.py index 20235b47d6..9d7dcdb341 100644 --- a/model/common/tests/grid_tests/test_grid_manager.py +++ b/model/common/tests/grid_tests/test_grid_manager.py @@ -88,9 +88,10 @@ "END": 31558, } -zero_base = gm.ToZeroBasedIndexTransformation() +zero_base = gm.ToZeroBasedIndexTransformation() vertical = v_grid.VerticalGridConfig(num_levels=80) + @pytest.fixture def simple_grid_gridfile(tmp_path): path = tmp_path.joinpath(SIMPLE_GRID_NC).absolute() @@ -299,7 +300,6 @@ def test_gridfile_index_fields(simple_grid_gridfile, caplog): caplog.set_level(logging.DEBUG) simple_grid = simple.SimpleGrid() with gm.GridFile(str(simple_grid_gridfile)) as parser: - assert np.allclose( parser.int_field(gm.ConnectivityName.C2E), simple_grid.connectivities[dims.C2EDim] ) @@ -312,7 +312,7 @@ def test_gridfile_index_fields(simple_grid_gridfile, caplog): assert np.allclose( parser.int_field(gm.ConnectivityName.V2C), simple_grid.connectivities[dims.V2CDim] ) - + # TODO @magdalena add test cases for hexagon vertices v2e2v # v2e2v: grid,??? @@ -782,7 +782,9 @@ def test_start_end_index(processor_props, caplog, dim, experiment, grid_file): partitioner = halo.SimpleMetisDecomposer() - with manager.with_decomposer(partitioner, processor_props) as partitioned: # add these args to __call__? + with manager.with_decomposer( + partitioner, processor_props + ) as partitioned: # add these args to __call__? partitioned(limited_area=limited_area) grid = partitioned.grid @@ -804,5 +806,3 @@ def test_start_end_index(processor_props, caplog, dim, experiment, grid_file): assert grid.end_index(domain) == single_node_grid.end_index( domain ), f"end index wrong for domain {domain}" - - From e01bc8d450f8aa7576d599fbce59cccd2f4a3e49 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 9 Jul 2025 13:27:24 +0200 Subject: [PATCH 037/240] fix grid_tests --- .../model/common/decomposition/halo.py | 122 ++++++++++-------- .../tests/grid_tests/mpi_tests/conftest.py | 8 ++ .../mpi_tests/test_parallel_grid_manager.py | 30 ++--- uv.lock | 46 +++++++ 4 files changed, 139 insertions(+), 67 deletions(-) create mode 100644 model/common/tests/grid_tests/mpi_tests/conftest.py diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 3ea2f5ed52..afaa127ecf 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -20,13 +20,14 @@ # SPDX-License-Identifier: GPL-3.0-or-later import logging -from typing import Protocol +from typing import Optional, Protocol import gt4py.next as gtx +import gt4py.next.backend as gtx_backend import icon4py.model.common.decomposition.definitions as defs from icon4py.model.common import dimension as dims, exceptions -from icon4py.model.common.settings import xp +from icon4py.model.common.utils import data_allocation as data_alloc log = logging.getLogger(__name__) @@ -41,9 +42,10 @@ class HaloGenerator: def __init__( self, run_properties: defs.ProcessProperties, - rank_mapping: xp.ndarray, - connectivities: dict[gtx.Dimension, xp.ndarray], + rank_mapping: data_alloc.NDArray, + connectivities: dict[gtx.Dimension, data_alloc.NDArray], num_levels: int, + backend: Optional[gtx_backend.Backend], ): """ @@ -57,6 +59,7 @@ def __init__( self._mapping = rank_mapping self._connectivities = connectivities self._num_levels = num_levels + self._xp = data_alloc.import_array_ns(backend) @property def face_face_connectivity(self): @@ -85,12 +88,12 @@ def face_node_connectivity(self): def _validate(self): assert self._mapping.ndim == 1 # the decomposition should match the communicator size - assert xp.max(self._mapping) == self._props.comm_size - 1 + assert self._xp.max(self._mapping) == self._props.comm_size - 1 - def _post_init(self): + def __post_init__(self): self._validate() - def _connectivity(self, dim: gtx.Dimension) -> xp.ndarray: + def _connectivity(self, dim: gtx.Dimension) -> data_alloc.NDArray: try: conn_table = self._connectivities[dim] return conn_table @@ -99,7 +102,7 @@ def _connectivity(self, dim: gtx.Dimension) -> xp.ndarray: f"Connectivity for offset {dim} is not available" ) from err - def next_halo_line(self, cell_line: xp.ndarray, depot=None): + def next_halo_line(self, cell_line: data_alloc.NDArray, depot=None): """Returns the global indices of the next halo line. Args: @@ -111,36 +114,42 @@ def next_halo_line(self, cell_line: xp.ndarray, depot=None): cell_neighbors = self._cell_neighbors(cell_line) if depot is not None: - cells_so_far = xp.hstack((depot, cell_line)) + cells_so_far = self._xp.hstack((depot, cell_line)) else: cells_so_far = cell_line - next_halo_cells = xp.setdiff1d(xp.unique(cell_neighbors), cells_so_far, assume_unique=True) + next_halo_cells = self._xp.setdiff1d( + self._xp.unique(cell_neighbors), cells_so_far, assume_unique=True + ) return next_halo_cells - def _cell_neighbors(self, cells: xp.ndarray): - return xp.unique(self.face_face_connectivity[cells, :]) + def _cell_neighbors(self, cells: data_alloc.NDArray): + return self._xp.unique(self.face_face_connectivity[cells, :]) - def _find_neighbors(self, source_indices: xp.ndarray, connectivity: xp.ndarray) -> xp.ndarray: + def _find_neighbors( + self, source_indices: data_alloc.NDArray, connectivity: data_alloc.NDArray + ) -> data_alloc.NDArray: """Get a flattened list of all (unique) neighbors to a given global index list""" neighbors = connectivity[source_indices, :] shp = neighbors.shape - unique_neighbors = xp.unique(neighbors.reshape(shp[0] * shp[1])) + unique_neighbors = self._xp.unique(neighbors.reshape(shp[0] * shp[1])) return unique_neighbors - def find_edge_neighbors_for_cells(self, cell_line: xp.ndarray) -> xp.ndarray: + def find_edge_neighbors_for_cells(self, cell_line: data_alloc.NDArray) -> data_alloc.NDArray: return self._find_neighbors(cell_line, connectivity=self.face_edge_connectivity) - def find_edge_neighbors_for_vertices(self, vertex_line: xp.ndarray) -> xp.ndarray: + def find_edge_neighbors_for_vertices( + self, vertex_line: data_alloc.NDArray + ) -> data_alloc.NDArray: return self._find_neighbors(vertex_line, connectivity=self.node_edge_connectivity) - def find_vertex_neighbors_for_cells(self, cell_line: xp.ndarray) -> xp.ndarray: + def find_vertex_neighbors_for_cells(self, cell_line: data_alloc.NDArray) -> data_alloc.NDArray: return self._find_neighbors(cell_line, connectivity=self.face_node_connectivity) - def owned_cells(self) -> xp.ndarray: + def owned_cells(self) -> data_alloc.NDArray: """Returns the global indices of the cells owned by this rank""" owned_cells = self._mapping == self._props.rank - return xp.asarray(owned_cells).nonzero()[0] + return self._xp.asarray(owned_cells).nonzero()[0] def _update_owner_mask_by_max_rank_convention( self, owner_mask, all_indices, indices_on_cutting_line, target_connectivity @@ -161,10 +170,10 @@ def _update_owner_mask_by_max_rank_convention( updated owner mask """ for index in indices_on_cutting_line: - local_index = xp.nonzero(all_indices == index)[0][0] + local_index = self._xp.nonzero(all_indices == index)[0][0] owning_ranks = self._mapping[target_connectivity[index]] assert ( - xp.unique(owning_ranks).size > 1 + self._xp.unique(owning_ranks).size > 1 ), f"rank {self._props.rank}: all neighboring cells are owned by the same rank" assert ( self._props.rank in owning_ranks @@ -190,17 +199,19 @@ def __call__(self) -> defs.DecompositionInfo: first_halo_cells = self.next_halo_line(owned_cells) second_halo_cells = self.next_halo_line(first_halo_cells, owned_cells) - total_halo_cells = xp.hstack((first_halo_cells, second_halo_cells)) - all_cells = xp.hstack((owned_cells, total_halo_cells)) + total_halo_cells = self._xp.hstack((first_halo_cells, second_halo_cells)) + all_cells = self._xp.hstack((owned_cells, total_halo_cells)) - cell_owner_mask = xp.isin(all_cells, owned_cells) - cell_halo_levels = defs.DecompositionFlag.UNDEFINED * xp.ones(all_cells.size, dtype=int) + cell_owner_mask = self._xp.isin(all_cells, owned_cells) + cell_halo_levels = defs.DecompositionFlag.UNDEFINED * self._xp.ones( + all_cells.size, dtype=int + ) cell_halo_levels[cell_owner_mask] = defs.DecompositionFlag.OWNED cell_halo_levels[ - xp.isin(all_cells, first_halo_cells) + self._xp.isin(all_cells, first_halo_cells) ] = defs.DecompositionFlag.FIRST_HALO_LINE cell_halo_levels[ - xp.isin(all_cells, second_halo_cells) + self._xp.isin(all_cells, second_halo_cells) ] = defs.DecompositionFlag.SECOND_HALO_LINE decomp_info = defs.DecompositionInfo(klevels=self._num_levels).with_dimension( dims.CellDim, all_cells, cell_owner_mask, cell_halo_levels @@ -213,76 +224,83 @@ def __call__(self) -> defs.DecompositionInfo: second_halo_cells ) # TODO (@halungge): do we need that at all? - vertex_on_cutting_line = xp.intersect1d(vertex_on_owned_cells, vertex_on_first_halo_line) + vertex_on_cutting_line = self._xp.intersect1d( + vertex_on_owned_cells, vertex_on_first_halo_line + ) # create decomposition_info for vertices - all_vertices = xp.unique(xp.hstack((vertex_on_owned_cells, vertex_on_first_halo_line))) - vertex_owner_mask = xp.isin(all_vertices, vertex_on_owned_cells) + all_vertices = self._xp.unique( + self._xp.hstack((vertex_on_owned_cells, vertex_on_first_halo_line)) + ) + vertex_owner_mask = self._xp.isin(all_vertices, vertex_on_owned_cells) vertex_owner_mask = self._update_owner_mask_by_max_rank_convention( vertex_owner_mask, all_vertices, vertex_on_cutting_line, self.node_face_connectivity, ) - vertex_second_level = xp.setdiff1d(vertex_on_first_halo_line, vertex_on_owned_cells) - vertex_halo_levels = defs.DecompositionFlag.UNDEFINED * xp.ones( + vertex_second_level = self._xp.setdiff1d(vertex_on_first_halo_line, vertex_on_owned_cells) + vertex_halo_levels = defs.DecompositionFlag.UNDEFINED * self._xp.ones( all_vertices.size, dtype=int ) vertex_halo_levels[vertex_owner_mask] = defs.DecompositionFlag.OWNED vertex_halo_levels[ - xp.logical_and( - xp.logical_not(vertex_owner_mask), - xp.isin(all_vertices, vertex_on_cutting_line), + self._xp.logical_and( + self._xp.logical_not(vertex_owner_mask), + self._xp.isin(all_vertices, vertex_on_cutting_line), ) ] = defs.DecompositionFlag.FIRST_HALO_LINE vertex_halo_levels[ - xp.isin(all_vertices, vertex_second_level) + self._xp.isin(all_vertices, vertex_second_level) ] = defs.DecompositionFlag.SECOND_HALO_LINE decomp_info.with_dimension( dims.VertexDim, all_vertices, vertex_owner_mask, vertex_halo_levels ) - #: edges + # edges edges_on_owned_cells = self.find_edge_neighbors_for_cells(owned_cells) edges_on_first_halo_line = self.find_edge_neighbors_for_cells(first_halo_cells) edges_on_second_halo_line = self.find_edge_neighbors_for_cells(second_halo_cells) - level_two_edges = xp.setdiff1d( + level_two_edges = self._xp.setdiff1d( self.find_edge_neighbors_for_vertices(vertex_on_cutting_line), edges_on_owned_cells ) # level_two_edges = xp.setdiff1d(edges_on_first_halo_line, edges_on_owned_cells) - all_edges = xp.hstack( + all_edges = self._xp.hstack( ( edges_on_owned_cells, level_two_edges, - xp.setdiff1d(edges_on_second_halo_line, edges_on_first_halo_line), + self._xp.setdiff1d(edges_on_second_halo_line, edges_on_first_halo_line), ) ) - all_edges = xp.unique(all_edges) + all_edges = self._xp.unique(all_edges) # We need to reduce the overlap: # `edges_on_owned_cells` and `edges_on_first_halo_line` both contain the edges on the cutting line. - edge_intersect_owned_first_line = xp.intersect1d( + edge_intersect_owned_first_line = self._xp.intersect1d( edges_on_owned_cells, edges_on_first_halo_line ) # construct the owner mask - edge_owner_mask = xp.isin(all_edges, edges_on_owned_cells) + edge_owner_mask = self._xp.isin(all_edges, edges_on_owned_cells) edge_owner_mask = self._update_owner_mask_by_max_rank_convention( edge_owner_mask, all_edges, edge_intersect_owned_first_line, self.edge_face_connectivity, ) - edge_halo_levels = defs.DecompositionFlag.UNDEFINED * xp.ones(all_edges.shape, dtype=int) + edge_halo_levels = defs.DecompositionFlag.UNDEFINED * self._xp.ones( + all_edges.shape, dtype=int + ) edge_halo_levels[edge_owner_mask] = defs.DecompositionFlag.OWNED edge_halo_levels[ - xp.logical_and( - xp.logical_not(edge_owner_mask), xp.isin(all_edges, edge_intersect_owned_first_line) + self._xp.logical_and( + self._xp.logical_not(edge_owner_mask), + self._xp.isin(all_edges, edge_intersect_owned_first_line), ) ] = defs.DecompositionFlag.FIRST_HALO_LINE edge_halo_levels[ - xp.isin(all_edges, level_two_edges) + self._xp.isin(all_edges, level_two_edges) ] = defs.DecompositionFlag.SECOND_HALO_LINE decomp_info.with_dimension(dims.EdgeDim, all_edges, edge_owner_mask, edge_halo_levels) @@ -294,7 +312,7 @@ def __call__(self) -> defs.DecompositionInfo: class Decomposer(Protocol): - def __call__(self, adjacency_matrix, n_part: int) -> xp.ndarray: + def __call__(self, adjacency_matrix, n_part: int) -> data_alloc.NDArray: ... @@ -307,7 +325,7 @@ class SimpleMetisDecomposer(Decomposer): https://documen.tician.de/pymetis/functionality.html """ - def __call__(self, adjacency_matrix: xp.ndarray, n_part: int) -> xp.ndarray: + def __call__(self, adjacency_matrix: data_alloc.NDArray, n_part: int) -> data_alloc.NDArray: """ Generate partition labesl for this grid topology using METIS: https://github.com/KarypisLab/METIS @@ -323,9 +341,9 @@ def __call__(self, adjacency_matrix: xp.ndarray, n_part: int) -> xp.ndarray: import pymetis cut_count, partition_index = pymetis.part_graph(nparts=n_part, adjacency=adjacency_matrix) - return xp.array(partition_index) + return self._xp.array(partition_index) class SingleNodeDecomposer(Decomposer): - def __call__(self, adjacency_matrix: xp.ndarray, n_part: int) -> xp.ndarray: - return xp.zeros(adjacency_matrix.shape[0], dtype=xp.int32) + def __call__(self, adjacency_matrix: data_alloc.NDArray, n_part: int) -> data_alloc.NDArray: + return self._xp.zeros(adjacency_matrix.shape[0], dtype=gtx.int32) diff --git a/model/common/tests/grid_tests/mpi_tests/conftest.py b/model/common/tests/grid_tests/mpi_tests/conftest.py new file mode 100644 index 0000000000..80b673df7e --- /dev/null +++ b/model/common/tests/grid_tests/mpi_tests/conftest.py @@ -0,0 +1,8 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + diff --git a/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py index e956097d13..6af7cf1dc1 100644 --- a/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py @@ -10,13 +10,10 @@ import pytest +import icon4py.model.testing.grid_utils as grid_utils from icon4py.model.common.decomposition import halo from icon4py.model.common.grid import grid_manager as gm, vertical as v_grid -from icon4py.model.common.test_utils import datatest_utils as dt_utils -from icon4py.model.common.test_utils.parallel_helpers import ( # noqa: F401 # import fixtures from test_utils package - check_comm_size, - processor_props, -) +from icon4py.model.testing import datatest_utils as dt_utils from .. import utils @@ -28,9 +25,9 @@ # mpi marker meses up mpi initialization -# @pytest.mark.mpi(min_size=2) +@pytest.mark.mpi(min_size=2) @pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_props(caplog, processor_props): # noqa: F811 # fixture +def test_props(caplog, processor_props): # fixture caplog.set_level(logging.DEBUG) """dummy test to check setup""" assert processor_props.comm_size > 1 @@ -41,27 +38,30 @@ def test_props(caplog, processor_props): # noqa: F811 # fixture @pytest.mark.parametrize( "grid_file, experiment", [ - (utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT), + (dt_utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT), # (dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT) ], ) -@pytest.mark.parametrize("dim", utils.horizontal_dim()) -def test_start_end_index(caplog, processor_props, grid_file, experiment, dim, icon_grid): # noqa: F811 # fixture +@pytest.mark.parametrize("dim", utils.horizontal_dims()) +def test_start_end_index( + caplog, backend, processor_props, grid_file, experiment, dim, icon_grid +): # fixture caplog.set_level(logging.INFO) - file = utils.resolve_file_from_gridfile_name(grid_file) - limited_area = experiment == dt_utils.REGIONAL_EXPERIMENT + file = grid_utils.resolve_full_grid_file_name(grid_file) + partitioner = halo.SimpleMetisDecomposer() manager = gm.GridManager( gm.ToZeroBasedIndexTransformation(), file, v_grid.VerticalGridConfig(1) ) + single_node_grid = utils.run_grid_manager(file, keep_skip_values=True).grid with manager.with_decomposer(partitioner, processor_props) as manage: - manage(limited_area=limited_area) + manage(backend=backend, keep_skip_values=True) grid = manage.grid for domain in utils.global_grid_domains(dim): - assert grid.start_index(domain) == utils.single_node_grid.start_index( + assert grid.start_index(domain) == single_node_grid.start_index( domain ), f"start index wrong for domain {domain}" - assert grid.end_index(domain) == utils.single_node_grid.end_index( + assert grid.end_index(domain) == single_node_grid.end_index( domain ), f"end index wrong for domain {domain}" diff --git a/uv.lock b/uv.lock index 882607e7f0..588a12246f 100644 --- a/uv.lock +++ b/uv.lock @@ -1804,6 +1804,7 @@ all = [ { name = "mpi4py" }, { name = "netcdf4" }, { name = "numpy" }, + { name = "pymetis" }, { name = "scikit-learn" }, { name = "uxarray" }, { name = "xarray", extra = ["complete"] }, @@ -1823,6 +1824,7 @@ dace = [ distributed = [ { name = "ghex" }, { name = "mpi4py" }, + { name = "pymetis" }, ] io = [ { name = "cartopy" }, @@ -1855,6 +1857,7 @@ requires-dist = [ { name = "netcdf4", marker = "extra == 'io'", specifier = ">=1.6.1" }, { name = "numpy", marker = "extra == 'io'", specifier = ">=1.23.3" }, { name = "packaging", specifier = ">=20.0" }, + { name = "pymetis", marker = "extra == 'distributed'", specifier = ">2022.1" }, { name = "scikit-learn", marker = "extra == 'io'", specifier = ">=1.4.0" }, { name = "serialbox4py", specifier = ">=2.6.2", index = "https://test.pypi.org/simple/" }, { name = "typing-extensions", specifier = ">=4.11.0" }, @@ -3328,6 +3331,49 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f7/3f/01c8b82017c199075f8f788d0d906b9ffbbc5a47dc9918a945e13d5a2bda/pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a", size = 1205513, upload-time = "2024-05-04T13:41:57.345Z" }, ] +[[package]] +name = "pymetis" +version = "2025.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/cd/e8f6b7004abf2c7587fc760d0b8830d7c23f76b88f72dd3b412fd9f69e3d/pymetis-2025.1.1.tar.gz", hash = "sha256:c02bf0fdd9483ff9dac031c73219ab9aa7a90e110f09d3c00f1799f334d813d8", size = 5033014, upload-time = "2025-05-05T19:23:08.011Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/71/3f916e0590fc0efa56c9bd6e4e0f2f9cf0e6da3c04b690cadebc997b1f3e/pymetis-2025.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6d857b37bffffe345c3b6314332dcd55ca9217255bba9e8941aa8a08b434662f", size = 244755, upload-time = "2025-05-05T19:22:17.317Z" }, + { url = "https://files.pythonhosted.org/packages/11/84/369a861c621fa91aa5a5f232dabf864fe349b0ea8daef7af5e63995ea626/pymetis-2025.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:00016ee75edac02da6c86df51dde1fb2e5abb815f1a334ecd273d33d9c4b2a0b", size = 217258, upload-time = "2025-05-05T19:22:19.177Z" }, + { url = "https://files.pythonhosted.org/packages/87/77/2eb3f059a784a478dcf142a8b941b46ed06baf921bbefdce72600a21f276/pymetis-2025.1.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76852cdb0fbea473df31fb995c13366fa41a4969853e96acb566e075556c0559", size = 336565, upload-time = "2025-05-05T19:22:20.8Z" }, + { url = "https://files.pythonhosted.org/packages/04/0d/c7a71e9fa74b6402b961ae523a806495f480d3c6ee348c583830bd84d7ad/pymetis-2025.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83372d22b015c017dae859a95974c1ead9800772411e0ed3ac106d127c21db23", size = 274742, upload-time = "2025-05-05T19:22:22.09Z" }, + { url = "https://files.pythonhosted.org/packages/3a/96/ed0862d0789dd54c429e5635cdb6d1bb286261666d8f5c77d523a5c1a900/pymetis-2025.1.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:64137431e0f41515d2c784a57a68269607fcc930691a6661ad818f5804c5a9a9", size = 1461516, upload-time = "2025-05-05T19:22:24.809Z" }, + { url = "https://files.pythonhosted.org/packages/97/fd/7eae7d6a2462ce63669ed8813c43e8d8b1ea93ff440ceab38b0130d6a8cb/pymetis-2025.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e0df438aca9fb832173644b2190b1c181bc10d8926ce8071752fb6c5cddf1117", size = 1290326, upload-time = "2025-05-05T19:22:26.29Z" }, + { url = "https://files.pythonhosted.org/packages/54/c5/567e26cb9be60b4f11eac3511d4889569a7fb7044dfc2392a3f905fb366c/pymetis-2025.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5bfbd3df3b3114c106054b15cc0455094526240fdd29c284084fdae3f4fdd289", size = 245886, upload-time = "2025-05-05T19:22:28.308Z" }, + { url = "https://files.pythonhosted.org/packages/ef/0b/35fc2fc5a16913af40a1eb2d85096117f2db2e2e82c477f7ab8fa91318ea/pymetis-2025.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:45c3478ff2a1d208e75cc5fe1189f0aac8d1efe6a61e12665be762347c91d6c0", size = 218446, upload-time = "2025-05-05T19:22:29.791Z" }, + { url = "https://files.pythonhosted.org/packages/d7/26/f86f9be6b62c23d7ac9141081b111d7215723063a0edc204951669f1d18d/pymetis-2025.1.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b74810074c602fedb0bcece8fdcbf58773e3bcfa1e28884271d069598405dc5", size = 337537, upload-time = "2025-05-05T19:22:31.512Z" }, + { url = "https://files.pythonhosted.org/packages/5f/c1/daf8797d7af40fe5748366118c9383e60bda339cac5e74c48dc8e7136931/pymetis-2025.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aeda5d9ee8df41db134c9cf0a10c51a805dae29c9dae7f9cb7ccad37e15d6ec", size = 276008, upload-time = "2025-05-05T19:22:33.332Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7f/ec0894623175ccc2d6de261c7f8fe7c4a99f7914ac6195bdff75883ad075/pymetis-2025.1.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:9b64ad48f45cb55d2077f965114f9818a10d24e2744d40c0ebc4d5a2065db10c", size = 1462382, upload-time = "2025-05-05T19:22:35.234Z" }, + { url = "https://files.pythonhosted.org/packages/66/8c/16cc223821ed25250270b6f50633b0fb0da43912007e068be6f52ff98185/pymetis-2025.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:408437b4aad82b67379f624b9faf9761b09caccdae083823a84b1bc7a470d86a", size = 1291900, upload-time = "2025-05-05T19:22:37.188Z" }, + { url = "https://files.pythonhosted.org/packages/84/8e/ac5d7110fea586fe660ef466e03abaca358dc2ed8282f01b4ae5cc93f5c1/pymetis-2025.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7dd5019162d8f19d03f1be9096547660d02a5925949e00529f8a5d7d4dfadb95", size = 246019, upload-time = "2025-05-05T19:22:39.039Z" }, + { url = "https://files.pythonhosted.org/packages/01/1e/74f3f9c9538a3435028b2bf00f659d84578861b6ca589acbd42d3623274c/pymetis-2025.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c65cb13972feda4f7f186c08c8cf619ac5f3f1264c9609466615b909219b9e58", size = 218388, upload-time = "2025-05-05T19:22:40.392Z" }, + { url = "https://files.pythonhosted.org/packages/a2/de/3462853dbb8ce25610db986cfc9f0e4fa3014f21412d7659bfa747596604/pymetis-2025.1.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d5b1d8c053dbd456bfc2f1fac5a2307c1c5cb614973a6485de966c049d48b18a", size = 337651, upload-time = "2025-05-05T19:22:42.145Z" }, + { url = "https://files.pythonhosted.org/packages/73/43/24c63cee0acd9d2ee9e1d657c933c33c16b83e638d7d36dca95437f4b979/pymetis-2025.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce64511e0fcb58b3012d6bd007303f55a4b3156025ff945ca5d4069349608334", size = 275477, upload-time = "2025-05-05T19:22:44.128Z" }, + { url = "https://files.pythonhosted.org/packages/07/b4/a6db323a67ac917ea902d1b52f64ca8defe1ae875890f55d7aefe55ceed5/pymetis-2025.1.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5fba99f74ea32e60abde9d8ad77c4ee9865fabe1fc2627b982d3bb359689360e", size = 1462678, upload-time = "2025-05-05T19:22:45.605Z" }, + { url = "https://files.pythonhosted.org/packages/c5/5f/0e94aa79362017f7464d249ab056245cc42b8cf2200e92fd593d89c77040/pymetis-2025.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ccc9f54edd33466a917ef5d2ea1c12e76787b4790e14754460daf427f1666247", size = 1292407, upload-time = "2025-05-05T19:22:47.504Z" }, + { url = "https://files.pythonhosted.org/packages/39/be/731625c087d720e533c2fd9350bfc1b581701e3ca507c6269d4c633b6653/pymetis-2025.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d4cdadbd49ba1f71a87c2a3c7e38241bff1cf16f9399ad05c1e856ce70c129d6", size = 246081, upload-time = "2025-05-05T19:22:48.826Z" }, + { url = "https://files.pythonhosted.org/packages/06/1e/d220c9090e0f2c81919a2c482bf912c59010a25558a5c3ef50ababe9cd7f/pymetis-2025.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d591bb9cd7d10b5f855b264ba713dc4a885492e61c5353c0a5ca34b9a28a4cf6", size = 218458, upload-time = "2025-05-05T19:22:50.209Z" }, + { url = "https://files.pythonhosted.org/packages/87/57/20fbab509ac524ec94bd608ddda41c0d027a5661549d7402e3204b52fbd1/pymetis-2025.1.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18704c986d76725ccb86288be24b069618cfc5916672450b967946f41a52cf2b", size = 337713, upload-time = "2025-05-05T19:22:51.525Z" }, + { url = "https://files.pythonhosted.org/packages/b9/fe/bb12c811cededf3b9b483851622e4622e2a27b64b0bca720cd966f522d16/pymetis-2025.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12ce2c20a0d30600b0d3fe049107ca76f3fdd9fdae1009b33b1ce168eb2aa089", size = 275622, upload-time = "2025-05-05T19:22:52.949Z" }, + { url = "https://files.pythonhosted.org/packages/1a/c8/87ac7974edf3940a7d3c6b8b1db20f2eb861c46634d8b5827e8b9c443d0f/pymetis-2025.1.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:33ac080d603a309cdaa235c83291a392f4c1e9e89d498b62a7c947ce6ab9f8f2", size = 1462651, upload-time = "2025-05-05T19:22:54.408Z" }, + { url = "https://files.pythonhosted.org/packages/57/18/763d5d4fa5da5c375b019d11251bc1955d005a4e4a6fccbf88e190b31aa6/pymetis-2025.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c389c822fe38db2b5bff60674250a3f97448b372d060d5d4789fae961d425223", size = 1292442, upload-time = "2025-05-05T19:22:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/d9/71/c7a486a55ee0efa3cd611b585ed4ac0c92241ba7ee488f3a5bafdf2ad8b8/pymetis-2025.1.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:625da502859f7d9449ccc5702c5228396ded95446ee61ab02f3ba84d58bb1a3b", size = 244783, upload-time = "2025-05-05T19:22:57.205Z" }, + { url = "https://files.pythonhosted.org/packages/10/8d/f1170b961645e5a33926314b62c3fc9f5a5e472a0d735bcf1a2dd0bdf656/pymetis-2025.1.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:05d7737ffa5594b622e6be53c9ae11855f798e3cd67b345bc8cdc286a501fab5", size = 217473, upload-time = "2025-05-05T19:22:58.517Z" }, + { url = "https://files.pythonhosted.org/packages/e7/cc/bc3c5190500648f7fe7b1427a68bdf7fc9fa3e2f03a740fe4ee4a1730d50/pymetis-2025.1.1-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f44b6cc411b395c04356305120b3fec19f4a0fc4e081d6f95c0acb3879fa1e24", size = 335596, upload-time = "2025-05-05T19:22:59.795Z" }, + { url = "https://files.pythonhosted.org/packages/65/97/fc54d7ac05cd4b3de8d9ff3ebacecee53249270c685f332d33f6aecfd79b/pymetis-2025.1.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3b9c5eeaf0a4be0c6e158fe0b43de6187580d9dc9b0c40c6569891c420e2731", size = 274340, upload-time = "2025-05-05T19:23:01.111Z" }, + { url = "https://files.pythonhosted.org/packages/a0/9f/2a5c32a99a80b4a8d6ee880c57e760167a8caa91a9f7c12566081612bfb5/pymetis-2025.1.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0e4363afc9aee6517cb3bd67978fa2bd5e923015f2e5630aad8451fd17cb24ef", size = 246012, upload-time = "2025-05-05T19:23:02.437Z" }, + { url = "https://files.pythonhosted.org/packages/e0/84/0e26a51fc1e6327e5f3d8bd67cfc78f47873d0f7f1c2f852464b56052d17/pymetis-2025.1.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:3880b7ec42b5d5dc1cee5170304962caf6349a347a882feb8156f082922badce", size = 218677, upload-time = "2025-05-05T19:23:03.775Z" }, + { url = "https://files.pythonhosted.org/packages/64/7d/e77d5c7771a8c31e3cf00a5abb116ec8d70db5c8116692feba1fa0cc12ec/pymetis-2025.1.1-pp311-pypy311_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:03d71a0cf3d18c76c0a98131876d41d7810b89f8dcb2a21ef11eff9f290de688", size = 336761, upload-time = "2025-05-05T19:23:05.113Z" }, + { url = "https://files.pythonhosted.org/packages/5b/29/22137aeaef2b472aabddea09fe7f512423ad50e405bb8da6c82089ca6720/pymetis-2025.1.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:daf4d7ea3f147fbb5328566abd4f493b1911ea911000036693bb5c639c7db4a5", size = 275642, upload-time = "2025-05-05T19:23:06.543Z" }, +] + [[package]] name = "pyparsing" version = "3.2.0" From 35521c2b2ac6b800848fffcb5ae94666fd13655d Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 9 Jul 2025 14:04:10 +0200 Subject: [PATCH 038/240] move test files in decomposition, fix some errors in halo.py --- .../model/common/decomposition/definitions.py | 9 +- .../model/common/decomposition/halo.py | 2 +- .../decomposition_tests/mpi_tests/conftest.py | 8 ++ .../{ => mpi_tests}/test_halo.py | 118 +++++++++--------- .../{ => mpi_tests}/test_mpi_decomposition.py | 0 .../src/icon4py/model/testing/serialbox.py | 6 +- 6 files changed, 78 insertions(+), 65 deletions(-) create mode 100644 model/common/tests/decomposition_tests/mpi_tests/conftest.py rename model/common/tests/decomposition_tests/{ => mpi_tests}/test_halo.py (78%) rename model/common/tests/decomposition_tests/{ => mpi_tests}/test_mpi_decomposition.py (100%) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 3e03dc1004..eeadc948fa 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -82,10 +82,15 @@ class EntryType(IntEnum): @utils.chainable def with_dimension( - self, dim: Dimension, global_index: data_alloc.NDArray, owner_mask: data_alloc.NDArray + self, + dim: Dimension, + global_index: data_alloc.NDArray, + owner_mask: data_alloc.NDArray, + halo_levels: data_alloc.NDArray, ): self._global_index[dim] = global_index self._owner_mask[dim] = owner_mask + self._halo_levels[dim] = halo_levels def __init__( self, @@ -414,7 +419,7 @@ class DecompositionFlag(enum.IntEnum): """ used for: - cells that share 1 edge with an OWNED cell - - vertices that are on OWNED cell + - vertices that are on OWNED cell - edges that are on OWNED cell """ diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index afaa127ecf..5265229724 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -45,7 +45,7 @@ def __init__( rank_mapping: data_alloc.NDArray, connectivities: dict[gtx.Dimension, data_alloc.NDArray], num_levels: int, - backend: Optional[gtx_backend.Backend], + backend: Optional[gtx_backend.Backend] = None, ): """ diff --git a/model/common/tests/decomposition_tests/mpi_tests/conftest.py b/model/common/tests/decomposition_tests/mpi_tests/conftest.py new file mode 100644 index 0000000000..80b673df7e --- /dev/null +++ b/model/common/tests/decomposition_tests/mpi_tests/conftest.py @@ -0,0 +1,8 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + diff --git a/model/common/tests/decomposition_tests/test_halo.py b/model/common/tests/decomposition_tests/mpi_tests/test_halo.py similarity index 78% rename from model/common/tests/decomposition_tests/test_halo.py rename to model/common/tests/decomposition_tests/mpi_tests/test_halo.py index bee554b24a..d1976bd4ac 100644 --- a/model/common/tests/decomposition_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/mpi_tests/test_halo.py @@ -27,20 +27,20 @@ import pytest import icon4py.model.common.dimension as dims -import icon4py.model.common.grid.grid_manager as gm from icon4py.model.common.decomposition import definitions as defs from icon4py.model.common.decomposition.definitions import DecompositionFlag from icon4py.model.common.decomposition.halo import ( HaloGenerator, SimpleMetisDecomposer, ) -from icon4py.model.common.grid import base as base_grid, simple, vertical as v_grid -from icon4py.model.common.settings import xp -from icon4py.model.common.test_utils import datatest_utils as dt_utils, helpers -from icon4py.model.common.test_utils.parallel_helpers import ( # noqa: F401 # import fixtures from test_utils package - check_comm_size, - processor_props, +from icon4py.model.common.grid import ( + base as base_grid, + grid_manager as gm, + gridfile as grid_file, + simple, + vertical as v_grid, ) +from icon4py.model.testing import datatest_utils as dt_utils, helpers UGRID_FILE = dt_utils.GRIDS_PATH.joinpath(dt_utils.R02B04_GLOBAL).joinpath( @@ -49,8 +49,9 @@ GRID_FILE = dt_utils.GRIDS_PATH.joinpath(dt_utils.R02B04_GLOBAL).joinpath( "icon_grid_0013_R02B04_R.nc" ) +backend = None -SIMPLE_DISTRIBUTION = xp.asarray( +SIMPLE_DISTRIBUTION = np.asarray( [ 0, # 0c 1, # 1c @@ -176,29 +177,30 @@ } -def test_halo_constructor_owned_cells(processor_props): # noqa F811 # fixture +def test_halo_constructor_owned_cells(processor_props): # F811 # fixture grid = simple.SimpleGrid() halo_generator = HaloGenerator( - connectivities=grid.connectivities, + connectivities=grid.neighbor_tables, run_properties=processor_props, rank_mapping=SIMPLE_DISTRIBUTION, num_levels=1, + backend=backend, ) my_owned_cells = halo_generator.owned_cells() print(f"rank {processor_props.rank} owns {my_owned_cells} ") assert my_owned_cells.size == len(_CELL_OWN[processor_props.rank]) - assert xp.setdiff1d(my_owned_cells, _CELL_OWN[processor_props.rank]).size == 0 + assert np.setdiff1d(my_owned_cells, _CELL_OWN[processor_props.rank]).size == 0 @pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) @pytest.mark.mpi(min_size=4) -def test_element_ownership_is_unique(dim, processor_props): # noqa F811 # fixture +def test_element_ownership_is_unique(dim, processor_props): # F811 # fixture if processor_props.comm_size != 4: pytest.skip("This test requires exactly 4 MPI ranks.") grid = simple.SimpleGrid() halo_generator = HaloGenerator( - connectivities=grid.connectivities, + connectivities=grid.neighbor_tables, run_properties=processor_props, rank_mapping=SIMPLE_DISTRIBUTION, num_levels=1, @@ -213,14 +215,14 @@ def test_element_ownership_is_unique(dim, processor_props): # noqa F811 # fixtu comm = processor_props.comm my_size = owned.shape[0] - local_sizes = xp.array(comm.gather(my_size, root=0)) + local_sizes = np.array(comm.gather(my_size, root=0)) buffer_size = 27 - send_buf = -1 * xp.ones(buffer_size, dtype=int) + send_buf = -1 * np.ones(buffer_size, dtype=int) send_buf[:my_size] = owned print(f"rank {processor_props.rank} send_buf: {send_buf}") if processor_props.rank == 0: print(f"local_sizes: {local_sizes}") - recv_buffer = -1 * xp.ones((4, buffer_size), dtype=int) + recv_buffer = -1 * np.ones((4, buffer_size), dtype=int) print(f"{recv_buffer.shape}") else: recv_buffer = None @@ -230,19 +232,19 @@ def test_element_ownership_is_unique(dim, processor_props): # noqa F811 # fixtu print(f"global indices: {recv_buffer}") # check there are no duplicates values = recv_buffer[recv_buffer != -1] - assert values.size == len(xp.unique(values)) + assert values.size == len(np.unique(values)) # check the buffer has all global indices - assert xp.all(xp.sort(values) == global_indices(dim)) + assert np.all(np.sort(values) == global_indices(dim)) @pytest.mark.mpi(min_size=4) @pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) -def test_halo_constructor_decomposition_info_global_indices(processor_props, dim): # noqa F811 # fixture +def test_halo_constructor_decomposition_info_global_indices(processor_props, dim): # F811 # fixture if processor_props.comm_size != 4: pytest.skip("This test requires exactly 4 MPI ranks.") grid = simple.SimpleGrid() halo_generator = HaloGenerator( - connectivities=grid.connectivities, + connectivities=grid.neighbor_tables, run_properties=processor_props, rank_mapping=SIMPLE_DISTRIBUTION, num_levels=1, @@ -252,24 +254,24 @@ def test_halo_constructor_decomposition_info_global_indices(processor_props, dim my_halo = decomp_info.global_index(dim, defs.DecompositionInfo.EntryType.HALO) print(f"rank {processor_props.rank} has halo {dim} : {my_halo}") assert my_halo.size == len(HALO[dim][processor_props.rank]) - assert xp.setdiff1d(my_halo, HALO[dim][processor_props.rank], assume_unique=True).size == 0 + assert np.setdiff1d(my_halo, HALO[dim][processor_props.rank], assume_unique=True).size == 0 my_owned = decomp_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED) print(f"rank {processor_props.rank} owns {dim} : {my_owned} ") assert_same_entries(dim, my_owned, OWNED, processor_props.rank) def assert_same_entries( - dim: gtx.Dimension, my_owned: np.ndarray, reference: dict[int, list], rank: int + dim: gtx.Dimension, my_owned: np.ndarray, reference: dict[gtx.Dimension, dict], rank: int ): assert my_owned.size == len(reference[dim][rank]) - assert xp.setdiff1d(my_owned, reference[dim][rank], assume_unique=True).size == 0 + assert np.setdiff1d(my_owned, reference[dim][rank], assume_unique=True).size == 0 @pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) -def test_halo_constructor_decomposition_info_halo_levels(processor_props, dim): # noqa F811 # fixture +def test_halo_constructor_decomposition_info_halo_levels(processor_props, dim): # F811 # fixture grid = simple.SimpleGrid() halo_generator = HaloGenerator( - connectivities=grid.connectivities, + connectivities=grid.neighbor_tables, run_properties=processor_props, rank_mapping=SIMPLE_DISTRIBUTION, num_levels=1, @@ -278,24 +280,24 @@ def test_halo_constructor_decomposition_info_halo_levels(processor_props, dim): my_halo_levels = decomp_info.halo_levels(dim) print(f"{dim.value}: rank {processor_props.rank} has halo levels {my_halo_levels} ") if dim != dims.EdgeDim: - assert xp.all( + assert np.all( my_halo_levels != DecompositionFlag.UNDEFINED ), ( "All indices should have a defined DecompositionFlag" ) # THIS WILL CURRENTLY FAIL FOR EDGES - assert xp.where(my_halo_levels == DecompositionFlag.OWNED)[0].size == len( + assert np.where(my_halo_levels == DecompositionFlag.OWNED)[0].size == len( OWNED[dim][processor_props.rank] ) owned_local_indices = decomp_info.local_index(dim, defs.DecompositionInfo.EntryType.OWNED) - assert xp.all( + assert np.all( my_halo_levels[owned_local_indices] == DecompositionFlag.OWNED ), "owned local indices should have DecompositionFlag.OWNED" - first_halo_line_local_index = xp.where(my_halo_levels == DecompositionFlag.FIRST_HALO_LINE)[0] + first_halo_line_local_index = np.where(my_halo_levels == DecompositionFlag.FIRST_HALO_LINE)[0] first_halo_line_global_index = decomp_info.global_index( dim, defs.DecompositionInfo.EntryType.ALL )[first_halo_line_local_index] assert_same_entries(dim, first_halo_line_global_index, FIRST_HALO_LINE, processor_props.rank) - second_halo_line_local_index = xp.where(my_halo_levels == DecompositionFlag.SECOND_HALO_LINE)[0] + second_halo_line_local_index = np.where(my_halo_levels == DecompositionFlag.SECOND_HALO_LINE)[0] second_halo_line_global_index = decomp_info.global_index( dim, defs.DecompositionInfo.EntryType.ALL )[second_halo_line_local_index] @@ -306,25 +308,25 @@ def grid_file_manager(file: pathlib.Path) -> gm.GridManager: manager = gm.GridManager( gm.ToZeroBasedIndexTransformation(), str(file), v_grid.VerticalGridConfig(num_levels=1) ) - manager() + manager(keep_skip_values=True) return manager -def global_indices(dim: dims.Dimension) -> int: +def global_indices(dim: gtx.Dimension) -> int: mesh = simple.SimpleGrid() - return xp.arange(mesh.size[dim], dtype=xp.int32) + return np.arange(mesh.size[dim], dtype=gtx.int32) # TODO unused - remove or fix and use? def icon_distribution( props: defs.ProcessProperties, decomposition_info: defs.DecompositionInfo -) -> xp.ndarray: +) -> np.ndarray: cell_index = decomposition_info.global_index( dims.CellDim, defs.DecompositionInfo.EntryType.OWNED ) comm = props.comm local_sizes, recv_buffer = gather_field(cell_index, comm) - distribution = xp.empty((sum(local_sizes)), dtype=int) + distribution = np.empty((sum(local_sizes)), dtype=int) if comm.rank == 0: start_index = 0 for s in comm.size: @@ -336,10 +338,10 @@ def icon_distribution( return distribution -def gather_field(field: xp.ndarray, comm: mpi4py.MPI.Comm) -> tuple: - local_sizes = xp.array(comm.gather(field.size, root=0)) +def gather_field(field: np.ndarray, comm: mpi4py.MPI.Comm) -> tuple: + local_sizes = np.array(comm.gather(field.size, root=0)) if comm.rank == 0: - recv_buffer = xp.empty(sum(local_sizes), dtype=field.dtype) + recv_buffer = np.empty(sum(local_sizes), dtype=field.dtype) else: recv_buffer = None comm.Gatherv(sendbuf=field, recvbuf=(recv_buffer, local_sizes), root=0) @@ -347,36 +349,34 @@ def gather_field(field: xp.ndarray, comm: mpi4py.MPI.Comm) -> tuple: @pytest.mark.mpi -def test_distributed_fields(processor_props): # noqa F811 # fixture +def test_distributed_fields(processor_props): # F811 # fixture grid_manager = grid_file_manager(GRID_FILE) global_grid = grid_manager.grid labels = decompose(global_grid, processor_props) halo_generator = HaloGenerator( - connectivities=global_grid.connectivities, + connectivities=global_grid.neighbor_tables, run_properties=processor_props, rank_mapping=labels, num_levels=1, ) decomposition_info = halo_generator() # distributed read: read one field per dimension - local_geometry_fields = grid_manager._read_geometry(decomposition_info) - local_cell_area = local_geometry_fields[gm.GridFile.GeometryName.CELL_AREA] - local_edge_length = local_geometry_fields[gm.GridFile.GeometryName.EDGE_LENGTH] - local_vlon = grid_manager.read_coordinates(decomposition_info)[ - gm.GridFile.CoordinateName.VERTEX_LONGITUDE - ] + local_geometry_fields = grid_manager.geometry + local_cell_area = local_geometry_fields[grid_file.GeometryName.CELL_AREA] + local_edge_length = local_geometry_fields[grid_file.GeometryName.EDGE_LENGTH] + local_vlon = grid_manager.coordinates[grid_file.CoordinateName.VERTEX_LONGITUDE] print( - f"rank = {processor_props.rank} has size(cell_area): {local_cell_area.shape}, has size(edge_length): {local_edge_length.shape}" + f"rank = {processor_props.rank} has size(cell_area): {local_cell_area.asnumpy().shape}, has size(edge_length): {local_edge_length.shape}" ) # the local number of cells must be at most the global number of cells (analytically computed) assert local_cell_area.size <= global_grid.global_properties.num_cells # global read: read the same (global fields) - global_geometry_fields = grid_manager._read_geometry() - global_cell_area = global_geometry_fields[gm.GridFile.GeometryName.CELL_AREA] - global_edge_length = global_geometry_fields[gm.GridFile.GeometryName.EDGE_LENGTH] - global_vlon = grid_manager.read_coordinates()[gm.GridFile.CoordinateName.VERTEX_LONGITUDE] + global_geometry_fields = grid_manager.geometry + global_cell_area = global_geometry_fields[grid_file.GeometryName.CELL_AREA] + global_edge_length = global_geometry_fields[grid_file.GeometryName.EDGE_LENGTH] + global_vlon = grid_manager.coordinates[dims.VertexDim]["lon"] assert_gathered_field_against_global( decomposition_info, processor_props, dims.CellDim, global_cell_area, local_cell_area @@ -390,18 +390,18 @@ def test_distributed_fields(processor_props): # noqa F811 # fixture ) -def decompose(grid: base_grid.BaseGrid, processor_props): # noqa F811 # fixture +def decompose(grid: base_grid.BaseGrid, processor_props): # F811 # fixture partitioner = SimpleMetisDecomposer() - labels = partitioner(grid.connectivities[dims.C2E2CDim], n_part=processor_props.comm_size) + labels = partitioner(grid.neighbor_tables[dims.C2E2CDim], n_part=processor_props.comm_size) return labels def assert_gathered_field_against_global( decomposition_info: defs.DecompositionInfo, - processor_props: defs.ProcessProperties, # noqa F811 # fixture - dim: dims.Dimension, - global_reference_field: xp.ndarray, - local_field: xp.ndarray, + processor_props: defs.ProcessProperties, # F811 # fixture + dim: gtx.Dimension, + global_reference_field: np.ndarray, + local_field: np.ndarray, ): assert ( local_field.size @@ -416,8 +416,8 @@ def assert_gathered_field_against_global( processor_props.comm, ) if processor_props.rank == 0: - assert xp.all(gathered_sizes == global_index_sizes) - sorted_ = xp.zeros(global_reference_field.shape, dtype=xp.float64) + assert np.all(gathered_sizes == global_index_sizes) + sorted_ = np.zeros(global_reference_field.shape, dtype=gtx.float64) sorted_[gathered_global_indices] = gathered_field assert helpers.dallclose(sorted_, global_reference_field) diff --git a/model/common/tests/decomposition_tests/test_mpi_decomposition.py b/model/common/tests/decomposition_tests/mpi_tests/test_mpi_decomposition.py similarity index 100% rename from model/common/tests/decomposition_tests/test_mpi_decomposition.py rename to model/common/tests/decomposition_tests/mpi_tests/test_mpi_decomposition.py diff --git a/model/testing/src/icon4py/model/testing/serialbox.py b/model/testing/src/icon4py/model/testing/serialbox.py index d5925021b6..edfa6f045d 100644 --- a/model/testing/src/icon4py/model/testing/serialbox.py +++ b/model/testing/src/icon4py/model/testing/serialbox.py @@ -446,9 +446,9 @@ def construct_decomposition_info(self): num_edges=self.num(dims.EdgeDim), num_vertices=self.num(dims.VertexDim), ) - .with_dimension(*self._get_decomp_fields(dims.CellDim)) - .with_dimension(*self._get_decomp_fields(dims.EdgeDim)) - .with_dimension(*self._get_decomp_fields(dims.VertexDim)) + .with_dimension(*self._get_decomposition_fields(dims.CellDim)) + .with_dimension(*self._get_decomposition_fields(dims.EdgeDim)) + .with_dimension(*self._get_decomposition_fields(dims.VertexDim)) ) def _get_decomposition_fields(self, dim: gtx.Dimension): From 5ebd07f697aba9864391afd434e4d3c28a3946ec Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 10 Jul 2025 16:35:00 +0200 Subject: [PATCH 039/240] fix tests --- .../model/common/decomposition/definitions.py | 3 + .../model/common/decomposition/halo.py | 45 +++-- .../common/decomposition/mpi_decomposition.py | 4 +- .../src/icon4py/model/common/exceptions.py | 7 +- .../src/icon4py/model/common/grid/simple.py | 1 + .../decomposition_tests/mpi_tests/conftest.py | 10 + .../mpi_tests/test_halo.py | 175 ++++++++++++------ .../mpi_tests/test_mpi_decomposition.py | 83 +++++---- 8 files changed, 217 insertions(+), 111 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index eeadc948fa..e96a2679ba 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -44,6 +44,9 @@ class ProcessProperties(Protocol): def single_node(self) -> bool: return self.comm_size == 1 + def __str__(self): + return f"Comm name={self.comm_name}: rank = {self.rank}/{self.comm_size}" + @dataclass(frozen=True, init=False) class SingleNodeProcessProperties(ProcessProperties): diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 5265229724..352c4127e3 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -33,18 +33,15 @@ log = logging.getLogger(__name__) -# TODO (@halungge) do we need three of those: one for each dimension? - - class HaloGenerator: """Creates necessary halo information for a given rank.""" def __init__( self, run_properties: defs.ProcessProperties, - rank_mapping: data_alloc.NDArray, + rank_mapping: data_alloc.NDArray, # TODO should be an argument to __call__ connectivities: dict[gtx.Dimension, data_alloc.NDArray], - num_levels: int, + num_levels, # TODO is currently needed for ghex, pass via a different struct that the decomposition info and remove backend: Optional[gtx_backend.Backend] = None, ): """ @@ -53,13 +50,14 @@ def __init__( run_properties: contains information on the communicator and local compute node. rank_mapping: array with shape (global_num_cells,): mapping of global cell indices to their rank in the distribution connectivities: connectivity arrays needed to construct the halos - num_levels: # TODO (@halungge): should not be needed here + backend: GT4Py (used to determine the array ns import) """ + self._xp = data_alloc.import_array_ns(backend) + self._num_levels = num_levels self._props = run_properties self._mapping = rank_mapping self._connectivities = connectivities - self._num_levels = num_levels - self._xp = data_alloc.import_array_ns(backend) + self._validate() @property def face_face_connectivity(self): @@ -86,12 +84,33 @@ def face_node_connectivity(self): return self._connectivity(dims.C2VDim) def _validate(self): - assert self._mapping.ndim == 1 - # the decomposition should match the communicator size - assert self._xp.max(self._mapping) == self._props.comm_size - 1 + # validate the distribution mapping: + num_cells = self.face_face_connectivity.shape[0] + expected_shape = (num_cells,) + if not self._mapping.shape == expected_shape: + raise exceptions.ValidationError( + "rank_mapping", f"should have shape {expected_shape} but is {self._mapping.shape}" + ) - def __post_init__(self): - self._validate() + # the decomposition should match the communicator size + if self._xp.max(self._mapping) > self._props.comm_size - 1: + raise exceptions.ValidationError( + "rank_mapping", + f"The distribution assumes more nodes than the current run is scheduled on {self._props} ", + ) + # make sure we have all connectivity arrays used in the halo construction + relevant_dimension = [ + dims.C2E2CDim, + dims.E2CDim, + dims.C2EDim, + dims.C2VDim, + dims.V2CDim, + dims.V2EDim, + ] + for d in relevant_dimension: + assert ( + d in self._connectivities.keys() + ), f"Table for {d} is missing from the neighbor table array." def _connectivity(self, dim: gtx.Dimension) -> data_alloc.NDArray: try: diff --git a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py index c64b04fbfd..fa5b508139 100644 --- a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py +++ b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py @@ -116,11 +116,11 @@ class MPICommProcessProperties(definitions.ProcessProperties): comm: mpi4py.MPI.Comm = None @functools.cached_property - def rank(self): + def rank(self) -> int: return self.comm.Get_rank() @functools.cached_property - def comm_name(self): + def comm_name(self) -> str: return self.comm.Get_name() @functools.cached_property diff --git a/model/common/src/icon4py/model/common/exceptions.py b/model/common/src/icon4py/model/common/exceptions.py index a968254760..85f9d28bb2 100644 --- a/model/common/src/icon4py/model/common/exceptions.py +++ b/model/common/src/icon4py/model/common/exceptions.py @@ -16,10 +16,15 @@ class InvalidConfigError(Exception): class IncompleteStateError(Exception): - def __init__(self, field_name): + def __init__(self, field_name: str): super().__init__(f"Field '{field_name}' is missing.") +class ValidationError(Exception): + def __init__(self, name: str, msg: str): + super().__init__(f"'{name}': {msg}.") + + class IconGridError(RuntimeError): pass diff --git a/model/common/src/icon4py/model/common/grid/simple.py b/model/common/src/icon4py/model/common/grid/simple.py index 56776979aa..c8e78e1ce6 100644 --- a/model/common/src/icon4py/model/common/grid/simple.py +++ b/model/common/src/icon4py/model/common/grid/simple.py @@ -424,6 +424,7 @@ def __init__(self, backend: gtx_backend.Backend | None = None): "E2C2EO": (self._construct_connectivity, dims.E2C2EODim, dims.EdgeDim, dims.EdgeDim), "E2C2E": (self._construct_connectivity, dims.E2C2EDim, dims.EdgeDim, dims.EdgeDim), "V2C": (self._construct_connectivity, dims.V2CDim, dims.VertexDim, dims.CellDim), + "C2V": (self._construct_connectivity, dims.C2VDim, dims.CellDim, dims.VertexDim), "V2E": (self._construct_connectivity, dims.V2EDim, dims.VertexDim, dims.EdgeDim), "E2C": (self._construct_connectivity, dims.E2CDim, dims.EdgeDim, dims.CellDim), "E2V": (self._construct_connectivity, dims.E2VDim, dims.EdgeDim, dims.VertexDim), diff --git a/model/common/tests/decomposition_tests/mpi_tests/conftest.py b/model/common/tests/decomposition_tests/mpi_tests/conftest.py index 80b673df7e..a0b781be2f 100644 --- a/model/common/tests/decomposition_tests/mpi_tests/conftest.py +++ b/model/common/tests/decomposition_tests/mpi_tests/conftest.py @@ -5,4 +5,14 @@ # # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause +try: + import sys + _ = sys.modules["icon4py.model.testing.pytest_config"] +except KeyError: + from icon4py.model.testing.pytest_config import * # noqa: F403 + + +from icon4py.model.testing.parallel_helpers import ( + processor_props, # noqa: F401 # import fixtures from test_utils package +) diff --git a/model/common/tests/decomposition_tests/mpi_tests/test_halo.py b/model/common/tests/decomposition_tests/mpi_tests/test_halo.py index d1976bd4ac..07592aafe6 100644 --- a/model/common/tests/decomposition_tests/mpi_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/mpi_tests/test_halo.py @@ -6,33 +6,26 @@ # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause -# ICON4Py - ICON inspired code in Python and GT4Py -# -# Copyright (c) 2022, ETH Zurich and MeteoSwiss -# All rights reserved. -# -# This file is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or any later -# version. See the LICENSE.txt file at the top-level directory of this -# distribution for a copy of the license or check . -# -# SPDX-License-Identifier: GPL-3.0-or-later import pathlib import gt4py.next as gtx -import mpi4py -import mpi4py.MPI import numpy as np import pytest import icon4py.model.common.dimension as dims -from icon4py.model.common.decomposition import definitions as defs -from icon4py.model.common.decomposition.definitions import DecompositionFlag -from icon4py.model.common.decomposition.halo import ( - HaloGenerator, - SimpleMetisDecomposer, -) +from icon4py.model.common import exceptions +from icon4py.model.common.decomposition import definitions as defs, mpi_decomposition + + +try: + import mpi4py # import mpi4py to check for optional mpi dependency + import mpi4py.MPI + + mpi_decomposition.init_mpi() +except ImportError: + pytest.skip("Skipping parallel on single node installation", allow_module_level=True) + +from icon4py.model.common.decomposition import halo from icon4py.model.common.grid import ( base as base_grid, grid_manager as gm, @@ -51,6 +44,27 @@ ) backend = None +""" +TESTDATA using the [SimpleGrid](../../../src/icon4py/model/common/grid/simple.py) +The distribution maps all of the 18 cells of the simple grid to ranks 0..3 + +the dictionaries contain the mapping from rank to global (in the simple grid) index of the dimension: +_CELL_OWN: rank -> owned cells, essentially the inversion of the SIMPLE_DISTRIBUTION +_EDGE_OWN: rank -> owned edges +_VERTEX_OWN: rank -> owned vertices + +the decision as to whether a "secondary" dimension (edge, vertices) is owned by a rank are made according to the +rules and conventions described in (../../../src/icon4py/model/common/decomposition/halo.py) + + +_CELL_FIRST_HALO_LINE: +_CELL_SECON_HALO_LINE: +_EDGE_FIRST_HALO_LINE: +_EDGE_SECOND_HALO_LINE: +_VERTEX_FIRST_HALO_LINE: +_VERTEX_SECOND_HALO_LINE: :mapping of rank to global indices that belongs to a ranks halo lines. +""" + SIMPLE_DISTRIBUTION = np.asarray( [ 0, # 0c @@ -177,9 +191,19 @@ } +def grid_file_manager(file: pathlib.Path) -> gm.GridManager: + manager = gm.GridManager( + gm.ToZeroBasedIndexTransformation(), str(file), v_grid.VerticalGridConfig(num_levels=1) + ) + manager(keep_skip_values=True) + return manager + + +@pytest.mark.mpi(min_size=4) def test_halo_constructor_owned_cells(processor_props): # F811 # fixture grid = simple.SimpleGrid() - halo_generator = HaloGenerator( + + halo_generator = halo.HaloGenerator( connectivities=grid.neighbor_tables, run_properties=processor_props, rank_mapping=SIMPLE_DISTRIBUTION, @@ -193,17 +217,49 @@ def test_halo_constructor_owned_cells(processor_props): # F811 # fixture assert np.setdiff1d(my_owned_cells, _CELL_OWN[processor_props.rank]).size == 0 +def test_halo_constructor_validate_number_of_node_mismatch(processor_props): + grid = simple.SimpleGrid() + distribution = (processor_props.comm_size + 1) * np.ones((grid.num_cells,), dtype=int) + with pytest.raises(expected_exception=exceptions.ValidationError) as e: + halo.HaloGenerator( + connectivities=grid.neighbor_tables, + run_properties=processor_props, + rank_mapping=distribution, + num_levels=1, + ) + assert "The distribution assumes more nodes than the current run" in e.value.args[0] + + +@pytest.mark.parametrize("shape", [(simple.SimpleGrid._CELLS, 3), (2,)]) +def test_halo_constructor_validate_rank_mapping_wrong_shape(processor_props, shape): + grid = simple.SimpleGrid() + with pytest.raises(exceptions.ValidationError) as e: + halo.HaloGenerator( + connectivities=grid.neighbor_tables, + run_properties=processor_props, + rank_mapping=np.zeros((grid.num_cells, 3), dtype=int), + num_levels=1, + ) + assert f"should have shape ({grid.num_cells},)" in e.value.args[0] + + +def global_indices(dim: gtx.Dimension) -> np.ndarray: + mesh = simple.SimpleGrid() + return np.arange(mesh.size[dim], dtype=gtx.int32) + + @pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) @pytest.mark.mpi(min_size=4) def test_element_ownership_is_unique(dim, processor_props): # F811 # fixture if processor_props.comm_size != 4: pytest.skip("This test requires exactly 4 MPI ranks.") grid = simple.SimpleGrid() - halo_generator = HaloGenerator( + halo_generator = halo.HaloGenerator( connectivities=grid.neighbor_tables, run_properties=processor_props, rank_mapping=SIMPLE_DISTRIBUTION, num_levels=1, + backend=backend, ) decomposition_info = halo_generator() @@ -243,7 +299,7 @@ def test_halo_constructor_decomposition_info_global_indices(processor_props, dim if processor_props.comm_size != 4: pytest.skip("This test requires exactly 4 MPI ranks.") grid = simple.SimpleGrid() - halo_generator = HaloGenerator( + halo_generator = halo.HaloGenerator( connectivities=grid.neighbor_tables, run_properties=processor_props, rank_mapping=SIMPLE_DISTRIBUTION, @@ -267,10 +323,11 @@ def assert_same_entries( assert np.setdiff1d(my_owned, reference[dim][rank], assume_unique=True).size == 0 +@pytest.mark.mpi(min_size=4) @pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) def test_halo_constructor_decomposition_info_halo_levels(processor_props, dim): # F811 # fixture grid = simple.SimpleGrid() - halo_generator = HaloGenerator( + halo_generator = halo.HaloGenerator( connectivities=grid.neighbor_tables, run_properties=processor_props, rank_mapping=SIMPLE_DISTRIBUTION, @@ -281,42 +338,33 @@ def test_halo_constructor_decomposition_info_halo_levels(processor_props, dim): print(f"{dim.value}: rank {processor_props.rank} has halo levels {my_halo_levels} ") if dim != dims.EdgeDim: assert np.all( - my_halo_levels != DecompositionFlag.UNDEFINED + my_halo_levels != defs.DecompositionFlag.UNDEFINED ), ( "All indices should have a defined DecompositionFlag" ) # THIS WILL CURRENTLY FAIL FOR EDGES - assert np.where(my_halo_levels == DecompositionFlag.OWNED)[0].size == len( + assert np.where(my_halo_levels == defs.DecompositionFlag.OWNED)[0].size == len( OWNED[dim][processor_props.rank] ) owned_local_indices = decomp_info.local_index(dim, defs.DecompositionInfo.EntryType.OWNED) assert np.all( - my_halo_levels[owned_local_indices] == DecompositionFlag.OWNED + my_halo_levels[owned_local_indices] == defs.DecompositionFlag.OWNED ), "owned local indices should have DecompositionFlag.OWNED" - first_halo_line_local_index = np.where(my_halo_levels == DecompositionFlag.FIRST_HALO_LINE)[0] + first_halo_line_local_index = np.where( + my_halo_levels == defs.DecompositionFlag.FIRST_HALO_LINE + )[0] first_halo_line_global_index = decomp_info.global_index( dim, defs.DecompositionInfo.EntryType.ALL )[first_halo_line_local_index] assert_same_entries(dim, first_halo_line_global_index, FIRST_HALO_LINE, processor_props.rank) - second_halo_line_local_index = np.where(my_halo_levels == DecompositionFlag.SECOND_HALO_LINE)[0] + second_halo_line_local_index = np.where( + my_halo_levels == defs.DecompositionFlag.SECOND_HALO_LINE + )[0] second_halo_line_global_index = decomp_info.global_index( dim, defs.DecompositionInfo.EntryType.ALL )[second_halo_line_local_index] assert_same_entries(dim, second_halo_line_global_index, SECOND_HALO_LINE, processor_props.rank) -def grid_file_manager(file: pathlib.Path) -> gm.GridManager: - manager = gm.GridManager( - gm.ToZeroBasedIndexTransformation(), str(file), v_grid.VerticalGridConfig(num_levels=1) - ) - manager(keep_skip_values=True) - return manager - - -def global_indices(dim: gtx.Dimension) -> int: - mesh = simple.SimpleGrid() - return np.arange(mesh.size[dim], dtype=gtx.int32) - - # TODO unused - remove or fix and use? def icon_distribution( props: defs.ProcessProperties, decomposition_info: defs.DecompositionInfo @@ -348,14 +396,25 @@ def gather_field(field: np.ndarray, comm: mpi4py.MPI.Comm) -> tuple: return local_sizes, recv_buffer +def decompose(grid: base_grid.BaseGrid, processor_props): # F811 # fixture + partitioner = halo.SimpleMetisDecomposer() + labels = partitioner(grid.neighbor_tables[dims.C2E2CDim], n_part=processor_props.comm_size) + return labels + + +@pytest.mark.xfail @pytest.mark.mpi def test_distributed_fields(processor_props): # F811 # fixture grid_manager = grid_file_manager(GRID_FILE) global_grid = grid_manager.grid + global_cell_area = grid_manager.geometry[grid_file.GeometryName.CELL_AREA] + global_edge_lat = grid_manager.coordinates[dims.EdgeDim]["lat"] + global_vertex_lon = grid_manager.coordinates[dims.VertexDim]["lon"] + labels = decompose(global_grid, processor_props) - halo_generator = HaloGenerator( + halo_generator = halo.HaloGenerator( connectivities=global_grid.neighbor_tables, run_properties=processor_props, rank_mapping=labels, @@ -363,39 +422,33 @@ def test_distributed_fields(processor_props): # F811 # fixture ) decomposition_info = halo_generator() # distributed read: read one field per dimension - local_geometry_fields = grid_manager.geometry - local_cell_area = local_geometry_fields[grid_file.GeometryName.CELL_AREA] - local_edge_length = local_geometry_fields[grid_file.GeometryName.EDGE_LENGTH] - local_vlon = grid_manager.coordinates[grid_file.CoordinateName.VERTEX_LONGITUDE] + + ## TODO why is this local?? + local_cell_area = grid_manager.geometry[grid_file.GeometryName.CELL_AREA] + local_edge_lat = grid_manager.coordinates[dims.EdgeDim]["lat"] + local_vertex_lon = grid_manager.coordinates[dims.VertexDim]["lon"] print( - f"rank = {processor_props.rank} has size(cell_area): {local_cell_area.asnumpy().shape}, has size(edge_length): {local_edge_length.shape}" + f"rank = {processor_props.rank} has size(cell_area): {local_cell_area.ndarray.shape}, " + f"has size(edge_length): {local_edge_lat.ndarray.shape}, has size(vertex_length): {local_vertex_lon.ndarray.shape}" ) # the local number of cells must be at most the global number of cells (analytically computed) - assert local_cell_area.size <= global_grid.global_properties.num_cells + assert ( + local_cell_area.asnumpy().shape[0] <= global_grid.global_properties.num_cells + ), "local field is larger than global field" # global read: read the same (global fields) - global_geometry_fields = grid_manager.geometry - global_cell_area = global_geometry_fields[grid_file.GeometryName.CELL_AREA] - global_edge_length = global_geometry_fields[grid_file.GeometryName.EDGE_LENGTH] - global_vlon = grid_manager.coordinates[dims.VertexDim]["lon"] assert_gathered_field_against_global( decomposition_info, processor_props, dims.CellDim, global_cell_area, local_cell_area ) assert_gathered_field_against_global( - decomposition_info, processor_props, dims.EdgeDim, global_edge_length, local_edge_length + decomposition_info, processor_props, dims.EdgeDim, global_edge_lat, local_edge_lat ) assert_gathered_field_against_global( - decomposition_info, processor_props, dims.VertexDim, global_vlon, local_vlon + decomposition_info, processor_props, dims.VertexDim, global_vertex_lon, local_vertex_lon ) -def decompose(grid: base_grid.BaseGrid, processor_props): # F811 # fixture - partitioner = SimpleMetisDecomposer() - labels = partitioner(grid.neighbor_tables[dims.C2E2CDim], n_part=processor_props.comm_size) - return labels - - def assert_gathered_field_against_global( decomposition_info: defs.DecompositionInfo, processor_props: defs.ProcessProperties, # F811 # fixture diff --git a/model/common/tests/decomposition_tests/mpi_tests/test_mpi_decomposition.py b/model/common/tests/decomposition_tests/mpi_tests/test_mpi_decomposition.py index 3d0340216d..2cee56a0af 100644 --- a/model/common/tests/decomposition_tests/mpi_tests/test_mpi_decomposition.py +++ b/model/common/tests/decomposition_tests/mpi_tests/test_mpi_decomposition.py @@ -9,7 +9,7 @@ import numpy as np import pytest -from icon4py.model.common.utils.data_allocation import constant_field +from icon4py.model.common.utils import data_allocation as data_alloc try: @@ -17,14 +17,10 @@ except ImportError: pytest.skip("Skipping parallel on single node installation", allow_module_level=True) +import logging + from icon4py.model.common import dimension as dims -from icon4py.model.common.decomposition.definitions import ( - DecompositionInfo, - DomainDescriptorIdGenerator, - SingleNodeExchange, - create_exchange, -) -from icon4py.model.common.decomposition.mpi_decomposition import GHexMultiNodeExchange +from icon4py.model.common.decomposition import definitions, mpi_decomposition from icon4py.model.testing.datatest_fixtures import ( # noqa: F401 # import fixtures from test_utils data_provider, decomposition_info, @@ -41,12 +37,15 @@ ) +_log = logging.getLogger(__name__) + + """ running tests with mpi: -mpirun -np 2 python -m pytest -v --with-mpi tests/mpi_tests/test_parallel_setup.py +mpirun -np 2 python -m pytest -v --with-mpi tests/mpi_tests/test_mpi_decomposition.py -mpirun -np 2 pytest -v --with-mpi tests/mpi_tests/ +mpirun -np 2 pytest -v --with-mpi -k mpi_tests/ """ @@ -79,15 +78,19 @@ def test_decomposition_info_masked( ): check_comm_size(processor_props, sizes=[2]) my_rank = processor_props.rank - all_indices = decomposition_info.global_index(dim, DecompositionInfo.EntryType.ALL) + all_indices = decomposition_info.global_index(dim, definitions.DecompositionInfo.EntryType.ALL) my_total = total[my_rank] my_owned = owned[my_rank] assert all_indices.shape[0] == my_total - owned_indices = decomposition_info.global_index(dim, DecompositionInfo.EntryType.OWNED) + owned_indices = decomposition_info.global_index( + dim, definitions.DecompositionInfo.EntryType.OWNED + ) assert owned_indices.shape[0] == my_owned - halo_indices = decomposition_info.global_index(dim, DecompositionInfo.EntryType.HALO) + halo_indices = decomposition_info.global_index( + dim, definitions.DecompositionInfo.EntryType.HALO + ) assert halo_indices.shape[0] == my_total - my_owned _assert_index_partitioning(all_indices, halo_indices, owned_indices) @@ -124,18 +127,20 @@ def test_decomposition_info_local_index( ): check_comm_size(processor_props, sizes=[2]) my_rank = processor_props.rank - all_indices = decomposition_info.local_index(dim, DecompositionInfo.EntryType.ALL) + all_indices = decomposition_info.local_index(dim, definitions.DecompositionInfo.EntryType.ALL) my_total = total[my_rank] my_owned = owned[my_rank] assert all_indices.shape[0] == my_total assert np.array_equal(all_indices, np.arange(0, my_total)) - halo_indices = decomposition_info.local_index(dim, DecompositionInfo.EntryType.HALO) + halo_indices = decomposition_info.local_index(dim, definitions.DecompositionInfo.EntryType.HALO) assert halo_indices.shape[0] == my_total - my_owned assert halo_indices.shape[0] < all_indices.shape[0] assert np.all(halo_indices <= np.max(all_indices)) - owned_indices = decomposition_info.local_index(dim, DecompositionInfo.EntryType.OWNED) + owned_indices = decomposition_info.local_index( + dim, definitions.DecompositionInfo.EntryType.OWNED + ) assert owned_indices.shape[0] == my_owned assert owned_indices.shape[0] <= all_indices.shape[0] assert np.all(owned_indices <= np.max(all_indices)) @@ -151,7 +156,7 @@ def test_domain_descriptor_id_are_globally_unique( ): props = processor_props size = props.comm_size - id_gen = DomainDescriptorIdGenerator(parallel_props=props) + id_gen = definitions.DomainDescriptorIdGenerator(parallel_props=props) id1 = id_gen() assert id1 == props.comm_size * props.rank assert id1 < props.comm_size * (props.rank + 2) @@ -181,16 +186,20 @@ def test_decomposition_info_matches_gridsize( check_comm_size(processor_props) assert ( decomposition_info.global_index( - dim=dims.CellDim, entry_type=DecompositionInfo.EntryType.ALL + dim=dims.CellDim, entry_type=definitions.DecompositionInfo.EntryType.ALL ).shape[0] == icon_grid.num_cells ) assert ( - decomposition_info.global_index(dims.VertexDim, DecompositionInfo.EntryType.ALL).shape[0] + decomposition_info.global_index( + dims.VertexDim, definitions.DecompositionInfo.EntryType.ALL + ).shape[0] == icon_grid.num_vertices ) assert ( - decomposition_info.global_index(dims.EdgeDim, DecompositionInfo.EntryType.ALL).shape[0] + decomposition_info.global_index( + dims.EdgeDim, definitions.DecompositionInfo.EntryType.ALL + ).shape[0] == icon_grid.num_edges ) @@ -202,11 +211,11 @@ def test_create_multi_node_runtime_with_mpi( processor_props, # noqa: F811 # fixture ): props = processor_props - exchange = create_exchange(props, decomposition_info) + exchange = definitions.create_exchange(props, decomposition_info) if props.comm_size > 1: - assert isinstance(exchange, GHexMultiNodeExchange) + assert isinstance(exchange, mpi_decomposition.GHexMultiNodeExchange) else: - assert isinstance(exchange, SingleNodeExchange) + assert isinstance(exchange, definitions.SingleNodeExchange) @pytest.mark.parametrize("processor_props", [False], indirect=True) @@ -215,8 +224,8 @@ def test_create_single_node_runtime_without_mpi( processor_props, # noqa: F811 # fixture decomposition_info, # noqa: F811 # fixture ): - exchange = create_exchange(processor_props, decomposition_info) - assert isinstance(exchange, SingleNodeExchange) + exchange = definitions.create_exchange(processor_props, decomposition_info) + assert isinstance(exchange, definitions.SingleNodeExchange) @pytest.mark.mpi @@ -228,33 +237,39 @@ def test_exchange_on_dummy_data( grid_savepoint, # noqa: F811 # fixture metrics_savepoint, # noqa: F811 # fixture dimension, + caplog, ): - exchange = create_exchange(processor_props, decomposition_info) + caplog.set_level(logging.WARN, __name__) + exchange = definitions.create_exchange(processor_props, decomposition_info) grid = grid_savepoint.construct_icon_grid(on_gpu=False) number = processor_props.rank + 10.0 - input_field = constant_field( + input_field = data_alloc.constant_field( grid, number, dimension, dims.KDim, ) - halo_points = decomposition_info.local_index(dimension, DecompositionInfo.EntryType.HALO) - local_points = decomposition_info.local_index(dimension, DecompositionInfo.EntryType.OWNED) + halo_points = decomposition_info.local_index( + dimension, definitions.DecompositionInfo.EntryType.HALO + ) + local_points = decomposition_info.local_index( + dimension, definitions.DecompositionInfo.EntryType.OWNED + ) assert np.all(input_field == number) exchange.exchange_and_wait(dimension, input_field) result = input_field.asnumpy() - print(f"rank={processor_props.rank} - num of halo points ={halo_points.shape}") - print( + _log.info(f"rank={processor_props.rank} - num of halo points ={halo_points.shape}") + _log.info( f" rank={processor_props.rank} - exchanged points: {np.sum(result != number)/grid.num_levels}" ) - print(f"rank={processor_props.rank} - halo points: {halo_points}") + _log.debug(f"rank={processor_props.rank} - halo points: {halo_points}") assert np.all(result[local_points, :] == number) assert np.all(result[halo_points, :] != number) changed_points = np.argwhere(result[:, 2] != number) - print(f"rank={processor_props.rank} - num changed points {changed_points.shape} ") + _log.info(f"rank={processor_props.rank} - num changed points {changed_points.shape} ") - print(f"rank={processor_props.rank} - changed points {changed_points} ") + _log.debug(f"rank={processor_props.rank} - changed points {changed_points} ") From 8596a8acd8f8a474980eec9997744ce1b0bd9386 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 10 Jul 2025 16:54:13 +0200 Subject: [PATCH 040/240] delete unused enum --- model/common/src/icon4py/model/common/grid/grid_manager.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 199e5a7e73..8c2865bb36 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -35,10 +35,6 @@ _single_node_properties = decomposition.SingleNodeProcessProperties() -class ReadType(enum.IntEnum): - FLOAT = 0 - INT = 1 - class IconGridError(RuntimeError): pass @@ -133,7 +129,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): # TODO # add args to __call__? @utils.chainable - def with_decomposer( + def set_decomposer( self, decomposer: Callable[[np.ndarray, int], np.ndarray], run_properties: decomposition.ProcessProperties, From e88bef91872f6a825185e60ba4d8b098a9b49cf8 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 10 Jul 2025 16:54:31 +0200 Subject: [PATCH 041/240] add docstrings for decomposer --- .../src/icon4py/model/common/decomposition/halo.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 352c4127e3..b243137f4a 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -331,7 +331,14 @@ def __call__(self) -> defs.DecompositionInfo: class Decomposer(Protocol): - def __call__(self, adjacency_matrix, n_part: int) -> data_alloc.NDArray: + def __call__(self, adjacency_matrix:data_alloc.NDArray, n_part: int) -> data_alloc.NDArray: + """ + Call the decomposition. + + Args: + adjacency_matrix: face-to-face connectivity matrix on the global (undecomposed) grid. In the Icon4py context this C2E2C + n_part: number of nodes + """ ... @@ -365,4 +372,5 @@ def __call__(self, adjacency_matrix: data_alloc.NDArray, n_part: int) -> data_al class SingleNodeDecomposer(Decomposer): def __call__(self, adjacency_matrix: data_alloc.NDArray, n_part: int) -> data_alloc.NDArray: + """Dummy decomposer for single node: assigns all cells to rank = 0""" return self._xp.zeros(adjacency_matrix.shape[0], dtype=gtx.int32) From 059add0555db7e5627172a4c4ead23ab1549e5a1 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 10 Jul 2025 16:54:52 +0200 Subject: [PATCH 042/240] rename with_decomposer to set_decomposer --- .../tests/grid_tests/mpi_tests/test_parallel_grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py index 6af7cf1dc1..d8bf2dcaf6 100644 --- a/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py @@ -54,7 +54,7 @@ def test_start_end_index( gm.ToZeroBasedIndexTransformation(), file, v_grid.VerticalGridConfig(1) ) single_node_grid = utils.run_grid_manager(file, keep_skip_values=True).grid - with manager.with_decomposer(partitioner, processor_props) as manage: + with manager.set_decomposer(partitioner, processor_props) as manage: manage(backend=backend, keep_skip_values=True) grid = manage.grid From a4adf6590699d04faca99b64a2a99812c827cd54 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 10 Jul 2025 19:08:37 +0200 Subject: [PATCH 043/240] fix Optional typeing --- model/common/src/icon4py/model/common/grid/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/base.py b/model/common/src/icon4py/model/common/grid/base.py index b194c3195d..688a1db625 100644 --- a/model/common/src/icon4py/model/common/grid/base.py +++ b/model/common/src/icon4py/model/common/grid/base.py @@ -13,7 +13,7 @@ import warnings from abc import ABC, abstractmethod from types import ModuleType -from typing import Callable, Dict, Sequence +from typing import Callable, Dict, Sequence, Optional import gt4py.next as gtx import numpy as np @@ -80,7 +80,7 @@ def num_cells(self): class BaseGrid(ABC): def __init__(self): - self.config: GridConfig = None + self.config: Optional[GridConfig] = None self._neighbor_tables: Dict[gtx.Dimension, data_alloc.NDArray] = {} self.size: Dict[gtx.Dimension, int] = {} self._connectivity_mapping: Dict[str, tuple[Callable, gtx.Dimension, ...]] = {} From 05591c37ea94f8ebf6d115fe22d6730bf7516f55 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 10 Jul 2025 19:08:55 +0200 Subject: [PATCH 044/240] add TODO --- .../common/src/icon4py/model/common/decomposition/definitions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index e96a2679ba..f56e8426be 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -98,6 +98,7 @@ def with_dimension( def __init__( self, klevels: int, + # TODO @halungge those were added for py2fgen, are they still needed num_cells: Optional[int] = None, num_edges: Optional[int] = None, num_vertices: Optional[int] = None, From 3f54b8d937df953cfbd7b7ea760258cb2dd4f846 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 10 Jul 2025 19:11:47 +0200 Subject: [PATCH 045/240] pass decomposition mapping in __call__ --- .../model/common/decomposition/halo.py | 76 ++++++++++++++----- .../src/icon4py/model/common/grid/base.py | 2 +- .../icon4py/model/common/grid/grid_manager.py | 36 +++++---- .../mpi_tests/test_halo.py | 33 ++++++-- 4 files changed, 107 insertions(+), 40 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index b243137f4a..f023416138 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -5,6 +5,7 @@ # # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause +import functools # ICON4Py - ICON inspired code in Python and GT4Py # @@ -18,28 +19,65 @@ # distribution for a copy of the license or check . # # SPDX-License-Identifier: GPL-3.0-or-later - import logging +from types import ModuleType from typing import Optional, Protocol import gt4py.next as gtx import gt4py.next.backend as gtx_backend +import numpy as np -import icon4py.model.common.decomposition.definitions as defs from icon4py.model.common import dimension as dims, exceptions +from icon4py.model.common.decomposition import definitions as defs +from icon4py.model.common.grid import base from icon4py.model.common.utils import data_allocation as data_alloc log = logging.getLogger(__name__) -class HaloGenerator: +class DecompositionFactory(Protocol): + """Callable that takes a mapping from faces (aka cells) to ranks""" + + def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: + ... + + +class NoHalos(DecompositionFactory): + def __init__( + self, + size: base.HorizontalGridSize, + num_levels: int, + backend: Optional[gtx_backend.Backend] = None, + ): + self._size = size + self._num_levels = num_levels + self._backend = backend + + def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: + xp = data_alloc.import_array_ns(self._backend) + create_arrays = functools.partial(_create_dummy_decomposition_arrays, array_ns=xp) + decomposition_info = defs.DecompositionInfo(klevels=self._num_levels) + + decomposition_info.with_dimension(dims.EdgeDim, *create_arrays(self._size.num_edges)) + decomposition_info.with_dimension(dims.CellDim, *create_arrays(self._size.num_cells)) + decomposition_info.with_dimension(dims.VertexDim, *create_arrays(self._size.num_vertices)) + return decomposition_info + + +def _create_dummy_decomposition_arrays(size: int, array_ns: ModuleType = np): + indices = array_ns.arange(size, dtype=gtx.int32) + owner_mask = array_ns.ones((size,), dtype=bool) + halo_levels = array_ns.ones((size,), dtype=gtx.int32) * defs.DecompositionFlag.OWNED + return indices, owner_mask, halo_levels + + +class HaloGenerator(DecompositionFactory): """Creates necessary halo information for a given rank.""" def __init__( self, run_properties: defs.ProcessProperties, - rank_mapping: data_alloc.NDArray, # TODO should be an argument to __call__ connectivities: dict[gtx.Dimension, data_alloc.NDArray], num_levels, # TODO is currently needed for ghex, pass via a different struct that the decomposition info and remove backend: Optional[gtx_backend.Backend] = None, @@ -48,16 +86,14 @@ def __init__( Args: run_properties: contains information on the communicator and local compute node. - rank_mapping: array with shape (global_num_cells,): mapping of global cell indices to their rank in the distribution connectivities: connectivity arrays needed to construct the halos backend: GT4Py (used to determine the array ns import) """ self._xp = data_alloc.import_array_ns(backend) self._num_levels = num_levels self._props = run_properties - self._mapping = rank_mapping self._connectivities = connectivities - self._validate() + self._assert_all_neighbor_tables() @property def face_face_connectivity(self): @@ -83,21 +119,24 @@ def node_face_connectivity(self): def face_node_connectivity(self): return self._connectivity(dims.C2VDim) - def _validate(self): + def _validate_mapping(self, face_to_rank_mapping: data_alloc.NDArray): # validate the distribution mapping: num_cells = self.face_face_connectivity.shape[0] expected_shape = (num_cells,) - if not self._mapping.shape == expected_shape: + if not face_to_rank_mapping.shape == expected_shape: raise exceptions.ValidationError( - "rank_mapping", f"should have shape {expected_shape} but is {self._mapping.shape}" + "rank_mapping", + f"should have shape {expected_shape} but is {face_to_rank_mapping.shape}", ) # the decomposition should match the communicator size - if self._xp.max(self._mapping) > self._props.comm_size - 1: + if self._xp.max(face_to_rank_mapping) > self._props.comm_size - 1: raise exceptions.ValidationError( "rank_mapping", f"The distribution assumes more nodes than the current run is scheduled on {self._props} ", ) + + def _assert_all_neighbor_tables(self): # make sure we have all connectivity arrays used in the halo construction relevant_dimension = [ dims.C2E2CDim, @@ -116,6 +155,7 @@ def _connectivity(self, dim: gtx.Dimension) -> data_alloc.NDArray: try: conn_table = self._connectivities[dim] return conn_table + return conn_table except KeyError as err: raise exceptions.MissingConnectivity( f"Connectivity for offset {dim} is not available" @@ -165,13 +205,13 @@ def find_edge_neighbors_for_vertices( def find_vertex_neighbors_for_cells(self, cell_line: data_alloc.NDArray) -> data_alloc.NDArray: return self._find_neighbors(cell_line, connectivity=self.face_node_connectivity) - def owned_cells(self) -> data_alloc.NDArray: + def owned_cells(self, face_to_rank: data_alloc.NDArray) -> data_alloc.NDArray: """Returns the global indices of the cells owned by this rank""" - owned_cells = self._mapping == self._props.rank + owned_cells = face_to_rank == self._props.rank return self._xp.asarray(owned_cells).nonzero()[0] def _update_owner_mask_by_max_rank_convention( - self, owner_mask, all_indices, indices_on_cutting_line, target_connectivity + self, face_to_rank, owner_mask, all_indices, indices_on_cutting_line, target_connectivity ): """ In order to have unique ownership of edges (and vertices) among nodes there needs to be @@ -190,7 +230,7 @@ def _update_owner_mask_by_max_rank_convention( """ for index in indices_on_cutting_line: local_index = self._xp.nonzero(all_indices == index)[0][0] - owning_ranks = self._mapping[target_connectivity[index]] + owning_ranks = face_to_rank[target_connectivity[index]] assert ( self._xp.unique(owning_ranks).size > 1 ), f"rank {self._props.rank}: all neighboring cells are owned by the same rank" @@ -205,14 +245,14 @@ def _update_owner_mask_by_max_rank_convention( return owner_mask # TODO (@halungge): move out of halo generator? - def __call__(self) -> defs.DecompositionInfo: + def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: """ Constructs the DecompositionInfo for the current rank. The DecompositionInfo object is constructed for all horizontal dimension starting from the cell distribution. Edges and vertices are then handled through their connectivity to the distributed cells. """ - + self._validate_mapping(face_to_rank) #: cells owned_cells = self.owned_cells() # global indices of owned cells first_halo_cells = self.next_halo_line(owned_cells) @@ -331,7 +371,7 @@ def __call__(self) -> defs.DecompositionInfo: class Decomposer(Protocol): - def __call__(self, adjacency_matrix:data_alloc.NDArray, n_part: int) -> data_alloc.NDArray: + def __call__(self, adjacency_matrix: data_alloc.NDArray, n_part: int) -> data_alloc.NDArray: """ Call the decomposition. diff --git a/model/common/src/icon4py/model/common/grid/base.py b/model/common/src/icon4py/model/common/grid/base.py index 688a1db625..a43e268959 100644 --- a/model/common/src/icon4py/model/common/grid/base.py +++ b/model/common/src/icon4py/model/common/grid/base.py @@ -13,7 +13,7 @@ import warnings from abc import ABC, abstractmethod from types import ModuleType -from typing import Callable, Dict, Sequence, Optional +from typing import Callable, Dict, Optional, Sequence import gt4py.next as gtx import numpy as np diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 8c2865bb36..46936df528 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -5,7 +5,6 @@ # # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause -import enum import functools import logging import pathlib @@ -35,7 +34,6 @@ _single_node_properties = decomposition.SingleNodeProcessProperties() - class IconGridError(RuntimeError): pass @@ -92,12 +90,14 @@ def __init__( grid_file: Union[pathlib.Path, str], config: v_grid.VerticalGridConfig, # TODO (@halungge) remove to separate vertical and horizontal grid decomposer: Callable[[np.ndarray, int], np.ndarray] = halo.SingleNodeDecomposer(), + halo_constructor: Optional[halo.HaloGenerator] = None, run_properties: decomposition.ProcessProperties = _single_node_properties, ): self._run_properties = run_properties self._transformation = transformation self._file_name = str(grid_file) self._decompose = decomposer + self._halo_constructor = halo_constructor self._vertical_config = config self._grid: Optional[icon.IconGrid] = None self._decomposition_info: Optional[decomposition.DecompositionInfo] = None @@ -348,6 +348,26 @@ def _construct_grid( """ xp = data_alloc.import_array_ns(backend) on_gpu = data_alloc.is_cupy_device(backend) + global_connectivities_for_halo_construction = { + dims.C2E2C: self._get_index_field(gridfile.ConnectivityName.C2E2C), + dims.C2E: self._get_index_field(gridfile.ConnectivityName.C2E), + dims.E2C: self._get_index_field(gridfile.ConnectivityName.E2C), + dims.V2E: self._get_index_field(gridfile.ConnectivityName.V2E), + dims.V2C: self._get_index_field(gridfile.ConnectivityName.V2C), + dims.C2V: self._get_index_field(gridfile.ConnectivityName.C2V), + } + if not self._run_properties.single_node(): + cells_to_rank_mapping = self._decompose( + global_connectivities_for_halo_construction[dims.C2E2C], + self._run_properties.comm_size, + ) + self._halo_constructor(cells_to_rank_mapping) + + global_connectivities = { + dims.E2V: self._get_index_field(gridfile.ConnectivityName.E2V), + dims.V2E2V: self._get_index_field(gridfile.ConnectivityName.V2E2V), + } + _determine_limited_area = functools.partial(refinement.is_limited_area_grid, array_ns=xp) _derived_connectivities = functools.partial( _add_derived_connectivities, @@ -360,16 +380,7 @@ def _construct_grid( ) grid.set_refinement_control(refinement_fields) - global_connectivities = { - dims.C2E2C: self._get_index_field(gridfile.ConnectivityName.C2E2C), - dims.C2E: self._get_index_field(gridfile.ConnectivityName.C2E), - dims.E2C: self._get_index_field(gridfile.ConnectivityName.E2C), - dims.V2E: self._get_index_field(gridfile.ConnectivityName.V2E), - dims.E2V: self._get_index_field(gridfile.ConnectivityName.E2V), - dims.V2C: self._get_index_field(gridfile.ConnectivityName.V2C), - dims.C2V: self._get_index_field(gridfile.ConnectivityName.C2V), - dims.V2E2V: self._get_index_field(gridfile.ConnectivityName.V2E2V), - } + global_connectivities.update(global_connectivities_for_halo_construction) grid.set_neighbor_tables( {o.target[1]: xp.asarray(c) for o, c in global_connectivities.items()} @@ -699,5 +710,4 @@ def construct_local_connectivity( positions = np.searchsorted(global_idx_sorted, local_connectivity[i, valid_neighbor_mask]) indices = sorted_index_of_global_idx[positions] local_connectivity[i, valid_neighbor_mask] = indices - # _log.debug(f"rank {self._props.rank} has local connectivity f: {local_connectivity}") return local_connectivity diff --git a/model/common/tests/decomposition_tests/mpi_tests/test_halo.py b/model/common/tests/decomposition_tests/mpi_tests/test_halo.py index 07592aafe6..9d81cda197 100644 --- a/model/common/tests/decomposition_tests/mpi_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/mpi_tests/test_halo.py @@ -206,11 +206,10 @@ def test_halo_constructor_owned_cells(processor_props): # F811 # fixture halo_generator = halo.HaloGenerator( connectivities=grid.neighbor_tables, run_properties=processor_props, - rank_mapping=SIMPLE_DISTRIBUTION, num_levels=1, backend=backend, ) - my_owned_cells = halo_generator.owned_cells() + my_owned_cells = halo_generator.owned_cells(SIMPLE_DISTRIBUTION) print(f"rank {processor_props.rank} owns {my_owned_cells} ") assert my_owned_cells.size == len(_CELL_OWN[processor_props.rank]) @@ -221,12 +220,12 @@ def test_halo_constructor_validate_number_of_node_mismatch(processor_props): grid = simple.SimpleGrid() distribution = (processor_props.comm_size + 1) * np.ones((grid.num_cells,), dtype=int) with pytest.raises(expected_exception=exceptions.ValidationError) as e: - halo.HaloGenerator( + halo_generator = halo.HaloGenerator( connectivities=grid.neighbor_tables, run_properties=processor_props, - rank_mapping=distribution, num_levels=1, ) + halo_generator(distribution) assert "The distribution assumes more nodes than the current run" in e.value.args[0] @@ -234,12 +233,12 @@ def test_halo_constructor_validate_number_of_node_mismatch(processor_props): def test_halo_constructor_validate_rank_mapping_wrong_shape(processor_props, shape): grid = simple.SimpleGrid() with pytest.raises(exceptions.ValidationError) as e: - halo.HaloGenerator( + halo_generator = halo.HaloGenerator( connectivities=grid.neighbor_tables, run_properties=processor_props, - rank_mapping=np.zeros((grid.num_cells, 3), dtype=int), num_levels=1, ) + halo_generator(np.zeros((grid.num_cells, 3), dtype=int)) assert f"should have shape ({grid.num_cells},)" in e.value.args[0] @@ -417,10 +416,9 @@ def test_distributed_fields(processor_props): # F811 # fixture halo_generator = halo.HaloGenerator( connectivities=global_grid.neighbor_tables, run_properties=processor_props, - rank_mapping=labels, num_levels=1, ) - decomposition_info = halo_generator() + decomposition_info = halo_generator(labels) # distributed read: read one field per dimension ## TODO why is this local?? @@ -489,3 +487,22 @@ def test_halo_neighbor_access_c2e(): # 3. compute geofac_div = primal_edge_length * edge_orientation / area # 4. gather geofac_div # 5 compare (possible reorder + + +def test_no_halo(): + grid = simple.SimpleGrid() + halo_generator = halo.NoHalos(grid.config.horizontal_config, num_levels=10, backend=None) + mapping = np.zeros((grid.num_cells), dtype=int) + decomposition_info = halo_generator(mapping) + np.testing.assert_allclose( + np.arange(grid.num_cells), decomposition_info.global_index(dims.CellDim) + ) + assert np.all(decomposition_info.owner_mask(dims.CellDim)) + np.testing.assert_allclose( + np.arange(grid.num_edges), decomposition_info.global_index(dims.EdgeDim) + ) + assert np.all(decomposition_info.owner_mask(dims.EdgeDim)) + np.testing.assert_allclose( + np.arange(grid.num_vertices), decomposition_info.global_index(dims.VertexDim) + ) + assert np.all(decomposition_info.owner_mask(dims.VertexDim)) From 00b05e7833a375043c6cd86d524db82f8050a848 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 11 Jul 2025 09:21:21 +0200 Subject: [PATCH 046/240] pass face to rank distributon to __call__ --- .../icon4py/model/common/decomposition/halo.py | 4 +++- .../decomposition_tests/mpi_tests/test_halo.py | 18 +++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index f023416138..f0af4956ff 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -254,7 +254,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: """ self._validate_mapping(face_to_rank) #: cells - owned_cells = self.owned_cells() # global indices of owned cells + owned_cells = self.owned_cells(face_to_rank) # global indices of owned cells first_halo_cells = self.next_halo_line(owned_cells) second_halo_cells = self.next_halo_line(first_halo_cells, owned_cells) @@ -293,6 +293,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: ) vertex_owner_mask = self._xp.isin(all_vertices, vertex_on_owned_cells) vertex_owner_mask = self._update_owner_mask_by_max_rank_convention( + face_to_rank, vertex_owner_mask, all_vertices, vertex_on_cutting_line, @@ -343,6 +344,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: # construct the owner mask edge_owner_mask = self._xp.isin(all_edges, edges_on_owned_cells) edge_owner_mask = self._update_owner_mask_by_max_rank_convention( + face_to_rank, edge_owner_mask, all_edges, edge_intersect_owned_first_line, diff --git a/model/common/tests/decomposition_tests/mpi_tests/test_halo.py b/model/common/tests/decomposition_tests/mpi_tests/test_halo.py index 9d81cda197..1a411cf963 100644 --- a/model/common/tests/decomposition_tests/mpi_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/mpi_tests/test_halo.py @@ -256,12 +256,11 @@ def test_element_ownership_is_unique(dim, processor_props): # F811 # fixture halo_generator = halo.HaloGenerator( connectivities=grid.neighbor_tables, run_properties=processor_props, - rank_mapping=SIMPLE_DISTRIBUTION, num_levels=1, backend=backend, ) - decomposition_info = halo_generator() + decomposition_info = halo_generator(SIMPLE_DISTRIBUTION) owned = decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED) print(f"\nrank {processor_props.rank} owns {dim} : {owned} ") if not mpi4py.MPI.Is_initialized(): @@ -301,11 +300,10 @@ def test_halo_constructor_decomposition_info_global_indices(processor_props, dim halo_generator = halo.HaloGenerator( connectivities=grid.neighbor_tables, run_properties=processor_props, - rank_mapping=SIMPLE_DISTRIBUTION, num_levels=1, ) - decomp_info = halo_generator() + decomp_info = halo_generator(SIMPLE_DISTRIBUTION) my_halo = decomp_info.global_index(dim, defs.DecompositionInfo.EntryType.HALO) print(f"rank {processor_props.rank} has halo {dim} : {my_halo}") assert my_halo.size == len(HALO[dim][processor_props.rank]) @@ -329,10 +327,9 @@ def test_halo_constructor_decomposition_info_halo_levels(processor_props, dim): halo_generator = halo.HaloGenerator( connectivities=grid.neighbor_tables, run_properties=processor_props, - rank_mapping=SIMPLE_DISTRIBUTION, num_levels=1, ) - decomp_info = halo_generator() + decomp_info = halo_generator(SIMPLE_DISTRIBUTION) my_halo_levels = decomp_info.halo_levels(dim) print(f"{dim.value}: rank {processor_props.rank} has halo levels {my_halo_levels} ") if dim != dims.EdgeDim: @@ -477,8 +474,9 @@ def assert_gathered_field_against_global( # Will uses geofac_div and geofac_n2s +@pytest.mark.xfail def test_halo_neighbor_access_c2e(): - ... + pytest.fail("TODO implement") # geofac_div = primal_edge_length(C2E) * edge_orientation / area # 1. read grid and distribue - GridManager @@ -494,15 +492,21 @@ def test_no_halo(): halo_generator = halo.NoHalos(grid.config.horizontal_config, num_levels=10, backend=None) mapping = np.zeros((grid.num_cells), dtype=int) decomposition_info = halo_generator(mapping) + # cells np.testing.assert_allclose( np.arange(grid.num_cells), decomposition_info.global_index(dims.CellDim) ) assert np.all(decomposition_info.owner_mask(dims.CellDim)) + assert np.all(decomposition_info.halo_levels(dims.CellDim) == defs.DecompositionFlag.OWNED) + # edges np.testing.assert_allclose( np.arange(grid.num_edges), decomposition_info.global_index(dims.EdgeDim) ) + assert np.all(decomposition_info.halo_levels(dims.EdgeDim) == defs.DecompositionFlag.OWNED) assert np.all(decomposition_info.owner_mask(dims.EdgeDim)) + # vertices np.testing.assert_allclose( np.arange(grid.num_vertices), decomposition_info.global_index(dims.VertexDim) ) + assert np.all(decomposition_info.halo_levels(dims.VertexDim) == defs.DecompositionFlag.OWNED) assert np.all(decomposition_info.owner_mask(dims.VertexDim)) From 2cdcea6dd62f1dc8440af20ceed343ef2ffe1ba3 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 11 Jul 2025 10:42:23 +0200 Subject: [PATCH 047/240] WIP (1) --- .../model/common/decomposition/halo.py | 34 ++++++------------- .../icon4py/model/common/grid/grid_manager.py | 27 ++++++++------- .../mpi_tests/test_halo.py | 28 +++++++-------- .../tests/grid_tests/test_grid_manager.py | 2 +- 4 files changed, 40 insertions(+), 51 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index f0af4956ff..c1e45406a1 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -6,22 +6,9 @@ # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause import functools - -# ICON4Py - ICON inspired code in Python and GT4Py -# -# Copyright (c) 2022, ETH Zurich and MeteoSwiss -# All rights reserved. -# -# This file is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or any later -# version. See the LICENSE.txt file at the top-level directory of this -# distribution for a copy of the license or check . -# -# SPDX-License-Identifier: GPL-3.0-or-later import logging from types import ModuleType -from typing import Optional, Protocol +from typing import Optional, Protocol, runtime_checkable import gt4py.next as gtx import gt4py.next.backend as gtx_backend @@ -35,22 +22,22 @@ log = logging.getLogger(__name__) - -class DecompositionFactory(Protocol): +@runtime_checkable +class HaloConstructor(Protocol): """Callable that takes a mapping from faces (aka cells) to ranks""" def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: ... -class NoHalos(DecompositionFactory): +class NoHalos(HaloConstructor): def __init__( self, - size: base.HorizontalGridSize, + horizontal_size: base.HorizontalGridSize, num_levels: int, backend: Optional[gtx_backend.Backend] = None, ): - self._size = size + self._size = horizontal_size self._num_levels = num_levels self._backend = backend @@ -71,8 +58,7 @@ def _create_dummy_decomposition_arrays(size: int, array_ns: ModuleType = np): halo_levels = array_ns.ones((size,), dtype=gtx.int32) * defs.DecompositionFlag.OWNED return indices, owner_mask, halo_levels - -class HaloGenerator(DecompositionFactory): +class IconLikeHaloConstructor(HaloConstructor): """Creates necessary halo information for a given rank.""" def __init__( @@ -371,7 +357,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: # TODO (@halungge): refine type hints: adjacency_matrix should be a connectivity matrix of C2E2C and # the return value an array of shape (n_cells,) - +@runtime_checkable class Decomposer(Protocol): def __call__(self, adjacency_matrix: data_alloc.NDArray, n_part: int) -> data_alloc.NDArray: """ @@ -409,10 +395,10 @@ def __call__(self, adjacency_matrix: data_alloc.NDArray, n_part: int) -> data_al import pymetis cut_count, partition_index = pymetis.part_graph(nparts=n_part, adjacency=adjacency_matrix) - return self._xp.array(partition_index) + return np.array(partition_index) class SingleNodeDecomposer(Decomposer): def __call__(self, adjacency_matrix: data_alloc.NDArray, n_part: int) -> data_alloc.NDArray: """Dummy decomposer for single node: assigns all cells to rank = 0""" - return self._xp.zeros(adjacency_matrix.shape[0], dtype=gtx.int32) + return np.zeros(adjacency_matrix.shape[0], dtype=gtx.int32) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 46936df528..2272e2f294 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -15,7 +15,7 @@ import gt4py.next.backend as gtx_backend import numpy as np -from icon4py.model.common import dimension as dims, type_alias as ta, utils +from icon4py.model.common import dimension as dims, type_alias as ta, utils, exceptions from icon4py.model.common.decomposition import ( definitions as decomposition, halo, @@ -84,20 +84,24 @@ class GridManager: """ + def open(self): + """Open the gridfile resource for reading.""" + self._reader = gridfile.GridFile(self._file_name) + self._reader.open() + def __init__( self, transformation: IndexTransformation, grid_file: Union[pathlib.Path, str], - config: v_grid.VerticalGridConfig, # TODO (@halungge) remove to separate vertical and horizontal grid + config: v_grid.VerticalGridConfig, # TODO (@halungge) remove: - separate vertical from horizontal grid decomposer: Callable[[np.ndarray, int], np.ndarray] = halo.SingleNodeDecomposer(), - halo_constructor: Optional[halo.HaloGenerator] = None, run_properties: decomposition.ProcessProperties = _single_node_properties, ): self._run_properties = run_properties self._transformation = transformation self._file_name = str(grid_file) self._decompose = decomposer - self._halo_constructor = halo_constructor + self._halo_constructor = None self._vertical_config = config self._grid: Optional[icon.IconGrid] = None self._decomposition_info: Optional[decomposition.DecompositionInfo] = None @@ -105,10 +109,6 @@ def __init__( self._reader = None self._coordinates: CoordinateDict = {} - def open(self): - """Open the gridfile resource for reading.""" - self._reader = gridfile.GridFile(self._file_name) - self._reader.open() def close(self): """close the gridfile resource.""" @@ -127,15 +127,18 @@ def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is FileNotFoundError: raise FileNotFoundError(f"gridfile {self._file_name} not found, aborting") - # TODO # add args to __call__? - @utils.chainable + @utils.chainable # TODO split into to functions def set_decomposer( self, decomposer: Callable[[np.ndarray, int], np.ndarray], - run_properties: decomposition.ProcessProperties, ): - self._run_properties = run_properties self._decompose = decomposer + self._validate_decomposer() + + def _validate_decomposer(self): + if not self._decompose or isinstance(self._decompose, + halo.SingleNodeDecomposer) and not self._run_properties.single_node(): + raise exceptions.InvalidConfigError(f"Need Decomposer for for multi node run.") def __call__(self, backend: Optional[gtx_backend.Backend], keep_skip_values: bool): if not self._reader: diff --git a/model/common/tests/decomposition_tests/mpi_tests/test_halo.py b/model/common/tests/decomposition_tests/mpi_tests/test_halo.py index 1a411cf963..677efd6272 100644 --- a/model/common/tests/decomposition_tests/mpi_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/mpi_tests/test_halo.py @@ -203,7 +203,7 @@ def grid_file_manager(file: pathlib.Path) -> gm.GridManager: def test_halo_constructor_owned_cells(processor_props): # F811 # fixture grid = simple.SimpleGrid() - halo_generator = halo.HaloGenerator( + halo_generator = halo.IconLikeHaloConstructor( connectivities=grid.neighbor_tables, run_properties=processor_props, num_levels=1, @@ -220,7 +220,7 @@ def test_halo_constructor_validate_number_of_node_mismatch(processor_props): grid = simple.SimpleGrid() distribution = (processor_props.comm_size + 1) * np.ones((grid.num_cells,), dtype=int) with pytest.raises(expected_exception=exceptions.ValidationError) as e: - halo_generator = halo.HaloGenerator( + halo_generator = halo.IconLikeHaloConstructor( connectivities=grid.neighbor_tables, run_properties=processor_props, num_levels=1, @@ -233,7 +233,7 @@ def test_halo_constructor_validate_number_of_node_mismatch(processor_props): def test_halo_constructor_validate_rank_mapping_wrong_shape(processor_props, shape): grid = simple.SimpleGrid() with pytest.raises(exceptions.ValidationError) as e: - halo_generator = halo.HaloGenerator( + halo_generator = halo.IconLikeHaloConstructor( connectivities=grid.neighbor_tables, run_properties=processor_props, num_levels=1, @@ -253,7 +253,7 @@ def test_element_ownership_is_unique(dim, processor_props): # F811 # fixture if processor_props.comm_size != 4: pytest.skip("This test requires exactly 4 MPI ranks.") grid = simple.SimpleGrid() - halo_generator = halo.HaloGenerator( + halo_generator = halo.IconLikeHaloConstructor( connectivities=grid.neighbor_tables, run_properties=processor_props, num_levels=1, @@ -297,7 +297,7 @@ def test_halo_constructor_decomposition_info_global_indices(processor_props, dim if processor_props.comm_size != 4: pytest.skip("This test requires exactly 4 MPI ranks.") grid = simple.SimpleGrid() - halo_generator = halo.HaloGenerator( + halo_generator = halo.IconLikeHaloConstructor( connectivities=grid.neighbor_tables, run_properties=processor_props, num_levels=1, @@ -324,7 +324,7 @@ def assert_same_entries( @pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) def test_halo_constructor_decomposition_info_halo_levels(processor_props, dim): # F811 # fixture grid = simple.SimpleGrid() - halo_generator = halo.HaloGenerator( + halo_generator = halo.IconLikeHaloConstructor( connectivities=grid.neighbor_tables, run_properties=processor_props, num_levels=1, @@ -410,7 +410,7 @@ def test_distributed_fields(processor_props): # F811 # fixture labels = decompose(global_grid, processor_props) - halo_generator = halo.HaloGenerator( + halo_generator = halo.IconLikeHaloConstructor( connectivities=global_grid.neighbor_tables, run_properties=processor_props, num_levels=1, @@ -488,25 +488,25 @@ def test_halo_neighbor_access_c2e(): def test_no_halo(): - grid = simple.SimpleGrid() - halo_generator = halo.NoHalos(grid.config.horizontal_config, num_levels=10, backend=None) - mapping = np.zeros((grid.num_cells), dtype=int) - decomposition_info = halo_generator(mapping) + grid_size = base_grid.HorizontalGridSize(num_cells=9, num_edges=14, num_vertices=6) + halo_generator = halo.NoHalos(horizontal_size=grid_size, num_levels=10, backend=None) + decomposition = halo.SingleNodeDecomposer() + decomposition_info = halo_generator(decomposition(np.arange(grid_size.num_cells), 1)) # cells np.testing.assert_allclose( - np.arange(grid.num_cells), decomposition_info.global_index(dims.CellDim) + np.arange(grid_size.num_cells), decomposition_info.global_index(dims.CellDim) ) assert np.all(decomposition_info.owner_mask(dims.CellDim)) assert np.all(decomposition_info.halo_levels(dims.CellDim) == defs.DecompositionFlag.OWNED) # edges np.testing.assert_allclose( - np.arange(grid.num_edges), decomposition_info.global_index(dims.EdgeDim) + np.arange(grid_size.num_edges), decomposition_info.global_index(dims.EdgeDim) ) assert np.all(decomposition_info.halo_levels(dims.EdgeDim) == defs.DecompositionFlag.OWNED) assert np.all(decomposition_info.owner_mask(dims.EdgeDim)) # vertices np.testing.assert_allclose( - np.arange(grid.num_vertices), decomposition_info.global_index(dims.VertexDim) + np.arange(grid_size.num_vertices), decomposition_info.global_index(dims.VertexDim) ) assert np.all(decomposition_info.halo_levels(dims.VertexDim) == defs.DecompositionFlag.OWNED) assert np.all(decomposition_info.owner_mask(dims.VertexDim)) diff --git a/model/common/tests/grid_tests/test_grid_manager.py b/model/common/tests/grid_tests/test_grid_manager.py index 42d3e96ff3..4be27ea7d7 100644 --- a/model/common/tests/grid_tests/test_grid_manager.py +++ b/model/common/tests/grid_tests/test_grid_manager.py @@ -621,7 +621,7 @@ def test_local_connectivities(processor_props, caplog, field_offset): # fixture partitioner = halo.SimpleMetisDecomposer() face_face_connectivity = grid.connectivities[dims.C2E2CDim] labels = partitioner(face_face_connectivity, n_part=processor_props.comm_size) - halo_generator = halo.HaloGenerator( + halo_generator = halo.IconLikeHaloConstructor( connectivities=grid.neighbor_tables, run_properties=processor_props, rank_mapping=labels, From c5975ff1f6f7c23e91fa5f0ec907a9bcbc24cc35 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 11 Jul 2025 11:32:57 +0200 Subject: [PATCH 048/240] validate decomposer and run_properties --- .../model/common/decomposition/halo.py | 3 ++ .../icon4py/model/common/grid/grid_manager.py | 14 ++++--- .../tests/grid_tests/mpi_tests/conftest.py | 10 +++++ .../mpi_tests/test_parallel_grid_manager.py | 37 ++++++++++++++++--- 4 files changed, 53 insertions(+), 11 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index c1e45406a1..863d2dd809 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -22,6 +22,7 @@ log = logging.getLogger(__name__) + @runtime_checkable class HaloConstructor(Protocol): """Callable that takes a mapping from faces (aka cells) to ranks""" @@ -58,6 +59,7 @@ def _create_dummy_decomposition_arrays(size: int, array_ns: ModuleType = np): halo_levels = array_ns.ones((size,), dtype=gtx.int32) * defs.DecompositionFlag.OWNED return indices, owner_mask, halo_levels + class IconLikeHaloConstructor(HaloConstructor): """Creates necessary halo information for a given rank.""" @@ -357,6 +359,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: # TODO (@halungge): refine type hints: adjacency_matrix should be a connectivity matrix of C2E2C and # the return value an array of shape (n_cells,) + @runtime_checkable class Decomposer(Protocol): def __call__(self, adjacency_matrix: data_alloc.NDArray, n_part: int) -> data_alloc.NDArray: diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 2272e2f294..f4c04411ed 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -15,7 +15,7 @@ import gt4py.next.backend as gtx_backend import numpy as np -from icon4py.model.common import dimension as dims, type_alias as ta, utils, exceptions +from icon4py.model.common import dimension as dims, exceptions, type_alias as ta, utils from icon4py.model.common.decomposition import ( definitions as decomposition, halo, @@ -109,7 +109,6 @@ def __init__( self._reader = None self._coordinates: CoordinateDict = {} - def close(self): """close the gridfile resource.""" self._reader.close() @@ -127,7 +126,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is FileNotFoundError: raise FileNotFoundError(f"gridfile {self._file_name} not found, aborting") - @utils.chainable # TODO split into to functions + @utils.chainable # TODO split into to functions def set_decomposer( self, decomposer: Callable[[np.ndarray, int], np.ndarray], @@ -136,9 +135,12 @@ def set_decomposer( self._validate_decomposer() def _validate_decomposer(self): - if not self._decompose or isinstance(self._decompose, - halo.SingleNodeDecomposer) and not self._run_properties.single_node(): - raise exceptions.InvalidConfigError(f"Need Decomposer for for multi node run.") + if ( + not self._decompose + or isinstance(self._decompose, halo.SingleNodeDecomposer) + and not self._run_properties.single_node() + ): + raise exceptions.InvalidConfigError("Need a Decomposer for for multi node run.") def __call__(self, backend: Optional[gtx_backend.Backend], keep_skip_values: bool): if not self._reader: diff --git a/model/common/tests/grid_tests/mpi_tests/conftest.py b/model/common/tests/grid_tests/mpi_tests/conftest.py index 80b673df7e..a0b781be2f 100644 --- a/model/common/tests/grid_tests/mpi_tests/conftest.py +++ b/model/common/tests/grid_tests/mpi_tests/conftest.py @@ -5,4 +5,14 @@ # # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause +try: + import sys + _ = sys.modules["icon4py.model.testing.pytest_config"] +except KeyError: + from icon4py.model.testing.pytest_config import * # noqa: F403 + + +from icon4py.model.testing.parallel_helpers import ( + processor_props, # noqa: F401 # import fixtures from test_utils package +) diff --git a/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py index d8bf2dcaf6..67ea46ac53 100644 --- a/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py @@ -11,7 +11,8 @@ import pytest import icon4py.model.testing.grid_utils as grid_utils -from icon4py.model.common.decomposition import halo +from icon4py.model.common import exceptions +from icon4py.model.common.decomposition import definitions, halo, mpi_decomposition from icon4py.model.common.grid import grid_manager as gm, vertical as v_grid from icon4py.model.testing import datatest_utils as dt_utils @@ -20,11 +21,12 @@ try: import mpi4py # noqa F401: import mpi4py to check for optional mpi dependency + + mpi_decomposition.init_mpi() except ImportError: pytest.skip("Skipping parallel on single node installation", allow_module_level=True) -# mpi marker meses up mpi initialization @pytest.mark.mpi(min_size=2) @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_props(caplog, processor_props): # fixture @@ -33,6 +35,8 @@ def test_props(caplog, processor_props): # fixture assert processor_props.comm_size > 1 +# TODO FIXME +@pytest.mark.xfail @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize( @@ -51,10 +55,18 @@ def test_start_end_index( partitioner = halo.SimpleMetisDecomposer() manager = gm.GridManager( - gm.ToZeroBasedIndexTransformation(), file, v_grid.VerticalGridConfig(1) + gm.ToZeroBasedIndexTransformation(), + file, + v_grid.VerticalGridConfig(1), + run_properties=processor_props, ) - single_node_grid = utils.run_grid_manager(file, keep_skip_values=True).grid - with manager.set_decomposer(partitioner, processor_props) as manage: + single_node_grid = gm.GridManager( + gm.ToZeroBasedIndexTransformation(), + file, + v_grid.VerticalGridConfig(1), + run_properties=definitions.get_processor_properties(definitions.SingleNodeRun()), + ).grid + with manager.set_decomposer(partitioner) as manage: manage(backend=backend, keep_skip_values=True) grid = manage.grid @@ -65,3 +77,18 @@ def test_start_end_index( assert grid.end_index(domain) == single_node_grid.end_index( domain ), f"end index wrong for domain {domain}" + + +@pytest.mark.mpi(min_size=2) +def test_grid_manager_validate_decomposer(processor_props): + file = grid_utils.resolve_full_grid_file_name(dt_utils.R02B04_GLOBAL) + manager = gm.GridManager( + gm.ToZeroBasedIndexTransformation(), + file, + v_grid.VerticalGridConfig(1), + run_properties=processor_props, + ) + with pytest.raises(exceptions.InvalidConfigError) as e: + manager.set_decomposer(halo.SingleNodeDecomposer()) + + assert "Need a Decomposer for for multi" in e.value.args[0] From dbe051f4750a179992f5071778be2dfd0129ff44 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 11 Jul 2025 14:47:24 +0200 Subject: [PATCH 049/240] fix processor_props parametrization --- .../diffusion_tests/mpi_tests/conftest.py | 13 ---- .../mpi_tests/test_parallel_diffusion.py | 3 +- .../tests/dycore_tests/mpi_tests/conftest.py | 11 --- .../mpi_tests/test_parallel_solve_nonhydro.py | 2 +- .../model/common/decomposition/halo.py | 3 +- .../common/decomposition/mpi_decomposition.py | 2 +- .../icon4py/model/common/grid/grid_manager.py | 69 ++++++++++++++----- .../{mpi_tests => }/conftest.py | 14 ++-- .../mpi_tests/test_halo.py | 3 + .../mpi_tests/test_mpi_decomposition.py | 58 ++++++---------- .../decomposition_tests/test_definitions.py | 11 +-- .../tests/grid_tests/mpi_tests/conftest.py | 18 ----- .../mpi_tests/test_parallel_grid_manager.py | 1 + .../mpi_tests/test_parallel_icon.py | 15 ++-- .../tests/grid_tests/test_grid_manager.py | 7 +- .../model/testing/datatest_fixtures.py | 6 +- .../icon4py/model/testing/datatest_utils.py | 6 -- .../src/icon4py/model/testing/grid_utils.py | 23 ++----- .../icon4py/model/testing/parallel_helpers.py | 9 +-- 19 files changed, 112 insertions(+), 162 deletions(-) delete mode 100644 model/atmosphere/diffusion/tests/diffusion_tests/mpi_tests/conftest.py delete mode 100644 model/atmosphere/dycore/tests/dycore_tests/mpi_tests/conftest.py rename model/common/tests/decomposition_tests/{mpi_tests => }/conftest.py (58%) delete mode 100644 model/common/tests/grid_tests/mpi_tests/conftest.py diff --git a/model/atmosphere/diffusion/tests/diffusion_tests/mpi_tests/conftest.py b/model/atmosphere/diffusion/tests/diffusion_tests/mpi_tests/conftest.py deleted file mode 100644 index e72ac2af47..0000000000 --- a/model/atmosphere/diffusion/tests/diffusion_tests/mpi_tests/conftest.py +++ /dev/null @@ -1,13 +0,0 @@ -# ICON4Py - ICON inspired code in Python and GT4Py -# -# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss -# All rights reserved. -# -# Please, refer to the LICENSE file in the root directory. -# SPDX-License-Identifier: BSD-3-Clause -from icon4py.model.testing.datatest_fixtures import ( - icon_grid, # noqa: F401 # import fixtures from test_utils package -) -from icon4py.model.testing.parallel_helpers import ( - processor_props, # noqa: F401 # import fixtures from test_utils package -) diff --git a/model/atmosphere/diffusion/tests/diffusion_tests/mpi_tests/test_parallel_diffusion.py b/model/atmosphere/diffusion/tests/diffusion_tests/mpi_tests/test_parallel_diffusion.py index 368e044215..f605d0081d 100644 --- a/model/atmosphere/diffusion/tests/diffusion_tests/mpi_tests/test_parallel_diffusion.py +++ b/model/atmosphere/diffusion/tests/diffusion_tests/mpi_tests/test_parallel_diffusion.py @@ -18,12 +18,12 @@ from .. import utils -@pytest.skip("FIXME: Need updated test data yet", allow_module_level=True) @pytest.mark.mpi @pytest.mark.parametrize("experiment", [datatest_utils.REGIONAL_EXPERIMENT]) @pytest.mark.parametrize("ndyn_substeps", [2]) @pytest.mark.parametrize("linit", [True, False]) @pytest.mark.parametrize("orchestration", [False, True]) +@pytest.mark.parametrize("processor_props", [True, False], indirect=True) def test_parallel_diffusion( experiment, step_date_init, @@ -161,6 +161,7 @@ def test_parallel_diffusion( @pytest.mark.parametrize("experiment", [datatest_utils.REGIONAL_EXPERIMENT]) @pytest.mark.parametrize("ndyn_substeps", [2]) @pytest.mark.parametrize("linit", [True]) +@pytest.mark.parametrize("processor_props", [True], indirect=True) def test_parallel_diffusion_multiple_steps( experiment, step_date_init, diff --git a/model/atmosphere/dycore/tests/dycore_tests/mpi_tests/conftest.py b/model/atmosphere/dycore/tests/dycore_tests/mpi_tests/conftest.py deleted file mode 100644 index 4817ced04b..0000000000 --- a/model/atmosphere/dycore/tests/dycore_tests/mpi_tests/conftest.py +++ /dev/null @@ -1,11 +0,0 @@ -# ICON4Py - ICON inspired code in Python and GT4Py -# -# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss -# All rights reserved. -# -# Please, refer to the LICENSE file in the root directory. -# SPDX-License-Identifier: BSD-3-Clause - -from icon4py.model.testing.parallel_helpers import ( - processor_props, # noqa: F401 # import fixtures from test_utils package -) diff --git a/model/atmosphere/dycore/tests/dycore_tests/mpi_tests/test_parallel_solve_nonhydro.py b/model/atmosphere/dycore/tests/dycore_tests/mpi_tests/test_parallel_solve_nonhydro.py index 426fcb28d2..6f3dd8be17 100644 --- a/model/atmosphere/dycore/tests/dycore_tests/mpi_tests/test_parallel_solve_nonhydro.py +++ b/model/atmosphere/dycore/tests/dycore_tests/mpi_tests/test_parallel_solve_nonhydro.py @@ -19,7 +19,7 @@ from .. import utils -@pytest.skip("FIXME: Need updated test data yet", allow_module_level=True) +@pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.datatest @pytest.mark.parametrize( "istep_init, jstep_init, step_date_init,istep_exit, jstep_exit, step_date_exit", diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 863d2dd809..5c9e8cb707 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -67,7 +67,7 @@ def __init__( self, run_properties: defs.ProcessProperties, connectivities: dict[gtx.Dimension, data_alloc.NDArray], - num_levels, # TODO is currently needed for ghex, pass via a different struct that the decomposition info and remove + num_levels, backend: Optional[gtx_backend.Backend] = None, ): """ @@ -75,6 +75,7 @@ def __init__( Args: run_properties: contains information on the communicator and local compute node. connectivities: connectivity arrays needed to construct the halos + num_levels: number of vertical levels, TODO: should be removed, it is needed in GHEX that why we have it no the DecompotionInfo backend: GT4Py (used to determine the array ns import) """ self._xp = data_alloc.import_array_ns(backend) diff --git a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py index fa5b508139..b2cd3c1ce3 100644 --- a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py +++ b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py @@ -105,7 +105,7 @@ def filter(self, record: logging.LogRecord) -> bool: @definitions.get_processor_properties.register(definitions.MultiNodeRun) def get_multinode_properties( - s: definitions.MultiNodeRun, comm_id: CommId = None + s: definitions.RunType, comm_id: CommId = None ) -> definitions.ProcessProperties: return _get_processor_properties(with_mpi=True, comm_id=comm_id) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index f4c04411ed..6d759b25e5 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -20,6 +20,7 @@ definitions as decomposition, halo, ) +from icon4py.model.common.decomposition.halo import HaloConstructor from icon4py.model.common.grid import ( base, gridfile, @@ -135,9 +136,8 @@ def set_decomposer( self._validate_decomposer() def _validate_decomposer(self): - if ( - not self._decompose - or isinstance(self._decompose, halo.SingleNodeDecomposer) + if not self._decompose or ( + isinstance(self._decompose, halo.SingleNodeDecomposer) and not self._run_properties.single_node() ): raise exceptions.InvalidConfigError("Need a Decomposer for for multi node run.") @@ -145,6 +145,7 @@ def _validate_decomposer(self): def __call__(self, backend: Optional[gtx_backend.Backend], keep_skip_values: bool): if not self._reader: self.open() + self._grid = self._construct_grid(backend=backend, with_skip_values=keep_skip_values) self._coordinates = self._read_coordinates(backend) self._geometry = self._read_geometry_fields(backend) @@ -353,6 +354,7 @@ def _construct_grid( """ xp = data_alloc.import_array_ns(backend) on_gpu = data_alloc.is_cupy_device(backend) + grid_size = self._read_horizontal_grid_size() global_connectivities_for_halo_construction = { dims.C2E2C: self._get_index_field(gridfile.ConnectivityName.C2E2C), dims.C2E: self._get_index_field(gridfile.ConnectivityName.C2E), @@ -360,17 +362,20 @@ def _construct_grid( dims.V2E: self._get_index_field(gridfile.ConnectivityName.V2E), dims.V2C: self._get_index_field(gridfile.ConnectivityName.V2C), dims.C2V: self._get_index_field(gridfile.ConnectivityName.C2V), + dims.V2E2V: self._get_index_field(gridfile.ConnectivityName.V2E2V), } - if not self._run_properties.single_node(): - cells_to_rank_mapping = self._decompose( - global_connectivities_for_halo_construction[dims.C2E2C], - self._run_properties.comm_size, - ) - self._halo_constructor(cells_to_rank_mapping) + cells_to_rank_mapping = self._decompose( + global_connectivities_for_halo_construction[dims.C2E2C], + self._run_properties.comm_size, + ) + halo_constructor = self._initialize_halo_constructor( + grid_size, global_connectivities_for_halo_construction, backend + ) + decomposition_info = halo_constructor(cells_to_rank_mapping) + ## TODO from here do local reads (and halo exchanges!!) global_connectivities = { dims.E2V: self._get_index_field(gridfile.ConnectivityName.E2V), - dims.V2E2V: self._get_index_field(gridfile.ConnectivityName.V2E2V), } _determine_limited_area = functools.partial(refinement.is_limited_area_grid, array_ns=xp) @@ -381,7 +386,7 @@ def _construct_grid( refinement_fields = functools.partial(self._read_grid_refinement_fields, backend=backend)() limited_area = _determine_limited_area(refinement_fields[dims.CellDim].ndarray) grid = self._initialize_global( - with_skip_values=with_skip_values, limited_area=limited_area, on_gpu=on_gpu + grid_size, with_skip_values=with_skip_values, limited_area=limited_area, on_gpu=on_gpu ) grid.set_refinement_control(refinement_fields) @@ -406,7 +411,11 @@ def _get_index_field(self, field: gridfile.GridFileName, transpose=True, apply_o return field def _initialize_global( - self, with_skip_values: bool, limited_area: bool, on_gpu: bool + self, + grid_size: base.HorizontalGridSize, + with_skip_values: bool, + limited_area: bool, + on_gpu: bool, ) -> icon.IconGrid: """ Read basic information from the grid file: @@ -423,16 +432,11 @@ def _initialize_global( IconGrid: basic grid, setup only with id and config information. """ - num_cells = self._reader.dimension(gridfile.DimensionName.CELL_NAME) - num_edges = self._reader.dimension(gridfile.DimensionName.EDGE_NAME) - num_vertices = self._reader.dimension(gridfile.DimensionName.VERTEX_NAME) + uuid = self._reader.attribute(gridfile.MandatoryPropertyName.GRID_UUID) grid_root = self._reader.attribute(gridfile.MandatoryPropertyName.ROOT) grid_level = self._reader.attribute(gridfile.MandatoryPropertyName.LEVEL) global_params = icon.GlobalGridParams(root=grid_root, level=grid_level) - grid_size = base.HorizontalGridSize( - num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells - ) config = base.GridConfig( horizontal_config=grid_size, vertical_size=self._vertical_config.num_levels, @@ -443,6 +447,35 @@ def _initialize_global( grid = icon.IconGrid(uuid).set_config(config).set_global_params(global_params) return grid + def _read_horizontal_grid_size(self): + num_cells = self._reader.dimension(gridfile.DimensionName.CELL_NAME) + num_edges = self._reader.dimension(gridfile.DimensionName.EDGE_NAME) + num_vertices = self._reader.dimension(gridfile.DimensionName.VERTEX_NAME) + grid_size = base.HorizontalGridSize( + num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells + ) + return grid_size + + def _initialize_halo_constructor( + self, + grid_size: base.HorizontalGridSize, + connectivities: dict, + backend=Optional[gtx_backend.Backend], + ) -> HaloConstructor: + if self._run_properties.single_node(): + return halo.NoHalos( + num_levels=self._vertical_config.num_levels, + horizontal_size=grid_size, + backend=backend, + ) + else: + return halo.IconLikeHaloConstructor( + run_properties=self._run_properties, + connectivities=connectivities, + num_levels=self._vertical_config.num_levels, + backend=backend, + ) + def _add_derived_connectivities(grid: icon.IconGrid, array_ns: ModuleType = np) -> icon.IconGrid: e2v_table = grid._neighbor_tables[dims.E2VDim] diff --git a/model/common/tests/decomposition_tests/mpi_tests/conftest.py b/model/common/tests/decomposition_tests/conftest.py similarity index 58% rename from model/common/tests/decomposition_tests/mpi_tests/conftest.py rename to model/common/tests/decomposition_tests/conftest.py index a0b781be2f..8220971738 100644 --- a/model/common/tests/decomposition_tests/mpi_tests/conftest.py +++ b/model/common/tests/decomposition_tests/conftest.py @@ -11,8 +11,14 @@ _ = sys.modules["icon4py.model.testing.pytest_config"] except KeyError: from icon4py.model.testing.pytest_config import * # noqa: F403 - - -from icon4py.model.testing.parallel_helpers import ( - processor_props, # noqa: F401 # import fixtures from test_utils package +from icon4py.model.testing.datatest_fixtures import ( # noqa: F401 # import fixtures from test_utils + data_provider, + decomposition_info, + download_ser_data, + experiment, + grid_savepoint, + icon_grid, + metrics_savepoint, + processor_props, + ranked_data_path, ) diff --git a/model/common/tests/decomposition_tests/mpi_tests/test_halo.py b/model/common/tests/decomposition_tests/mpi_tests/test_halo.py index 677efd6272..3cf1535361 100644 --- a/model/common/tests/decomposition_tests/mpi_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/mpi_tests/test_halo.py @@ -216,6 +216,7 @@ def test_halo_constructor_owned_cells(processor_props): # F811 # fixture assert np.setdiff1d(my_owned_cells, _CELL_OWN[processor_props.rank]).size == 0 +@pytest.mark.parametrize("processor_props", [True, False], indirect=True) def test_halo_constructor_validate_number_of_node_mismatch(processor_props): grid = simple.SimpleGrid() distribution = (processor_props.comm_size + 1) * np.ones((grid.num_cells,), dtype=int) @@ -229,6 +230,7 @@ def test_halo_constructor_validate_number_of_node_mismatch(processor_props): assert "The distribution assumes more nodes than the current run" in e.value.args[0] +@pytest.mark.parametrize("processor_props", [True, False], indirect=True) @pytest.mark.parametrize("shape", [(simple.SimpleGrid._CELLS, 3), (2,)]) def test_halo_constructor_validate_rank_mapping_wrong_shape(processor_props, shape): grid = simple.SimpleGrid() @@ -322,6 +324,7 @@ def assert_same_entries( @pytest.mark.mpi(min_size=4) @pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) +@pytest.mark.parametrize("processor_props", [True], indirect=True) def test_halo_constructor_decomposition_info_halo_levels(processor_props, dim): # F811 # fixture grid = simple.SimpleGrid() halo_generator = halo.IconLikeHaloConstructor( diff --git a/model/common/tests/decomposition_tests/mpi_tests/test_mpi_decomposition.py b/model/common/tests/decomposition_tests/mpi_tests/test_mpi_decomposition.py index 2cee56a0af..3b0c31843d 100644 --- a/model/common/tests/decomposition_tests/mpi_tests/test_mpi_decomposition.py +++ b/model/common/tests/decomposition_tests/mpi_tests/test_mpi_decomposition.py @@ -10,6 +10,7 @@ import pytest from icon4py.model.common.utils import data_allocation as data_alloc +from icon4py.model.testing import parallel_helpers try: @@ -21,20 +22,6 @@ from icon4py.model.common import dimension as dims from icon4py.model.common.decomposition import definitions, mpi_decomposition -from icon4py.model.testing.datatest_fixtures import ( # noqa: F401 # import fixtures from test_utils - data_provider, - decomposition_info, - download_ser_data, - experiment, - grid_savepoint, - icon_grid, - metrics_savepoint, - ranked_data_path, -) -from icon4py.model.testing.parallel_helpers import ( # noqa: F401 # import fixtures from test_utils package - check_comm_size, - processor_props, -) _log = logging.getLogger(__name__) @@ -52,7 +39,7 @@ @pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_props(processor_props): # noqa: F811 # fixture +def test_props(processor_props): # fixture assert processor_props.comm @@ -72,11 +59,10 @@ def test_decomposition_info_masked( owned, total, caplog, - download_ser_data, # noqa: F811 # fixture - decomposition_info, # noqa: F811 # fixture - processor_props, # noqa: F811 # fixture + decomposition_info, + processor_props, ): - check_comm_size(processor_props, sizes=[2]) + parallel_helpers.check_comm_size(processor_props, sizes=[2]) my_rank = processor_props.rank all_indices = decomposition_info.global_index(dim, definitions.DecompositionInfo.EntryType.ALL) my_total = total[my_rank] @@ -121,11 +107,10 @@ def test_decomposition_info_local_index( owned, total, caplog, - download_ser_data, # noqa: F811 #fixture - decomposition_info, # noqa: F811 #fixture - processor_props, # noqa: F811 #fixture + decomposition_info, + processor_props, ): - check_comm_size(processor_props, sizes=[2]) + parallel_helpers.check_comm_size(processor_props, sizes=[2]) my_rank = processor_props.rank all_indices = decomposition_info.local_index(dim, definitions.DecompositionInfo.EntryType.ALL) my_total = total[my_rank] @@ -152,7 +137,7 @@ def test_decomposition_info_local_index( @pytest.mark.parametrize("num", [1, 2, 3, 4, 5, 6, 7, 8]) def test_domain_descriptor_id_are_globally_unique( num, - processor_props, # noqa F811 #fixture + processor_props, ): props = processor_props size = props.comm_size @@ -178,12 +163,11 @@ def test_domain_descriptor_id_are_globally_unique( @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_decomposition_info_matches_gridsize( caplog, - download_ser_data, # noqa: F811 #fixture - decomposition_info, # noqa: F811 #fixture - icon_grid, # noqa: F811 #fixture - processor_props, # noqa: F811 #fixture + decomposition_info, + icon_grid, + processor_props, ): - check_comm_size(processor_props) + parallel_helpers.check_comm_size(processor_props) assert ( decomposition_info.global_index( dim=dims.CellDim, entry_type=definitions.DecompositionInfo.EntryType.ALL @@ -207,8 +191,8 @@ def test_decomposition_info_matches_gridsize( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_create_multi_node_runtime_with_mpi( - decomposition_info, # noqa: F811 # fixture - processor_props, # noqa: F811 # fixture + decomposition_info, + processor_props, ): props = processor_props exchange = definitions.create_exchange(props, decomposition_info) @@ -221,8 +205,8 @@ def test_create_multi_node_runtime_with_mpi( @pytest.mark.parametrize("processor_props", [False], indirect=True) @pytest.mark.mpi_skip() def test_create_single_node_runtime_without_mpi( - processor_props, # noqa: F811 # fixture - decomposition_info, # noqa: F811 # fixture + processor_props, + decomposition_info, ): exchange = definitions.create_exchange(processor_props, decomposition_info) assert isinstance(exchange, definitions.SingleNodeExchange) @@ -232,10 +216,10 @@ def test_create_single_node_runtime_without_mpi( @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize("dimension", (dims.CellDim, dims.VertexDim, dims.EdgeDim)) def test_exchange_on_dummy_data( - processor_props, # noqa: F811 # fixture - decomposition_info, # noqa: F811 # fixture - grid_savepoint, # noqa: F811 # fixture - metrics_savepoint, # noqa: F811 # fixture + processor_props, # fixture + decomposition_info, # fixture + grid_savepoint, # fixture + metrics_savepoint, # fixture dimension, caplog, ): diff --git a/model/common/tests/decomposition_tests/test_definitions.py b/model/common/tests/decomposition_tests/test_definitions.py index 58774345d5..4fa1aec491 100644 --- a/model/common/tests/decomposition_tests/test_definitions.py +++ b/model/common/tests/decomposition_tests/test_definitions.py @@ -12,19 +12,10 @@ SingleNodeExchange, create_exchange, ) -from icon4py.model.testing.datatest_fixtures import ( # noqa: F401 # import fixtures form test_utils - data_provider, - download_ser_data, - experiment, - grid_savepoint, - icon_grid, - processor_props, - ranked_data_path, -) @pytest.mark.datatest -def test_create_single_node_runtime_without_mpi(icon_grid, processor_props): # noqa: F811 # fixture +def test_create_single_node_runtime_without_mpi(icon_grid, processor_props): # fixture decomposition_info = DecompositionInfo( klevels=10, num_cells=icon_grid.num_cells, diff --git a/model/common/tests/grid_tests/mpi_tests/conftest.py b/model/common/tests/grid_tests/mpi_tests/conftest.py deleted file mode 100644 index a0b781be2f..0000000000 --- a/model/common/tests/grid_tests/mpi_tests/conftest.py +++ /dev/null @@ -1,18 +0,0 @@ -# ICON4Py - ICON inspired code in Python and GT4Py -# -# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss -# All rights reserved. -# -# Please, refer to the LICENSE file in the root directory. -# SPDX-License-Identifier: BSD-3-Clause -try: - import sys - - _ = sys.modules["icon4py.model.testing.pytest_config"] -except KeyError: - from icon4py.model.testing.pytest_config import * # noqa: F403 - - -from icon4py.model.testing.parallel_helpers import ( - processor_props, # noqa: F401 # import fixtures from test_utils package -) diff --git a/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py index 67ea46ac53..bf37aadcf9 100644 --- a/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py @@ -79,6 +79,7 @@ def test_start_end_index( ), f"end index wrong for domain {domain}" +@pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi(min_size=2) def test_grid_manager_validate_decomposer(processor_props): file = grid_utils.resolve_full_grid_file_name(dt_utils.R02B04_GLOBAL) diff --git a/model/common/tests/grid_tests/mpi_tests/test_parallel_icon.py b/model/common/tests/grid_tests/mpi_tests/test_parallel_icon.py index 08463dd2f6..c607d05ba4 100644 --- a/model/common/tests/grid_tests/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/grid_tests/mpi_tests/test_parallel_icon.py @@ -12,10 +12,7 @@ import icon4py.model.common.dimension as dims import icon4py.model.common.grid.horizontal as h_grid -from icon4py.model.testing.parallel_helpers import ( # noqa: F401 # import fixtures from test_utils package - check_comm_size, - processor_props, -) +from icon4py.model.testing import parallel_helpers from .. import utils @@ -28,7 +25,7 @@ @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_props(processor_props): # noqa: F811 # fixture +def test_props(processor_props): """dummy test to check whether the MPI initialization and GHEX setup works.""" import ghex.context as ghex @@ -59,9 +56,9 @@ def test_props(processor_props): # noqa: F811 # fixture @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize("dim", utils.main_horizontal_dims()) -def test_distributed_local(processor_props, dim, icon_grid, caplog): # noqa: F811 # fixture +def test_distributed_local(processor_props, dim, icon_grid, caplog): caplog.set_level(logging.INFO) - check_comm_size(processor_props) + parallel_helpers.check_comm_size(processor_props) domain = h_grid.domain(dim)(h_grid.Zone.LOCAL) # local still runs entire field: assert icon_grid.start_index(domain) == 0 @@ -117,8 +114,8 @@ def test_distributed_local(processor_props, dim, icon_grid, caplog): # noqa: F8 @pytest.mark.mpi @pytest.mark.parametrize("dim", utils.main_horizontal_dims()) @pytest.mark.parametrize("marker", [h_grid.Zone.HALO, h_grid.Zone.HALO_LEVEL_2]) -def test_distributed_halo(processor_props, dim, marker, icon_grid): # noqa: F811 # fixture - check_comm_size(processor_props) +def test_distributed_halo(processor_props, dim, marker, icon_grid): + parallel_helpers.check_comm_size(processor_props) num = int(next(iter(re.findall(r"\d+", marker.value)))) domain = h_grid.domain(dim)(marker) start_index = icon_grid.start_index(domain) diff --git a/model/common/tests/grid_tests/test_grid_manager.py b/model/common/tests/grid_tests/test_grid_manager.py index 4be27ea7d7..eab7b5b11f 100644 --- a/model/common/tests/grid_tests/test_grid_manager.py +++ b/model/common/tests/grid_tests/test_grid_manager.py @@ -617,18 +617,17 @@ def test_limited_area_on_grid(grid_file, expected): ) def test_local_connectivities(processor_props, caplog, field_offset): # fixture caplog.set_level(logging.INFO) - grid = utils.run_grid_manager(dt_utils.R02B04_GLOBAL, backend=None).grid + grid = utils.run_grid_manager(dt_utils.R02B04_GLOBAL, keep_skip_values=True, backend=None).grid partitioner = halo.SimpleMetisDecomposer() - face_face_connectivity = grid.connectivities[dims.C2E2CDim] + face_face_connectivity = grid.neighbor_tables[dims.C2E2CDim] labels = partitioner(face_face_connectivity, n_part=processor_props.comm_size) halo_generator = halo.IconLikeHaloConstructor( connectivities=grid.neighbor_tables, run_properties=processor_props, - rank_mapping=labels, num_levels=1, ) - decomposition_info = halo_generator() + decomposition_info = halo_generator(labels) connectivity = gm.construct_local_connectivity( field_offset, decomposition_info, connectivity=grid.neighbor_tables[field_offset.target[1]] diff --git a/model/testing/src/icon4py/model/testing/datatest_fixtures.py b/model/testing/src/icon4py/model/testing/datatest_fixtures.py index cd6d00a12e..7384b8c38f 100644 --- a/model/testing/src/icon4py/model/testing/datatest_fixtures.py +++ b/model/testing/src/icon4py/model/testing/datatest_fixtures.py @@ -18,9 +18,11 @@ def experiment(): return dt_utils.REGIONAL_EXPERIMENT -@pytest.fixture(params=[False], scope="session") +@pytest.fixture(scope="session", params=[False]) def processor_props(request): - return dt_utils.get_processor_properties_for_run(decomposition.SingleNodeRun()) + with_mpi = request.param + runtype = decomposition.get_runtype(with_mpi=with_mpi) + yield decomposition.get_processor_properties(runtype) @pytest.fixture(scope="session") diff --git a/model/testing/src/icon4py/model/testing/datatest_utils.py b/model/testing/src/icon4py/model/testing/datatest_utils.py index 513da4b5cd..fa1a0ddfe8 100644 --- a/model/testing/src/icon4py/model/testing/datatest_utils.py +++ b/model/testing/src/icon4py/model/testing/datatest_utils.py @@ -14,8 +14,6 @@ from gt4py.next import backend as gtx_backend -from icon4py.model.common.decomposition import definitions as decomposition - DEFAULT_TEST_DATA_FOLDER = "testdata" GLOBAL_EXPERIMENT = "exclaim_ape_R02B04" @@ -138,10 +136,6 @@ def get_grid_id_for_experiment(experiment) -> uuid.UUID: raise ValueError(f"Experiment '{experiment}' has no grid id ") from err -def get_processor_properties_for_run(run_instance): - return decomposition.get_processor_properties(run_instance) - - def get_ranked_data_path(base_path, processor_properties): return base_path.absolute().joinpath(f"mpitask{processor_properties.comm_size}") diff --git a/model/testing/src/icon4py/model/testing/grid_utils.py b/model/testing/src/icon4py/model/testing/grid_utils.py index 781c91c735..f9643c0f02 100644 --- a/model/testing/src/icon4py/model/testing/grid_utils.py +++ b/model/testing/src/icon4py/model/testing/grid_utils.py @@ -8,16 +8,13 @@ import pathlib from typing import Optional -import gt4py.next as gtx import gt4py.next.backend as gtx_backend -from icon4py.model.common import dimension as dims -from icon4py.model.common.decomposition import definitions +from icon4py.model.common.decomposition import halo from icon4py.model.common.grid import ( geometry, geometry_attributes as geometry_attrs, grid_manager as gm, - icon, vertical as v_grid, ) from icon4py.model.common.utils import data_allocation as data_alloc @@ -133,25 +130,15 @@ def get_grid_geometry( num_levels = get_num_levels(experiment) register_name = "_".join((experiment, data_alloc.backend_name(backend))) - def _construct_dummy_decomposition_info(grid: icon.IconGrid) -> definitions.DecompositionInfo: - def _add_dimension(dim: gtx.Dimension): - indices = data_alloc.index_field(grid, dim, backend=backend) - owner_mask = xp.ones((grid.size[dim],), dtype=bool) - decomposition_info.with_dimension(dim, indices.ndarray, owner_mask) - - decomposition_info = definitions.DecompositionInfo(klevels=grid.num_levels) - _add_dimension(dims.EdgeDim) - _add_dimension(dims.VertexDim) - _add_dimension(dims.CellDim) - - return decomposition_info - def _construct_grid_geometry(): gm = _download_and_load_gridfile( grid_file, keep_skip_values=True, num_levels=num_levels, backend=backend ) grid = gm.grid - decomposition_info = _construct_dummy_decomposition_info(grid) + dummy_halo_constructor = halo.NoHalos( + horizontal_size=grid.config.horizontal_config, num_levels=num_levels, backend=backend + ) + decomposition_info = dummy_halo_constructor(xp.zeros((grid.num_levels,), dtype=int)) geometry_source = geometry.GridGeometry( grid, decomposition_info, backend, gm.coordinates, gm.geometry, geometry_attrs.attrs ) diff --git a/model/testing/src/icon4py/model/testing/parallel_helpers.py b/model/testing/src/icon4py/model/testing/parallel_helpers.py index de0cbc736e..a4e0f6aa42 100644 --- a/model/testing/src/icon4py/model/testing/parallel_helpers.py +++ b/model/testing/src/icon4py/model/testing/parallel_helpers.py @@ -8,16 +8,9 @@ import pytest -from icon4py.model.common.decomposition.definitions import ProcessProperties, get_runtype -from icon4py.model.common.decomposition.mpi_decomposition import get_multinode_properties +from icon4py.model.common.decomposition.definitions import ProcessProperties def check_comm_size(props: ProcessProperties, sizes=(1, 2, 4)): if props.comm_size not in sizes: pytest.xfail(f"wrong comm size: {props.comm_size}: test only works for comm-sizes: {sizes}") - - -@pytest.fixture(scope="session") -def processor_props(request): - runtype = get_runtype(with_mpi=True) - yield get_multinode_properties(runtype) From f51b9b89bf1c7e28b98eaecc06672514a4d544c8 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 25 Jul 2025 14:26:06 +0200 Subject: [PATCH 050/240] rename GridConfig.horizontal_config to GridConfig.horizontal_size fix test_grid_manager --- .../src/icon4py/model/common/grid/base.py | 10 +- .../icon4py/model/common/grid/grid_manager.py | 100 ++++++++++-------- .../src/icon4py/model/common/grid/simple.py | 2 +- model/common/tests/io_tests/test_io.py | 14 +-- model/common/tests/io_tests/test_writers.py | 2 +- .../src/icon4py/model/testing/grid_utils.py | 2 +- .../src/icon4py/model/testing/serialbox.py | 2 +- .../icon4py/tools/py2fgen/wrappers/common.py | 2 +- 8 files changed, 72 insertions(+), 62 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/base.py b/model/common/src/icon4py/model/common/grid/base.py index 9ce6932d05..e415903dff 100644 --- a/model/common/src/icon4py/model/common/grid/base.py +++ b/model/common/src/icon4py/model/common/grid/base.py @@ -12,7 +12,7 @@ import math import uuid from types import ModuleType -from typing import Callable, Dict, Mapping, Sequence, Optional +from typing import Callable, Dict, Mapping, Sequence import gt4py.next as gtx import numpy as np @@ -50,7 +50,7 @@ class HorizontalGridSize: @dataclasses.dataclass(frozen=True, kw_only=True) class GridConfig: - horizontal_config: HorizontalGridSize + horizontal_size: HorizontalGridSize # TODO (Magdalena): Decouple the vertical from horizontal grid. vertical_size: int limited_area: bool = True @@ -65,15 +65,15 @@ def num_levels(self): @property def num_vertices(self): - return self.horizontal_config.num_vertices + return self.horizontal_size.num_vertices @property def num_edges(self): - return self.horizontal_config.num_edges + return self.horizontal_size.num_edges @property def num_cells(self): - return self.horizontal_config.num_cells + return self.horizontal_size.num_cells def _1d_size(connectivity: gtx_common.NeighborTable) -> int: diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 4a4979c991..9af1b2b16a 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -9,7 +9,7 @@ import logging import pathlib from types import ModuleType -from typing import Literal, Optional, Protocol, TypeAlias, Union +from typing import Callable, Literal, Optional, Protocol, TypeAlias, Union import gt4py.next as gtx import gt4py.next.backend as gtx_backend @@ -28,6 +28,7 @@ refinement, vertical as v_grid, ) +from icon4py.model.common.grid.base import HorizontalGridSize from icon4py.model.common.utils import data_allocation as data_alloc @@ -107,8 +108,8 @@ def __init__( self._grid: Optional[icon.IconGrid] = None self._decomposition_info: Optional[decomposition.DecompositionInfo] = None self._geometry: GeometryDict = {} - self._reader = None self._coordinates: CoordinateDict = {} + self._reader = None def close(self): """close the gridfile resource.""" @@ -353,9 +354,21 @@ def _construct_grid( """ xp = data_alloc.import_array_ns(backend) - grid_size = self._read_horizontal_grid_size() - global_connectivities_for_halo_construction = { - dims.C2E2C: self._get_index_field(gridfile.ConnectivityName.C2E2C), + + uuid_ = self._reader.attribute(gridfile.MandatoryPropertyName.GRID_UUID) + global_grid_size = self._read_full_grid_size() + grid_root = self._reader.attribute(gridfile.MandatoryPropertyName.ROOT) + grid_level = self._reader.attribute(gridfile.MandatoryPropertyName.LEVEL) + + cell_to_cell_neighbors = self._get_index_field(gridfile.ConnectivityName.C2E2C) + cells_to_rank_mapping = self._decompose( + cell_to_cell_neighbors, + self._run_properties.comm_size, + ) + # TODO: (magdalena) reduce the set of neighbor tables used in the halo construction + # TODO: (magdalena) this is all numpy currently! + neighbor_tables_for_halo_construction = { + dims.C2E2C: cell_to_cell_neighbors, dims.C2E: self._get_index_field(gridfile.ConnectivityName.C2E), dims.E2C: self._get_index_field(gridfile.ConnectivityName.E2C), dims.V2E: self._get_index_field(gridfile.ConnectivityName.V2E), @@ -363,55 +376,47 @@ def _construct_grid( dims.C2V: self._get_index_field(gridfile.ConnectivityName.C2V), dims.V2E2V: self._get_index_field(gridfile.ConnectivityName.V2E2V), } - cells_to_rank_mapping = self._decompose( - global_connectivities_for_halo_construction[dims.C2E2C], - self._run_properties.comm_size, - ) + # halo_constructor - creates the decomposition info, which can then be used to generate the local patches on each rank halo_constructor = self._initialize_halo_constructor( - grid_size, global_connectivities_for_halo_construction, backend + global_grid_size, neighbor_tables_for_halo_construction, backend ) decomposition_info = halo_constructor(cells_to_rank_mapping) ## TODO from here do local reads (and halo exchanges!!) - uuid_ = self._reader.attribute(gridfile.MandatoryPropertyName.GRID_UUID) - global_connectivities = { - dims.E2V: self._get_index_field(gridfile.ConnectivityName.E2V), - } + neighbor_tables = neighbor_tables_for_halo_construction + neighbor_tables[dims.E2V] = self._get_index_field(gridfile.ConnectivityName.E2V) + neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) _determine_limited_area = functools.partial(refinement.is_limited_area_grid, array_ns=xp) _derived_connectivities = functools.partial( _get_derived_connectivities, array_ns=xp, ) + refinement_fields = functools.partial(self._read_grid_refinement_fields, backend=backend)() limited_area = _determine_limited_area(refinement_fields[dims.CellDim].ndarray) - grid = self._initialize_global( - grid_size, with_skip_values=with_skip_values, limited_area=limited_area, on_gpu=on_gpu + global_params = icon.GlobalGridParams(root=grid_root, level=grid_level) + start, end, _ = self._read_start_end_indices() + grid_config = base.GridConfig( + horizontal_size=global_grid_size, + vertical_size=self._vertical_config.num_levels, + limited_area=limited_area, ) - grid.set_refinement_control(refinement_fields) - - global_connectivities.update(global_connectivities_for_halo_construction) - - grid.set_neighbor_tables( - {o.target[1]: xp.asarray(c) for o, c in global_connectivities.items()} + grid = icon.icon_grid( + uuid_, + allocator=backend, + config=grid_config, + neighbor_tables=neighbor_tables, + start_indices=start, + end_indices=end, + global_properties=global_params, + refinement_control=refinement_fields, ) - - _derived_connectivities(grid) - start, end, _ = self._read_start_end_indices() - for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values(): - grid.set_start_end_indices(dim, start[dim], end[dim]) - return grid - def _get_index_field(self, field: gridfile.GridFileName, transpose=True, apply_offset=True): - field = self._reader.int_variable(field, transpose=transpose) - if apply_offset: - field = field + self._transformation(field) - return field - def _initialize_global( - self, with_skip_values: bool, limited_area: bool, backend:gtx_backend.Backend + self, with_skip_values: bool, limited_area: bool, backend: gtx_backend.Backend ) -> icon.IconGrid: """ Read basic information from the grid file: @@ -429,16 +434,13 @@ def _initialize_global( """ xp = data_alloc.import_array_ns(backend) - grid_size = self._read_horizontal_grid_size() - num_cells = self._reader.dimension(gridfile.DimensionName.CELL_NAME) - num_edges = self._reader.dimension(gridfile.DimensionName.EDGE_NAME) - num_vertices = self._reader.dimension(gridfile.DimensionName.VERTEX_NAME) + grid_size = self._read_full_grid_size() uuid_ = self._reader.attribute(gridfile.MandatoryPropertyName.GRID_UUID) grid_root = self._reader.attribute(gridfile.MandatoryPropertyName.ROOT) grid_level = self._reader.attribute(gridfile.MandatoryPropertyName.LEVEL) global_params = icon.GlobalGridParams(root=grid_root, level=grid_level) config = base.GridConfig( - horizontal_config=grid_size, + horizontal_size=grid_size, vertical_size=self._vertical_config.num_levels, limited_area=limited_area, keep_skip_values=with_skip_values, @@ -457,8 +459,10 @@ def _initialize_global( neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) start, end, _ = self._read_start_end_indices() + # TODO is this necessary? start_indices = {dim: start[dim] for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values()} end_indices = {dim: end[dim] for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values()} + refinement_fields = self._read_grid_refinement_fields() return icon.icon_grid( id_=uuid_, @@ -476,19 +480,26 @@ def _get_index_field(self, field: gridfile.GridFileName, transpose=True, apply_o if apply_offset: field = field + self._transformation(field) return field - def _read_horizontal_grid_size(self): + + def _read_full_grid_size(self) -> HorizontalGridSize: + """ + Read the grid size propertes (cells, edges, vertices) from the grid file. + + As the grid file contains the _full_ (non-distributed) grid, these are the sizes of prior to distribution. + + """ num_cells = self._reader.dimension(gridfile.DimensionName.CELL_NAME) num_edges = self._reader.dimension(gridfile.DimensionName.EDGE_NAME) num_vertices = self._reader.dimension(gridfile.DimensionName.VERTEX_NAME) - grid_size = base.HorizontalGridSize( + full_grid_size = base.HorizontalGridSize( num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells ) - return grid_size + return full_grid_size def _initialize_halo_constructor( self, grid_size: base.HorizontalGridSize, - connectivities: dict, + connectivities: dict[gtx.FieldOffset, data_alloc.NDArray], backend=Optional[gtx_backend.Backend], ) -> HaloConstructor: if self._run_properties.single_node(): @@ -506,7 +517,6 @@ def _initialize_halo_constructor( ) - def _get_derived_connectivities( neighbor_tables: dict[gtx.FieldOffset, data_alloc.NDArray], array_ns: ModuleType = np ) -> dict[gtx.FieldOffset, data_alloc.NDArray]: diff --git a/model/common/src/icon4py/model/common/grid/simple.py b/model/common/src/icon4py/model/common/grid/simple.py index b5d85c158d..a22f3e9029 100644 --- a/model/common/src/icon4py/model/common/grid/simple.py +++ b/model/common/src/icon4py/model/common/grid/simple.py @@ -422,7 +422,7 @@ def simple_grid(backend: gtx_backend.Backend | None = None) -> base.Grid: ) vertical_grid_config = VerticalGridConfig(num_levels=10) config = base.GridConfig( - horizontal_config=horizontal_grid_size, + horizontal_size=horizontal_grid_size, vertical_size=vertical_grid_config.num_levels, limited_area=False, ) diff --git a/model/common/tests/io_tests/test_io.py b/model/common/tests/io_tests/test_io.py index dd2d04770b..d01e034481 100644 --- a/model/common/tests/io_tests/test_io.py +++ b/model/common/tests/io_tests/test_io.py @@ -143,7 +143,7 @@ def test_io_monitor_create_output_path(test_path): monitor = IOMonitor( config, vertical_params, - simple_grid.config.horizontal_config, + simple_grid.config.horizontal_size, grid_file, simple_grid.id, ) @@ -164,7 +164,7 @@ def test_io_monitor_write_ugrid_file(test_path): monitor = IOMonitor( config, vertical_params, - simple_grid.config.horizontal_config, + simple_grid.config.horizontal_size, grid_file, "simple_grid", ) @@ -207,7 +207,7 @@ def test_io_monitor_write_and_read_ugrid_dataset(test_path, variables): monitor = IOMonitor( config, vertical_params, - grid.config.horizontal_config, + grid.config.horizontal_size, grid_file, grid.id, ) @@ -257,7 +257,7 @@ def test_fieldgroup_monitor_write_dataset_file_roll(test_path): monitor = FieldGroupMonitor( config, vertical=vertical_params, - horizontal=grid.config.horizontal_config, + horizontal=grid.config.horizontal_size, grid_id=grid.id, output_path=test_path, ) @@ -371,7 +371,7 @@ def create_field_group_monitor(test_path, grid, start_time="2024-01-01T00:00:00" group_monitor = FieldGroupMonitor( config, vertical=vertical_params, - horizontal=grid.config.horizontal_config, + horizontal=grid.config.horizontal_size, grid_id=grid.id, output_path=test_path, ) @@ -430,7 +430,7 @@ def test_fieldgroup_monitor_constructs_output_path_and_filepattern(test_path): variables=["exner_function", "air_density"], ) vertical_size = simple_grid.config.vertical_size - horizontal_size = simple_grid.config.horizontal_config + horizontal_size = simple_grid.config.horizontal_size group_monitor = FieldGroupMonitor( config, vertical=vertical_size, @@ -452,7 +452,7 @@ def test_fieldgroup_monitor_throw_exception_on_missing_field(test_path): variables=["exner_function", "air_density", "foo"], ) vertical_size = simple_grid.config.vertical_size - horizontal_size = simple_grid.config.horizontal_config + horizontal_size = simple_grid.config.horizontal_size group_monitor = FieldGroupMonitor( config, vertical=vertical_size, diff --git a/model/common/tests/io_tests/test_writers.py b/model/common/tests/io_tests/test_writers.py index a5b1461e61..ea84ecc063 100644 --- a/model/common/tests/io_tests/test_writers.py +++ b/model/common/tests/io_tests/test_writers.py @@ -54,7 +54,7 @@ def initialized_writer( vct_a=gtx.as_field((dims.KDim,), heights), vct_b=None, ) - horizontal = grid.config.horizontal_config + horizontal = grid.config.horizontal_size fname = str(test_path.absolute()) + "/" + random_name + ".nc" writer = NETCDFWriter( fname, diff --git a/model/testing/src/icon4py/model/testing/grid_utils.py b/model/testing/src/icon4py/model/testing/grid_utils.py index f9643c0f02..76dfb5868a 100644 --- a/model/testing/src/icon4py/model/testing/grid_utils.py +++ b/model/testing/src/icon4py/model/testing/grid_utils.py @@ -136,7 +136,7 @@ def _construct_grid_geometry(): ) grid = gm.grid dummy_halo_constructor = halo.NoHalos( - horizontal_size=grid.config.horizontal_config, num_levels=num_levels, backend=backend + horizontal_size=grid.config.horizontal_size, num_levels=num_levels, backend=backend ) decomposition_info = dummy_halo_constructor(xp.zeros((grid.num_levels,), dtype=int)) geometry_source = geometry.GridGeometry( diff --git a/model/testing/src/icon4py/model/testing/serialbox.py b/model/testing/src/icon4py/model/testing/serialbox.py index 22ea1ca4aa..26f7912a7c 100644 --- a/model/testing/src/icon4py/model/testing/serialbox.py +++ b/model/testing/src/icon4py/model/testing/serialbox.py @@ -468,7 +468,7 @@ def construct_icon_grid( edge_ends = self.edge_end_index() config = base.GridConfig( - horizontal_config=base.HorizontalGridSize( + horizontal_size=base.HorizontalGridSize( num_vertices=self.num(dims.VertexDim), num_cells=self.num(dims.CellDim), num_edges=self.num(dims.EdgeDim), diff --git a/tools/src/icon4py/tools/py2fgen/wrappers/common.py b/tools/src/icon4py/tools/py2fgen/wrappers/common.py index 0b52a521cb..9bffd0cb74 100644 --- a/tools/src/icon4py/tools/py2fgen/wrappers/common.py +++ b/tools/src/icon4py/tools/py2fgen/wrappers/common.py @@ -209,7 +209,7 @@ def construct_icon_grid( e2c2e0 = add_origin(xp, e2c2e) config = base.GridConfig( - horizontal_config=base.HorizontalGridSize( + horizontal_size=base.HorizontalGridSize( num_vertices=num_vertices, num_cells=num_cells, num_edges=num_edges, From 683ec594ada1f132efc94d418d82d62182115c5c Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 25 Jul 2025 17:26:54 +0200 Subject: [PATCH 051/240] fix test_halo.py --- .../model/common/decomposition/definitions.py | 11 +- .../model/common/decomposition/halo.py | 103 +++++++++++------- .../mpi_tests/test_halo.py | 69 +++++++----- .../src/icon4py/model/testing/serialbox.py | 6 +- .../icon4py/tools/py2fgen/wrappers/common.py | 6 +- 5 files changed, 114 insertions(+), 81 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index f56e8426be..6ea69c262e 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -84,7 +84,7 @@ class EntryType(IntEnum): HALO = 2 @utils.chainable - def with_dimension( + def set_dimension( self, dim: Dimension, global_index: data_alloc.NDArray, @@ -98,18 +98,11 @@ def with_dimension( def __init__( self, klevels: int, - # TODO @halungge those were added for py2fgen, are they still needed - num_cells: Optional[int] = None, - num_edges: Optional[int] = None, - num_vertices: Optional[int] = None, ): + self._klevels = klevels self._global_index = {} self._halo_levels = {} - self._klevels = klevels self._owner_mask = {} - self._num_vertices = num_vertices - self._num_cells = num_cells - self._num_edges = num_edges @property def klevels(self): diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 5c9e8cb707..0f39bf0213 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -23,6 +23,10 @@ log = logging.getLogger(__name__) +def _value(k: gtx.FieldOffset | str): + return k.value if isinstance(k, gtx.FieldOffset) else k + + @runtime_checkable class HaloConstructor(Protocol): """Callable that takes a mapping from faces (aka cells) to ranks""" @@ -47,9 +51,9 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: create_arrays = functools.partial(_create_dummy_decomposition_arrays, array_ns=xp) decomposition_info = defs.DecompositionInfo(klevels=self._num_levels) - decomposition_info.with_dimension(dims.EdgeDim, *create_arrays(self._size.num_edges)) - decomposition_info.with_dimension(dims.CellDim, *create_arrays(self._size.num_cells)) - decomposition_info.with_dimension(dims.VertexDim, *create_arrays(self._size.num_vertices)) + decomposition_info.set_dimension(dims.EdgeDim, *create_arrays(self._size.num_edges)) + decomposition_info.set_dimension(dims.CellDim, *create_arrays(self._size.num_cells)) + decomposition_info.set_dimension(dims.VertexDim, *create_arrays(self._size.num_vertices)) return decomposition_info @@ -66,7 +70,7 @@ class IconLikeHaloConstructor(HaloConstructor): def __init__( self, run_properties: defs.ProcessProperties, - connectivities: dict[gtx.Dimension, data_alloc.NDArray], + connectivities: dict[gtx.FieldOffset | str, data_alloc.NDArray], num_levels, backend: Optional[gtx_backend.Backend] = None, ): @@ -81,32 +85,33 @@ def __init__( self._xp = data_alloc.import_array_ns(backend) self._num_levels = num_levels self._props = run_properties - self._connectivities = connectivities + + self._connectivities = {_value(k): v for k, v in connectivities.items()} self._assert_all_neighbor_tables() @property def face_face_connectivity(self): - return self._connectivity(dims.C2E2CDim) + return self._connectivity(dims.C2E2C.value) @property def edge_face_connectivity(self): - return self._connectivity(dims.E2CDim) + return self._connectivity(dims.E2C) @property def face_edge_connectivity(self): - return self._connectivity(dims.C2EDim) + return self._connectivity(dims.C2E) @property def node_edge_connectivity(self): - return self._connectivity(dims.V2EDim) + return self._connectivity(dims.V2E) @property def node_face_connectivity(self): - return self._connectivity(dims.V2CDim) + return self._connectivity(dims.V2C) @property def face_node_connectivity(self): - return self._connectivity(dims.C2VDim) + return self._connectivity(dims.C2V) def _validate_mapping(self, face_to_rank_mapping: data_alloc.NDArray): # validate the distribution mapping: @@ -128,52 +133,51 @@ def _validate_mapping(self, face_to_rank_mapping: data_alloc.NDArray): def _assert_all_neighbor_tables(self): # make sure we have all connectivity arrays used in the halo construction relevant_dimension = [ - dims.C2E2CDim, - dims.E2CDim, - dims.C2EDim, - dims.C2VDim, - dims.V2CDim, - dims.V2EDim, + dims.C2E2C, + dims.E2C, + dims.C2E, + dims.C2V, + dims.V2C, + dims.V2E, ] for d in relevant_dimension: assert ( - d in self._connectivities.keys() + d.value in self._connectivities.keys() ), f"Table for {d} is missing from the neighbor table array." - def _connectivity(self, dim: gtx.Dimension) -> data_alloc.NDArray: + def _connectivity(self, offset: gtx.FieldOffset | str) -> data_alloc.NDArray: try: - conn_table = self._connectivities[dim] - return conn_table + conn_table = self._connectivities.get(_value(offset)) return conn_table except KeyError as err: raise exceptions.MissingConnectivity( - f"Connectivity for offset {dim} is not available" + f"Connectivity for offset {offset} is not available" ) from err - def next_halo_line(self, cell_line: data_alloc.NDArray, depot=None): - """Returns the global indices of the next halo line. + def next_halo_line(self, cells: data_alloc.NDArray, depot: data_alloc.NDArray | None = None): + """Returns the full-grid indices of the next halo line. + + If a depot is given the function only return indices that are not in the depot Args: - cell_line: global indices of cells we want to find the neighbors of - depot: global indices that have already been collected + cells: 1d array, full-grid indices of cells we want to find the neighbors of + depot: full-grid indices that have already been collected Returns: - next_halo_cells: global indices of the next halo line + next_halo_cells: full-grid indices of the next halo line """ - cell_neighbors = self._cell_neighbors(cell_line) + assert cells.ndim == 1, "input should be 1d array" # TODO: otherwise reshape instead + cell_neighbors = self._find_cell_neighbors(cells) if depot is not None: - cells_so_far = self._xp.hstack((depot, cell_line)) + cells_so_far = self._xp.hstack((depot, cells)) else: - cells_so_far = cell_line + cells_so_far = cells next_halo_cells = self._xp.setdiff1d( self._xp.unique(cell_neighbors), cells_so_far, assume_unique=True ) return next_halo_cells - def _cell_neighbors(self, cells: data_alloc.NDArray): - return self._xp.unique(self.face_face_connectivity[cells, :]) - def _find_neighbors( self, source_indices: data_alloc.NDArray, connectivity: data_alloc.NDArray ) -> data_alloc.NDArray: @@ -183,6 +187,10 @@ def _find_neighbors( unique_neighbors = self._xp.unique(neighbors.reshape(shp[0] * shp[1])) return unique_neighbors + def _find_cell_neighbors(self, cells: data_alloc.NDArray): + """Find all neighboring cells of a list of cells.""" + return self._find_neighbors(cells, connectivity=self.face_face_connectivity) + def find_edge_neighbors_for_cells(self, cell_line: data_alloc.NDArray) -> data_alloc.NDArray: return self._find_neighbors(cell_line, connectivity=self.face_edge_connectivity) @@ -195,7 +203,7 @@ def find_vertex_neighbors_for_cells(self, cell_line: data_alloc.NDArray) -> data return self._find_neighbors(cell_line, connectivity=self.face_node_connectivity) def owned_cells(self, face_to_rank: data_alloc.NDArray) -> data_alloc.NDArray: - """Returns the global indices of the cells owned by this rank""" + """Returns the full-grid indices of the cells owned by this rank""" owned_cells = face_to_rank == self._props.rank return self._xp.asarray(owned_cells).nonzero()[0] @@ -233,24 +241,39 @@ def _update_owner_mask_by_max_rank_convention( owner_mask[local_index] = True return owner_mask - # TODO (@halungge): move out of halo generator? def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: """ Constructs the DecompositionInfo for the current rank. The DecompositionInfo object is constructed for all horizontal dimension starting from the cell distribution. Edges and vertices are then handled through their connectivity to the distributed cells. + """ + #: icon does hard coding of 2 halo lines for cells, make this dynamic! + + num_cell_halo_lines = 2 + self._validate_mapping(face_to_rank) #: cells + owned_cells = self.owned_cells(face_to_rank) # global indices of owned cells + # cell_halos = [] + # current = owned_cells + # depot = None + # for i in range(num_cell_halo_lines): + # cell_halos[i] = self.next_halo_line(current, depot) + # depot = self._xp.union1d(depot, current) + # current = cell_halos[i] + # first_halo_cells = cell_halos[0] + # second_halo_cells = cell_halos[1] + first_halo_cells = self.next_halo_line(owned_cells) second_halo_cells = self.next_halo_line(first_halo_cells, owned_cells) - total_halo_cells = self._xp.hstack((first_halo_cells, second_halo_cells)) all_cells = self._xp.hstack((owned_cells, total_halo_cells)) cell_owner_mask = self._xp.isin(all_cells, owned_cells) + # initialize cell halo levels cell_halo_levels = defs.DecompositionFlag.UNDEFINED * self._xp.ones( all_cells.size, dtype=int ) @@ -261,7 +284,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: cell_halo_levels[ self._xp.isin(all_cells, second_halo_cells) ] = defs.DecompositionFlag.SECOND_HALO_LINE - decomp_info = defs.DecompositionInfo(klevels=self._num_levels).with_dimension( + decomp_info = defs.DecompositionInfo(klevels=self._num_levels).set_dimension( dims.CellDim, all_cells, cell_owner_mask, cell_halo_levels ) @@ -281,7 +304,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: self._xp.hstack((vertex_on_owned_cells, vertex_on_first_halo_line)) ) vertex_owner_mask = self._xp.isin(all_vertices, vertex_on_owned_cells) - vertex_owner_mask = self._update_owner_mask_by_max_rank_convention( + vertex_owner_mask = self._update_owner_mask_by_max_rank_convention( # icon specific face_to_rank, vertex_owner_mask, all_vertices, @@ -302,7 +325,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: vertex_halo_levels[ self._xp.isin(all_vertices, vertex_second_level) ] = defs.DecompositionFlag.SECOND_HALO_LINE - decomp_info.with_dimension( + decomp_info.set_dimension( dims.VertexDim, all_vertices, vertex_owner_mask, vertex_halo_levels ) @@ -352,7 +375,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: edge_halo_levels[ self._xp.isin(all_edges, level_two_edges) ] = defs.DecompositionFlag.SECOND_HALO_LINE - decomp_info.with_dimension(dims.EdgeDim, all_edges, edge_owner_mask, edge_halo_levels) + decomp_info.set_dimension(dims.EdgeDim, all_edges, edge_owner_mask, edge_halo_levels) return decomp_info diff --git a/model/common/tests/decomposition_tests/mpi_tests/test_halo.py b/model/common/tests/decomposition_tests/mpi_tests/test_halo.py index 3cf1535361..2339578582 100644 --- a/model/common/tests/decomposition_tests/mpi_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/mpi_tests/test_halo.py @@ -25,6 +25,8 @@ except ImportError: pytest.skip("Skipping parallel on single node installation", allow_module_level=True) +from gt4py.next import common as gtx_common + from icon4py.model.common.decomposition import halo from icon4py.model.common.grid import ( base as base_grid, @@ -191,6 +193,15 @@ } +@pytest.fixture(scope="session") +def simple_neighbor_tables(): + grid = simple.simple_grid() + neighbor_tables = { + k: v.ndarray for k, v in grid.connectivities.items() if gtx_common.is_neighbor_table(v) + } + return neighbor_tables + + def grid_file_manager(file: pathlib.Path) -> gm.GridManager: manager = gm.GridManager( gm.ToZeroBasedIndexTransformation(), str(file), v_grid.VerticalGridConfig(num_levels=1) @@ -200,11 +211,9 @@ def grid_file_manager(file: pathlib.Path) -> gm.GridManager: @pytest.mark.mpi(min_size=4) -def test_halo_constructor_owned_cells(processor_props): # F811 # fixture - grid = simple.SimpleGrid() - +def test_halo_constructor_owned_cells(processor_props, simple_neighbor_tables): # F811 # fixture halo_generator = halo.IconLikeHaloConstructor( - connectivities=grid.neighbor_tables, + connectivities=simple_neighbor_tables, run_properties=processor_props, num_levels=1, backend=backend, @@ -217,12 +226,12 @@ def test_halo_constructor_owned_cells(processor_props): # F811 # fixture @pytest.mark.parametrize("processor_props", [True, False], indirect=True) -def test_halo_constructor_validate_number_of_node_mismatch(processor_props): - grid = simple.SimpleGrid() - distribution = (processor_props.comm_size + 1) * np.ones((grid.num_cells,), dtype=int) +def test_halo_constructor_validate_number_of_node_mismatch(processor_props, simple_neighbor_tables): + num_cells = simple_neighbor_tables["C2E2C"].shape[0] + distribution = (processor_props.comm_size + 1) * np.ones((num_cells,), dtype=int) with pytest.raises(expected_exception=exceptions.ValidationError) as e: halo_generator = halo.IconLikeHaloConstructor( - connectivities=grid.neighbor_tables, + connectivities=simple_neighbor_tables, run_properties=processor_props, num_levels=1, ) @@ -231,32 +240,35 @@ def test_halo_constructor_validate_number_of_node_mismatch(processor_props): @pytest.mark.parametrize("processor_props", [True, False], indirect=True) -@pytest.mark.parametrize("shape", [(simple.SimpleGrid._CELLS, 3), (2,)]) -def test_halo_constructor_validate_rank_mapping_wrong_shape(processor_props, shape): - grid = simple.SimpleGrid() +def test_halo_constructor_validate_rank_mapping_wrong_shape( + processor_props, simple_neighbor_tables +): + num_cells = simple_neighbor_tables["C2E2C"].shape[0] with pytest.raises(exceptions.ValidationError) as e: halo_generator = halo.IconLikeHaloConstructor( - connectivities=grid.neighbor_tables, + connectivities=simple_neighbor_tables, run_properties=processor_props, num_levels=1, ) - halo_generator(np.zeros((grid.num_cells, 3), dtype=int)) - assert f"should have shape ({grid.num_cells},)" in e.value.args[0] + halo_generator(np.zeros((num_cells, 3), dtype=int)) + assert f"should have shape ({num_cells},)" in e.value.args[0] def global_indices(dim: gtx.Dimension) -> np.ndarray: - mesh = simple.SimpleGrid() + mesh = simple.simple_grid() return np.arange(mesh.size[dim], dtype=gtx.int32) @pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) @pytest.mark.mpi(min_size=4) -def test_element_ownership_is_unique(dim, processor_props): # F811 # fixture +def test_element_ownership_is_unique( + dim, processor_props, simple_neighbor_tables +): # F811 # fixture if processor_props.comm_size != 4: pytest.skip("This test requires exactly 4 MPI ranks.") - grid = simple.SimpleGrid() + halo_generator = halo.IconLikeHaloConstructor( - connectivities=grid.neighbor_tables, + connectivities=simple_neighbor_tables, run_properties=processor_props, num_levels=1, backend=backend, @@ -295,12 +307,13 @@ def test_element_ownership_is_unique(dim, processor_props): # F811 # fixture @pytest.mark.mpi(min_size=4) @pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) -def test_halo_constructor_decomposition_info_global_indices(processor_props, dim): # F811 # fixture +def test_halo_constructor_decomposition_info_global_indices( + processor_props, simple_neighbor_tables, dim +): # F811 # fixture if processor_props.comm_size != 4: pytest.skip("This test requires exactly 4 MPI ranks.") - grid = simple.SimpleGrid() halo_generator = halo.IconLikeHaloConstructor( - connectivities=grid.neighbor_tables, + connectivities=simple_neighbor_tables, run_properties=processor_props, num_levels=1, ) @@ -325,10 +338,11 @@ def assert_same_entries( @pytest.mark.mpi(min_size=4) @pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) @pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_halo_constructor_decomposition_info_halo_levels(processor_props, dim): # F811 # fixture - grid = simple.SimpleGrid() +def test_halo_constructor_decomposition_info_halo_levels( + processor_props, dim, simple_neighbor_tables +): # F811 # fixture halo_generator = halo.IconLikeHaloConstructor( - connectivities=grid.neighbor_tables, + connectivities=simple_neighbor_tables, run_properties=processor_props, num_levels=1, ) @@ -395,9 +409,11 @@ def gather_field(field: np.ndarray, comm: mpi4py.MPI.Comm) -> tuple: return local_sizes, recv_buffer -def decompose(grid: base_grid.BaseGrid, processor_props): # F811 # fixture +def decompose(grid: base_grid.Grid, processor_props): # F811 # fixture partitioner = halo.SimpleMetisDecomposer() - labels = partitioner(grid.neighbor_tables[dims.C2E2CDim], n_part=processor_props.comm_size) + labels = partitioner( + grid.connectivities[dims.C2E2C].asnumpy(), n_part=processor_props.comm_size + ) return labels @@ -407,6 +423,7 @@ def test_distributed_fields(processor_props): # F811 # fixture grid_manager = grid_file_manager(GRID_FILE) global_grid = grid_manager.grid + global_cell_area = grid_manager.geometry[grid_file.GeometryName.CELL_AREA] global_edge_lat = grid_manager.coordinates[dims.EdgeDim]["lat"] global_vertex_lon = grid_manager.coordinates[dims.VertexDim]["lon"] diff --git a/model/testing/src/icon4py/model/testing/serialbox.py b/model/testing/src/icon4py/model/testing/serialbox.py index 26f7912a7c..2e683a14a1 100644 --- a/model/testing/src/icon4py/model/testing/serialbox.py +++ b/model/testing/src/icon4py/model/testing/serialbox.py @@ -446,9 +446,9 @@ def construct_decomposition_info(self): num_edges=self.num(dims.EdgeDim), num_vertices=self.num(dims.VertexDim), ) - .with_dimension(*self._get_decomposition_fields(dims.CellDim)) - .with_dimension(*self._get_decomposition_fields(dims.EdgeDim)) - .with_dimension(*self._get_decomposition_fields(dims.VertexDim)) + .set_dimension(*self._get_decomposition_fields(dims.CellDim)) + .set_dimension(*self._get_decomposition_fields(dims.EdgeDim)) + .set_dimension(*self._get_decomposition_fields(dims.VertexDim)) ) def _get_decomposition_fields(self, dim: gtx.Dimension): diff --git a/tools/src/icon4py/tools/py2fgen/wrappers/common.py b/tools/src/icon4py/tools/py2fgen/wrappers/common.py index 9bffd0cb74..b33f17044a 100644 --- a/tools/src/icon4py/tools/py2fgen/wrappers/common.py +++ b/tools/src/icon4py/tools/py2fgen/wrappers/common.py @@ -294,9 +294,9 @@ def construct_decomposition( definitions.DecompositionInfo( klevels=num_levels, num_cells=num_cells, num_edges=num_edges, num_vertices=num_vertices ) - .with_dimension(dims.CellDim, c_glb_index, c_owner_mask) - .with_dimension(dims.EdgeDim, e_glb_index, e_owner_mask) - .with_dimension(dims.VertexDim, v_glb_index, v_owner_mask) + .set_dimension(dims.CellDim, c_glb_index, c_owner_mask) + .set_dimension(dims.EdgeDim, e_glb_index, e_owner_mask) + .set_dimension(dims.VertexDim, v_glb_index, v_owner_mask) ) processor_props = definitions.get_processor_properties(definitions.MultiNodeRun(), comm_id) exchange = definitions.create_exchange(processor_props, decomposition_info) From 562ff50deff8871ae7d494555f63a564487d4f7d Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 25 Jul 2025 18:25:16 +0200 Subject: [PATCH 052/240] construct local patch (I) --- .../model/common/decomposition/definitions.py | 24 ++++-------- .../icon4py/model/common/grid/grid_manager.py | 38 +++++++++++++++---- .../src/icon4py/model/common/grid/gridfile.py | 7 ++-- 3 files changed, 41 insertions(+), 28 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 6ea69c262e..6163fdf56a 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -16,7 +16,7 @@ from typing import Any, Optional, Protocol, Sequence, Union, runtime_checkable import numpy as np -from gt4py.next import Dimension +import gt4py.next as gtx from icon4py.model.common import utils from icon4py.model.common.utils import data_allocation as data_alloc @@ -108,18 +108,6 @@ def __init__( def klevels(self): return self._klevels - @property - def num_cells(self): - return self._num_cells - - @property - def num_edges(self): - return self._num_edges - - @property - def num_vertices(self): - return self._num_vertices - def local_index(self, dim: Dimension, entry_type: EntryType = EntryType.ALL): match entry_type: case DecompositionInfo.EntryType.ALL: @@ -144,10 +132,12 @@ def _to_local_index(self, dim): xp.arange(data.shape[0]) return xp.arange(data.shape[0]) - def owner_mask(self, dim: Dimension) -> data_alloc.NDArray: + def global_to_local(self, dim:gtx.Dimension): + + def owner_mask(self, dim: gtx.Dimension) -> data_alloc.NDArray: return self._owner_mask[dim] - def global_index(self, dim: Dimension, entry_type: EntryType = EntryType.ALL): + def global_index(self, dim: gtx.Dimension, entry_type: EntryType = EntryType.ALL): match entry_type: case DecompositionInfo.EntryType.ALL: return self._global_index[dim] @@ -158,10 +148,10 @@ def global_index(self, dim: Dimension, entry_type: EntryType = EntryType.ALL): case _: raise NotImplementedError() - def halo_levels(self, dim: Dimension): + def halo_levels(self, dim: gtx.Dimension): return self._halo_levels[dim] - def halo_level_mask(self, dim: Dimension, level: DecompositionFlag): + def halo_level_mask(self, dim: gtx.Dimension, level: DecompositionFlag): return np.where(self._halo_levels[dim] == level, True, False) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 9af1b2b16a..bd864e3863 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -74,6 +74,21 @@ def __call__(self, array: data_alloc.NDArray): GeometryDict: TypeAlias = dict[gridfile.GeometryName, gtx.Field] +def _reduce_to_rank_local_size(full_size_neighbor_tables: dict[dims.FieldOffset, data_alloc.NDArray], decomposition_info:decomposition.DecompositionInfo)->dict[dims.FieldOffset, data_alloc.NDArray]: + + def get_rank_local_values(k:gtx.FieldOffset, v:data_alloc.NDArray): + index_target_dim = k.source + index_source_dim = k.target[0] + + index = decomposition_info.global_index(index_source_dim) + return v[index, :] + + return {k: get_rank_local_values(k, v) for k, v in full_size_neighbor_tables.items()} + + + + + class GridManager: """ Read ICON grid file and set up grid topology, refinement information and geometry fields. @@ -354,17 +369,22 @@ def _construct_grid( """ xp = data_alloc.import_array_ns(backend) - + ## FULL GRID PROPERTIES uuid_ = self._reader.attribute(gridfile.MandatoryPropertyName.GRID_UUID) global_grid_size = self._read_full_grid_size() grid_root = self._reader.attribute(gridfile.MandatoryPropertyName.ROOT) grid_level = self._reader.attribute(gridfile.MandatoryPropertyName.LEVEL) + _determine_limited_area = functools.partial(refinement.is_limited_area_grid, array_ns=xp) + refinement_fields = functools.partial(self._read_grid_refinement_fields, backend=backend)() + limited_area = _determine_limited_area(refinement_fields[dims.CellDim].ndarray) + # DECOMPOSITION cell_to_cell_neighbors = self._get_index_field(gridfile.ConnectivityName.C2E2C) cells_to_rank_mapping = self._decompose( cell_to_cell_neighbors, self._run_properties.comm_size, ) + # HALO CONSTRUCTION # TODO: (magdalena) reduce the set of neighbor tables used in the halo construction # TODO: (magdalena) this is all numpy currently! neighbor_tables_for_halo_construction = { @@ -383,19 +403,21 @@ def _construct_grid( decomposition_info = halo_constructor(cells_to_rank_mapping) ## TODO from here do local reads (and halo exchanges!!) + # CONSTRUCT LOCAL PATCH - neighbor_tables = neighbor_tables_for_halo_construction - neighbor_tables[dims.E2V] = self._get_index_field(gridfile.ConnectivityName.E2V) + neighbor_tables = _reduce_to_rank_local_size(neighbor_tables_for_halo_construction) # reduce locally? or read again + edge_index = decomposition_info.global_index(dims.EdgeDim) + neighbor_tables[dims.E2V] = self._get_index_field(gridfile.ConnectivityName.E2V, indices = edge_index) neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) - _determine_limited_area = functools.partial(refinement.is_limited_area_grid, array_ns=xp) + + _derived_connectivities = functools.partial( _get_derived_connectivities, array_ns=xp, ) - refinement_fields = functools.partial(self._read_grid_refinement_fields, backend=backend)() - limited_area = _determine_limited_area(refinement_fields[dims.CellDim].ndarray) + global_params = icon.GlobalGridParams(root=grid_root, level=grid_level) start, end, _ = self._read_start_end_indices() grid_config = base.GridConfig( @@ -475,8 +497,8 @@ def _initialize_global( refinement_control=refinement_fields, ) - def _get_index_field(self, field: gridfile.GridFileName, transpose=True, apply_offset=True): - field = self._reader.int_variable(field, transpose=transpose) + def _get_index_field(self, field: gridfile.GridFileName, indices: np.ndarray|None = None, transpose=True, apply_offset=True): + field = self._reader.int_variable(field, indices=indices, transpose=transpose) if apply_offset: field = field + self._transformation(field) return field diff --git a/model/common/src/icon4py/model/common/grid/gridfile.py b/model/common/src/icon4py/model/common/grid/gridfile.py index 574cc68cd5..fc822555cd 100644 --- a/model/common/src/icon4py/model/common/grid/gridfile.py +++ b/model/common/src/icon4py/model/common/grid/gridfile.py @@ -253,7 +253,7 @@ def attribute(self, name: PropertyName) -> Union[str, int, float]: return self._dataset.getncattr(name) def int_variable( - self, name: FieldName, indices: np.ndarray = None, transpose: bool = True + self, name: FieldName, indices: np.ndarray|None = None, transpose: bool = True ) -> np.ndarray: """Read a integer field from the grid file. @@ -261,6 +261,7 @@ def int_variable( Args: name: name of the field to read + indices: list of indices to read transpose: flag to indicate whether the file should be transposed (for 2d fields) Returns: NDArray: field data @@ -272,8 +273,8 @@ def int_variable( def variable( self, name: FieldName, - indices: np.ndarray = None, - transpose=False, + indices: np.ndarray|None = None, + transpose = False, dtype: np.dtype = gtx.float64, ) -> np.ndarray: """Read a field from the grid file. From 704cebc86a4496d952b9d5d1769c5d003cbf2a55 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 29 Jul 2025 09:04:04 +0200 Subject: [PATCH 053/240] global_to_local (1) --- .../model/common/decomposition/definitions.py | 5 ++-- .../icon4py/model/common/grid/grid_manager.py | 30 +++++++++++-------- .../src/icon4py/model/common/grid/gridfile.py | 6 ++-- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 6163fdf56a..6750a8b3dc 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -15,8 +15,8 @@ from enum import IntEnum from typing import Any, Optional, Protocol, Sequence, Union, runtime_checkable +import gt4py.next as gtx import numpy as np -import gt4py.next as gtx from icon4py.model.common import utils from icon4py.model.common.utils import data_allocation as data_alloc @@ -132,7 +132,8 @@ def _to_local_index(self, dim): xp.arange(data.shape[0]) return xp.arange(data.shape[0]) - def global_to_local(self, dim:gtx.Dimension): + def global_to_local(self, dim: gtx.Dimension): + ... def owner_mask(self, dim: gtx.Dimension) -> data_alloc.NDArray: return self._owner_mask[dim] diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index bd864e3863..3c5ac5fe4a 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -74,9 +74,11 @@ def __call__(self, array: data_alloc.NDArray): GeometryDict: TypeAlias = dict[gridfile.GeometryName, gtx.Field] -def _reduce_to_rank_local_size(full_size_neighbor_tables: dict[dims.FieldOffset, data_alloc.NDArray], decomposition_info:decomposition.DecompositionInfo)->dict[dims.FieldOffset, data_alloc.NDArray]: - - def get_rank_local_values(k:gtx.FieldOffset, v:data_alloc.NDArray): +def _reduce_to_rank_local_size( + full_size_neighbor_tables: dict[dims.FieldOffset, data_alloc.NDArray], + decomposition_info: decomposition.DecompositionInfo, +) -> dict[dims.FieldOffset, data_alloc.NDArray]: + def get_rank_local_values(k: gtx.FieldOffset, v: data_alloc.NDArray): index_target_dim = k.source index_source_dim = k.target[0] @@ -86,9 +88,6 @@ def get_rank_local_values(k:gtx.FieldOffset, v:data_alloc.NDArray): return {k: get_rank_local_values(k, v) for k, v in full_size_neighbor_tables.items()} - - - class GridManager: """ Read ICON grid file and set up grid topology, refinement information and geometry fields. @@ -405,19 +404,20 @@ def _construct_grid( ## TODO from here do local reads (and halo exchanges!!) # CONSTRUCT LOCAL PATCH - neighbor_tables = _reduce_to_rank_local_size(neighbor_tables_for_halo_construction) # reduce locally? or read again + neighbor_tables = _reduce_to_rank_local_size( + neighbor_tables_for_halo_construction + ) # reduce locally? or read again edge_index = decomposition_info.global_index(dims.EdgeDim) - neighbor_tables[dims.E2V] = self._get_index_field(gridfile.ConnectivityName.E2V, indices = edge_index) + neighbor_tables[dims.E2V] = self._get_index_field( + gridfile.ConnectivityName.E2V, indices=edge_index + ) neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) - - _derived_connectivities = functools.partial( _get_derived_connectivities, array_ns=xp, ) - global_params = icon.GlobalGridParams(root=grid_root, level=grid_level) start, end, _ = self._read_start_end_indices() grid_config = base.GridConfig( @@ -497,7 +497,13 @@ def _initialize_global( refinement_control=refinement_fields, ) - def _get_index_field(self, field: gridfile.GridFileName, indices: np.ndarray|None = None, transpose=True, apply_offset=True): + def _get_index_field( + self, + field: gridfile.GridFileName, + indices: np.ndarray | None = None, + transpose=True, + apply_offset=True, + ): field = self._reader.int_variable(field, indices=indices, transpose=transpose) if apply_offset: field = field + self._transformation(field) diff --git a/model/common/src/icon4py/model/common/grid/gridfile.py b/model/common/src/icon4py/model/common/grid/gridfile.py index fc822555cd..d52dde48b8 100644 --- a/model/common/src/icon4py/model/common/grid/gridfile.py +++ b/model/common/src/icon4py/model/common/grid/gridfile.py @@ -253,7 +253,7 @@ def attribute(self, name: PropertyName) -> Union[str, int, float]: return self._dataset.getncattr(name) def int_variable( - self, name: FieldName, indices: np.ndarray|None = None, transpose: bool = True + self, name: FieldName, indices: np.ndarray | None = None, transpose: bool = True ) -> np.ndarray: """Read a integer field from the grid file. @@ -273,8 +273,8 @@ def int_variable( def variable( self, name: FieldName, - indices: np.ndarray|None = None, - transpose = False, + indices: np.ndarray | None = None, + transpose=False, dtype: np.dtype = gtx.float64, ) -> np.ndarray: """Read a field from the grid file. From 66cd76c3de84674967b2538fedc47c185d80b7d1 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 6 Aug 2025 13:40:16 +0200 Subject: [PATCH 054/240] add global to local function --- .../model/common/decomposition/definitions.py | 39 +++- .../model/common/decomposition/halo.py | 30 ++- .../icon4py/model/common/grid/grid_manager.py | 6 +- .../tests/decomposition_tests/__init__.py | 8 + .../decomposition_tests/mpi_tests/__init__.py | 8 + .../mpi_tests/test_halo.py | 180 ++---------------- .../decomposition_tests/test_definitions.py | 108 +++++++++-- .../two_ranks_distribution.py | 16 ++ .../common/tests/decomposition_tests/utils.py | 148 ++++++++++++++ 9 files changed, 344 insertions(+), 199 deletions(-) create mode 100644 model/common/tests/decomposition_tests/__init__.py create mode 100644 model/common/tests/decomposition_tests/mpi_tests/__init__.py create mode 100644 model/common/tests/decomposition_tests/two_ranks_distribution.py create mode 100644 model/common/tests/decomposition_tests/utils.py diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 6750a8b3dc..e61765cb55 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -19,6 +19,7 @@ import numpy as np from icon4py.model.common import utils +from icon4py.model.common.grid import gridfile from icon4py.model.common.utils import data_allocation as data_alloc @@ -86,7 +87,7 @@ class EntryType(IntEnum): @utils.chainable def set_dimension( self, - dim: Dimension, + dim: gtx.Dimension, global_index: data_alloc.NDArray, owner_mask: data_alloc.NDArray, halo_levels: data_alloc.NDArray, @@ -108,7 +109,7 @@ def __init__( def klevels(self): return self._klevels - def local_index(self, dim: Dimension, entry_type: EntryType = EntryType.ALL): + def local_index(self, dim: gtx.Dimension, entry_type: EntryType = EntryType.ALL): match entry_type: case DecompositionInfo.EntryType.ALL: return self._to_local_index(dim) @@ -132,8 +133,22 @@ def _to_local_index(self, dim): xp.arange(data.shape[0]) return xp.arange(data.shape[0]) - def global_to_local(self, dim: gtx.Dimension): - ... + def global_to_local( + self, dim: gtx.Dimension, indices_to_translate: data_alloc.NDArray + ) -> data_alloc.NDArray: + global_indices = self.global_index(dim) + sorter = np.argsort(global_indices) + + mask = np.isin(indices_to_translate, global_indices) + # Find the positions of `values_to_find` in the sorted `global_indices`. + # The `sorter` argument tells searchsorted to work with the sorted version + # of `global_indices` without creating an explicit sorted copy. + local_neighbors = np.where( + mask, + sorter[np.searchsorted(global_indices, indices_to_translate, sorter=sorter)], + gridfile.GridFile.INVALID_INDEX, + ) + return local_neighbors def owner_mask(self, dim: gtx.Dimension) -> data_alloc.NDArray: return self._owner_mask[dim] @@ -155,6 +170,10 @@ def halo_levels(self, dim: gtx.Dimension): def halo_level_mask(self, dim: gtx.Dimension, level: DecompositionFlag): return np.where(self._halo_levels[dim] == level, True, False) + # TODO unused - delete + def is_on_node(self, dim, index: int, entryType: EntryType = EntryType.ALL) -> bool: + return np.isin(index, self.global_index(dim, entry_type=entryType)).item() + class ExchangeResult(Protocol): def wait(self): @@ -166,10 +185,10 @@ def is_ready(self) -> bool: @runtime_checkable class ExchangeRuntime(Protocol): - def exchange(self, dim: Dimension, *fields: tuple) -> ExchangeResult: + def exchange(self, dim: gtx.Dimension, *fields: tuple) -> ExchangeResult: ... - def exchange_and_wait(self, dim: Dimension, *fields: tuple): + def exchange_and_wait(self, dim: gtx.Dimension, *fields: tuple): ... def get_size(self): @@ -181,10 +200,10 @@ def my_rank(self): @dataclass class SingleNodeExchange: - def exchange(self, dim: Dimension, *fields: tuple) -> ExchangeResult: + def exchange(self, dim: gtx.Dimension, *fields: tuple) -> ExchangeResult: return SingleNodeResult() - def exchange_and_wait(self, dim: Dimension, *fields: tuple): + def exchange_and_wait(self, dim: gtx.Dimension, *fields: tuple): return def my_rank(self): @@ -407,8 +426,8 @@ class DecompositionFlag(enum.IntEnum): """ used for: - cells that share 1 edge with an OWNED cell - - vertices that are on OWNED cell - - edges that are on OWNED cell + - vertices that are on OWNED cell, but not owned + - edges that are on OWNED cell, but not owned """ SECOND_HALO_LINE = 2 diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 0f39bf0213..abe5775e85 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -332,26 +332,19 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: # edges edges_on_owned_cells = self.find_edge_neighbors_for_cells(owned_cells) edges_on_first_halo_line = self.find_edge_neighbors_for_cells(first_halo_cells) - edges_on_second_halo_line = self.find_edge_neighbors_for_cells(second_halo_cells) - + edges_on_cutting_line = self._xp.intersect1d(edges_on_owned_cells, edges_on_first_halo_line) level_two_edges = self._xp.setdiff1d( self.find_edge_neighbors_for_vertices(vertex_on_cutting_line), edges_on_owned_cells ) - # level_two_edges = xp.setdiff1d(edges_on_first_halo_line, edges_on_owned_cells) - all_edges = self._xp.hstack( - ( - edges_on_owned_cells, - level_two_edges, - self._xp.setdiff1d(edges_on_second_halo_line, edges_on_first_halo_line), + all_edges = self._xp.unique( + self._xp.hstack( + ( + edges_on_owned_cells, + level_two_edges, + ) ) ) - all_edges = self._xp.unique(all_edges) - # We need to reduce the overlap: - # `edges_on_owned_cells` and `edges_on_first_halo_line` both contain the edges on the cutting line. - edge_intersect_owned_first_line = self._xp.intersect1d( - edges_on_owned_cells, edges_on_first_halo_line - ) # construct the owner mask edge_owner_mask = self._xp.isin(all_edges, edges_on_owned_cells) @@ -359,19 +352,24 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: face_to_rank, edge_owner_mask, all_edges, - edge_intersect_owned_first_line, + edges_on_cutting_line, self.edge_face_connectivity, ) + edge_halo_levels = defs.DecompositionFlag.UNDEFINED * self._xp.ones( all_edges.shape, dtype=int ) edge_halo_levels[edge_owner_mask] = defs.DecompositionFlag.OWNED + # LEVEL_ONE edges are on a owned cell but are not owned: these are all edges on the cutting line that are not owned (by the convention) + edge_halo_levels[ self._xp.logical_and( self._xp.logical_not(edge_owner_mask), - self._xp.isin(all_edges, edge_intersect_owned_first_line), + self._xp.isin(all_edges, edges_on_cutting_line), ) ] = defs.DecompositionFlag.FIRST_HALO_LINE + + # LEVEL_TWO edges share exactly one vertext with an owned cell, they are on the first halo-line cells, but not on the cutting line edge_halo_levels[ self._xp.isin(all_edges, level_two_edges) ] = defs.DecompositionFlag.SECOND_HALO_LINE diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 3c5ac5fe4a..194599863c 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -75,9 +75,9 @@ def __call__(self, array: data_alloc.NDArray): def _reduce_to_rank_local_size( - full_size_neighbor_tables: dict[dims.FieldOffset, data_alloc.NDArray], + full_size_neighbor_tables: dict[gtx.FieldOffset, data_alloc.NDArray], decomposition_info: decomposition.DecompositionInfo, -) -> dict[dims.FieldOffset, data_alloc.NDArray]: +) -> dict[gtx.FieldOffset, data_alloc.NDArray]: def get_rank_local_values(k: gtx.FieldOffset, v: data_alloc.NDArray): index_target_dim = k.source index_source_dim = k.target[0] @@ -404,6 +404,8 @@ def _construct_grid( ## TODO from here do local reads (and halo exchanges!!) # CONSTRUCT LOCAL PATCH + # TODO first: read local neighbor tables and convert global to local indices + # TODO: instead of reading shring existing one to local size and to global to local indices neighbor_tables = _reduce_to_rank_local_size( neighbor_tables_for_halo_construction ) # reduce locally? or read again diff --git a/model/common/tests/decomposition_tests/__init__.py b/model/common/tests/decomposition_tests/__init__.py new file mode 100644 index 0000000000..80b673df7e --- /dev/null +++ b/model/common/tests/decomposition_tests/__init__.py @@ -0,0 +1,8 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + diff --git a/model/common/tests/decomposition_tests/mpi_tests/__init__.py b/model/common/tests/decomposition_tests/mpi_tests/__init__.py new file mode 100644 index 0000000000..80b673df7e --- /dev/null +++ b/model/common/tests/decomposition_tests/mpi_tests/__init__.py @@ -0,0 +1,8 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + diff --git a/model/common/tests/decomposition_tests/mpi_tests/test_halo.py b/model/common/tests/decomposition_tests/mpi_tests/test_halo.py index 2339578582..9a5add5d39 100644 --- a/model/common/tests/decomposition_tests/mpi_tests/test_halo.py +++ b/model/common/tests/decomposition_tests/mpi_tests/test_halo.py @@ -16,6 +16,8 @@ from icon4py.model.common import exceptions from icon4py.model.common.decomposition import definitions as defs, mpi_decomposition +from .. import utils + try: import mpi4py # import mpi4py to check for optional mpi dependency @@ -46,152 +48,6 @@ ) backend = None -""" -TESTDATA using the [SimpleGrid](../../../src/icon4py/model/common/grid/simple.py) -The distribution maps all of the 18 cells of the simple grid to ranks 0..3 - -the dictionaries contain the mapping from rank to global (in the simple grid) index of the dimension: -_CELL_OWN: rank -> owned cells, essentially the inversion of the SIMPLE_DISTRIBUTION -_EDGE_OWN: rank -> owned edges -_VERTEX_OWN: rank -> owned vertices - -the decision as to whether a "secondary" dimension (edge, vertices) is owned by a rank are made according to the -rules and conventions described in (../../../src/icon4py/model/common/decomposition/halo.py) - - -_CELL_FIRST_HALO_LINE: -_CELL_SECON_HALO_LINE: -_EDGE_FIRST_HALO_LINE: -_EDGE_SECOND_HALO_LINE: -_VERTEX_FIRST_HALO_LINE: -_VERTEX_SECOND_HALO_LINE: :mapping of rank to global indices that belongs to a ranks halo lines. -""" - -SIMPLE_DISTRIBUTION = np.asarray( - [ - 0, # 0c - 1, # 1c - 1, # 2c - 0, # 3c - 0, # 4c - 1, # 5c - 0, # 6c - 0, # 7c - 2, # 8c - 2, # 9c - 0, # 10c - 2, # 11c - 3, # 12c - 3, # 13c - 1, # 14c - 3, # 15c - 3, # 16c - 1, # 17c - ] -) -_CELL_OWN = {0: [0, 3, 4, 6, 7, 10], 1: [1, 2, 5, 14, 17], 2: [8, 9, 11], 3: [12, 13, 15, 16]} - -_CELL_FIRST_HALO_LINE = { - 0: [1, 11, 13, 9, 2, 15], - 1: [3, 8, 4, 11, 16, 13, 15], - 2: [5, 7, 6, 12, 14], - 3: [9, 10, 17, 14, 0, 1], -} - -_CELL_SECOND_HALO_LINE = { - 0: [17, 5, 12, 14, 8, 16], - 1: [0, 7, 6, 9, 10, 12], - 2: [2, 1, 4, 3, 10, 15, 16, 17], - 3: [6, 7, 8, 2, 3, 4, 5, 11], -} - -_CELL_HALO = { - 0: _CELL_FIRST_HALO_LINE[0] + _CELL_SECOND_HALO_LINE[0], - 1: _CELL_FIRST_HALO_LINE[1] + _CELL_SECOND_HALO_LINE[1], - 2: _CELL_FIRST_HALO_LINE[2] + _CELL_SECOND_HALO_LINE[2], - 3: _CELL_FIRST_HALO_LINE[3] + _CELL_SECOND_HALO_LINE[3], -} - -_EDGE_OWN = { - 0: [1, 5, 12, 13, 14, 9], - 1: [8, 7, 6, 25, 4, 2], - 2: [16, 11, 15, 17, 10, 24], - 3: [19, 23, 22, 26, 0, 3, 20, 18, 21], -} - -_EDGE_FIRST_HALO_LINE = {0: [0, 4, 17, 21, 10, 2], 1: [3, 15, 20, 26, 24], 2: [18], 3: []} - -_EDGE_SECOND_HALO_LINE = { - 0: [3, 6, 7, 8, 15, 24, 25, 26, 16, 22, 23, 18, 19, 20, 11], - 1: [0, 1, 5, 9, 12, 11, 10, 13, 16, 17, 18, 19, 21, 22, 23], - 2: [2, 9, 12, 4, 8, 7, 14, 21, 13, 19, 20, 22, 23, 25, 26], - 3: [11, 10, 14, 13, 16, 17, 24, 25, 6, 2, 1, 5, 4, 8, 7], -} - -_EDGE_THIRD_HALO_LINE = { - 0: [], - 1: [14], - 2: [0, 1, 3, 5, 6], - 3: [9, 12, 15], -} -_EDGE_HALO = { - 0: _EDGE_FIRST_HALO_LINE[0] + _EDGE_SECOND_HALO_LINE[0] + _EDGE_THIRD_HALO_LINE[0], - 1: _EDGE_FIRST_HALO_LINE[1] + _EDGE_SECOND_HALO_LINE[1] + _EDGE_THIRD_HALO_LINE[1], - 2: _EDGE_FIRST_HALO_LINE[2] + _EDGE_SECOND_HALO_LINE[2] + _EDGE_THIRD_HALO_LINE[2], - 3: _EDGE_FIRST_HALO_LINE[3] + _EDGE_SECOND_HALO_LINE[3] + _EDGE_THIRD_HALO_LINE[3], -} - -_VERTEX_OWN = { - 0: [4], - 1: [], - 2: [3, 5], - 3: [ - 0, - 1, - 2, - 6, - 7, - 8, - ], -} - -_VERTEX_FIRST_HALO_LINE = { - 0: [0, 1, 5, 8, 7, 3], - 1: [1, 2, 0, 5, 3, 8, 6], - 2: [ - 6, - 8, - 7, - ], - 3: [], -} - -_VERTEX_SECOND_HALO_LINE = { - 0: [2, 6], - 1: [7, 4], - 2: [4, 0, 2, 1], - 3: [3, 4, 5], -} -_VERTEX_HALO = { - 0: _VERTEX_FIRST_HALO_LINE[0] + _VERTEX_SECOND_HALO_LINE[0], - 1: _VERTEX_FIRST_HALO_LINE[1] + _VERTEX_SECOND_HALO_LINE[1], - 2: _VERTEX_FIRST_HALO_LINE[2] + _VERTEX_SECOND_HALO_LINE[2], - 3: _VERTEX_FIRST_HALO_LINE[3] + _VERTEX_SECOND_HALO_LINE[3], -} - -OWNED = {dims.CellDim: _CELL_OWN, dims.EdgeDim: _EDGE_OWN, dims.VertexDim: _VERTEX_OWN} -HALO = {dims.CellDim: _CELL_HALO, dims.EdgeDim: _EDGE_HALO, dims.VertexDim: _VERTEX_HALO} -FIRST_HALO_LINE = { - dims.CellDim: _CELL_FIRST_HALO_LINE, - dims.VertexDim: _VERTEX_FIRST_HALO_LINE, - dims.EdgeDim: _EDGE_FIRST_HALO_LINE, -} -SECOND_HALO_LINE = { - dims.CellDim: _CELL_SECOND_HALO_LINE, - dims.VertexDim: _VERTEX_SECOND_HALO_LINE, - dims.EdgeDim: _EDGE_SECOND_HALO_LINE, -} - @pytest.fixture(scope="session") def simple_neighbor_tables(): @@ -218,11 +74,11 @@ def test_halo_constructor_owned_cells(processor_props, simple_neighbor_tables): num_levels=1, backend=backend, ) - my_owned_cells = halo_generator.owned_cells(SIMPLE_DISTRIBUTION) + my_owned_cells = halo_generator.owned_cells(utils.SIMPLE_DISTRIBUTION) print(f"rank {processor_props.rank} owns {my_owned_cells} ") - assert my_owned_cells.size == len(_CELL_OWN[processor_props.rank]) - assert np.setdiff1d(my_owned_cells, _CELL_OWN[processor_props.rank]).size == 0 + assert my_owned_cells.size == len(utils._CELL_OWN[processor_props.rank]) + assert np.setdiff1d(my_owned_cells, utils._CELL_OWN[processor_props.rank]).size == 0 @pytest.mark.parametrize("processor_props", [True, False], indirect=True) @@ -274,11 +130,9 @@ def test_element_ownership_is_unique( backend=backend, ) - decomposition_info = halo_generator(SIMPLE_DISTRIBUTION) + decomposition_info = halo_generator(utils.SIMPLE_DISTRIBUTION) owned = decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED) print(f"\nrank {processor_props.rank} owns {dim} : {owned} ") - if not mpi4py.MPI.Is_initialized(): - mpi4py.MPI.Init() # assert that each cell is only owned by one rank comm = processor_props.comm @@ -318,14 +172,16 @@ def test_halo_constructor_decomposition_info_global_indices( num_levels=1, ) - decomp_info = halo_generator(SIMPLE_DISTRIBUTION) + decomp_info = halo_generator(utils.SIMPLE_DISTRIBUTION) my_halo = decomp_info.global_index(dim, defs.DecompositionInfo.EntryType.HALO) print(f"rank {processor_props.rank} has halo {dim} : {my_halo}") - assert my_halo.size == len(HALO[dim][processor_props.rank]) - assert np.setdiff1d(my_halo, HALO[dim][processor_props.rank], assume_unique=True).size == 0 + assert my_halo.size == len(utils.HALO[dim][processor_props.rank]) + assert ( + np.setdiff1d(my_halo, utils.HALO[dim][processor_props.rank], assume_unique=True).size == 0 + ) my_owned = decomp_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED) print(f"rank {processor_props.rank} owns {dim} : {my_owned} ") - assert_same_entries(dim, my_owned, OWNED, processor_props.rank) + assert_same_entries(dim, my_owned, utils.OWNED, processor_props.rank) def assert_same_entries( @@ -346,7 +202,7 @@ def test_halo_constructor_decomposition_info_halo_levels( run_properties=processor_props, num_levels=1, ) - decomp_info = halo_generator(SIMPLE_DISTRIBUTION) + decomp_info = halo_generator(utils.SIMPLE_DISTRIBUTION) my_halo_levels = decomp_info.halo_levels(dim) print(f"{dim.value}: rank {processor_props.rank} has halo levels {my_halo_levels} ") if dim != dims.EdgeDim: @@ -356,7 +212,7 @@ def test_halo_constructor_decomposition_info_halo_levels( "All indices should have a defined DecompositionFlag" ) # THIS WILL CURRENTLY FAIL FOR EDGES assert np.where(my_halo_levels == defs.DecompositionFlag.OWNED)[0].size == len( - OWNED[dim][processor_props.rank] + utils.OWNED[dim][processor_props.rank] ) owned_local_indices = decomp_info.local_index(dim, defs.DecompositionInfo.EntryType.OWNED) assert np.all( @@ -368,14 +224,18 @@ def test_halo_constructor_decomposition_info_halo_levels( first_halo_line_global_index = decomp_info.global_index( dim, defs.DecompositionInfo.EntryType.ALL )[first_halo_line_local_index] - assert_same_entries(dim, first_halo_line_global_index, FIRST_HALO_LINE, processor_props.rank) + assert_same_entries( + dim, first_halo_line_global_index, utils.FIRST_HALO_LINE, processor_props.rank + ) second_halo_line_local_index = np.where( my_halo_levels == defs.DecompositionFlag.SECOND_HALO_LINE )[0] second_halo_line_global_index = decomp_info.global_index( dim, defs.DecompositionInfo.EntryType.ALL )[second_halo_line_local_index] - assert_same_entries(dim, second_halo_line_global_index, SECOND_HALO_LINE, processor_props.rank) + assert_same_entries( + dim, second_halo_line_global_index, utils.SECOND_HALO_LINE, processor_props.rank + ) # TODO unused - remove or fix and use? diff --git a/model/common/tests/decomposition_tests/test_definitions.py b/model/common/tests/decomposition_tests/test_definitions.py index 4fa1aec491..116f16671b 100644 --- a/model/common/tests/decomposition_tests/test_definitions.py +++ b/model/common/tests/decomposition_tests/test_definitions.py @@ -5,23 +5,109 @@ # # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause +import dataclasses + +import numpy as np import pytest +from gt4py.next import common as gtx_common + +import icon4py.model.common.dimension as dims +import icon4py.model.common.utils.data_allocation as data_alloc +from icon4py.model.common.decomposition import definitions, halo +from icon4py.model.common.grid import simple -from icon4py.model.common.decomposition.definitions import ( - DecompositionInfo, - SingleNodeExchange, - create_exchange, -) +from . import utils +from .mpi_tests.test_halo import assert_same_entries @pytest.mark.datatest def test_create_single_node_runtime_without_mpi(icon_grid, processor_props): # fixture - decomposition_info = DecompositionInfo( + decomposition_info = definitions.DecompositionInfo( klevels=10, - num_cells=icon_grid.num_cells, - num_edges=icon_grid.num_edges, - num_vertices=icon_grid.num_vertices, ) - exchange = create_exchange(processor_props, decomposition_info) + exchange = definitions.create_exchange(processor_props, decomposition_info) + + assert isinstance(exchange, definitions.SingleNodeExchange) + + +@dataclasses.dataclass(frozen=True) +class DummyProps(definitions.ProcessProperties): + def __init__(self, rank: int): + object.__setattr__(self, "rank", rank % 4) + object.__setattr__(self, "comm", None) + object.__setattr__(self, "comm_name", "dummy on 4") + object.__setattr__(self, "comm_size", 4) + + +def dummy_four_ranks(rank) -> definitions.ProcessProperties: + return DummyProps(rank=rank) + + +def get_neighbor_tables_for_simple_grid() -> dict[str, data_alloc.NDArray]: + grid = simple.simple_grid() + neighbor_tables = { + k: v.ndarray + for k, v in grid.connectivities.items() + if gtx_common.is_neighbor_connectivity(v) + } + return neighbor_tables + + +offsets = [dims.E2C, dims.E2V, dims.C2E, dims.C2E2C, dims.V2C, dims.V2E, dims.C2V, dims.E2C2V] + + +@pytest.mark.parametrize("offset", offsets) +@pytest.mark.parametrize("rank", [0, 1, 2, 3]) +def test_global_to_local_index(offset, rank): + grid = simple.simple_grid() + neighbor_tables = { + k: v.ndarray + for k, v in grid.connectivities.items() + if gtx_common.is_neighbor_connectivity(v) + } + props = dummy_four_ranks(rank) + halo_constructor = halo.IconLikeHaloConstructor(props, neighbor_tables, 1) + decomposition_info = halo_constructor(utils.SIMPLE_DISTRIBUTION) + source_indices_on_local_grid = decomposition_info.global_index(offset.target[0]) + + offset_full_grid = grid.connectivities[offset.value].ndarray[source_indices_on_local_grid] + neighbor_dim = offset.source + neighbor_index_full_grid = decomposition_info.global_index(neighbor_dim) + + local_offset = decomposition_info.global_to_local(neighbor_dim, offset_full_grid) + + ## assert by backmapping + + for i in range(local_offset.shape[0]): + for k in range(local_offset.shape[1]): + k_ = local_offset[i][k] + if k_ == -1: + # global index is not on this local patch: + assert not np.isin(offset_full_grid[i][k], neighbor_index_full_grid) + else: + ( + neighbor_index_full_grid[k_] == offset_full_grid[i][k], + f"failed to map [{offset_full_grid[i]}] to local: [{local_offset[i]}]", + ) + + +# TODO this duplicates and serializes a test from mpi_tests/test_halo.py +@pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) +@pytest.mark.parametrize("rank", [0, 1, 2, 3]) +def test_halo_constructor_decomposition_info_global_indices(dim, rank): + simple_neighbor_tables = get_neighbor_tables_for_simple_grid() + props = dummy_four_ranks(rank) + halo_generator = halo.IconLikeHaloConstructor( + connectivities=simple_neighbor_tables, + run_properties=props, + num_levels=1, + ) - assert isinstance(exchange, SingleNodeExchange) + decomp_info = halo_generator(utils.SIMPLE_DISTRIBUTION) + my_halo = decomp_info.global_index(dim, definitions.DecompositionInfo.EntryType.HALO) + print(f"rank {props.rank} has halo {dim} : {my_halo}") + assert my_halo.size == len(utils.HALO[dim][props.rank]) + assert np.setdiff1d(my_halo, utils.HALO[dim][props.rank], assume_unique=True).size == 0 + my_owned = decomp_info.global_index(dim, definitions.DecompositionInfo.EntryType.OWNED) + print(f"rank {props.rank} owns {dim} : {my_owned} ") + assert_same_entries(dim, my_owned, utils.OWNED, props.rank) diff --git a/model/common/tests/decomposition_tests/two_ranks_distribution.py b/model/common/tests/decomposition_tests/two_ranks_distribution.py new file mode 100644 index 0000000000..3585a75603 --- /dev/null +++ b/model/common/tests/decomposition_tests/two_ranks_distribution.py @@ -0,0 +1,16 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +import numpy as np + + +TWO_RANKS_DISTRIBUTION: np.ndarray = np.ones(10) +TWO_RANKS_DISTRIBUTION[5:7, 10] = 0 + + +# TODO define all the rest diff --git a/model/common/tests/decomposition_tests/utils.py b/model/common/tests/decomposition_tests/utils.py new file mode 100644 index 0000000000..77cf237b66 --- /dev/null +++ b/model/common/tests/decomposition_tests/utils.py @@ -0,0 +1,148 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +import numpy as np + +from icon4py.model.common import dimension as dims + + +""" +TESTDATA using the [SimpleGrid](../../../src/icon4py/model/common/grid/simple.py) +The distribution maps all of the 18 cells of the simple grid to ranks 0..3 + +the dictionaries contain the mapping from rank to global (in the simple grid) index of the dimension: +_CELL_OWN: rank -> owned cells, essentially the inversion of the SIMPLE_DISTRIBUTION +_EDGE_OWN: rank -> owned edges +_VERTEX_OWN: rank -> owned vertices + +the decision as to whether a "secondary" dimension (edge, vertices) is owned by a rank are made according to the +rules and conventions described in (../../../src/icon4py/model/common/decomposition/halo.py) + + +_CELL_FIRST_HALO_LINE: +_CELL_SECON_HALO_LINE: +_EDGE_FIRST_HALO_LINE: +_EDGE_SECOND_HALO_LINE: +_VERTEX_FIRST_HALO_LINE: +_VERTEX_SECOND_HALO_LINE: :mapping of rank to global indices that belongs to a ranks halo lines. +""" + + +SIMPLE_DISTRIBUTION = np.asarray( + [ + 0, # 0c + 1, # 1c + 1, # 2c + 0, # 3c + 0, # 4c + 1, # 5c + 0, # 6c + 0, # 7c + 2, # 8c + 2, # 9c + 0, # 10c + 2, # 11c + 3, # 12c + 3, # 13c + 1, # 14c + 3, # 15c + 3, # 16c + 1, # 17c + ] +) +_CELL_OWN = {0: [0, 3, 4, 6, 7, 10], 1: [1, 2, 5, 14, 17], 2: [8, 9, 11], 3: [12, 13, 15, 16]} +_CELL_FIRST_HALO_LINE = { + 0: [1, 11, 13, 9, 2, 15], + 1: [3, 8, 4, 11, 16, 13, 15], + 2: [5, 7, 6, 12, 14], + 3: [9, 10, 17, 14, 0, 1], +} +_CELL_SECOND_HALO_LINE = { + 0: [17, 5, 12, 14, 8, 16], + 1: [0, 7, 6, 9, 10, 12], + 2: [2, 1, 4, 3, 10, 15, 16, 17], + 3: [6, 7, 8, 2, 3, 4, 5, 11], +} +_CELL_HALO = { + 0: _CELL_FIRST_HALO_LINE[0] + _CELL_SECOND_HALO_LINE[0], + 1: _CELL_FIRST_HALO_LINE[1] + _CELL_SECOND_HALO_LINE[1], + 2: _CELL_FIRST_HALO_LINE[2] + _CELL_SECOND_HALO_LINE[2], + 3: _CELL_FIRST_HALO_LINE[3] + _CELL_SECOND_HALO_LINE[3], +} +_EDGE_OWN = { + 0: [1, 5, 12, 13, 14, 9], + 1: [8, 7, 6, 25, 4, 2], + 2: [16, 11, 15, 17, 10, 24], + 3: [19, 23, 22, 26, 0, 3, 20, 18, 21], +} +_EDGE_FIRST_HALO_LINE = {0: [0, 4, 17, 21, 10, 2], 1: [3, 15, 20, 26, 24], 2: [18], 3: []} +_EDGE_SECOND_HALO_LINE = { + 0: [3, 6, 7, 8, 15, 24, 25, 26, 16, 22, 23, 18, 19, 20, 11], + 1: [0, 1, 5, 9, 12, 11, 10, 13, 16, 17, 18, 19, 21, 22, 23], + 2: [2, 9, 12, 4, 8, 7, 14, 21, 13, 19, 20, 22, 23, 25, 26], + 3: [11, 10, 14, 13, 16, 17, 24, 25, 6, 2, 1, 5, 4, 8, 7], +} +_EDGE_THIRD_HALO_LINE = { + 0: [], + 1: [14], + 2: [0, 1, 3, 5, 6], + 3: [9, 12, 15], +} +_EDGE_HALO = { + 0: _EDGE_FIRST_HALO_LINE[0] + _EDGE_SECOND_HALO_LINE[0], + 1: _EDGE_FIRST_HALO_LINE[1] + _EDGE_SECOND_HALO_LINE[1], + 2: _EDGE_FIRST_HALO_LINE[2] + _EDGE_SECOND_HALO_LINE[2], + 3: _EDGE_FIRST_HALO_LINE[3] + _EDGE_SECOND_HALO_LINE[3], +} +_VERTEX_OWN = { + 0: [4], + 1: [], + 2: [3, 5], + 3: [ + 0, + 1, + 2, + 6, + 7, + 8, + ], +} +_VERTEX_FIRST_HALO_LINE = { + 0: [0, 1, 5, 8, 7, 3], + 1: [1, 2, 0, 5, 3, 8, 6], + 2: [ + 6, + 8, + 7, + ], + 3: [], +} +_VERTEX_SECOND_HALO_LINE = { + 0: [2, 6], + 1: [7, 4], + 2: [4, 0, 2, 1], + 3: [3, 4, 5], +} +_VERTEX_HALO = { + 0: _VERTEX_FIRST_HALO_LINE[0] + _VERTEX_SECOND_HALO_LINE[0], + 1: _VERTEX_FIRST_HALO_LINE[1] + _VERTEX_SECOND_HALO_LINE[1], + 2: _VERTEX_FIRST_HALO_LINE[2] + _VERTEX_SECOND_HALO_LINE[2], + 3: _VERTEX_FIRST_HALO_LINE[3] + _VERTEX_SECOND_HALO_LINE[3], +} +OWNED = {dims.CellDim: _CELL_OWN, dims.EdgeDim: _EDGE_OWN, dims.VertexDim: _VERTEX_OWN} +HALO = {dims.CellDim: _CELL_HALO, dims.EdgeDim: _EDGE_HALO, dims.VertexDim: _VERTEX_HALO} +FIRST_HALO_LINE = { + dims.CellDim: _CELL_FIRST_HALO_LINE, + dims.VertexDim: _VERTEX_FIRST_HALO_LINE, + dims.EdgeDim: _EDGE_FIRST_HALO_LINE, +} +SECOND_HALO_LINE = { + dims.CellDim: _CELL_SECOND_HALO_LINE, + dims.VertexDim: _VERTEX_SECOND_HALO_LINE, + dims.EdgeDim: _EDGE_SECOND_HALO_LINE, +} From 90cbda1d34bc0769640846f5fe85282cd2cf8d1d Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 6 Aug 2025 17:02:39 +0200 Subject: [PATCH 055/240] run pre-commit --- .../model/common/decomposition/definitions.py | 8 ++--- .../model/common/decomposition/halo.py | 29 +++++++++---------- .../icon4py/model/common/grid/grid_manager.py | 14 ++------- .../tests/decomposition_tests/__init__.py | 1 - .../icon4py/model/testing/datatest_utils.py | 5 +++- .../src/icon4py/model/testing/grid_utils.py | 4 +-- tach.toml | 2 +- 7 files changed, 25 insertions(+), 38 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 7d6a19fc15..342a668b20 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -182,13 +182,9 @@ def is_ready(self) -> bool: ... @runtime_checkable class ExchangeRuntime(Protocol): - def exchange(self, dim: Dimension, *fields: tuple) -> ExchangeResult: ... - def exchange(self, dim: gtx.Dimension, *fields: tuple) -> ExchangeResult: - ... + def exchange(self, dim: gtx.Dimension, *fields: tuple) -> ExchangeResult: ... - def exchange_and_wait(self, dim: Dimension, *fields: tuple): ... - def exchange_and_wait(self, dim: gtx.Dimension, *fields: tuple): - ... + def exchange_and_wait(self, dim: gtx.Dimension, *fields: tuple): ... def get_size(self): ... diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index abe5775e85..77080e4dbb 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -31,8 +31,7 @@ def _value(k: gtx.FieldOffset | str): class HaloConstructor(Protocol): """Callable that takes a mapping from faces (aka cells) to ranks""" - def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: - ... + def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: ... class NoHalos(HaloConstructor): @@ -251,7 +250,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: """ #: icon does hard coding of 2 halo lines for cells, make this dynamic! - num_cell_halo_lines = 2 + # TODO make number of halo lines a parameter self._validate_mapping(face_to_rank) #: cells @@ -278,12 +277,12 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: all_cells.size, dtype=int ) cell_halo_levels[cell_owner_mask] = defs.DecompositionFlag.OWNED - cell_halo_levels[ - self._xp.isin(all_cells, first_halo_cells) - ] = defs.DecompositionFlag.FIRST_HALO_LINE - cell_halo_levels[ - self._xp.isin(all_cells, second_halo_cells) - ] = defs.DecompositionFlag.SECOND_HALO_LINE + cell_halo_levels[self._xp.isin(all_cells, first_halo_cells)] = ( + defs.DecompositionFlag.FIRST_HALO_LINE + ) + cell_halo_levels[self._xp.isin(all_cells, second_halo_cells)] = ( + defs.DecompositionFlag.SECOND_HALO_LINE + ) decomp_info = defs.DecompositionInfo(klevels=self._num_levels).set_dimension( dims.CellDim, all_cells, cell_owner_mask, cell_halo_levels ) @@ -322,9 +321,9 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: self._xp.isin(all_vertices, vertex_on_cutting_line), ) ] = defs.DecompositionFlag.FIRST_HALO_LINE - vertex_halo_levels[ - self._xp.isin(all_vertices, vertex_second_level) - ] = defs.DecompositionFlag.SECOND_HALO_LINE + vertex_halo_levels[self._xp.isin(all_vertices, vertex_second_level)] = ( + defs.DecompositionFlag.SECOND_HALO_LINE + ) decomp_info.set_dimension( dims.VertexDim, all_vertices, vertex_owner_mask, vertex_halo_levels ) @@ -370,9 +369,9 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: ] = defs.DecompositionFlag.FIRST_HALO_LINE # LEVEL_TWO edges share exactly one vertext with an owned cell, they are on the first halo-line cells, but not on the cutting line - edge_halo_levels[ - self._xp.isin(all_edges, level_two_edges) - ] = defs.DecompositionFlag.SECOND_HALO_LINE + edge_halo_levels[self._xp.isin(all_edges, level_two_edges)] = ( + defs.DecompositionFlag.SECOND_HALO_LINE + ) decomp_info.set_dimension(dims.EdgeDim, all_edges, edge_owner_mask, edge_halo_levels) return decomp_info diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index dcf579e2a5..d13e22c46e 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -16,18 +16,9 @@ import numpy as np from icon4py.model.common import dimension as dims, exceptions, type_alias as ta, utils -from icon4py.model.common.decomposition import ( - definitions as decomposition, - halo, -) +from icon4py.model.common.decomposition import definitions as decomposition, halo from icon4py.model.common.decomposition.halo import HaloConstructor -from icon4py.model.common.grid import ( - base, - gridfile, - icon, - refinement, - vertical as v_grid, -) +from icon4py.model.common.grid import base, gridfile, icon, refinement, vertical as v_grid from icon4py.model.common.grid.base import HorizontalGridSize from icon4py.model.common.utils import data_allocation as data_alloc @@ -72,6 +63,7 @@ def __call__(self, array: data_alloc.NDArray): CoordinateDict: TypeAlias = dict[gtx.Dimension, dict[Literal["lat", "lon"], gtx.Field]] GeometryDict: TypeAlias = dict[gridfile.GeometryName, gtx.Field] + # TODO delete? def _reduce_to_rank_local_size( full_size_neighbor_tables: dict[gtx.FieldOffset, data_alloc.NDArray], diff --git a/model/common/tests/decomposition_tests/__init__.py b/model/common/tests/decomposition_tests/__init__.py index 80b673df7e..de9850de36 100644 --- a/model/common/tests/decomposition_tests/__init__.py +++ b/model/common/tests/decomposition_tests/__init__.py @@ -5,4 +5,3 @@ # # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause - diff --git a/model/testing/src/icon4py/model/testing/datatest_utils.py b/model/testing/src/icon4py/model/testing/datatest_utils.py index 4b2d5f786c..6c00c83df9 100644 --- a/model/testing/src/icon4py/model/testing/datatest_utils.py +++ b/model/testing/src/icon4py/model/testing/datatest_utils.py @@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Optional from gt4py.next import backend as gtx_backend + from icon4py.model.common.decomposition import definitions as decomposition @@ -130,7 +131,9 @@ def get_processor_properties_for_run( return decomposition.get_processor_properties(run_instance) -def get_ranked_data_path(base_path, processor_properties:decomposition.ProcessProperties): +def get_ranked_data_path( + base_path: pathlib.Path, processor_properties: decomposition.ProcessProperties +) -> pathlib.Path: return base_path.absolute().joinpath(f"mpitask{processor_properties.comm_size}") diff --git a/model/testing/src/icon4py/model/testing/grid_utils.py b/model/testing/src/icon4py/model/testing/grid_utils.py index 4fcdd21c12..c8385aba2c 100644 --- a/model/testing/src/icon4py/model/testing/grid_utils.py +++ b/model/testing/src/icon4py/model/testing/grid_utils.py @@ -11,8 +11,6 @@ import gt4py.next.backend as gtx_backend from icon4py.model.common.decomposition import halo -from icon4py.model.common import dimension as dims -from icon4py.model.common.decomposition import definitions as decomposition_defs from icon4py.model.common.grid import ( geometry, geometry_attributes as geometry_attrs, @@ -149,7 +147,7 @@ def get_grid_geometry( num_levels = get_num_levels(experiment) register_name = "_".join((experiment, data_alloc.backend_name(backend))) - def _construct_grid_geometry(): + def _construct_grid_geometry() -> geometry.GridGeometry: gm = _download_and_load_gridfile( grid_file, keep_skip_values=True, num_levels=num_levels, backend=backend ) diff --git a/tach.toml b/tach.toml index 91065136b8..f6ccc87f15 100644 --- a/tach.toml +++ b/tach.toml @@ -16,7 +16,7 @@ forbid_circular_dependencies = true # exclude optional external dependencies [external] exclude = ["fprettify", "configargparse", "cupy","ghex", "dace", "mpi4py", "netcdf4","xarray", "uxarray", - "cftime"] + "cftime", "pymetis"] rename = ["serialbox:serialbox4py"] [[modules]] From 875b267e54fcce2ae1a24b081d347cf31d3b8dfc Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 7 Aug 2025 10:55:22 +0200 Subject: [PATCH 056/240] fix issues from merge (WIP 1) --- model/common/src/icon4py/model/common/grid/grid_manager.py | 2 +- model/common/tests/common/grid/unit_tests/test_icon.py | 2 +- model/testing/src/icon4py/model/testing/datatest_utils.py | 6 ++---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index d13e22c46e..07f5bb5602 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -398,7 +398,7 @@ def _construct_grid( # TODO first: read local neighbor tables and convert global to local indices # TODO: instead of reading shring existing one to local size and to global to local indices neighbor_tables = _reduce_to_rank_local_size( - neighbor_tables_for_halo_construction + neighbor_tables_for_halo_construction, decomposition_info ) # reduce locally? or read again edge_index = decomposition_info.global_index(dims.EdgeDim) neighbor_tables[dims.E2V] = self._get_index_field( diff --git a/model/common/tests/common/grid/unit_tests/test_icon.py b/model/common/tests/common/grid/unit_tests/test_icon.py index 6dee82873c..5bcf26889f 100644 --- a/model/common/tests/common/grid/unit_tests/test_icon.py +++ b/model/common/tests/common/grid/unit_tests/test_icon.py @@ -85,7 +85,7 @@ def grid(icon_grid, request): @pytest.mark.datatest @pytest.mark.parametrize("dim", utils.main_horizontal_dims()) @pytest.mark.parametrize("marker", [h_grid.Zone.HALO, h_grid.Zone.HALO_LEVEL_2]) -def test_halo(grid, dim, marker): +def test_halo(icon_grid, dim, marker): # For single node this returns an empty region - start and end index are the same see also ./mpi_tests/test_icon.py domain = h_grid.domain(dim)(marker) assert grid.start_index(domain) == HALO_IDX[dim][0] diff --git a/model/testing/src/icon4py/model/testing/datatest_utils.py b/model/testing/src/icon4py/model/testing/datatest_utils.py index 6c00c83df9..8f1ee18411 100644 --- a/model/testing/src/icon4py/model/testing/datatest_utils.py +++ b/model/testing/src/icon4py/model/testing/datatest_utils.py @@ -131,10 +131,8 @@ def get_processor_properties_for_run( return decomposition.get_processor_properties(run_instance) -def get_ranked_data_path( - base_path: pathlib.Path, processor_properties: decomposition.ProcessProperties -) -> pathlib.Path: - return base_path.absolute().joinpath(f"mpitask{processor_properties.comm_size}") +def get_ranked_data_path(base_path: pathlib.Path, comm_size: int) -> pathlib.Path: + return base_path.absolute().joinpath(f"mpitask{comm_size}") def get_datapath_for_experiment( From 2df11a748181a468393e02d48907fa7a3f5ce0ac Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 8 Aug 2025 09:16:36 +0200 Subject: [PATCH 057/240] move test files --- .../decomposition}/two_ranks_distribution.py | 0 .../{decomposition_tests => common/decomposition}/utils.py | 0 .../grid}/mpi_tests/test_parallel_grid_manager.py | 0 model/common/tests/decomposition_tests/__init__.py | 7 ------- 4 files changed, 7 deletions(-) rename model/common/tests/{decomposition_tests => common/decomposition}/two_ranks_distribution.py (100%) rename model/common/tests/{decomposition_tests => common/decomposition}/utils.py (100%) rename model/common/tests/{grid_tests => common/grid}/mpi_tests/test_parallel_grid_manager.py (100%) delete mode 100644 model/common/tests/decomposition_tests/__init__.py diff --git a/model/common/tests/decomposition_tests/two_ranks_distribution.py b/model/common/tests/common/decomposition/two_ranks_distribution.py similarity index 100% rename from model/common/tests/decomposition_tests/two_ranks_distribution.py rename to model/common/tests/common/decomposition/two_ranks_distribution.py diff --git a/model/common/tests/decomposition_tests/utils.py b/model/common/tests/common/decomposition/utils.py similarity index 100% rename from model/common/tests/decomposition_tests/utils.py rename to model/common/tests/common/decomposition/utils.py diff --git a/model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py similarity index 100% rename from model/common/tests/grid_tests/mpi_tests/test_parallel_grid_manager.py rename to model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py diff --git a/model/common/tests/decomposition_tests/__init__.py b/model/common/tests/decomposition_tests/__init__.py deleted file mode 100644 index de9850de36..0000000000 --- a/model/common/tests/decomposition_tests/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# ICON4Py - ICON inspired code in Python and GT4Py -# -# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss -# All rights reserved. -# -# Please, refer to the LICENSE file in the root directory. -# SPDX-License-Identifier: BSD-3-Clause From 361b582487d35b7e5399417a0826c00dcee5f96a Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 8 Aug 2025 11:02:48 +0200 Subject: [PATCH 058/240] remove grid size form decomposition info --- model/testing/src/icon4py/model/testing/serialbox.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/model/testing/src/icon4py/model/testing/serialbox.py b/model/testing/src/icon4py/model/testing/serialbox.py index 4223dc685a..ffddf67c0f 100644 --- a/model/testing/src/icon4py/model/testing/serialbox.py +++ b/model/testing/src/icon4py/model/testing/serialbox.py @@ -443,12 +443,8 @@ def decomp_domain(self, dim): def construct_decomposition_info(self): return ( - # TODO: @halungge why are these sizes needed? decomposition.DecompositionInfo( klevels=self.num(dims.KDim), - num_cells=self.num(dims.CellDim), - num_edges=self.num(dims.EdgeDim), - num_vertices=self.num(dims.VertexDim), ) .set_dimension(*self._get_decomposition_fields(dims.CellDim)) .set_dimension(*self._get_decomposition_fields(dims.EdgeDim)) From ffe702354f66c8b34a189365c39fafe70a9b7ea0 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 8 Aug 2025 11:03:27 +0200 Subject: [PATCH 059/240] duplicate _construct_grid to make single node tests run --- .../icon4py/model/common/grid/grid_manager.py | 56 ++++++++++--------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 07f5bb5602..14da0050d3 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -349,7 +349,9 @@ def geometry(self) -> GeometryDict: def coordinates(self) -> CoordinateDict: return self._coordinates - def _construct_grid( + + + def _construct_grid_distributed( self, backend: Optional[gtx_backend.Backend], with_skip_values: bool ) -> icon.IconGrid: """Construct the grid topology from the icon grid file. @@ -385,6 +387,7 @@ def _construct_grid( dims.V2C: self._get_index_field(gridfile.ConnectivityName.V2C), dims.C2V: self._get_index_field(gridfile.ConnectivityName.C2V), dims.V2E2V: self._get_index_field(gridfile.ConnectivityName.V2E2V), + dims.E2V: self._get_index_field(gridfile.ConnectivityName.E2V) # TODO this one is not used in the halo construction } # halo_constructor - creates the decomposition info, which can then be used to generate the local patches on each rank halo_constructor = self._initialize_halo_constructor( @@ -397,19 +400,21 @@ def _construct_grid( # TODO first: read local neighbor tables and convert global to local indices # TODO: instead of reading shring existing one to local size and to global to local indices - neighbor_tables = _reduce_to_rank_local_size( - neighbor_tables_for_halo_construction, decomposition_info - ) # reduce locally? or read again - edge_index = decomposition_info.global_index(dims.EdgeDim) - neighbor_tables[dims.E2V] = self._get_index_field( - gridfile.ConnectivityName.E2V, indices=edge_index - ) - neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) + #neighbor_tables = _reduce_to_rank_local_size( + # neighbor_tables_for_halo_construction, decomposition_info + #) # reduce locally? or read again + #edge_index = decomposition_info.global_index(dims.EdgeDim) + #neighbor_tables[dims.E2V] = self._get_index_field( + # gridfile.ConnectivityName.E2V, indices=edge_index + #) + neighbor_tables = neighbor_tables_for_halo_construction + _derived_connectivities = functools.partial( _get_derived_connectivities, array_ns=xp, ) + neighbor_tables.update(_derived_connectivities(neighbor_tables, array_ns=xp)) global_params = icon.GlobalGridParams(root=grid_root, level=grid_level) start, end, _ = self._read_start_end_indices() @@ -430,30 +435,31 @@ def _construct_grid( ) return grid - def _initialize_global( - self, with_skip_values: bool, limited_area: bool, backend: gtx_backend.Backend + def _construct_grid( + self, backend: Optional[gtx_backend.Backend], with_skip_values: bool ) -> icon.IconGrid: - """ - Read basic information from the grid file: - Mostly reads global grid file parameters and dimensions. - - Args: - with_skip_values: bool whether or not to remove skip values in neighbor tables - limited_area: bool whether or not the produced grid is a limited area grid. - # TODO (@halungge) this is not directly encoded in the grid, which is why we passed it in. It could be determined from the refinement fields though. - - on_gpu: bool, whether or not we run on GPU. # TODO (@halungge) can this be removed and defined differently. + """Construct the grid topology from the icon grid file. - Returns: - IconGrid: basic grid, setup only with id and config information. + Reads connectivity fields from the grid file and constructs derived connectivities needed in + Icon4py from them. Adds constructed start/end index information to the grid. """ xp = data_alloc.import_array_ns(backend) - grid_size = self._read_full_grid_size() + refinement_fields = self._read_grid_refinement_fields(backend=backend) + limited_area = refinement.is_limited_area_grid( + refinement_fields[dims.CellDim].ndarray, array_ns=xp + ) + + num_cells = self._reader.dimension(gridfile.DimensionName.CELL_NAME) + num_edges = self._reader.dimension(gridfile.DimensionName.EDGE_NAME) + num_vertices = self._reader.dimension(gridfile.DimensionName.VERTEX_NAME) uuid_ = self._reader.attribute(gridfile.MandatoryPropertyName.GRID_UUID) grid_root = self._reader.attribute(gridfile.MandatoryPropertyName.ROOT) grid_level = self._reader.attribute(gridfile.MandatoryPropertyName.LEVEL) global_params = icon.GlobalGridParams(root=grid_root, level=grid_level) + grid_size = base.HorizontalGridSize( + num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells + ) config = base.GridConfig( horizontal_size=grid_size, vertical_size=self._vertical_config.num_levels, @@ -474,10 +480,8 @@ def _initialize_global( neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) start, end, _ = self._read_start_end_indices() - # TODO is this necessary? start_indices = {dim: start[dim] for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values()} end_indices = {dim: end[dim] for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values()} - refinement_fields = self._read_grid_refinement_fields() return icon.icon_grid( id_=uuid_, From 514b14d84578e99772c5a9cce18b7403bd0e0094 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 8 Aug 2025 11:16:48 +0200 Subject: [PATCH 060/240] test fix --- .../icon4py/model/common/grid/grid_manager.py | 19 +++++++++---------- .../tests/common/grid/unit_tests/test_icon.py | 2 +- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 14da0050d3..fca8ad4620 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -349,8 +349,6 @@ def geometry(self) -> GeometryDict: def coordinates(self) -> CoordinateDict: return self._coordinates - - def _construct_grid_distributed( self, backend: Optional[gtx_backend.Backend], with_skip_values: bool ) -> icon.IconGrid: @@ -387,7 +385,9 @@ def _construct_grid_distributed( dims.V2C: self._get_index_field(gridfile.ConnectivityName.V2C), dims.C2V: self._get_index_field(gridfile.ConnectivityName.C2V), dims.V2E2V: self._get_index_field(gridfile.ConnectivityName.V2E2V), - dims.E2V: self._get_index_field(gridfile.ConnectivityName.E2V) # TODO this one is not used in the halo construction + dims.E2V: self._get_index_field( + gridfile.ConnectivityName.E2V + ), # TODO this one is not used in the halo construction } # halo_constructor - creates the decomposition info, which can then be used to generate the local patches on each rank halo_constructor = self._initialize_halo_constructor( @@ -400,16 +400,15 @@ def _construct_grid_distributed( # TODO first: read local neighbor tables and convert global to local indices # TODO: instead of reading shring existing one to local size and to global to local indices - #neighbor_tables = _reduce_to_rank_local_size( - # neighbor_tables_for_halo_construction, decomposition_info - #) # reduce locally? or read again - #edge_index = decomposition_info.global_index(dims.EdgeDim) - #neighbor_tables[dims.E2V] = self._get_index_field( + # neighbor_tables = _reduce_to_rank_local_size( + # neighbor_tables_for_halo_construction, decomposition_info + # ) # reduce locally? or read again + # edge_index = decomposition_info.global_index(dims.EdgeDim) + # neighbor_tables[dims.E2V] = self._get_index_field( # gridfile.ConnectivityName.E2V, indices=edge_index - #) + # ) neighbor_tables = neighbor_tables_for_halo_construction - _derived_connectivities = functools.partial( _get_derived_connectivities, array_ns=xp, diff --git a/model/common/tests/common/grid/unit_tests/test_icon.py b/model/common/tests/common/grid/unit_tests/test_icon.py index 5bcf26889f..6dee82873c 100644 --- a/model/common/tests/common/grid/unit_tests/test_icon.py +++ b/model/common/tests/common/grid/unit_tests/test_icon.py @@ -85,7 +85,7 @@ def grid(icon_grid, request): @pytest.mark.datatest @pytest.mark.parametrize("dim", utils.main_horizontal_dims()) @pytest.mark.parametrize("marker", [h_grid.Zone.HALO, h_grid.Zone.HALO_LEVEL_2]) -def test_halo(icon_grid, dim, marker): +def test_halo(grid, dim, marker): # For single node this returns an empty region - start and end index are the same see also ./mpi_tests/test_icon.py domain = h_grid.domain(dim)(marker) assert grid.start_index(domain) == HALO_IDX[dim][0] From 6c7d35e81d307eba50c2dea3e895b2ebd807e007 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 8 Aug 2025 11:34:21 +0200 Subject: [PATCH 061/240] test fix (3) --- .../decomposition/mpi_tests/test_halo.py | 15 ++++---------- .../unit_tests/test_definitions.py | 20 +++++-------------- .../tests/common/decomposition/utils.py | 8 ++++++++ .../tests/common/io/unit_tests/test_io.py | 6 +++--- 4 files changed, 20 insertions(+), 29 deletions(-) diff --git a/model/common/tests/common/decomposition/mpi_tests/test_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_halo.py index 9a5add5d39..6882113387 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_halo.py @@ -17,7 +17,7 @@ from icon4py.model.common.decomposition import definitions as defs, mpi_decomposition from .. import utils - +from ..utils import assert_same_entries try: import mpi4py # import mpi4py to check for optional mpi dependency @@ -37,13 +37,13 @@ simple, vertical as v_grid, ) -from icon4py.model.testing import datatest_utils as dt_utils, helpers +from icon4py.model.testing import datatest_utils as dt_utils, helpers, definitions as test_defs -UGRID_FILE = dt_utils.GRIDS_PATH.joinpath(dt_utils.R02B04_GLOBAL).joinpath( +UGRID_FILE = test_defs.grids_path().joinpath(dt_utils.R02B04_GLOBAL).joinpath( "icon_grid_0013_R02B04_R_ugrid.nc" ) -GRID_FILE = dt_utils.GRIDS_PATH.joinpath(dt_utils.R02B04_GLOBAL).joinpath( +GRID_FILE = test_defs.grids_path().joinpath(dt_utils.R02B04_GLOBAL).joinpath( "icon_grid_0013_R02B04_R.nc" ) backend = None @@ -184,13 +184,6 @@ def test_halo_constructor_decomposition_info_global_indices( assert_same_entries(dim, my_owned, utils.OWNED, processor_props.rank) -def assert_same_entries( - dim: gtx.Dimension, my_owned: np.ndarray, reference: dict[gtx.Dimension, dict], rank: int -): - assert my_owned.size == len(reference[dim][rank]) - assert np.setdiff1d(my_owned, reference[dim][rank], assume_unique=True).size == 0 - - @pytest.mark.mpi(min_size=4) @pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) @pytest.mark.parametrize("processor_props", [True], indirect=True) diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index 02759579e8..0826610406 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -16,23 +16,13 @@ from icon4py.model.common.decomposition import definitions, halo from icon4py.model.common.grid import simple -from . import utils -from .mpi_tests.test_halo import assert_same_entries -from icon4py.model.common.decomposition.definitions import ( - DecompositionInfo, - SingleNodeExchange, - create_exchange, -) -from icon4py.model.testing.fixtures.datatest import ( # import fixtures form test_utils - backend, - data_provider, - download_ser_data, - experiment, - grid_savepoint, + +from icon4py.model.testing.fixtures.datatest import ( icon_grid, processor_props, - ranked_data_path, + ) +from .. import utils @pytest.mark.datatest @@ -125,4 +115,4 @@ def test_halo_constructor_decomposition_info_global_indices(dim, rank): assert np.setdiff1d(my_halo, utils.HALO[dim][props.rank], assume_unique=True).size == 0 my_owned = decomp_info.global_index(dim, definitions.DecompositionInfo.EntryType.OWNED) print(f"rank {props.rank} owns {dim} : {my_owned} ") - assert_same_entries(dim, my_owned, utils.OWNED, props.rank) + utils.assert_same_entries(dim, my_owned, utils.OWNED, props.rank) diff --git a/model/common/tests/common/decomposition/utils.py b/model/common/tests/common/decomposition/utils.py index 77cf237b66..41fba0917b 100644 --- a/model/common/tests/common/decomposition/utils.py +++ b/model/common/tests/common/decomposition/utils.py @@ -7,6 +7,7 @@ # SPDX-License-Identifier: BSD-3-Clause import numpy as np +from gt4py import next as gtx from icon4py.model.common import dimension as dims @@ -146,3 +147,10 @@ dims.VertexDim: _VERTEX_SECOND_HALO_LINE, dims.EdgeDim: _EDGE_SECOND_HALO_LINE, } + + +def assert_same_entries( + dim: gtx.Dimension, my_owned: np.ndarray, reference: dict[gtx.Dimension, dict], rank: int +): + assert my_owned.size == len(reference[dim][rank]) + assert np.setdiff1d(my_owned, reference[dim][rank], assume_unique=True).size == 0 diff --git a/model/common/tests/common/io/unit_tests/test_io.py b/model/common/tests/common/io/unit_tests/test_io.py index 5b2cae8f6f..88fdc490bf 100644 --- a/model/common/tests/common/io/unit_tests/test_io.py +++ b/model/common/tests/common/io/unit_tests/test_io.py @@ -106,7 +106,7 @@ def test_io_monitor_create_output_path(test_path): monitor = IOMonitor( config, vertical_params, - test_io_utils.simple_grid.config.horizontal_config, + test_io_utils.simple_grid.config.horizontal_size, test_io_utils.grid_file, test_io_utils.simple_grid.id, ) @@ -129,7 +129,7 @@ def test_io_monitor_write_ugrid_file(test_path): monitor = IOMonitor( config, vertical_params, - test_io_utils.simple_grid.config.horizontal_config, + test_io_utils.simple_grid.config.horizontal_size, test_io_utils.grid_file, "simple_grid", ) @@ -172,7 +172,7 @@ def test_io_monitor_write_and_read_ugrid_dataset(test_path, variables): monitor = IOMonitor( config, vertical_params, - grid.config.horizontal_config, + grid.config.horizontal_size, test_io_utils.grid_file, grid.id, ) From f6c68ca9cd8c3fdad7c553c50087c6478089e3ad Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 8 Aug 2025 12:54:53 +0200 Subject: [PATCH 062/240] test fix (3) offset provider for KDim --- .../icon4py/model/common/decomposition/definitions.py | 1 + model/common/tests/common/decomposition/fixtures.py | 8 ++++++++ .../tests/common/decomposition/mpi_tests/test_halo.py | 1 + .../decomposition/mpi_tests/test_mpi_decomposition.py | 11 +---------- .../decomposition/unit_tests/test_definitions.py | 5 +---- .../metric/unit_tests/test_compute_zdiff_gradp_dsl.py | 4 ++-- .../src/icon4py/model/testing/fixtures/datatest.py | 2 +- .../src/icon4py/model/testing/parallel_helpers.py | 5 ----- 8 files changed, 15 insertions(+), 22 deletions(-) create mode 100644 model/common/tests/common/decomposition/fixtures.py diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 342a668b20..2da96217d9 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -94,6 +94,7 @@ def set_dimension( ): self._global_index[dim] = global_index self._owner_mask[dim] = owner_mask + self._halo_levels[dim] = halo_levels def __init__( self, diff --git a/model/common/tests/common/decomposition/fixtures.py b/model/common/tests/common/decomposition/fixtures.py new file mode 100644 index 0000000000..7fa5fc8bc2 --- /dev/null +++ b/model/common/tests/common/decomposition/fixtures.py @@ -0,0 +1,8 @@ +from icon4py.model.testing.fixtures.datatest import ( + data_provider, + decomposition_info, + grid_savepoint, + icon_grid, + metrics_savepoint, + processor_props, +) diff --git a/model/common/tests/common/decomposition/mpi_tests/test_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_halo.py index 6882113387..366d3aac62 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_halo.py @@ -11,6 +11,7 @@ import gt4py.next as gtx import numpy as np import pytest +from ..fixtures import * # noqa: F403 import icon4py.model.common.dimension as dims from icon4py.model.common import exceptions diff --git a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py index b83e63e751..ce59ef04f4 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py @@ -11,6 +11,7 @@ from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import parallel_helpers +from ..fixtures import * # noqa: F403 try: @@ -28,16 +29,6 @@ _log = logging.getLogger(__name__) -from icon4py.model.testing.fixtures.datatest import ( # import fixtures from test_utils - decomposition_info, - grid_savepoint, - icon_grid, - metrics_savepoint, -) -from icon4py.model.testing.parallel_helpers import ( - processor_props, -) - """ running tests with mpi: diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index 0826610406..1d45f23f81 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -17,12 +17,9 @@ from icon4py.model.common.grid import simple -from icon4py.model.testing.fixtures.datatest import ( - icon_grid, - processor_props, -) from .. import utils +from ..fixtures import * # noqa: F403 @pytest.mark.datatest diff --git a/model/common/tests/common/metric/unit_tests/test_compute_zdiff_gradp_dsl.py b/model/common/tests/common/metric/unit_tests/test_compute_zdiff_gradp_dsl.py index 3ef357ea23..f0f1f3d0be 100644 --- a/model/common/tests/common/metric/unit_tests/test_compute_zdiff_gradp_dsl.py +++ b/model/common/tests/common/metric/unit_tests/test_compute_zdiff_gradp_dsl.py @@ -65,7 +65,7 @@ def test_compute_zdiff_gradp_dsl( vertical_end=icon_grid.num_levels - 1, offset_provider={ "E2C": icon_grid.get_connectivity("E2C"), - "Koff": icon_grid.get_connectivity("Koff"), + "Koff": dims.KDim, }, ) @@ -77,7 +77,7 @@ def test_compute_zdiff_gradp_dsl( c_lin_e=c_lin_e.ndarray, z_ifc=metrics_savepoint.z_ifc().ndarray, flat_idx=flat_idx_np, - z_ifc_sliced=z_ifc_ground_level, + topography=z_ifc_ground_level, nlev=icon_grid.num_levels, horizontal_start=horizontal_start_edge, horizontal_start_1=start_nudging, diff --git a/model/testing/src/icon4py/model/testing/fixtures/datatest.py b/model/testing/src/icon4py/model/testing/fixtures/datatest.py index 2f6804a7ee..438cf4fe80 100644 --- a/model/testing/src/icon4py/model/testing/fixtures/datatest.py +++ b/model/testing/src/icon4py/model/testing/fixtures/datatest.py @@ -210,7 +210,7 @@ def icon_grid( @pytest.fixture -def decomposition_info(data_provider, experiment): +def decomposition_info(data_provider:serialbox.IconSerialDataProvider, experiment:str)->definitions.DecompositionInfo: root, level = dt_utils.get_global_grid_params(experiment) grid_id = dt_utils.get_grid_id_for_experiment(experiment) return data_provider.from_savepoint_grid( diff --git a/model/testing/src/icon4py/model/testing/parallel_helpers.py b/model/testing/src/icon4py/model/testing/parallel_helpers.py index 4b7d3d3fea..ec98c2989d 100644 --- a/model/testing/src/icon4py/model/testing/parallel_helpers.py +++ b/model/testing/src/icon4py/model/testing/parallel_helpers.py @@ -18,8 +18,3 @@ def check_comm_size(props: ProcessProperties, sizes: tuple[int, ...] = (1, 2, 4) if props.comm_size not in sizes: pytest.xfail(f"wrong comm size: {props.comm_size}: test only works for comm-sizes: {sizes}") - -@pytest.fixture(scope="session") -def processor_props(request: pytest.FixtureRequest) -> Iterable[ProcessProperties]: - runtype = get_runtype(with_mpi=True) - yield get_multinode_properties(runtype) From ed2336d847bce8b84d20a6d70aaf3b094615d1db Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 8 Aug 2025 13:22:58 +0200 Subject: [PATCH 063/240] remove duplicated test --- model/common/tests/common/decomposition/fixtures.py | 8 ++++++++ .../tests/common/decomposition/mpi_tests/test_halo.py | 10 ++++++---- .../decomposition/mpi_tests/test_mpi_decomposition.py | 11 ----------- .../decomposition/unit_tests/test_definitions.py | 5 ++--- model/common/tests/common/grid/fixtures.py | 11 +++++++++++ .../tests/common/grid/mpi_tests/test_parallel_icon.py | 2 +- .../src/icon4py/model/testing/fixtures/datatest.py | 4 +++- .../src/icon4py/model/testing/parallel_helpers.py | 5 +---- 8 files changed, 32 insertions(+), 24 deletions(-) create mode 100644 model/common/tests/common/grid/fixtures.py diff --git a/model/common/tests/common/decomposition/fixtures.py b/model/common/tests/common/decomposition/fixtures.py index 7fa5fc8bc2..0ebc2233fc 100644 --- a/model/common/tests/common/decomposition/fixtures.py +++ b/model/common/tests/common/decomposition/fixtures.py @@ -1,3 +1,11 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + from icon4py.model.testing.fixtures.datatest import ( data_provider, decomposition_info, diff --git a/model/common/tests/common/decomposition/mpi_tests/test_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_halo.py index 366d3aac62..9fb30fc1de 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_halo.py @@ -41,11 +41,13 @@ from icon4py.model.testing import datatest_utils as dt_utils, helpers, definitions as test_defs -UGRID_FILE = test_defs.grids_path().joinpath(dt_utils.R02B04_GLOBAL).joinpath( - "icon_grid_0013_R02B04_R_ugrid.nc" +UGRID_FILE = ( + test_defs.grids_path() + .joinpath(dt_utils.R02B04_GLOBAL) + .joinpath("icon_grid_0013_R02B04_R_ugrid.nc") ) -GRID_FILE = test_defs.grids_path().joinpath(dt_utils.R02B04_GLOBAL).joinpath( - "icon_grid_0013_R02B04_R.nc" +GRID_FILE = ( + test_defs.grids_path().joinpath(dt_utils.R02B04_GLOBAL).joinpath("icon_grid_0013_R02B04_R.nc") ) backend = None diff --git a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py index ce59ef04f4..19b1cf94e5 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py @@ -29,7 +29,6 @@ _log = logging.getLogger(__name__) - """ running tests with mpi: @@ -205,16 +204,6 @@ def test_create_multi_node_runtime_with_mpi( assert isinstance(exchange, definitions.SingleNodeExchange) -@pytest.mark.parametrize("processor_props", [False], indirect=True) -@pytest.mark.mpi_skip() -def test_create_single_node_runtime_without_mpi( - processor_props, - decomposition_info, -): - exchange = definitions.create_exchange(processor_props, decomposition_info) - assert isinstance(exchange, definitions.SingleNodeExchange) - - @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize("dimension", (dims.CellDim, dims.VertexDim, dims.EdgeDim)) diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index 1d45f23f81..5fc47e06c2 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -17,13 +17,12 @@ from icon4py.model.common.grid import simple - from .. import utils from ..fixtures import * # noqa: F403 -@pytest.mark.datatest -def test_create_single_node_runtime_without_mpi(icon_grid, processor_props): # fixture +@pytest.mark.parametrize("processor_props", [False], indirect=True) +def test_create_single_node_runtime_without_mpi(processor_props): # fixture decomposition_info = definitions.DecompositionInfo( klevels=10, ) diff --git a/model/common/tests/common/grid/fixtures.py b/model/common/tests/common/grid/fixtures.py new file mode 100644 index 0000000000..f82928069d --- /dev/null +++ b/model/common/tests/common/grid/fixtures.py @@ -0,0 +1,11 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +from icon4py.model.testing.fixtures.datatest import ( + processor_props, +) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py index f38593997a..3f96bd1513 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py @@ -14,11 +14,11 @@ import icon4py.model.common.grid.horizontal as h_grid from icon4py.model.testing.parallel_helpers import ( check_comm_size, - processor_props, ) from icon4py.model.testing import parallel_helpers from .. import utils +from .. import fixtures # noqa F403 try: diff --git a/model/testing/src/icon4py/model/testing/fixtures/datatest.py b/model/testing/src/icon4py/model/testing/fixtures/datatest.py index 438cf4fe80..8ee70b7de9 100644 --- a/model/testing/src/icon4py/model/testing/fixtures/datatest.py +++ b/model/testing/src/icon4py/model/testing/fixtures/datatest.py @@ -210,7 +210,9 @@ def icon_grid( @pytest.fixture -def decomposition_info(data_provider:serialbox.IconSerialDataProvider, experiment:str)->definitions.DecompositionInfo: +def decomposition_info( + data_provider: serialbox.IconSerialDataProvider, experiment: str +) -> definitions.DecompositionInfo: root, level = dt_utils.get_global_grid_params(experiment) grid_id = dt_utils.get_grid_id_for_experiment(experiment) return data_provider.from_savepoint_grid( diff --git a/model/testing/src/icon4py/model/testing/parallel_helpers.py b/model/testing/src/icon4py/model/testing/parallel_helpers.py index ec98c2989d..d5ca267629 100644 --- a/model/testing/src/icon4py/model/testing/parallel_helpers.py +++ b/model/testing/src/icon4py/model/testing/parallel_helpers.py @@ -6,15 +6,12 @@ # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause -from typing import Iterable import pytest -from icon4py.model.common.decomposition.definitions import ProcessProperties, get_runtype -from icon4py.model.common.decomposition.mpi_decomposition import get_multinode_properties +from icon4py.model.common.decomposition.definitions import ProcessProperties def check_comm_size(props: ProcessProperties, sizes: tuple[int, ...] = (1, 2, 4)) -> None: if props.comm_size not in sizes: pytest.xfail(f"wrong comm size: {props.comm_size}: test only works for comm-sizes: {sizes}") - From fc65d6d4804bbbbe86fd2212fe8ce96d4e2cb9a1 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 8 Aug 2025 17:47:40 +0200 Subject: [PATCH 064/240] apply global to local --- .../icon4py/model/common/grid/grid_manager.py | 121 +++++------------- model/common/tests/common/grid/fixtures.py | 2 + .../grid/unit_tests/test_grid_manager.py | 11 +- 3 files changed, 37 insertions(+), 97 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index fca8ad4620..2291b6b284 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -349,7 +349,9 @@ def geometry(self) -> GeometryDict: def coordinates(self) -> CoordinateDict: return self._coordinates - def _construct_grid_distributed( + + + def _construct_grid( self, backend: Optional[gtx_backend.Backend], with_skip_values: bool ) -> icon.IconGrid: """Construct the grid topology from the icon grid file. @@ -364,9 +366,16 @@ def _construct_grid_distributed( global_grid_size = self._read_full_grid_size() grid_root = self._reader.attribute(gridfile.MandatoryPropertyName.ROOT) grid_level = self._reader.attribute(gridfile.MandatoryPropertyName.LEVEL) + global_params = icon.GlobalGridParams(root=grid_root, level=grid_level) _determine_limited_area = functools.partial(refinement.is_limited_area_grid, array_ns=xp) refinement_fields = functools.partial(self._read_grid_refinement_fields, backend=backend)() limited_area = _determine_limited_area(refinement_fields[dims.CellDim].ndarray) + grid_config = base.GridConfig( + horizontal_size=global_grid_size, + vertical_size=self._vertical_config.num_levels, + limited_area=limited_area, + keep_skip_values=with_skip_values, + ) # DECOMPOSITION cell_to_cell_neighbors = self._get_index_field(gridfile.ConnectivityName.C2E2C) @@ -376,7 +385,7 @@ def _construct_grid_distributed( ) # HALO CONSTRUCTION # TODO: (magdalena) reduce the set of neighbor tables used in the halo construction - # TODO: (magdalena) this is all numpy currently! + # TODO: (magdalena) figure out where to do the host to device copies (xp.asarray...) neighbor_tables_for_halo_construction = { dims.C2E2C: cell_to_cell_neighbors, dims.C2E: self._get_index_field(gridfile.ConnectivityName.C2E), @@ -385,43 +394,39 @@ def _construct_grid_distributed( dims.V2C: self._get_index_field(gridfile.ConnectivityName.V2C), dims.C2V: self._get_index_field(gridfile.ConnectivityName.C2V), dims.V2E2V: self._get_index_field(gridfile.ConnectivityName.V2E2V), - dims.E2V: self._get_index_field( - gridfile.ConnectivityName.E2V - ), # TODO this one is not used in the halo construction - } + dims.E2V: self._get_index_field(gridfile.ConnectivityName.E2V), + } # halo_constructor - creates the decomposition info, which can then be used to generate the local patches on each rank halo_constructor = self._initialize_halo_constructor( global_grid_size, neighbor_tables_for_halo_construction, backend ) decomposition_info = halo_constructor(cells_to_rank_mapping) + self._decomposition_info = decomposition_info + my_cells = decomposition_info.global_index(dims.CellDim) + my_edges = decomposition_info.global_index(dims.EdgeDim) + my_vertices = decomposition_info.global_index(dims.VertexDim) + - ## TODO from here do local reads (and halo exchanges!!) + ## TODO do local reads (and halo exchanges!!) FIX: my_cells etc are in 0 base python coding - reading from file fails... + ## # CONSTRUCT LOCAL PATCH - # TODO first: read local neighbor tables and convert global to local indices - # TODO: instead of reading shring existing one to local size and to global to local indices - # neighbor_tables = _reduce_to_rank_local_size( - # neighbor_tables_for_halo_construction, decomposition_info - # ) # reduce locally? or read again - # edge_index = decomposition_info.global_index(dims.EdgeDim) - # neighbor_tables[dims.E2V] = self._get_index_field( - # gridfile.ConnectivityName.E2V, indices=edge_index - # ) - neighbor_tables = neighbor_tables_for_halo_construction - - _derived_connectivities = functools.partial( - _get_derived_connectivities, - array_ns=xp, - ) - neighbor_tables.update(_derived_connectivities(neighbor_tables, array_ns=xp)) + if not self._run_properties.single_node(): + neighbor_tables = { + k: decomposition_info.global_to_local( + k.target[0], v[decomposition_info.global_index(k.target[0])] + ) for k,v in neighbor_tables_for_halo_construction.items() + } + else: + neighbor_tables = neighbor_tables_for_halo_construction - global_params = icon.GlobalGridParams(root=grid_root, level=grid_level) + + # COMPUTE remaining derived connectivities + + neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) + # TODO compute for local patch start, end, _ = self._read_start_end_indices() - grid_config = base.GridConfig( - horizontal_size=global_grid_size, - vertical_size=self._vertical_config.num_levels, - limited_area=limited_area, - ) + grid = icon.icon_grid( uuid_, allocator=backend, @@ -434,64 +439,6 @@ def _construct_grid_distributed( ) return grid - def _construct_grid( - self, backend: Optional[gtx_backend.Backend], with_skip_values: bool - ) -> icon.IconGrid: - """Construct the grid topology from the icon grid file. - - Reads connectivity fields from the grid file and constructs derived connectivities needed in - Icon4py from them. Adds constructed start/end index information to the grid. - - """ - xp = data_alloc.import_array_ns(backend) - refinement_fields = self._read_grid_refinement_fields(backend=backend) - limited_area = refinement.is_limited_area_grid( - refinement_fields[dims.CellDim].ndarray, array_ns=xp - ) - - num_cells = self._reader.dimension(gridfile.DimensionName.CELL_NAME) - num_edges = self._reader.dimension(gridfile.DimensionName.EDGE_NAME) - num_vertices = self._reader.dimension(gridfile.DimensionName.VERTEX_NAME) - uuid_ = self._reader.attribute(gridfile.MandatoryPropertyName.GRID_UUID) - grid_root = self._reader.attribute(gridfile.MandatoryPropertyName.ROOT) - grid_level = self._reader.attribute(gridfile.MandatoryPropertyName.LEVEL) - global_params = icon.GlobalGridParams(root=grid_root, level=grid_level) - grid_size = base.HorizontalGridSize( - num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells - ) - config = base.GridConfig( - horizontal_size=grid_size, - vertical_size=self._vertical_config.num_levels, - limited_area=limited_area, - keep_skip_values=with_skip_values, - ) - - neighbor_tables = { - dims.C2E2C: xp.asarray(self._get_index_field(gridfile.ConnectivityName.C2E2C)), - dims.C2E: xp.asarray(self._get_index_field(gridfile.ConnectivityName.C2E)), - dims.E2C: xp.asarray(self._get_index_field(gridfile.ConnectivityName.E2C)), - dims.V2E: xp.asarray(self._get_index_field(gridfile.ConnectivityName.V2E)), - dims.E2V: xp.asarray(self._get_index_field(gridfile.ConnectivityName.E2V)), - dims.V2C: xp.asarray(self._get_index_field(gridfile.ConnectivityName.V2C)), - dims.C2V: xp.asarray(self._get_index_field(gridfile.ConnectivityName.C2V)), - dims.V2E2V: xp.asarray(self._get_index_field(gridfile.ConnectivityName.V2E2V)), - } - neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) - - start, end, _ = self._read_start_end_indices() - start_indices = {dim: start[dim] for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values()} - end_indices = {dim: end[dim] for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values()} - - return icon.icon_grid( - id_=uuid_, - allocator=backend, - config=config, - neighbor_tables=neighbor_tables, - start_indices=start_indices, - end_indices=end_indices, - global_properties=global_params, - refinement_control=refinement_fields, - ) def _get_index_field( self, diff --git a/model/common/tests/common/grid/fixtures.py b/model/common/tests/common/grid/fixtures.py index f82928069d..b2c7b68e42 100644 --- a/model/common/tests/common/grid/fixtures.py +++ b/model/common/tests/common/grid/fixtures.py @@ -8,4 +8,6 @@ from icon4py.model.testing.fixtures.datatest import ( processor_props, + grid_savepoint, +data_provider,download_ser_data, ranked_data_path ) diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index d3b4b6b77c..f20cb037cb 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -36,17 +36,8 @@ pytest.skip("optional netcdf dependency not installed", allow_module_level=True) -from icon4py.model.testing.fixtures import ( - backend, - data_provider, - download_ser_data, - grid_savepoint, - processor_props, - ranked_data_path, -) - from .. import utils - +from ..fixtures import * MCH_CH_RO4B09_GLOBAL_NUM_CELLS = 83886080 From 2bec8d89561afdf43c24015de4f337ff847dda5b Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Mon, 11 Aug 2025 14:51:11 +0200 Subject: [PATCH 065/240] fix global to local transformation in grid_manager --- .../model/common/decomposition/halo.py | 2 +- .../icon4py/model/common/grid/grid_manager.py | 40 ++++--------------- model/common/tests/common/grid/fixtures.py | 4 +- 3 files changed, 12 insertions(+), 34 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 77080e4dbb..34a08c7db2 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -418,7 +418,7 @@ def __call__(self, adjacency_matrix: data_alloc.NDArray, n_part: int) -> data_al import pymetis - cut_count, partition_index = pymetis.part_graph(nparts=n_part, adjacency=adjacency_matrix) + _, partition_index = pymetis.part_graph(nparts=n_part, adjacency=adjacency_matrix) return np.array(partition_index) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 2291b6b284..3384e0933a 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -64,21 +64,6 @@ def __call__(self, array: data_alloc.NDArray): GeometryDict: TypeAlias = dict[gridfile.GeometryName, gtx.Field] -# TODO delete? -def _reduce_to_rank_local_size( - full_size_neighbor_tables: dict[gtx.FieldOffset, data_alloc.NDArray], - decomposition_info: decomposition.DecompositionInfo, -) -> dict[gtx.FieldOffset, data_alloc.NDArray]: - def get_rank_local_values(k: gtx.FieldOffset, v: data_alloc.NDArray): - index_target_dim = k.source - index_source_dim = k.target[0] - - index = decomposition_info.global_index(index_source_dim) - return v[index, :] - - return {k: get_rank_local_values(k, v) for k, v in full_size_neighbor_tables.items()} - - class GridManager: """ Read ICON grid file and set up grid topology, refinement information and geometry fields. @@ -349,8 +334,6 @@ def geometry(self) -> GeometryDict: def coordinates(self) -> CoordinateDict: return self._coordinates - - def _construct_grid( self, backend: Optional[gtx_backend.Backend], with_skip_values: bool ) -> icon.IconGrid: @@ -395,31 +378,25 @@ def _construct_grid( dims.C2V: self._get_index_field(gridfile.ConnectivityName.C2V), dims.V2E2V: self._get_index_field(gridfile.ConnectivityName.V2E2V), dims.E2V: self._get_index_field(gridfile.ConnectivityName.E2V), - } + } # halo_constructor - creates the decomposition info, which can then be used to generate the local patches on each rank halo_constructor = self._initialize_halo_constructor( global_grid_size, neighbor_tables_for_halo_construction, backend ) decomposition_info = halo_constructor(cells_to_rank_mapping) self._decomposition_info = decomposition_info - my_cells = decomposition_info.global_index(dims.CellDim) - my_edges = decomposition_info.global_index(dims.EdgeDim) - my_vertices = decomposition_info.global_index(dims.VertexDim) - ## TODO do local reads (and halo exchanges!!) FIX: my_cells etc are in 0 base python coding - reading from file fails... ## # CONSTRUCT LOCAL PATCH - if not self._run_properties.single_node(): - neighbor_tables = { - k: decomposition_info.global_to_local( - k.target[0], v[decomposition_info.global_index(k.target[0])] - ) for k,v in neighbor_tables_for_halo_construction.items() - } - else: - neighbor_tables = neighbor_tables_for_halo_construction - + # TODO run this onlz for distrbuted grids otherwise to nothing internally + neighbor_tables = { + k: decomposition_info.global_to_local( + k.source, v[decomposition_info.global_index(k.target[0])] + ) + for k, v in neighbor_tables_for_halo_construction.items() + } # COMPUTE remaining derived connectivities @@ -439,7 +416,6 @@ def _construct_grid( ) return grid - def _get_index_field( self, field: gridfile.GridFileName, diff --git a/model/common/tests/common/grid/fixtures.py b/model/common/tests/common/grid/fixtures.py index b2c7b68e42..3e550a25f3 100644 --- a/model/common/tests/common/grid/fixtures.py +++ b/model/common/tests/common/grid/fixtures.py @@ -9,5 +9,7 @@ from icon4py.model.testing.fixtures.datatest import ( processor_props, grid_savepoint, -data_provider,download_ser_data, ranked_data_path + data_provider, + download_ser_data, + ranked_data_path, ) From 7b0d4d87469b9a3b3584a3f631f3becb479661ad Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 13 Aug 2025 10:07:01 +0200 Subject: [PATCH 066/240] compute start end indices WIP (i) --- .../icon4py/model/common/grid/horizontal.py | 25 +++++++++------- .../icon4py/model/common/grid/refinement.py | 21 ++++++++++++-- .../grid/unit_tests/test_grid_manager.py | 2 +- .../common/grid/unit_tests/test_refinement.py | 29 +++++++++++++++++-- 4 files changed, 62 insertions(+), 15 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/horizontal.py b/model/common/src/icon4py/model/common/grid/horizontal.py index 8e1dcfc1ef..6827eb57ff 100644 --- a/model/common/src/icon4py/model/common/grid/horizontal.py +++ b/model/common/src/icon4py/model/common/grid/horizontal.py @@ -37,7 +37,7 @@ import enum import functools from abc import abstractmethod -from typing import Final, Protocol +from typing import Final, Protocol, TypedDict import gt4py.next as gtx @@ -79,51 +79,56 @@ _LOCAL_CELLS: Final[int] = _MIN_RL_CELL_INT + _ICON_INDEX_OFFSET_CELLS # 4 _END_CELLS: Final[int] = 0 -_LATERAL_BOUNDARY_VERTICES = 1 + _ICON_INDEX_OFFSET_VERTEX # 8 +_LATERAL_BOUNDARY_VERTICES:Final[int] = 1 + _ICON_INDEX_OFFSET_VERTEX # 8 _INTERIOR_VERTICES: Final[int] = _ICON_INDEX_OFFSET_VERTEX # 7 _NUDGING_VERTICES: Final[int] = 0 _HALO_VERTICES: Final[int] = _MIN_RL_VERTEX_INT - 1 + _ICON_INDEX_OFFSET_VERTEX # 2 _LOCAL_VERTICES: Final[int] = _MIN_RL_VERTEX_INT + _ICON_INDEX_OFFSET_VERTEX # 3 _END_VERTICES: Final[int] = 0 - _EDGE_GRF: Final[int] = 24 _CELL_GRF: Final[int] = 14 _VERTEX_GRF: Final[int] = 13 +GRID_REFINEMENT_SIZE: Final[dict[gtx.Dimension, int]] = { + dims.CellDim: _CELL_GRF, dims.EdgeDim: _EDGE_GRF, dims.VertexDim: _VERTEX_GRF +} + -_LATERAL_BOUNDARY = { + +_LATERAL_BOUNDARY:Final[dict[gtx.Dimension, int]] = { dims.CellDim: _LATERAL_BOUNDARY_CELLS, dims.EdgeDim: _LATERAL_BOUNDARY_EDGES, dims.VertexDim: _LATERAL_BOUNDARY_VERTICES, } -_LOCAL = { +_LOCAL:Final[dict[gtx.Dimension, int]] = { dims.CellDim: _LOCAL_CELLS, dims.EdgeDim: _LOCAL_EDGES, dims.VertexDim: _LOCAL_VERTICES, } -_HALO = { +_HALO:Final[dict[gtx.Dimension, int]] = { dims.CellDim: _HALO_CELLS, dims.EdgeDim: _HALO_EDGES, dims.VertexDim: _HALO_VERTICES, } -_INTERIOR = { +_INTERIOR:Final[dict[gtx.Dimension, int]] = { dims.CellDim: _INTERIOR_CELLS, dims.EdgeDim: _INTERIOR_EDGES, dims.VertexDim: _INTERIOR_VERTICES, } -_NUDGING = { + +_NUDGING:Final[dict[gtx.Dimension, int]] = { dims.CellDim: _NUDGING_CELLS, dims.EdgeDim: _NUDGING_EDGES, dims.VertexDim: _NUDGING_VERTICES, } -_END = { +_END:Final[dict[gtx.Dimension, int]] = { dims.CellDim: _END_CELLS, dims.EdgeDim: _END_EDGES, dims.VertexDim: _END_VERTICES, } -_BOUNDS = { +_BOUNDS: Final[dict[gtx.Dimension, tuple[int, int]]]= { dims.CellDim: (0, _CELL_GRF - 1), dims.EdgeDim: (0, _EDGE_GRF - 1), dims.VertexDim: (0, _VERTEX_GRF - 1), diff --git a/model/common/src/icon4py/model/common/grid/refinement.py b/model/common/src/icon4py/model/common/grid/refinement.py index a16c39b765..ed6c6fcc26 100644 --- a/model/common/src/icon4py/model/common/grid/refinement.py +++ b/model/common/src/icon4py/model/common/grid/refinement.py @@ -16,7 +16,7 @@ import icon4py.model.common.grid.horizontal as h_grid from icon4py.model.common import dimension as dims from icon4py.model.common.utils import data_allocation as data_alloc - +from icon4py.model.common.utils.data_allocation import zero_field """ Refinement control for ICON grid. @@ -33,6 +33,7 @@ """ _log = logging.getLogger(__name__) +#TODO get these from grid file cell_grf, edge_grf, vertex_grf _MAX_ORDERED: Final[dict[gtx.Dimension, int]] = { dims.CellDim: 14, dims.EdgeDim: 24, @@ -40,7 +41,6 @@ } """Lateral boundary points are ordered and have an index indicating the (cell) s distance to the boundary, generally the number of ordered rows can be defined in the grid generator, but it will never exceed 14 for cells. -TODO: Are these the x_grf dimension in the netcdf grid file? """ @@ -63,6 +63,8 @@ """Start refin_ctrl levels for boundary nudging (as seen from the child domain).""" + + @dataclasses.dataclass(frozen=True) class RefinementValue: dim: gtx.Dimension @@ -81,6 +83,7 @@ def is_ordered(self) -> bool: return self.value not in _UNORDERED[self.dim] + def is_unordered_field( field: data_alloc.NDArray, dim: gtx.Dimension, array_ns: ModuleType = np ) -> data_alloc.NDArray: @@ -124,3 +127,17 @@ def is_limited_area_grid( The .item() call is needed to get a scalar return for cupy arrays. """ return array_ns.any(refinement_field > 0).item() + + +def compute_start_index(dim: gtx.Dimension, refinement_ctrl: data_alloc.NDArray, array_ns:ModuleType = np) -> data_alloc.NDArray: + """Compute the start index for the refinement control field for a given dimension.""" + assert dim.kind == gtx.DimensionKind.HORIZONTAL, f"dim = {dim=} refinement control values only exist for horizontal dimensions" + shape = (_MAX_ORDERED[dim],) + + starts = array_ns.zeros(shape, dtype=gtx.int32) + index = array_ns.min(array_ns.where(refinement_ctrl == h_grid.LineNumber.FIRST)) + starts[h_grid._LATERAL_BOUNDARY[dim]] = index + index = array_ns.min(array_ns.where(refinement_ctrl == h_grid.LineNumber.SECOND)) + starts[h_grid._LATERAL_BOUNDARY[dim] + 1] = index + + return starts diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index f20cb037cb..2d9cdfa50f 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -90,7 +90,7 @@ def test_grid_manager_refin_ctrl(grid_savepoint, grid_file, experiment, dim, bac ).grid.refinement_control refin_ctrl_serialized = grid_savepoint.refin_ctrl(dim) assert np.all( - refin_ctrl_serialized.ndarray + refin_ctrl_serialized.ndarrayg == refin.convert_to_unnested_refinement_values(refin_ctrl[dim].ndarray, dim) ) diff --git a/model/common/tests/common/grid/unit_tests/test_refinement.py b/model/common/tests/common/grid/unit_tests/test_refinement.py index 0aa17ae11f..7247d594d5 100644 --- a/model/common/tests/common/grid/unit_tests/test_refinement.py +++ b/model/common/tests/common/grid/unit_tests/test_refinement.py @@ -9,13 +9,14 @@ import gt4py.next as gtx import pytest -import icon4py.model.common.grid.refinement as refin +from icon4py.model.common.grid import refinement as refin, horizontal as h_grid +from icon4py.model.common import dimension as dims from icon4py.model.common.utils import data_allocation as data_alloc, device_utils from icon4py.model.testing import datatest_utils as dt_utils, grid_utils from icon4py.model.testing.fixtures import backend from .. import utils - +from ..fixtures import * def out_of_range(dim: gtx.Dimension): lower = range(-36, refin._UNORDERED[dim][1]) @@ -73,3 +74,27 @@ def test_is_local_area_grid_for_grid_files(grid_file, expected, dim, backend): limited_area = refin.is_limited_area_grid(refinement_field.ndarray, array_ns=xp) assert isinstance(limited_area, bool) assert expected == limited_area + +@pytest.fixture +def start_indices(grid_savepoint)->dict: + return {dims.CellDim:grid_savepoint.cells_start_index(),dims.EdgeDim:grid_savepoint.edge_start_index(),dims.VertexDim:grid_savepoint.vertex_start_index()} + +@pytest.mark.parametrize("dim", (dims.CellDim, dims.EdgeDim, dims.VertexDim)) +@pytest.mark.parametrize( + "grid_file, experiment", [(dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT)] +) +def test_compute_start_index(dim, grid_file, experiment, start_indices): + reference_start = start_indices.get(dim) + grid = grid_utils.get_grid_manager(grid_file, num_levels=1, keep_skip_values=True, backend=None).grid + refinement_control_field = grid.refinement_control[dim] + start_index = refin.compute_start_index(dim, refinement_control_field.ndarray) + assert start_index.ndim == 1 + assert start_index.shape[0] == h_grid.GRID_REFINEMENT_SIZE[dim] + domain = h_grid.domain(dim) + assert start_index[domain(h_grid.Zone.LATERAL_BOUNDARY)()] == reference_start[domain(h_grid.Zone.LATERAL_BOUNDARY)()] + assert start_index[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2)()] == reference_start[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2)()] + assert start_index[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_3)()] == reference_start[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_3)()] + assert start_index[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_4)()] == reference_start[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_4)()] + assert start_index[domain(h_grid.Zone.NUDGING)()] == reference_start[domain(h_grid.Zone.NUDGING)()] + assert start_index[domain(h_grid.Zone.INTERIOR)()] == reference_start[domain(h_grid.Zone.INTERIOR)()] + From b9472a2a1838473e4e5d9daea0d414b37bc5abb7 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 13 Aug 2025 13:12:40 +0200 Subject: [PATCH 067/240] change start_indices and end_indices to map from domain to index --- .../src/icon4py/model/common/grid/base.py | 9 ++--- .../icon4py/model/common/grid/grid_manager.py | 26 ++++++++++--- .../icon4py/model/common/grid/horizontal.py | 37 ++++++++++++++++++- .../src/icon4py/model/common/grid/icon.py | 8 ++-- .../grid/unit_tests/test_grid_manager.py | 15 ++------ .../common/grid/unit_tests/test_horizontal.py | 30 +++++++++++++++ .../src/icon4py/model/testing/serialbox.py | 15 ++++---- 7 files changed, 105 insertions(+), 35 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/base.py b/model/common/src/icon4py/model/common/grid/base.py index 0113eb26ec..2d7e966bcc 100644 --- a/model/common/src/icon4py/model/common/grid/base.py +++ b/model/common/src/icon4py/model/common/grid/base.py @@ -98,9 +98,8 @@ class Grid: connectivities: gtx_common.OffsetProvider geometry_type: GeometryType # only used internally for `start_index` and `end_index` public interface: - # TODO(havogt): consider refactoring to `Mapping[h_grid.Zone, gtx.int32]` - _start_indices: dict[gtx.Dimension, Mapping[int, gtx.int32]] - _end_indices: dict[gtx.Dimension, Mapping[int, gtx.int32]] + _start_indices: dict[gtx.Dimension, Mapping[h_grid.Domain, gtx.int32]] + _end_indices: dict[gtx.Dimension, Mapping[h_grid.Domain, gtx.int32]] def __post_init__(self): # TODO(havogt): replace `Koff[k]` by `KDim + k` syntax and remove the following line. @@ -176,7 +175,7 @@ def start_index(self, domain: h_grid.Domain) -> gtx.int32: if domain.local: # special treatment because this value is not set properly in the underlying data. return gtx.int32(0) - return gtx.int32(self._start_indices[domain.dim][domain()]) + return self._start_indices[domain.dim][domain] def end_index(self, domain: h_grid.Domain) -> gtx.int32: """ @@ -188,7 +187,7 @@ def end_index(self, domain: h_grid.Domain) -> gtx.int32: if domain.zone == h_grid.Zone.INTERIOR and not self.limited_area: # special treatment because this value is not set properly in the underlying data, for a global grid return gtx.int32(self.size[domain.dim]) - return gtx.int32(self._end_indices[domain.dim][domain()]) + return gtx.int32(self._end_indices[domain.dim][domain]) def construct_connectivity( diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index f2b80c2eb6..693e6b1bdb 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -16,7 +16,14 @@ from icon4py.model.common import dimension as dims, type_alias as ta from icon4py.model.common.decomposition import definitions as decomposition -from icon4py.model.common.grid import base, gridfile, icon, refinement, vertical as v_grid +from icon4py.model.common.grid import ( + base, + gridfile, + horizontal as h_grid, + icon, + refinement, + vertical as v_grid, +) from icon4py.model.common.utils import data_allocation as data_alloc @@ -245,9 +252,9 @@ def _read_grid_refinement_fields( def _read_start_end_indices( self, ) -> tuple[ - dict[gtx.Dimension : data_alloc.NDArray], - dict[gtx.Dimension : data_alloc.NDArray], - dict[gtx.Dimension : gtx.int32], + dict[gtx.Dimension, data_alloc.NDArray], + dict[gtx.Dimension, data_alloc.NDArray], + dict[gtx.Dimension, gtx.int32], ]: """ " Read the start/end indices from the grid file. @@ -354,8 +361,15 @@ def _construct_grid( neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) start, end, _ = self._read_start_end_indices() - start_indices = {dim: start[dim] for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values()} - end_indices = {dim: end[dim] for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values()} + + start_indices = { + dim: h_grid.map_domain_bounds(dim, start[dim]) + for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values() + } + end_indices = { + dim: h_grid.map_domain_bounds(dim, end[dim]) + for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values() + } return icon.icon_grid( id_=uuid_, diff --git a/model/common/src/icon4py/model/common/grid/horizontal.py b/model/common/src/icon4py/model/common/grid/horizontal.py index 8e1dcfc1ef..9be810680d 100644 --- a/model/common/src/icon4py/model/common/grid/horizontal.py +++ b/model/common/src/icon4py/model/common/grid/horizontal.py @@ -37,9 +37,10 @@ import enum import functools from abc import abstractmethod -from typing import Final, Protocol +from typing import Final, Protocol, runtime_checkable import gt4py.next as gtx +import numpy as np from icon4py.model.common import dimension as dims @@ -337,6 +338,7 @@ def _map_to_index(dim: gtx.Dimension, marker: Zone) -> int: raise ValueError(f"Unknown marker {marker}") +@runtime_checkable class Domain(Protocol): """ Interface for a domain object. @@ -348,6 +350,14 @@ class Domain(Protocol): _marker: Zone _index: int + def __eq__(self, other): + if isinstance(other, Domain): + return self.dim == other.dim and self.zone == other.zone + return False + + def __hash__(self): + return hash((self.dim, self.zone)) + def __str__(self): return f"Domain (dim = {self.dim}: zone = {self._marker} /[ {self._index}])" @@ -460,3 +470,28 @@ class CellDomain(Domain): def _valid(self, marker: Zone): return marker in CELL_ZONES + + +def get_zones_for_dim(dim: gtx.Dimension) -> tuple[Zone, ...]: + """ + Get the grid zones valid for a given horizontal dimension in ICON . + """ + match dim: + case dims.CellDim: + return CELL_ZONES + case dims.EdgeDim: + return tuple(Zone) + case dims.VertexDim: + return VERTEX_ZONES + case _: + raise ValueError( + f"Dimension should be one of {(dims.MAIN_HORIZONTAL_DIMENSIONS.values())} but was {dim}" + ) + + +def map_domain_bounds( + dim: gtx.Dimension, pre_computed_bounds: np.ndarray +) -> dict[Domain, gtx.int32]: + get_domain = domain(dim) + domains = (get_domain(zone) for zone in get_zones_for_dim(dim)) + return {d: gtx.int32(pre_computed_bounds[d._index].item()) for d in domains} diff --git a/model/common/src/icon4py/model/common/grid/icon.py b/model/common/src/icon4py/model/common/grid/icon.py index 78796c2563..08b9a030e4 100644 --- a/model/common/src/icon4py/model/common/grid/icon.py +++ b/model/common/src/icon4py/model/common/grid/icon.py @@ -10,13 +10,13 @@ import logging import math import uuid -from typing import Final, Optional +from typing import Final, Mapping, Optional import gt4py.next as gtx from gt4py.next import allocators as gtx_allocators from icon4py.model.common import constants, dimension as dims -from icon4py.model.common.grid import base +from icon4py.model.common.grid import base, horizontal as h_grid from icon4py.model.common.utils import data_allocation as data_alloc @@ -163,8 +163,8 @@ def icon_grid( allocator: gtx_allocators.FieldBufferAllocationUtil | None, config: base.GridConfig, neighbor_tables: dict[gtx.FieldOffset, data_alloc.NDArray], - start_indices: dict[gtx.Dimension, data_alloc.NDArray], - end_indices: dict[gtx.Dimension, data_alloc.NDArray], + start_indices: dict[gtx.Dimension, Mapping[h_grid.Domain, gtx.int32]], + end_indices: dict[gtx.Dimension, Mapping[h_grid.Domain, gtx.int32]], global_properties: GlobalGridParams, refinement_control: dict[gtx.Dimension, gtx.Field] | None = None, ) -> IconGrid: diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index 9026b50b75..cb79c52f46 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -8,8 +8,7 @@ import logging import typing -import gt4py.next as gtx -import numpy as np + import pytest from icon4py.model.common import dimension as dims @@ -21,13 +20,12 @@ vertical as v_grid, ) from icon4py.model.testing import ( - datatest_utils as dt_utils, helpers, ) if typing.TYPE_CHECKING: - import netCDF4 + pass try: import netCDF4 @@ -35,14 +33,7 @@ pytest.skip("optional netcdf dependency not installed", allow_module_level=True) -from icon4py.model.testing.fixtures import ( - backend, - data_provider, - download_ser_data, - grid_savepoint, - processor_props, - ranked_data_path, -) +from icon4py.model.testing.fixtures import * # noqa: F401 from .. import utils diff --git a/model/common/tests/common/grid/unit_tests/test_horizontal.py b/model/common/tests/common/grid/unit_tests/test_horizontal.py index 5bbc8a9981..52b86e1bee 100644 --- a/model/common/tests/common/grid/unit_tests/test_horizontal.py +++ b/model/common/tests/common/grid/unit_tests/test_horizontal.py @@ -9,8 +9,11 @@ import pytest +import icon4py.model import icon4py.model.common.dimension as dims import icon4py.model.common.grid.horizontal as h_grid +from icon4py.model.testing import datatest_utils as dt_utils +import gt4py.next as gtx from .. import utils @@ -65,3 +68,30 @@ def test_halo_zones(zone): assert zone.is_halo() else: assert not zone.is_halo() + + +@pytest.mark.datatest +@pytest.mark.parametrize("grid_file", [dt_utils.REGIONAL_EXPERIMENT, dt_utils.R02B04_GLOBAL]) +@pytest.mark.parametrize("dim", utils.main_horizontal_dims()) +def test_map_domain_bounds_start_index(grid_file, dim): + grid = utils.run_grid_manager(dt_utils.R02B04_GLOBAL, keep_skip_values=True, backend=None).grid + + start_index_array = grid._start_indices[dim] + _map_and_assert_array(dim, start_index_array) + + +@pytest.mark.datatest +@pytest.mark.parametrize("grid_file", [dt_utils.REGIONAL_EXPERIMENT, dt_utils.R02B04_GLOBAL]) +@pytest.mark.parametrize("dim", utils.main_horizontal_dims()) +def test_map_domain_bounds_end_index(grid_file, dim): + grid = utils.run_grid_manager(dt_utils.R02B04_GLOBAL, keep_skip_values=True, backend=None).grid + + end_index_array = grid._end_indices[dim] + _map_and_assert_array(dim, end_index_array) + + +def _map_and_assert_array(dim, index_array): + index_map = icon4py.model.common.grid.horizontal.map_domain_bounds(dim, index_array) + for d, index in index_map.items(): + assert index == index_array[d._index] + assert isinstance(index, gtx.int32) diff --git a/model/testing/src/icon4py/model/testing/serialbox.py b/model/testing/src/icon4py/model/testing/serialbox.py index e44256b0b7..4fd7775a84 100644 --- a/model/testing/src/icon4py/model/testing/serialbox.py +++ b/model/testing/src/icon4py/model/testing/serialbox.py @@ -19,7 +19,7 @@ import icon4py.model.common.field_type_aliases as fa import icon4py.model.common.grid.states as grid_states from icon4py.model.common import dimension as dims, type_alias -from icon4py.model.common.grid import base, icon +from icon4py.model.common.grid import base, horizontal as h_grid, icon from icon4py.model.common.states import prognostic_state from icon4py.model.common.utils import data_allocation as data_alloc @@ -488,14 +488,15 @@ def construct_icon_grid( e2c2e0 = np.column_stack((range(e2c2e.shape[0]), e2c2e)) start_indices = { - dims.VertexDim: vertex_starts, - dims.EdgeDim: edge_starts, - dims.CellDim: cell_starts, + dims.VertexDim: h_grid.map_domain_bounds(dims.VertexDim, vertex_starts), + dims.EdgeDim: h_grid.map_domain_bounds(dims.EdgeDim, edge_starts), + dims.CellDim: h_grid.map_domain_bounds(dims.CellDim, cell_starts), } + end_indices = { - dims.VertexDim: vertex_ends, - dims.EdgeDim: edge_ends, - dims.CellDim: cell_ends, + dims.VertexDim: h_grid.map_domain_bounds(dims.VertexDim, vertex_ends), + dims.EdgeDim: h_grid.map_domain_bounds(dims.EdgeDim, edge_ends), + dims.CellDim: h_grid.map_domain_bounds(dims.CellDim, cell_ends), } neighbor_tables = { From 04c359f58750890c7553193e154f2190633395ee Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 13 Aug 2025 13:38:33 +0200 Subject: [PATCH 068/240] fix setup of simple grid --- .../icon4py/model/common/grid/horizontal.py | 10 +++----- .../src/icon4py/model/common/grid/simple.py | 25 ++++++++----------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/horizontal.py b/model/common/src/icon4py/model/common/grid/horizontal.py index 9be810680d..f82434138b 100644 --- a/model/common/src/icon4py/model/common/grid/horizontal.py +++ b/model/common/src/icon4py/model/common/grid/horizontal.py @@ -143,7 +143,7 @@ class LineNumber(enum.IntEnum): EIGHTH = 7 -def _lateral_boundary(dim: gtx.Dimension, offset=LineNumber.FIRST) -> int: +def _lateral_boundary(dim: gtx.Dimension, offset: LineNumber = LineNumber.FIRST) -> int: """Indicate lateral boundary. These points correspond to the sorted points in ICON, the marker can be incremented in order @@ -170,16 +170,16 @@ def _local(dim: gtx.Dimension, offset=LineNumber.FIRST) -> int: return _domain_index(_LOCAL, dim, offset) -def _halo(dim: gtx.Dimension, offset=LineNumber.FIRST) -> int: +def _halo(dim: gtx.Dimension, offset: LineNumber = LineNumber.FIRST) -> int: return _domain_index(_HALO, dim, offset) -def _nudging(dim: gtx.Dimension, offset=LineNumber.FIRST) -> int: +def _nudging(dim: gtx.Dimension, offset: LineNumber = LineNumber.FIRST) -> int: """Indicate the nudging zone.""" return _domain_index(_NUDGING, dim, offset) -def _interior(dim: gtx.Dimension, offset=LineNumber.FIRST) -> int: +def _interior(dim: gtx.Dimension, offset: LineNumber = LineNumber.FIRST) -> int: """Indicate interior i.e. unordered prognostic cells in ICON.""" return _domain_index(_INTERIOR, dim, offset) @@ -334,8 +334,6 @@ def _map_to_index(dim: gtx.Dimension, marker: Zone) -> int: return _nudging(dim, LineNumber.FIRST) case Zone.NUDGING_LEVEL_2: return _nudging(dim, LineNumber.SECOND) - case _: - raise ValueError(f"Unknown marker {marker}") @runtime_checkable diff --git a/model/common/src/icon4py/model/common/grid/simple.py b/model/common/src/icon4py/model/common/grid/simple.py index ea7bd5537f..401eec1c03 100644 --- a/model/common/src/icon4py/model/common/grid/simple.py +++ b/model/common/src/icon4py/model/common/grid/simple.py @@ -453,31 +453,28 @@ def simple_grid(backend: gtx_backend.Backend | None = None) -> base.Grid: start_indices = { dims.CellDim: { - h_grid._map_to_index(dims.CellDim, zone): (0 if not zone.is_halo() else _CELLS) - for zone in h_grid.Zone - if zone in h_grid.CELL_ZONES + h_grid.domain(dims.CellDim)(zone): gtx.int32(0 if not zone.is_halo() else _CELLS) + for zone in h_grid.CELL_ZONES }, dims.EdgeDim: { - h_grid._map_to_index(dims.EdgeDim, zone): (0 if not zone.is_halo() else _EDGES) + h_grid.domain(dims.EdgeDim)(zone): gtx.int32(0 if not zone.is_halo() else _EDGES) for zone in h_grid.Zone }, dims.VertexDim: { - h_grid._map_to_index(dims.VertexDim, zone): (0 if not zone.is_halo() else _VERTICES) - for zone in h_grid.Zone - if zone in h_grid.VERTEX_ZONES + h_grid.domain(dims.VertexDim)(zone): gtx.int32(0 if not zone.is_halo() else _VERTICES) + for zone in h_grid.VERTEX_ZONES }, } end_indices = { dims.CellDim: { - h_grid._map_to_index(dims.CellDim, zone): _CELLS - for zone in h_grid.Zone - if zone in h_grid.CELL_ZONES + h_grid.domain(dims.CellDim)(zone): gtx.int32(_CELLS) for zone in h_grid.CELL_ZONES + }, + dims.EdgeDim: { + h_grid.domain(dims.EdgeDim)(zone): gtx.int32(_EDGES) for zone in h_grid.Zone }, - dims.EdgeDim: {h_grid._map_to_index(dims.EdgeDim, zone): _EDGES for zone in h_grid.Zone}, dims.VertexDim: { - h_grid._map_to_index(dims.VertexDim, zone): _VERTICES - for zone in h_grid.Zone - if zone in h_grid.VERTEX_ZONES + h_grid.domain(dims.VertexDim)(zone): gtx.int32(_VERTICES) + for zone in h_grid.VERTEX_ZONES }, } From bfbbf3e4622aad7e4a249d23961eb55622907dc6 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 13 Aug 2025 14:46:36 +0200 Subject: [PATCH 069/240] remove _index from Domain --- .../src/icon4py/model/common/grid/base.py | 2 +- .../icon4py/model/common/grid/grid_manager.py | 4 +- .../icon4py/model/common/grid/horizontal.py | 35 +++++++-------- model/common/tests/common/fixtures.py | 7 +++ .../common/grid/integration_tests/__init__.py | 7 +++ .../grid/integration_tests/test_horizontal.py | 40 +++++++++++++++++ .../common/grid/unit_tests/test_horizontal.py | 45 ------------------- .../src/icon4py/model/testing/serialbox.py | 40 ++++++++++++++--- 8 files changed, 107 insertions(+), 73 deletions(-) create mode 100644 model/common/tests/common/grid/integration_tests/__init__.py create mode 100644 model/common/tests/common/grid/integration_tests/test_horizontal.py diff --git a/model/common/src/icon4py/model/common/grid/base.py b/model/common/src/icon4py/model/common/grid/base.py index 2d7e966bcc..84bb0fadd9 100644 --- a/model/common/src/icon4py/model/common/grid/base.py +++ b/model/common/src/icon4py/model/common/grid/base.py @@ -172,7 +172,7 @@ def start_index(self, domain: h_grid.Domain) -> gtx.int32: For a given dimension, returns the start index of the horizontal region in a field given by the marker. """ - if domain.local: + if domain.is_local: # special treatment because this value is not set properly in the underlying data. return gtx.int32(0) return self._start_indices[domain.dim][domain] diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 693e6b1bdb..f645206a5b 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -363,11 +363,11 @@ def _construct_grid( start, end, _ = self._read_start_end_indices() start_indices = { - dim: h_grid.map_domain_bounds(dim, start[dim]) + dim: h_grid.map_icon_domain_bounds(dim, start[dim]) for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values() } end_indices = { - dim: h_grid.map_domain_bounds(dim, end[dim]) + dim: h_grid.map_icon_domain_bounds(dim, end[dim]) for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values() } diff --git a/model/common/src/icon4py/model/common/grid/horizontal.py b/model/common/src/icon4py/model/common/grid/horizontal.py index f82434138b..c3bd6694c9 100644 --- a/model/common/src/icon4py/model/common/grid/horizontal.py +++ b/model/common/src/icon4py/model/common/grid/horizontal.py @@ -302,7 +302,7 @@ def is_halo(self) -> bool: return self in (Zone.HALO, Zone.HALO_LEVEL_2) -def _map_to_index(dim: gtx.Dimension, marker: Zone) -> int: +def _map_to_icon_index(dim: gtx.Dimension, marker: Zone) -> int: match marker: case Zone.END: return _end(dim) @@ -345,8 +345,7 @@ class Domain(Protocol): """ _dim: gtx.Dimension - _marker: Zone - _index: int + _zone: Zone def __eq__(self, other): if isinstance(other, Domain): @@ -357,19 +356,18 @@ def __hash__(self): return hash((self.dim, self.zone)) def __str__(self): - return f"Domain (dim = {self.dim}: zone = {self._marker} /[ {self._index}])" + return f"Domain (dim = {self.dim}: zone = {self._zone} /[ {_map_to_icon_index(self.dim, self.zone)} ])" @abstractmethod def _valid(self, marker: Zone) -> bool: ... @property def zone(self) -> Zone: - return self._marker + return self._zone - def marker(self, marker: Zone): + def set_marker(self, marker: Zone): assert self._valid(marker), f" Domain `{marker}` not a valid zone for use with '{self.dim}'" - self._marker = marker - self._index = _map_to_index(self.dim, marker) + self._zone = marker return self @property @@ -377,11 +375,8 @@ def dim(self) -> gtx.Dimension: return self._dim @functools.cached_property - def local(self) -> bool: - return self._marker == Zone.LOCAL - - def __call__(self) -> int: - return self._index + def is_local(self) -> bool: + return self._zone == Zone.LOCAL def domain(dim: gtx.Dimension): @@ -407,13 +402,13 @@ def _domain(marker: Zone): return _domain -def _domain_factory(dim: gtx.Dimension, marker: Zone): +def _domain_factory(dim: gtx.Dimension, zone: Zone): if dim == dims.CellDim: - return CellDomain().marker(marker) + return CellDomain().set_marker(zone) elif dim == dims.EdgeDim: - return EdgeDomain().marker(marker) + return EdgeDomain().set_marker(zone) else: - return VertexDomain().marker(marker) + return VertexDomain().set_marker(zone) class EdgeDomain(Domain): @@ -487,9 +482,11 @@ def get_zones_for_dim(dim: gtx.Dimension) -> tuple[Zone, ...]: ) -def map_domain_bounds( +def map_icon_domain_bounds( dim: gtx.Dimension, pre_computed_bounds: np.ndarray ) -> dict[Domain, gtx.int32]: get_domain = domain(dim) domains = (get_domain(zone) for zone in get_zones_for_dim(dim)) - return {d: gtx.int32(pre_computed_bounds[d._index].item()) for d in domains} + return { + d: gtx.int32(pre_computed_bounds[_map_to_icon_index(dim, d.zone)].item()) for d in domains + } diff --git a/model/common/tests/common/fixtures.py b/model/common/tests/common/fixtures.py index f1b9e8b408..d51664459c 100644 --- a/model/common/tests/common/fixtures.py +++ b/model/common/tests/common/fixtures.py @@ -12,6 +12,13 @@ import pytest from icon4py.model.testing.datatest_utils import REGIONAL_EXPERIMENT +from icon4py.model.testing.fixtures.datatest import ( + grid_savepoint, + data_provider, + download_ser_data, + processor_props, + ranked_data_path, +) @pytest.fixture diff --git a/model/common/tests/common/grid/integration_tests/__init__.py b/model/common/tests/common/grid/integration_tests/__init__.py new file mode 100644 index 0000000000..de9850de36 --- /dev/null +++ b/model/common/tests/common/grid/integration_tests/__init__.py @@ -0,0 +1,7 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause diff --git a/model/common/tests/common/grid/integration_tests/test_horizontal.py b/model/common/tests/common/grid/integration_tests/test_horizontal.py new file mode 100644 index 0000000000..281dcc7bef --- /dev/null +++ b/model/common/tests/common/grid/integration_tests/test_horizontal.py @@ -0,0 +1,40 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +import pytest +from gt4py import next as gtx + +from icon4py.model.common.grid import horizontal as h_grid +from icon4py.model.testing import datatest_utils as dt_utils +from .. import utils +from ...fixtures import * # noqa: F401, F403 + + +@pytest.mark.datatest +@pytest.mark.parametrize("experiment", [dt_utils.REGIONAL_EXPERIMENT, dt_utils.GLOBAL_EXPERIMENT]) +@pytest.mark.parametrize("dim", utils.main_horizontal_dims()) +def test_map_domain_bounds_start_index(experiment, dim, grid_savepoint): + grid_savepoint.start_index(dim) + start_index_array = grid_savepoint.start_index(dim) + _map_and_assert_array(dim, start_index_array) + + +@pytest.mark.datatest +@pytest.mark.parametrize("experiment", [dt_utils.REGIONAL_EXPERIMENT, dt_utils.GLOBAL_EXPERIMENT]) +@pytest.mark.parametrize("dim", utils.main_horizontal_dims()) +def test_map_domain_bounds_end_index(experiment, dim, grid_savepoint): + end_index_array = grid_savepoint.end_index(dim) + _map_and_assert_array(dim, end_index_array) + + +def _map_and_assert_array(dim, index_array): + index_map = h_grid.map_icon_domain_bounds(dim, index_array) + for d, index in index_map.items(): + icon_index = h_grid._map_to_icon_index(d.dim, d.zone) + assert index == index_array[icon_index] + assert isinstance(index, gtx.int32) diff --git a/model/common/tests/common/grid/unit_tests/test_horizontal.py b/model/common/tests/common/grid/unit_tests/test_horizontal.py index 52b86e1bee..3482690ce5 100644 --- a/model/common/tests/common/grid/unit_tests/test_horizontal.py +++ b/model/common/tests/common/grid/unit_tests/test_horizontal.py @@ -6,14 +6,9 @@ # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause import logging - import pytest - -import icon4py.model import icon4py.model.common.dimension as dims import icon4py.model.common.grid.horizontal as h_grid -from icon4py.model.testing import datatest_utils as dt_utils -import gt4py.next as gtx from .. import utils @@ -49,49 +44,9 @@ def test_domain_raises_for_invalid_zones(dim, zone, caplog): e.match("not a valid zone") -@pytest.mark.parametrize("dim", utils.main_horizontal_dims()) -def test_zone_and_domain_index(dim, caplog): - """test mostly used for documentation purposes""" - caplog.set_level(logging.INFO) - for zone in zones(): - try: - domain = h_grid.domain(dim)(zone) - log.info(f"dim={dim}: zone={zone:16}: index={domain():3}") - assert domain() <= h_grid._BOUNDS[dim][1] - except AssertionError: - log.info(f"dim={dim}: zone={zone:16}: invalid") - - @pytest.mark.parametrize("zone", zones()) def test_halo_zones(zone): if zone in (h_grid.Zone.HALO, h_grid.Zone.HALO_LEVEL_2): assert zone.is_halo() else: assert not zone.is_halo() - - -@pytest.mark.datatest -@pytest.mark.parametrize("grid_file", [dt_utils.REGIONAL_EXPERIMENT, dt_utils.R02B04_GLOBAL]) -@pytest.mark.parametrize("dim", utils.main_horizontal_dims()) -def test_map_domain_bounds_start_index(grid_file, dim): - grid = utils.run_grid_manager(dt_utils.R02B04_GLOBAL, keep_skip_values=True, backend=None).grid - - start_index_array = grid._start_indices[dim] - _map_and_assert_array(dim, start_index_array) - - -@pytest.mark.datatest -@pytest.mark.parametrize("grid_file", [dt_utils.REGIONAL_EXPERIMENT, dt_utils.R02B04_GLOBAL]) -@pytest.mark.parametrize("dim", utils.main_horizontal_dims()) -def test_map_domain_bounds_end_index(grid_file, dim): - grid = utils.run_grid_manager(dt_utils.R02B04_GLOBAL, keep_skip_values=True, backend=None).grid - - end_index_array = grid._end_indices[dim] - _map_and_assert_array(dim, end_index_array) - - -def _map_and_assert_array(dim, index_array): - index_map = icon4py.model.common.grid.horizontal.map_domain_bounds(dim, index_array) - for d, index in index_map.items(): - assert index == index_array[d._index] - assert isinstance(index, gtx.int32) diff --git a/model/testing/src/icon4py/model/testing/serialbox.py b/model/testing/src/icon4py/model/testing/serialbox.py index 4fd7775a84..33c6ad8171 100644 --- a/model/testing/src/icon4py/model/testing/serialbox.py +++ b/model/testing/src/icon4py/model/testing/serialbox.py @@ -326,6 +326,34 @@ def vertex_end_index(self): def edge_start_index(self): return self._read_int32_shift1("e_start_index") + def start_index(self, dim: gtx.Dimension) -> np.ndarray: + """ + Use to specify lower end of domains of a field for field_operators. + """ + match dim: + case dims.CellDim: + return self.cells_start_index() + case dims.EdgeDim: + return self.edge_start_index() + case dims.VertexDim: + return self.vertex_start_index() + case _: + raise ValueError(f"Unsupported dimension {dim}") + + def end_index(self, dim: gtx.Dimension) -> np.ndarray: + """ + Use to specify upper end of domains of a field for field_operators. + """ + match dim: + case dims.CellDim: + return self.cells_end_index() + case dims.EdgeDim: + return self.edge_end_index() + case dims.VertexDim: + return self.vertex_end_index() + case _: + raise ValueError(f"Unsupported dimension {dim}") + def nflatlev(self): return self._read_int32_shift1("nflatlev").item() @@ -488,15 +516,15 @@ def construct_icon_grid( e2c2e0 = np.column_stack((range(e2c2e.shape[0]), e2c2e)) start_indices = { - dims.VertexDim: h_grid.map_domain_bounds(dims.VertexDim, vertex_starts), - dims.EdgeDim: h_grid.map_domain_bounds(dims.EdgeDim, edge_starts), - dims.CellDim: h_grid.map_domain_bounds(dims.CellDim, cell_starts), + dims.VertexDim: h_grid.map_icon_domain_bounds(dims.VertexDim, vertex_starts), + dims.EdgeDim: h_grid.map_icon_domain_bounds(dims.EdgeDim, edge_starts), + dims.CellDim: h_grid.map_icon_domain_bounds(dims.CellDim, cell_starts), } end_indices = { - dims.VertexDim: h_grid.map_domain_bounds(dims.VertexDim, vertex_ends), - dims.EdgeDim: h_grid.map_domain_bounds(dims.EdgeDim, edge_ends), - dims.CellDim: h_grid.map_domain_bounds(dims.CellDim, cell_ends), + dims.VertexDim: h_grid.map_icon_domain_bounds(dims.VertexDim, vertex_ends), + dims.EdgeDim: h_grid.map_icon_domain_bounds(dims.EdgeDim, edge_ends), + dims.CellDim: h_grid.map_icon_domain_bounds(dims.CellDim, cell_ends), } neighbor_tables = { From 2c7a9c67e9d466754328a3335548f68bdecbe187 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 13 Aug 2025 17:07:28 +0200 Subject: [PATCH 070/240] simple map Domain -> index frozen Domain --- .../src/icon4py/model/common/grid/base.py | 8 +- .../icon4py/model/common/grid/grid_manager.py | 7 +- .../icon4py/model/common/grid/horizontal.py | 123 +++++++----------- .../src/icon4py/model/common/grid/icon.py | 4 +- .../src/icon4py/model/common/grid/simple.py | 30 ++--- .../common/grid/unit_tests/test_horizontal.py | 2 +- .../src/icon4py/model/testing/serialbox.py | 12 +- 7 files changed, 78 insertions(+), 108 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/base.py b/model/common/src/icon4py/model/common/grid/base.py index 84bb0fadd9..d7c6690ba1 100644 --- a/model/common/src/icon4py/model/common/grid/base.py +++ b/model/common/src/icon4py/model/common/grid/base.py @@ -98,8 +98,8 @@ class Grid: connectivities: gtx_common.OffsetProvider geometry_type: GeometryType # only used internally for `start_index` and `end_index` public interface: - _start_indices: dict[gtx.Dimension, Mapping[h_grid.Domain, gtx.int32]] - _end_indices: dict[gtx.Dimension, Mapping[h_grid.Domain, gtx.int32]] + _start_indices: Mapping[h_grid.Domain, gtx.int32] + _end_indices: Mapping[h_grid.Domain, gtx.int32] def __post_init__(self): # TODO(havogt): replace `Koff[k]` by `KDim + k` syntax and remove the following line. @@ -175,7 +175,7 @@ def start_index(self, domain: h_grid.Domain) -> gtx.int32: if domain.is_local: # special treatment because this value is not set properly in the underlying data. return gtx.int32(0) - return self._start_indices[domain.dim][domain] + return self._start_indices[domain] def end_index(self, domain: h_grid.Domain) -> gtx.int32: """ @@ -187,7 +187,7 @@ def end_index(self, domain: h_grid.Domain) -> gtx.int32: if domain.zone == h_grid.Zone.INTERIOR and not self.limited_area: # special treatment because this value is not set properly in the underlying data, for a global grid return gtx.int32(self.size[domain.dim]) - return gtx.int32(self._end_indices[domain.dim][domain]) + return gtx.int32(self._end_indices[domain]) def construct_connectivity( diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index f645206a5b..0bd7a1ffba 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -361,14 +361,15 @@ def _construct_grid( neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) start, end, _ = self._read_start_end_indices() - start_indices = { - dim: h_grid.map_icon_domain_bounds(dim, start[dim]) + k: v for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values() + for k, v in h_grid.map_icon_domain_bounds(dim, start[dim]).items() } end_indices = { - dim: h_grid.map_icon_domain_bounds(dim, end[dim]) + k: v for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values() + for k, v in h_grid.map_icon_domain_bounds(dim, end[dim]).items() } return icon.icon_grid( diff --git a/model/common/src/icon4py/model/common/grid/horizontal.py b/model/common/src/icon4py/model/common/grid/horizontal.py index c3bd6694c9..52c2a66562 100644 --- a/model/common/src/icon4py/model/common/grid/horizontal.py +++ b/model/common/src/icon4py/model/common/grid/horizontal.py @@ -34,10 +34,10 @@ """ +import dataclasses import enum import functools -from abc import abstractmethod -from typing import Final, Protocol, runtime_checkable +from typing import Callable, Final import gt4py.next as gtx import numpy as np @@ -302,6 +302,34 @@ def is_halo(self) -> bool: return self in (Zone.HALO, Zone.HALO_LEVEL_2) +VERTEX_ZONES = ( + Zone.END, + Zone.INTERIOR, + Zone.HALO, + Zone.HALO_LEVEL_2, + Zone.LOCAL, + Zone.LATERAL_BOUNDARY, + Zone.LATERAL_BOUNDARY_LEVEL_2, + Zone.LATERAL_BOUNDARY_LEVEL_3, + Zone.LATERAL_BOUNDARY_LEVEL_4, +) + + +CELL_ZONES = ( + Zone.END, + Zone.INTERIOR, + Zone.HALO, + Zone.HALO_LEVEL_2, + Zone.LOCAL, + Zone.LATERAL_BOUNDARY, + Zone.LATERAL_BOUNDARY_LEVEL_2, + Zone.LATERAL_BOUNDARY_LEVEL_3, + Zone.LATERAL_BOUNDARY_LEVEL_4, + Zone.NUDGING, +) + +EDGE_ZONES = tuple(Zone) + def _map_to_icon_index(dim: gtx.Dimension, marker: Zone) -> int: match marker: case Zone.END: @@ -336,12 +364,11 @@ def _map_to_icon_index(dim: gtx.Dimension, marker: Zone) -> int: return _nudging(dim, LineNumber.SECOND) -@runtime_checkable -class Domain(Protocol): +@dataclasses.dataclass(frozen=True) +class Domain: """ - Interface for a domain object. - - Used to access horizontal domain zones in the ICON grid. + Domain Description on the horizontal grid + Used to access domain bounds in concrete the ICON grid. """ _dim: gtx.Dimension @@ -358,18 +385,10 @@ def __hash__(self): def __str__(self): return f"Domain (dim = {self.dim}: zone = {self._zone} /[ {_map_to_icon_index(self.dim, self.zone)} ])" - @abstractmethod - def _valid(self, marker: Zone) -> bool: ... - @property def zone(self) -> Zone: return self._zone - def set_marker(self, marker: Zone): - assert self._valid(marker), f" Domain `{marker}` not a valid zone for use with '{self.dim}'" - self._zone = marker - return self - @property def dim(self) -> gtx.Dimension: return self._dim @@ -379,7 +398,7 @@ def is_local(self) -> bool: return self._zone == Zone.LOCAL -def domain(dim: gtx.Dimension): +def domain(dim: gtx.Dimension) -> Callable[[Zone], Domain]: """ Factory function to create a domain object for a given dimension. @@ -395,7 +414,7 @@ def domain(dim: gtx.Dimension): """ - def _domain(marker: Zone): + def _domain(marker: Zone) -> Domain: return _domain_factory(dim, marker) assert dim.kind == gtx.DimensionKind.HORIZONTAL, "Only defined for horizontal dimensions" @@ -403,66 +422,20 @@ def _domain(marker: Zone): def _domain_factory(dim: gtx.Dimension, zone: Zone): - if dim == dims.CellDim: - return CellDomain().set_marker(zone) - elif dim == dims.EdgeDim: - return EdgeDomain().set_marker(zone) - else: - return VertexDomain().set_marker(zone) - - -class EdgeDomain(Domain): - """Domain object for the Edge dimension.""" + assert _validate( + dim, zone + ), f"Invalid zone {zone} for dimension {dim}. Valid zones are: {get_zones_for_dim(dim)}" + return Domain(dim, zone) - _dim = dims.EdgeDim - def _valid(self, marker: Zone): - return True - - -VERTEX_ZONES = ( - Zone.END, - Zone.INTERIOR, - Zone.HALO, - Zone.HALO_LEVEL_2, - Zone.LOCAL, - Zone.LATERAL_BOUNDARY, - Zone.LATERAL_BOUNDARY_LEVEL_2, - Zone.LATERAL_BOUNDARY_LEVEL_3, - Zone.LATERAL_BOUNDARY_LEVEL_4, -) - - -class VertexDomain(Domain): - """Domain object for the Vertex dimension.""" - - _dim = dims.VertexDim - - def _valid(self, marker: Zone): - return marker in VERTEX_ZONES - - -CELL_ZONES = ( - Zone.END, - Zone.INTERIOR, - Zone.HALO, - Zone.HALO_LEVEL_2, - Zone.LOCAL, - Zone.LATERAL_BOUNDARY, - Zone.LATERAL_BOUNDARY_LEVEL_2, - Zone.LATERAL_BOUNDARY_LEVEL_3, - Zone.LATERAL_BOUNDARY_LEVEL_4, - Zone.NUDGING, -) - - -class CellDomain(Domain): - """Domain object for the Cell dimension.""" - - _dim = dims.CellDim - - def _valid(self, marker: Zone): - return marker in CELL_ZONES +def _validate(dim: gtx.Dimension, marker: Zone) -> bool: + match dim: + case dims.CellDim: + return marker in CELL_ZONES + case dims.EdgeDim: + return marker in tuple(Zone) + case dims.VertexDim: + return marker in VERTEX_ZONES def get_zones_for_dim(dim: gtx.Dimension) -> tuple[Zone, ...]: diff --git a/model/common/src/icon4py/model/common/grid/icon.py b/model/common/src/icon4py/model/common/grid/icon.py index 08b9a030e4..9e184a4e71 100644 --- a/model/common/src/icon4py/model/common/grid/icon.py +++ b/model/common/src/icon4py/model/common/grid/icon.py @@ -163,8 +163,8 @@ def icon_grid( allocator: gtx_allocators.FieldBufferAllocationUtil | None, config: base.GridConfig, neighbor_tables: dict[gtx.FieldOffset, data_alloc.NDArray], - start_indices: dict[gtx.Dimension, Mapping[h_grid.Domain, gtx.int32]], - end_indices: dict[gtx.Dimension, Mapping[h_grid.Domain, gtx.int32]], + start_indices: Mapping[h_grid.Domain, gtx.int32], + end_indices: Mapping[h_grid.Domain, gtx.int32], global_properties: GlobalGridParams, refinement_control: dict[gtx.Dimension, gtx.Field] | None = None, ) -> IconGrid: diff --git a/model/common/src/icon4py/model/common/grid/simple.py b/model/common/src/icon4py/model/common/grid/simple.py index 401eec1c03..6a54dd25e2 100644 --- a/model/common/src/icon4py/model/common/grid/simple.py +++ b/model/common/src/icon4py/model/common/grid/simple.py @@ -451,31 +451,27 @@ def simple_grid(backend: gtx_backend.Backend | None = None) -> base.Grid: for offset, table in neighbor_tables.items() } + cell_domain = h_grid.domain(dims.CellDim) + edge_domain = h_grid.domain(dims.EdgeDim) + vertex_domain = h_grid.domain(dims.VertexDim) start_indices = { - dims.CellDim: { - h_grid.domain(dims.CellDim)(zone): gtx.int32(0 if not zone.is_halo() else _CELLS) + **{ + cell_domain(zone): gtx.int32(0 if not zone.is_halo() else _CELLS) for zone in h_grid.CELL_ZONES }, - dims.EdgeDim: { - h_grid.domain(dims.EdgeDim)(zone): gtx.int32(0 if not zone.is_halo() else _EDGES) - for zone in h_grid.Zone + **{ + edge_domain(zone): gtx.int32(0 if not zone.is_halo() else _EDGES) + for zone in h_grid.EDGE_ZONES }, - dims.VertexDim: { - h_grid.domain(dims.VertexDim)(zone): gtx.int32(0 if not zone.is_halo() else _VERTICES) + **{ + vertex_domain(zone): gtx.int32(0 if not zone.is_halo() else _VERTICES) for zone in h_grid.VERTEX_ZONES }, } end_indices = { - dims.CellDim: { - h_grid.domain(dims.CellDim)(zone): gtx.int32(_CELLS) for zone in h_grid.CELL_ZONES - }, - dims.EdgeDim: { - h_grid.domain(dims.EdgeDim)(zone): gtx.int32(_EDGES) for zone in h_grid.Zone - }, - dims.VertexDim: { - h_grid.domain(dims.VertexDim)(zone): gtx.int32(_VERTICES) - for zone in h_grid.VERTEX_ZONES - }, + **{cell_domain(zone): gtx.int32(_CELLS) for zone in h_grid.CELL_ZONES}, + **{edge_domain(zone): gtx.int32(_EDGES) for zone in h_grid.Zone}, + **{vertex_domain(zone): gtx.int32(_VERTICES) for zone in h_grid.VERTEX_ZONES}, } return base.Grid( diff --git a/model/common/tests/common/grid/unit_tests/test_horizontal.py b/model/common/tests/common/grid/unit_tests/test_horizontal.py index 3482690ce5..c288202237 100644 --- a/model/common/tests/common/grid/unit_tests/test_horizontal.py +++ b/model/common/tests/common/grid/unit_tests/test_horizontal.py @@ -41,7 +41,7 @@ def test_domain_raises_for_invalid_zones(dim, zone, caplog): ): with pytest.raises(AssertionError) as e: h_grid.domain(dim)(zone) - e.match("not a valid zone") + e.match("Invalid zone") @pytest.mark.parametrize("zone", zones()) diff --git a/model/testing/src/icon4py/model/testing/serialbox.py b/model/testing/src/icon4py/model/testing/serialbox.py index 33c6ad8171..1761d21378 100644 --- a/model/testing/src/icon4py/model/testing/serialbox.py +++ b/model/testing/src/icon4py/model/testing/serialbox.py @@ -516,15 +516,15 @@ def construct_icon_grid( e2c2e0 = np.column_stack((range(e2c2e.shape[0]), e2c2e)) start_indices = { - dims.VertexDim: h_grid.map_icon_domain_bounds(dims.VertexDim, vertex_starts), - dims.EdgeDim: h_grid.map_icon_domain_bounds(dims.EdgeDim, edge_starts), - dims.CellDim: h_grid.map_icon_domain_bounds(dims.CellDim, cell_starts), + **h_grid.map_icon_domain_bounds(dims.VertexDim, vertex_starts), + **h_grid.map_icon_domain_bounds(dims.EdgeDim, edge_starts), + **h_grid.map_icon_domain_bounds(dims.CellDim, cell_starts), } end_indices = { - dims.VertexDim: h_grid.map_icon_domain_bounds(dims.VertexDim, vertex_ends), - dims.EdgeDim: h_grid.map_icon_domain_bounds(dims.EdgeDim, edge_ends), - dims.CellDim: h_grid.map_icon_domain_bounds(dims.CellDim, cell_ends), + **h_grid.map_icon_domain_bounds(dims.VertexDim, vertex_ends), + **h_grid.map_icon_domain_bounds(dims.EdgeDim, edge_ends), + **h_grid.map_icon_domain_bounds(dims.CellDim, cell_ends), } neighbor_tables = { From 01dfddb93254ee5d05cd289b6ea0fe6a369e54cd Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 13 Aug 2025 17:54:53 +0200 Subject: [PATCH 071/240] fix grid wrapper --- .../icon4py/model/common/grid/horizontal.py | 1 + .../icon4py/tools/py2fgen/wrappers/common.py | 15 ++++++------ .../tools/py2fgen/wrappers/debug_utils.py | 24 ++++++++++++++----- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/horizontal.py b/model/common/src/icon4py/model/common/grid/horizontal.py index 52c2a66562..39105ddbdc 100644 --- a/model/common/src/icon4py/model/common/grid/horizontal.py +++ b/model/common/src/icon4py/model/common/grid/horizontal.py @@ -330,6 +330,7 @@ def is_halo(self) -> bool: EDGE_ZONES = tuple(Zone) + def _map_to_icon_index(dim: gtx.Dimension, marker: Zone) -> int: match marker: case Zone.END: diff --git a/tools/src/icon4py/tools/py2fgen/wrappers/common.py b/tools/src/icon4py/tools/py2fgen/wrappers/common.py index b98bb85d7d..d6a67da290 100644 --- a/tools/src/icon4py/tools/py2fgen/wrappers/common.py +++ b/tools/src/icon4py/tools/py2fgen/wrappers/common.py @@ -21,7 +21,7 @@ from icon4py.model.common import dimension as dims, model_backends from icon4py.model.common.decomposition import definitions, mpi_decomposition -from icon4py.model.common.grid import base, icon +from icon4py.model.common.grid import base, horizontal as h_grid, icon from icon4py.model.common.utils import data_allocation as data_alloc @@ -220,14 +220,15 @@ def construct_icon_grid( ) start_indices = { - dims.CellDim: cells_start_index, - dims.EdgeDim: edge_start_index, - dims.VertexDim: vertex_start_index, + **h_grid.map_icon_domain_bounds(dims.CellDim, cells_start_index), + **h_grid.map_icon_domain_bounds(dims.EdgeDim, edge_start_index), + **h_grid.map_icon_domain_bounds(dims.VertexDim, vertex_start_index), } + end_indices = { - dims.CellDim: cells_end_index, - dims.EdgeDim: edge_end_index, - dims.VertexDim: vertex_end_index, + **h_grid.map_icon_domain_bounds(dims.CellDim, cells_end_index), + **h_grid.map_icon_domain_bounds(dims.EdgeDim, edge_end_index), + **h_grid.map_icon_domain_bounds(dims.VertexDim, vertex_end_index), } return icon.icon_grid( diff --git a/tools/src/icon4py/tools/py2fgen/wrappers/debug_utils.py b/tools/src/icon4py/tools/py2fgen/wrappers/debug_utils.py index e86f935ab3..386bf1e89f 100644 --- a/tools/src/icon4py/tools/py2fgen/wrappers/debug_utils.py +++ b/tools/src/icon4py/tools/py2fgen/wrappers/debug_utils.py @@ -28,32 +28,44 @@ def print_grid_decomp_info( log.info( "icon_grid:cell_start for rank %s is.... %s", processor_props.rank, - icon_grid._start_indices[CellDim], + "\n".join( + [f"{k:<10} - {v}" for k, v in icon_grid._start_indices.items() if k.dim == CellDim] + ), ) log.info( "icon_grid:cell_end for rank %s is.... %s", processor_props.rank, - icon_grid._end_indices[CellDim], + "\n".join( + [f"{k:<10} - {v}" for k, v in icon_grid._end_indices.items() if k.dim == CellDim] + ), ) log.info( "icon_grid:vert_start for rank %s is.... %s", processor_props.rank, - icon_grid._start_indices[VertexDim], + "\n".join( + [f"{k:<10} - {v}" for k, v in icon_grid._start_indices.items() if k.dim == VertexDim] + ), ) log.info( "icon_grid:vert_end for rank %s is.... %s", processor_props.rank, - icon_grid._end_indices[VertexDim], + "\n".join( + [f"{k:<10} - {v}" for k, v in icon_grid._end_indices.items() if k.dim == VertexDim] + ), ) log.info( "icon_grid:edge_start for rank %s is.... %s", processor_props.rank, - icon_grid._start_indices[EdgeDim], + "\n".join( + [f"{k:<10} - {v}" for k, v in icon_grid._start_indices.items() if k.dim == EdgeDim] + ), ) log.info( "icon_grid:edge_end for rank %s is.... %s", processor_props.rank, - icon_grid._end_indices[EdgeDim], + "\n".join( + [f"{k:<10} - {v}" for k, v in icon_grid._end_indices.items() if k.dim == EdgeDim] + ), ) for offset, connectivity in icon_grid.connectivities.items(): From 1c990ebbe82612cbfabf443d32e03787585d7e1f Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 13 Aug 2025 18:05:17 +0200 Subject: [PATCH 072/240] revert unnecessary changes --- .../common/src/icon4py/model/common/grid/base.py | 2 +- .../src/icon4py/model/common/grid/horizontal.py | 2 +- .../common/grid/unit_tests/test_grid_manager.py | 15 ++++++++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/base.py b/model/common/src/icon4py/model/common/grid/base.py index d7c6690ba1..986875fa6e 100644 --- a/model/common/src/icon4py/model/common/grid/base.py +++ b/model/common/src/icon4py/model/common/grid/base.py @@ -187,7 +187,7 @@ def end_index(self, domain: h_grid.Domain) -> gtx.int32: if domain.zone == h_grid.Zone.INTERIOR and not self.limited_area: # special treatment because this value is not set properly in the underlying data, for a global grid return gtx.int32(self.size[domain.dim]) - return gtx.int32(self._end_indices[domain]) + return self._end_indices[domain] def construct_connectivity( diff --git a/model/common/src/icon4py/model/common/grid/horizontal.py b/model/common/src/icon4py/model/common/grid/horizontal.py index 39105ddbdc..cb09414b5a 100644 --- a/model/common/src/icon4py/model/common/grid/horizontal.py +++ b/model/common/src/icon4py/model/common/grid/horizontal.py @@ -384,7 +384,7 @@ def __hash__(self): return hash((self.dim, self.zone)) def __str__(self): - return f"Domain (dim = {self.dim}: zone = {self._zone} /[ {_map_to_icon_index(self.dim, self.zone)} ])" + return f"Domain (dim = {self.dim}: zone = {self._zone} /ICON index[ {_map_to_icon_index(self.dim, self.zone)} ])" @property def zone(self) -> Zone: diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index cb79c52f46..9026b50b75 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -8,7 +8,8 @@ import logging import typing - +import gt4py.next as gtx +import numpy as np import pytest from icon4py.model.common import dimension as dims @@ -20,12 +21,13 @@ vertical as v_grid, ) from icon4py.model.testing import ( + datatest_utils as dt_utils, helpers, ) if typing.TYPE_CHECKING: - pass + import netCDF4 try: import netCDF4 @@ -33,7 +35,14 @@ pytest.skip("optional netcdf dependency not installed", allow_module_level=True) -from icon4py.model.testing.fixtures import * # noqa: F401 +from icon4py.model.testing.fixtures import ( + backend, + data_provider, + download_ser_data, + grid_savepoint, + processor_props, + ranked_data_path, +) from .. import utils From f507594c32efc9a8265f5aedda93fdc819f77f7d Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 14 Aug 2025 09:27:07 +0200 Subject: [PATCH 073/240] do not use function under test in test assertion --- .../grid/integration_tests/test_horizontal.py | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/model/common/tests/common/grid/integration_tests/test_horizontal.py b/model/common/tests/common/grid/integration_tests/test_horizontal.py index 281dcc7bef..73c3fafcc8 100644 --- a/model/common/tests/common/grid/integration_tests/test_horizontal.py +++ b/model/common/tests/common/grid/integration_tests/test_horizontal.py @@ -34,7 +34,37 @@ def test_map_domain_bounds_end_index(experiment, dim, grid_savepoint): def _map_and_assert_array(dim, index_array): index_map = h_grid.map_icon_domain_bounds(dim, index_array) + same_index = False for d, index in index_map.items(): - icon_index = h_grid._map_to_icon_index(d.dim, d.zone) - assert index == index_array[icon_index] - assert isinstance(index, gtx.int32) + if d.zone == h_grid.Zone.INTERIOR: + same_index = index == index_array[h_grid._INTERIOR[d.dim]] + if d.zone == h_grid.Zone.LOCAL: + same_index = index == index_array[h_grid._LOCAL[d.dim]] + if d.zone == h_grid.Zone.END: + same_index = index == index_array[h_grid._END[d.dim]] + if d.zone == h_grid.Zone.LATERAL_BOUNDARY: + same_index = index == index_array[h_grid._LATERAL_BOUNDARY[d.dim]] + if d.zone == h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2: + same_index = index == index_array[h_grid._LATERAL_BOUNDARY[d.dim] + 1] + if d.zone == h_grid.Zone.LATERAL_BOUNDARY_LEVEL_3: + same_index = index == index_array[h_grid._LATERAL_BOUNDARY[d.dim] + 2] + if d.zone == h_grid.Zone.LATERAL_BOUNDARY_LEVEL_4: + same_index = index == index_array[h_grid._LATERAL_BOUNDARY[d.dim] + 3] + if d.zone == h_grid.Zone.LATERAL_BOUNDARY_LEVEL_5: + same_index = index == index_array[h_grid._LATERAL_BOUNDARY[d.dim] + 4] + if d.zone == h_grid.Zone.LATERAL_BOUNDARY_LEVEL_6: + same_index = index == index_array[h_grid._LATERAL_BOUNDARY[d.dim] + 5] + if d.zone == h_grid.Zone.LATERAL_BOUNDARY_LEVEL_7: + same_index = index == index_array[h_grid._LATERAL_BOUNDARY[d.dim] + 6] + if d.zone == h_grid.Zone.LATERAL_BOUNDARY_LEVEL_8: + same_index = index == index_array[h_grid._LATERAL_BOUNDARY[d.dim] + 7] + if d.zone == h_grid.Zone.NUDGING: + same_index = index == index_array[h_grid._NUDGING[d.dim]] + if d.zone == h_grid.Zone.NUDGING_LEVEL_2: + same_index = index == index_array[h_grid._NUDGING[d.dim] + 1] + if d.zone == h_grid.Zone.HALO: + same_index = index == index_array[h_grid._HALO[d.dim]] + if d.zone == h_grid.Zone.HALO_LEVEL_2: + same_index = index == index_array[h_grid._HALO[d.dim] + 1] + if not same_index: + raise AssertionError(f"Wrong index for {d.zone} zone in dimension {d.dim}: ") From 7045a4fb43a139b41474f1b1665ca6bf73c29643 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 14 Aug 2025 09:51:37 +0200 Subject: [PATCH 074/240] fix typing in horizontal.py --- .../icon4py/model/common/grid/horizontal.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/horizontal.py b/model/common/src/icon4py/model/common/grid/horizontal.py index cb09414b5a..1198affa73 100644 --- a/model/common/src/icon4py/model/common/grid/horizontal.py +++ b/model/common/src/icon4py/model/common/grid/horizontal.py @@ -37,7 +37,7 @@ import dataclasses import enum import functools -from typing import Callable, Final +from typing import Any, Callable, Final import gt4py.next as gtx import numpy as np @@ -152,14 +152,14 @@ def _lateral_boundary(dim: gtx.Dimension, offset: LineNumber = LineNumber.FIRST) return _domain_index(_LATERAL_BOUNDARY, dim, offset) -def _domain_index(value_dict, dim: gtx.Dimension, offset: LineNumber) -> int: +def _domain_index(value_dict: dict, dim: gtx.Dimension, offset: LineNumber) -> int: index = value_dict[dim] + offset assert index <= _BOUNDS[dim][1], f"Index {index} out of bounds for {dim}: {_BOUNDS[dim]}" assert index >= _BOUNDS[dim][0], f"Index {index} out of bounds for {dim}: {_BOUNDS[dim]}" return index -def _local(dim: gtx.Dimension, offset=LineNumber.FIRST) -> int: +def _local(dim: gtx.Dimension, offset: LineNumber = LineNumber.FIRST) -> int: """ Indicate points that are owned by the processing unit, i.e. non halo points. @@ -375,15 +375,15 @@ class Domain: _dim: gtx.Dimension _zone: Zone - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: if isinstance(other, Domain): return self.dim == other.dim and self.zone == other.zone return False - def __hash__(self): + def __hash__(self) -> int: return hash((self.dim, self.zone)) - def __str__(self): + def __str__(self) -> str: return f"Domain (dim = {self.dim}: zone = {self._zone} /ICON index[ {_map_to_icon_index(self.dim, self.zone)} ])" @property @@ -422,7 +422,7 @@ def _domain(marker: Zone) -> Domain: return _domain -def _domain_factory(dim: gtx.Dimension, zone: Zone): +def _domain_factory(dim: gtx.Dimension, zone: Zone) -> Domain: assert _validate( dim, zone ), f"Invalid zone {zone} for dimension {dim}. Valid zones are: {get_zones_for_dim(dim)}" @@ -434,9 +434,11 @@ def _validate(dim: gtx.Dimension, marker: Zone) -> bool: case dims.CellDim: return marker in CELL_ZONES case dims.EdgeDim: - return marker in tuple(Zone) + return marker in EDGE_ZONES case dims.VertexDim: return marker in VERTEX_ZONES + case _: + return False def get_zones_for_dim(dim: gtx.Dimension) -> tuple[Zone, ...]: @@ -458,9 +460,10 @@ def get_zones_for_dim(dim: gtx.Dimension) -> tuple[Zone, ...]: def map_icon_domain_bounds( dim: gtx.Dimension, pre_computed_bounds: np.ndarray -) -> dict[Domain, gtx.int32]: +) -> dict[Domain, gtx.int32]: # type: ignore [name-defined] get_domain = domain(dim) domains = (get_domain(zone) for zone in get_zones_for_dim(dim)) return { - d: gtx.int32(pre_computed_bounds[_map_to_icon_index(dim, d.zone)].item()) for d in domains + d: gtx.int32(pre_computed_bounds[_map_to_icon_index(dim, d.zone)].item()) + for d in domains # type: ignore [attr-defined] } From 4c95d9274b4680d0f391c43a000feab4deb6bd9d Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 14 Aug 2025 11:05:28 +0200 Subject: [PATCH 075/240] some typing fixes --- .../model/common/decomposition/definitions.py | 14 +++++++------- .../src/icon4py/model/common/grid/grid_manager.py | 3 ++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 2da96217d9..a5a6287d6c 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -170,7 +170,7 @@ def halo_levels(self, dim: gtx.Dimension): def halo_level_mask(self, dim: gtx.Dimension, level: DecompositionFlag): return np.where(self._halo_levels[dim] == level, True, False) - # TODO unused - delete + # TODO (@halungge) unused - delete def is_on_node(self, dim, index: int, entryType: EntryType = EntryType.ALL) -> bool: return np.isin(index, self.global_index(dim, entry_type=entryType)).item() @@ -185,11 +185,11 @@ def is_ready(self) -> bool: ... class ExchangeRuntime(Protocol): def exchange(self, dim: gtx.Dimension, *fields: tuple) -> ExchangeResult: ... - def exchange_and_wait(self, dim: gtx.Dimension, *fields: tuple): ... + def exchange_and_wait(self, dim: gtx.Dimension, *fields: tuple) -> None: ... - def get_size(self): ... + def get_size(self) -> int: ... - def my_rank(self): ... + def my_rank(self) -> int: ... @dataclass @@ -197,13 +197,13 @@ class SingleNodeExchange: def exchange(self, dim: gtx.Dimension, *fields: tuple) -> ExchangeResult: return SingleNodeResult() - def exchange_and_wait(self, dim: gtx.Dimension, *fields: tuple): + def exchange_and_wait(self, dim: gtx.Dimension, *fields: tuple) -> None: return - def my_rank(self): + def my_rank(self) -> int: return 0 - def get_size(self): + def get_size(self) -> int: return 1 def __call__(self, *args, **kwargs) -> Optional[ExchangeResult]: diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index e1939637b6..f32d242bf3 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -395,7 +395,7 @@ def _construct_grid( ## # CONSTRUCT LOCAL PATCH - # TODO run this onlz for distrbuted grids otherwise to nothing internally + # TODO run this only for distrbuted grids otherwise to nothing internally neighbor_tables = { k: decomposition_info.global_to_local( k.source, v[decomposition_info.global_index(k.target[0])] @@ -406,6 +406,7 @@ def _construct_grid( # COMPUTE remaining derived connectivities neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) + # TODO compute for local patch start, end, _ = self._read_start_end_indices() start_indices = { From 324a2c0794f1f17455182d61d77aa69fc9d02f62 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 14 Aug 2025 12:06:28 +0200 Subject: [PATCH 076/240] move transformation to gridfile.py --- .../icon4py/model/common/grid/grid_manager.py | 54 ++++++------------- .../src/icon4py/model/common/grid/gridfile.py | 52 +++++++++++++++--- .../icon4py/model/common/grid/refinement.py | 13 ++++- .../decomposition/mpi_tests/test_halo.py | 6 ++- .../grid/integration_tests/test_horizontal.py | 2 +- .../mpi_tests/test_parallel_grid_manager.py | 7 +-- .../grid/unit_tests/test_grid_manager.py | 11 ++-- .../common/grid/unit_tests/test_gridfile.py | 4 +- .../common/grid/unit_tests/test_refinement.py | 23 ++++---- .../src/icon4py/model/testing/grid_utils.py | 3 +- 10 files changed, 104 insertions(+), 71 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index f32d242bf3..a9e02d969c 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -9,7 +9,7 @@ import logging import pathlib from types import ModuleType -from typing import Callable, Literal, Optional, Protocol, TypeAlias, Union +from typing import Callable, Literal, Optional, TypeAlias, Union import gt4py.next as gtx import gt4py.next.backend as gtx_backend @@ -36,35 +36,6 @@ class IconGridError(RuntimeError): pass -class IndexTransformation(Protocol): - """Return a transformation field to be applied to index fields""" - - def __call__( - self, - array: data_alloc.NDArray, - ) -> data_alloc.NDArray: ... - - -class NoTransformation(IndexTransformation): - """Empty implementation of the Protocol. Just return zeros.""" - - def __call__(self, array: data_alloc.NDArray): - return np.zeros_like(array) - - -class ToZeroBasedIndexTransformation(IndexTransformation): - def __call__(self, array: data_alloc.NDArray): - """ - Calculate the index offset needed for usage with python. - - Fortran indices are 1-based, hence the offset is -1 for 0-based ness of python except for - INVALID values which are marked with -1 in the grid file and are kept such. - """ - return np.asarray( - np.where(array == gridfile.GridFile.INVALID_INDEX, 0, -1), dtype=gtx.int32 - ) - - CoordinateDict: TypeAlias = dict[gtx.Dimension, dict[Literal["lat", "lon"], gtx.Field]] GeometryDict: TypeAlias = dict[gridfile.GeometryName, gtx.Field] @@ -83,12 +54,12 @@ class GridManager: def open(self): """Open the gridfile resource for reading.""" - self._reader = gridfile.GridFile(self._file_name) + self._reader = gridfile.GridFile(self._file_name, self._transformation) self._reader.open() def __init__( self, - transformation: IndexTransformation, + transformation: gridfile.IndexTransformation, grid_file: Union[pathlib.Path, str], config: v_grid.VerticalGridConfig, # TODO (@halungge) remove: - separate vertical from horizontal grid decomposer: Callable[[np.ndarray, int], np.ndarray] = halo.SingleNodeDecomposer(), @@ -227,14 +198,18 @@ def _read_geometry_fields(self, backend: Optional[gtx_backend.Backend]): gridfile.GeometryName.CELL_NORMAL_ORIENTATION.value: gtx.as_field( (dims.CellDim, dims.C2EDim), self._reader.int_variable( - gridfile.GeometryName.CELL_NORMAL_ORIENTATION, transpose=True + gridfile.GeometryName.CELL_NORMAL_ORIENTATION, + transpose=True, + apply_transformation=False, ), allocator=backend, ), gridfile.GeometryName.EDGE_ORIENTATION_ON_VERTEX.value: gtx.as_field( (dims.VertexDim, dims.V2EDim), self._reader.int_variable( - gridfile.GeometryName.EDGE_ORIENTATION_ON_VERTEX, transpose=True + gridfile.GeometryName.EDGE_ORIENTATION_ON_VERTEX, + transpose=True, + apply_transformation=False, ), allocator=backend, ), @@ -265,7 +240,9 @@ def _read_grid_refinement_fields( refinement_control_fields = { dim: gtx.as_field( (dim,), - self._reader.int_variable(name, decomposition_info, transpose=False), + self._reader.int_variable( + name, decomposition_info, transpose=False, apply_transformation=False + ), allocator=backend, ) for dim, name in refinement_control_names.items() @@ -439,10 +416,9 @@ def _get_index_field( transpose=True, apply_offset=True, ): - field = self._reader.int_variable(field, indices=indices, transpose=transpose) - if apply_offset: - field = field + self._transformation(field) - return field + return self._reader.int_variable( + field, indices=indices, transpose=transpose, apply_transformation=apply_offset + ) def _read_full_grid_size(self) -> base.HorizontalGridSize: """ diff --git a/model/common/src/icon4py/model/common/grid/gridfile.py b/model/common/src/icon4py/model/common/grid/gridfile.py index 5dbf27d392..2ca92b48f1 100644 --- a/model/common/src/icon4py/model/common/grid/gridfile.py +++ b/model/common/src/icon4py/model/common/grid/gridfile.py @@ -8,12 +8,13 @@ import enum import logging -from typing import Union +from typing import Protocol, Union import numpy as np from gt4py import next as gtx from icon4py.model.common import exceptions +from icon4py.model.common.utils import data_allocation as data_alloc _log = logging.getLogger(__name__) @@ -30,6 +31,33 @@ def __init__(self, *args, **kwargs): raise ModuleNotFoundError("NetCDF4 is not installed.") +class IndexTransformation(Protocol): + """Return a transformation field to be applied to index fields""" + + def __call__( + self, + array: data_alloc.NDArray, + ) -> data_alloc.NDArray: ... + + +class NoTransformation(IndexTransformation): + """Empty implementation of the Protocol. Just return zeros.""" + + def __call__(self, array: data_alloc.NDArray): + return np.zeros_like(array) + + +class ToZeroBasedIndexTransformation(IndexTransformation): + def __call__(self, array: data_alloc.NDArray): + """ + Calculate the index offset needed for usage with python. + + Fortran indices are 1-based, hence the offset is -1 for 0-based ness of python except for + INVALID values which are marked with -1 in the grid file and are kept such. + """ + return np.asarray(np.where(array == GridFile.INVALID_INDEX, 0, -1), dtype=gtx.int32) + + class GridFileName(str, enum.Enum): pass @@ -238,8 +266,9 @@ class GridFile: INVALID_INDEX = -1 - def __init__(self, file_name: str): + def __init__(self, file_name: str, transformation: IndexTransformation): self._filename = file_name + self.transformation = transformation self._dataset = None def dimension(self, name: DimensionName) -> int: @@ -251,7 +280,11 @@ def attribute(self, name: PropertyName) -> Union[str, int, float]: return self._dataset.getncattr(name) def int_variable( - self, name: FieldName, indices: np.ndarray | None = None, transpose: bool = True + self, + name: FieldName, + indices: np.ndarray | None = None, + transpose: bool = True, + apply_transformation: bool = True, ) -> np.ndarray: """Read a integer field from the grid file. @@ -261,18 +294,25 @@ def int_variable( name: name of the field to read indices: list of indices to read transpose: flag to indicate whether the file should be transposed (for 2d fields) + apply_transformation: flag to indicate whether the transformation should be applied + to the indices, defaults to True Returns: NDArray: field data """ - _log.debug(f"reading {name}: transposing = {transpose}") - return self.variable(name, indices, transpose=transpose, dtype=gtx.int32) + _log.debug( + f"reading {name}: transposing = {transpose} apply_transformation={apply_transformation}" + ) + variable = self.variable(name, indices, transpose=transpose, dtype=gtx.int32) + if apply_transformation: + variable = variable + self.transformation(indices) + return variable def variable( self, name: FieldName, indices: np.ndarray | None = None, - transpose=False, + transpose: bool = False, dtype: np.dtype = gtx.float64, ) -> np.ndarray: """Read a field from the grid file. diff --git a/model/common/src/icon4py/model/common/grid/refinement.py b/model/common/src/icon4py/model/common/grid/refinement.py index e79c9e840c..298b2d8572 100644 --- a/model/common/src/icon4py/model/common/grid/refinement.py +++ b/model/common/src/icon4py/model/common/grid/refinement.py @@ -129,7 +129,18 @@ def is_limited_area_grid( def compute_start_index( dim: gtx.Dimension, refinement_ctrl: data_alloc.NDArray, array_ns: ModuleType = np ) -> data_alloc.NDArray: - """Compute the start index for the refinement control field for a given dimension.""" + """ + Compute the start index for the refinement control field for a given dimension. + + Args: + dim: Dimension to handle, one out of CellDim, EdgeDim, VertexDim + refinement_ctrl: refinement control array for the given dimension + array_ns: numpy or cupy module to use for array operations + + Returns: + + """ + assert ( dim.kind == gtx.DimensionKind.HORIZONTAL ), f"dim = {dim=} refinement control values only exist for horizontal dimensions" diff --git a/model/common/tests/common/decomposition/mpi_tests/test_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_halo.py index 9fb30fc1de..d7193a8052 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_halo.py @@ -11,6 +11,8 @@ import gt4py.next as gtx import numpy as np import pytest + +import icon4py.model.common.grid.gridfile from ..fixtures import * # noqa: F403 import icon4py.model.common.dimension as dims @@ -63,7 +65,9 @@ def simple_neighbor_tables(): def grid_file_manager(file: pathlib.Path) -> gm.GridManager: manager = gm.GridManager( - gm.ToZeroBasedIndexTransformation(), str(file), v_grid.VerticalGridConfig(num_levels=1) + icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation(), + str(file), + v_grid.VerticalGridConfig(num_levels=1), ) manager(keep_skip_values=True) return manager diff --git a/model/common/tests/common/grid/integration_tests/test_horizontal.py b/model/common/tests/common/grid/integration_tests/test_horizontal.py index 73c3fafcc8..55bd6647c9 100644 --- a/model/common/tests/common/grid/integration_tests/test_horizontal.py +++ b/model/common/tests/common/grid/integration_tests/test_horizontal.py @@ -65,6 +65,6 @@ def _map_and_assert_array(dim, index_array): if d.zone == h_grid.Zone.HALO: same_index = index == index_array[h_grid._HALO[d.dim]] if d.zone == h_grid.Zone.HALO_LEVEL_2: - same_index = index == index_array[h_grid._HALO[d.dim] + 1] + same_index = index == index_array[h_grid._HALO[d.dim] - 1] if not same_index: raise AssertionError(f"Wrong index for {d.zone} zone in dimension {d.dim}: ") diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index bf37aadcf9..52907fa977 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -10,6 +10,7 @@ import pytest +import icon4py.model.common.grid.gridfile import icon4py.model.testing.grid_utils as grid_utils from icon4py.model.common import exceptions from icon4py.model.common.decomposition import definitions, halo, mpi_decomposition @@ -55,13 +56,13 @@ def test_start_end_index( partitioner = halo.SimpleMetisDecomposer() manager = gm.GridManager( - gm.ToZeroBasedIndexTransformation(), + icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation(), file, v_grid.VerticalGridConfig(1), run_properties=processor_props, ) single_node_grid = gm.GridManager( - gm.ToZeroBasedIndexTransformation(), + icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation(), file, v_grid.VerticalGridConfig(1), run_properties=definitions.get_processor_properties(definitions.SingleNodeRun()), @@ -84,7 +85,7 @@ def test_start_end_index( def test_grid_manager_validate_decomposer(processor_props): file = grid_utils.resolve_full_grid_file_name(dt_utils.R02B04_GLOBAL) manager = gm.GridManager( - gm.ToZeroBasedIndexTransformation(), + icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation(), file, v_grid.VerticalGridConfig(1), run_properties=processor_props, diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index 2d9cdfa50f..b5f164277e 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -12,6 +12,7 @@ import numpy as np import pytest +import icon4py.model.common.grid.gridfile from icon4py.model.common import dimension as dims from icon4py.model.common.decomposition import definitions as defs, halo from icon4py.model.common.grid import ( @@ -42,7 +43,7 @@ MCH_CH_RO4B09_GLOBAL_NUM_CELLS = 83886080 -ZERO_BASE = gm.ToZeroBasedIndexTransformation() +ZERO_BASE = icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation() vertical = v_grid.VerticalGridConfig(num_levels=80) @@ -90,7 +91,7 @@ def test_grid_manager_refin_ctrl(grid_savepoint, grid_file, experiment, dim, bac ).grid.refinement_control refin_ctrl_serialized = grid_savepoint.refin_ctrl(dim) assert np.all( - refin_ctrl_serialized.ndarrayg + refin_ctrl_serialized.ndarray == refin.convert_to_unnested_refinement_values(refin_ctrl[dim].ndarray, dim) ) @@ -364,7 +365,9 @@ def test_gridmanager_given_file_not_found_then_abort(): fname = "./unknown_grid.nc" with pytest.raises(FileNotFoundError) as error: manager = gm.GridManager( - gm.NoTransformation(), fname, v_grid.VerticalGridConfig(num_levels=80) + icon4py.model.common.grid.gridfile.NoTransformation(), + fname, + v_grid.VerticalGridConfig(num_levels=80), ) manager(backend=None, keep_skip_values=True) assert error.value == 1 @@ -373,7 +376,7 @@ def test_gridmanager_given_file_not_found_then_abort(): @pytest.mark.parametrize("size", [100, 1500, 20000]) @pytest.mark.with_netcdf def test_gt4py_transform_offset_by_1_where_valid(size): - trafo = gm.ToZeroBasedIndexTransformation() + trafo = icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation() rng = np.random.default_rng() input_field = rng.integers(-1, size, size) offset = trafo(input_field) diff --git a/model/common/tests/common/grid/unit_tests/test_gridfile.py b/model/common/tests/common/grid/unit_tests/test_gridfile.py index 5a2bcb5a6b..bce9f1a828 100644 --- a/model/common/tests/common/grid/unit_tests/test_gridfile.py +++ b/model/common/tests/common/grid/unit_tests/test_gridfile.py @@ -26,7 +26,7 @@ @pytest.mark.with_netcdf def test_grid_file_dimension(): global_grid_file = str(gridtest_utils.resolve_full_grid_file_name(dt_utils.R02B04_GLOBAL)) - parser = gridfile.GridFile(global_grid_file) + parser = gridfile.GridFile(global_grid_file, gridfile.NoTransformation()) try: parser.open() assert parser.dimension(gridfile.DimensionName.CELL_NAME) == utils.R02B04_GLOBAL_NUM_CELLS @@ -51,7 +51,7 @@ def test_grid_file_dimension(): ) def test_grid_file_vertex_cell_edge_dimensions(grid_savepoint, grid_file): file = gridtest_utils.resolve_full_grid_file_name(grid_file) - parser = gridfile.GridFile(str(file)) + parser = gridfile.GridFile(str(file), gridfile.NoTransformation()) try: parser.open() assert parser.dimension(gridfile.DimensionName.CELL_NAME) == grid_savepoint.num( diff --git a/model/common/tests/common/grid/unit_tests/test_refinement.py b/model/common/tests/common/grid/unit_tests/test_refinement.py index 0adbb87ac0..26010480a6 100644 --- a/model/common/tests/common/grid/unit_tests/test_refinement.py +++ b/model/common/tests/common/grid/unit_tests/test_refinement.py @@ -101,25 +101,22 @@ def test_compute_start_index(dim, grid_file, experiment, start_indices): assert start_index.shape[0] == h_grid.GRID_REFINEMENT_SIZE[dim] domain = h_grid.domain(dim) assert ( - start_index[domain(h_grid.Zone.LATERAL_BOUNDARY)()] - == reference_start[domain(h_grid.Zone.LATERAL_BOUNDARY)()] + start_index[domain(h_grid.Zone.LATERAL_BOUNDARY)] + == reference_start[domain(h_grid.Zone.LATERAL_BOUNDARY)] ) assert ( - start_index[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2)()] - == reference_start[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2)()] + start_index[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2)] + == reference_start[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2)] ) assert ( - start_index[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_3)()] - == reference_start[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_3)()] + start_index[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_3)] + == reference_start[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_3)] ) assert ( - start_index[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_4)()] - == reference_start[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_4)()] + start_index[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_4)] + == reference_start[domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_4)] ) + assert start_index[domain(h_grid.Zone.NUDGING)] == reference_start[domain(h_grid.Zone.NUDGING)] assert ( - start_index[domain(h_grid.Zone.NUDGING)()] == reference_start[domain(h_grid.Zone.NUDGING)()] - ) - assert ( - start_index[domain(h_grid.Zone.INTERIOR)()] - == reference_start[domain(h_grid.Zone.INTERIOR)()] + start_index[domain(h_grid.Zone.INTERIOR)] == reference_start[domain(h_grid.Zone.INTERIOR)] ) diff --git a/model/testing/src/icon4py/model/testing/grid_utils.py b/model/testing/src/icon4py/model/testing/grid_utils.py index c8385aba2c..6e8564eedb 100644 --- a/model/testing/src/icon4py/model/testing/grid_utils.py +++ b/model/testing/src/icon4py/model/testing/grid_utils.py @@ -10,6 +10,7 @@ import gt4py.next.backend as gtx_backend +import icon4py.model.common.grid.gridfile from icon4py.model.common.decomposition import halo from icon4py.model.common.grid import ( geometry, @@ -127,7 +128,7 @@ def _download_and_load_gridfile( """ grid_file = _download_grid_file(file_path) manager = gm.GridManager( - gm.ToZeroBasedIndexTransformation(), + icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation(), grid_file, v_grid.VerticalGridConfig(num_levels=num_levels), ) From dde8759df78fcd789f80b2973f5890b0051306cf Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 26 Aug 2025 10:57:45 +0200 Subject: [PATCH 077/240] pass decomposer to GridManager.__call__ --- .../icon4py/model/common/grid/grid_manager.py | 77 +++++++++---------- .../decomposition/mpi_tests/test_halo.py | 21 +++-- 2 files changed, 46 insertions(+), 52 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 052ffa7c29..b4deb1dc4f 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -96,22 +96,9 @@ def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is FileNotFoundError: raise FileNotFoundError(f"gridfile {self._file_name} not found, aborting") - @utils.chainable # TODO(halungge): split into to functions - def set_decomposer( - self, - decomposer: Callable[[np.ndarray, int], np.ndarray], - ): - self._decompose = decomposer - self._validate_decomposer() - def _validate_decomposer(self): - if not self._decompose or ( - isinstance(self._decompose, halo.SingleNodeDecomposer) - and not self._run_properties.single_node() - ): - raise exceptions.InvalidConfigError("Need a Decomposer for for multi node run.") - def __call__(self, backend: gtx_backend.Backend | None, keep_skip_values: bool): + def __call__(self, backend: gtx_backend.Backend | None, keep_skip_values: bool, decomposer:Callable[[np.ndarray, int], np.ndarray]): if not self._reader: self.open() @@ -319,7 +306,7 @@ def coordinates(self) -> CoordinateDict: return self._coordinates def _construct_grid( - self, backend: gtx_backend.Backend | None, with_skip_values: bool + self, backend: gtx_backend.Backend | None, with_skip_values: bool, decomposer:Callable[[np.ndarray, int], np.ndarray] = None ) -> icon.IconGrid: """Construct the grid topology from the icon grid file. @@ -343,17 +330,8 @@ def _construct_grid( limited_area=limited_area, keep_skip_values=with_skip_values, ) - - # DECOMPOSITION cell_to_cell_neighbors = self._get_index_field(gridfile.ConnectivityName.C2E2C) - cells_to_rank_mapping = self._decompose( - cell_to_cell_neighbors, - self._run_properties.comm_size, - ) - # HALO CONSTRUCTION - # TODO(halungge): reduce the set of neighbor tables used in the halo construction - # TODO (halungge): figure out where to do the host to device copies (xp.asarray...) - neighbor_tables_for_halo_construction = { + neighbor_tables = { dims.C2E2C: cell_to_cell_neighbors, dims.C2E: self._get_index_field(gridfile.ConnectivityName.C2E), dims.E2C: self._get_index_field(gridfile.ConnectivityName.E2C), @@ -363,30 +341,47 @@ def _construct_grid( dims.V2E2V: self._get_index_field(gridfile.ConnectivityName.V2E2V), dims.E2V: self._get_index_field(gridfile.ConnectivityName.E2V), } - # halo_constructor - creates the decomposition info, which can then be used to generate the local patches on each rank - halo_constructor = self._initialize_halo_constructor( - global_grid_size, neighbor_tables_for_halo_construction, backend - ) - decomposition_info = halo_constructor(cells_to_rank_mapping) - self._decomposition_info = decomposition_info - ## TODO(halungge): do local reads (and halo exchanges!!) FIX: my_cells etc are in 0 base python coding - reading from file fails... - ## - # CONSTRUCT LOCAL PATCH + if decomposer is not None: + # DECOMPOSITION - # TODO(halungge): run this only for distrbuted grids otherwise to nothing internally - neighbor_tables = { - k: decomposition_info.global_to_local( - k.source, v[decomposition_info.global_index(k.target[0])] + cells_to_rank_mapping = self._decompose( + cell_to_cell_neighbors, + self._run_properties.comm_size, ) - for k, v in neighbor_tables_for_halo_construction.items() - } + # HALO CONSTRUCTION + # TODO(halungge): reduce the set of neighbor tables used in the halo construction + # TODO (halungge): figure out where to do the host to device copies (xp.asarray...) + neighbor_tables_for_halo_construction = neighbor_tables + # halo_constructor - creates the decomposition info, which can then be used to generate the local patches on each rank + halo_constructor = self._initialize_halo_constructor( + global_grid_size, neighbor_tables_for_halo_construction, backend + ) + decomposition_info = halo_constructor(cells_to_rank_mapping) + + self._decomposition_info = decomposition_info + + ## TODO(halungge): do local reads (and halo exchanges!!) FIX: my_cells etc are in 0 base python coding - reading from file fails... + ## + # CONSTRUCT LOCAL PATCH + + # TODO(halungge): run this only for distrbuted grids otherwise to nothing internally + neighbor_tables = { + k: decomposition_info.global_to_local( + k.source, v[decomposition_info.global_index(k.target[0])] + ) + for k, v in neighbor_tables_for_halo_construction.items() + } # COMPUTE remaining derived connectivities + else: + # TODO no decomposition + ... + # JOINT functionality neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) - # TODO(halungge): compute for local patch + # TODO(halungge): needs to be moved out of the joint section start, end, _ = self._read_start_end_indices() start_indices = { k: v diff --git a/model/common/tests/common/decomposition/mpi_tests/test_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_halo.py index d7193a8052..991b8661d3 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_halo.py @@ -40,7 +40,7 @@ simple, vertical as v_grid, ) -from icon4py.model.testing import datatest_utils as dt_utils, helpers, definitions as test_defs +from icon4py.model.testing import datatest_utils as dt_utils, definitions as test_defs UGRID_FILE = ( @@ -49,7 +49,7 @@ .joinpath("icon_grid_0013_R02B04_R_ugrid.nc") ) GRID_FILE = ( - test_defs.grids_path().joinpath(dt_utils.R02B04_GLOBAL).joinpath("icon_grid_0013_R02B04_R.nc") + test_defs.grids_path().joinpath(test_defs.Grids.R02B04_GLOBAL.name).joinpath("icon_grid_0012_R02B04_G.nc") ) backend = None @@ -63,7 +63,7 @@ def simple_neighbor_tables(): return neighbor_tables -def grid_file_manager(file: pathlib.Path) -> gm.GridManager: +def single_node_gridfile_manager(file: pathlib.Path) -> gm.GridManager: manager = gm.GridManager( icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation(), str(file), @@ -280,18 +280,18 @@ def decompose(grid: base_grid.Grid, processor_props): # F811 # fixture @pytest.mark.xfail @pytest.mark.mpi def test_distributed_fields(processor_props): # F811 # fixture - grid_manager = grid_file_manager(GRID_FILE) + grid_manager = single_node_gridfile_manager(GRID_FILE) - global_grid = grid_manager.grid + single_node_grid = grid_manager.grid global_cell_area = grid_manager.geometry[grid_file.GeometryName.CELL_AREA] global_edge_lat = grid_manager.coordinates[dims.EdgeDim]["lat"] global_vertex_lon = grid_manager.coordinates[dims.VertexDim]["lon"] - labels = decompose(global_grid, processor_props) - + labels = decompose(single_node_grid, processor_props) + neighbor_tables = {k:v.ndarray for k, v in single_node_grid.connectivities.items()} halo_generator = halo.IconLikeHaloConstructor( - connectivities=global_grid.neighbor_tables, + connectivities=neighbor_tables, run_properties=processor_props, num_levels=1, ) @@ -308,7 +308,7 @@ def test_distributed_fields(processor_props): # F811 # fixture ) # the local number of cells must be at most the global number of cells (analytically computed) assert ( - local_cell_area.asnumpy().shape[0] <= global_grid.global_properties.num_cells + local_cell_area.asnumpy().shape[0] <= single_node_grid.global_properties.num_cells ), "local field is larger than global field" # global read: read the same (global fields) @@ -349,12 +349,11 @@ def assert_gathered_field_against_global( sorted_[gathered_global_indices] = gathered_field assert helpers.dallclose(sorted_, global_reference_field) - # TODO add test including halo access: # Will uses geofac_div and geofac_n2s - @pytest.mark.xfail +@pytest.mark.mpi def test_halo_neighbor_access_c2e(): pytest.fail("TODO implement") # geofac_div = primal_edge_length(C2E) * edge_orientation / area From 1f61a40393dd37274ffe79424e090b804022a298 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 26 Aug 2025 13:10:28 +0200 Subject: [PATCH 078/240] move fixture import to conftests.py in model/common/tests/common/grid --- .../tests/common/grid/unit_tests/conftest.py | 22 +++++++++++++++++++ .../common/grid/unit_tests/test_geometry.py | 9 +------- .../common/grid/unit_tests/test_gridfile.py | 9 +------- .../tests/common/grid/unit_tests/test_icon.py | 11 +--------- .../common/grid/unit_tests/test_refinement.py | 2 +- .../common/grid/unit_tests/test_topography.py | 1 - .../common/grid/unit_tests/test_vertical.py | 22 +------------------ 7 files changed, 27 insertions(+), 49 deletions(-) create mode 100644 model/common/tests/common/grid/unit_tests/conftest.py diff --git a/model/common/tests/common/grid/unit_tests/conftest.py b/model/common/tests/common/grid/unit_tests/conftest.py new file mode 100644 index 0000000000..0ad54a5edf --- /dev/null +++ b/model/common/tests/common/grid/unit_tests/conftest.py @@ -0,0 +1,22 @@ +from icon4py.model.testing.fixtures import ( + backend, + damping_height, + flat_height, + data_provider, + download_ser_data, + experiment, + grid_savepoint, + icon_grid, + processor_props, + model_top_height, + stretch_factor, + htop_moist_proc, + metrics_savepoint, + ranked_data_path, + interpolation_savepoint, + lowest_layer_thickness, + maximal_layer_thickness, + topography_savepoint, + top_height_limit_for_maximal_layer_thickness, +) + diff --git a/model/common/tests/common/grid/unit_tests/test_geometry.py b/model/common/tests/common/grid/unit_tests/test_geometry.py index 64e3baf52e..9298885985 100644 --- a/model/common/tests/common/grid/unit_tests/test_geometry.py +++ b/model/common/tests/common/grid/unit_tests/test_geometry.py @@ -21,14 +21,7 @@ from icon4py.model.common.math import helpers as math_helpers from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import datatest_utils as dt_utils, grid_utils, test_utils -from icon4py.model.testing.fixtures import ( - backend, - data_provider, - download_ser_data, - grid_savepoint, - processor_props, - ranked_data_path, -) + def test_geometry_raises_for_unknown_field(backend): diff --git a/model/common/tests/common/grid/unit_tests/test_gridfile.py b/model/common/tests/common/grid/unit_tests/test_gridfile.py index bce9f1a828..b44bfab5ac 100644 --- a/model/common/tests/common/grid/unit_tests/test_gridfile.py +++ b/model/common/tests/common/grid/unit_tests/test_gridfile.py @@ -11,14 +11,7 @@ from icon4py.model.common import dimension as dims from icon4py.model.common.grid import gridfile from icon4py.model.testing import datatest_utils as dt_utils, grid_utils as gridtest_utils -from icon4py.model.testing.fixtures import ( - backend, - data_provider, - download_ser_data, - grid_savepoint, - processor_props, - ranked_data_path, -) + from .. import utils diff --git a/model/common/tests/common/grid/unit_tests/test_icon.py b/model/common/tests/common/grid/unit_tests/test_icon.py index cd63023206..f67a2ad967 100644 --- a/model/common/tests/common/grid/unit_tests/test_icon.py +++ b/model/common/tests/common/grid/unit_tests/test_icon.py @@ -19,16 +19,7 @@ icon, ) from icon4py.model.testing import datatest_utils as dt_utils, grid_utils as gridtest_utils -from icon4py.model.testing.fixtures import ( - backend, - data_provider, - download_ser_data, - experiment, - grid_savepoint, - icon_grid, - processor_props, - ranked_data_path, -) + from .. import utils diff --git a/model/common/tests/common/grid/unit_tests/test_refinement.py b/model/common/tests/common/grid/unit_tests/test_refinement.py index 5a8d04e05d..efa6012254 100644 --- a/model/common/tests/common/grid/unit_tests/test_refinement.py +++ b/model/common/tests/common/grid/unit_tests/test_refinement.py @@ -16,7 +16,7 @@ from icon4py.model.testing.fixtures import backend from .. import utils -from ..fixtures import * + def out_of_range(dim: gtx.Dimension): diff --git a/model/common/tests/common/grid/unit_tests/test_topography.py b/model/common/tests/common/grid/unit_tests/test_topography.py index 2cfffd58bb..99568b4d56 100644 --- a/model/common/tests/common/grid/unit_tests/test_topography.py +++ b/model/common/tests/common/grid/unit_tests/test_topography.py @@ -11,7 +11,6 @@ from icon4py.model.common.grid import geometry, topography as topo from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import datatest_utils as dt_utils, test_utils -from icon4py.model.testing.fixtures import * # noqa: F403 @pytest.mark.embedded_remap_error diff --git a/model/common/tests/common/grid/unit_tests/test_vertical.py b/model/common/tests/common/grid/unit_tests/test_vertical.py index ef25af9bb0..0b43bcfaaa 100644 --- a/model/common/tests/common/grid/unit_tests/test_vertical.py +++ b/model/common/tests/common/grid/unit_tests/test_vertical.py @@ -15,27 +15,7 @@ from icon4py.model.common.grid import vertical as v_grid from icon4py.model.common.utils import data_allocation as data_alloc, device_utils from icon4py.model.testing import datatest_utils as dt_utils, grid_utils, test_utils -from icon4py.model.testing.fixtures import ( - backend, - damping_height, - data_provider, - download_ser_data, - experiment, - flat_height, - grid_savepoint, - htop_moist_proc, - icon_grid, - interpolation_savepoint, - lowest_layer_thickness, - maximal_layer_thickness, - metrics_savepoint, - model_top_height, - processor_props, - ranked_data_path, - stretch_factor, - top_height_limit_for_maximal_layer_thickness, - topography_savepoint, -) + NUM_LEVELS = grid_utils.MCH_CH_R04B09_LEVELS From a9d9d339e0ef165c9f4756cbd7621a0d55f67271 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 26 Aug 2025 13:12:40 +0200 Subject: [PATCH 079/240] fix import --- model/testing/src/icon4py/model/testing/fixtures/datatest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/testing/src/icon4py/model/testing/fixtures/datatest.py b/model/testing/src/icon4py/model/testing/fixtures/datatest.py index 0254b04e7d..e0fecbe681 100644 --- a/model/testing/src/icon4py/model/testing/fixtures/datatest.py +++ b/model/testing/src/icon4py/model/testing/fixtures/datatest.py @@ -175,7 +175,7 @@ def icon_grid( @pytest.fixture def decomposition_info( data_provider: serialbox.IconSerialDataProvider, experiment: str -) -> definitions.DecompositionInfo: +) -> decomposition.DecompositionInfo: root, level = dt_utils.get_global_grid_params(experiment) grid_id = dt_utils.get_grid_id_for_experiment(experiment) return data_provider.from_savepoint_grid( From e5018657c4a9d58c9f3cbc733142658a73721760 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 26 Aug 2025 16:50:22 +0200 Subject: [PATCH 080/240] fix selective read from gridfile --- .../icon4py/model/common/grid/grid_manager.py | 16 ++-- .../src/icon4py/model/common/grid/gridfile.py | 21 ++++- .../decomposition/mpi_tests/test_halo.py | 8 +- model/common/tests/common/grid/fixtures.py | 19 ++++- .../grid/integration_tests/test_horizontal.py | 2 +- .../tests/common/grid/unit_tests/conftest.py | 22 ------ .../common/grid/unit_tests/test_geometry.py | 4 +- .../grid/unit_tests/test_grid_manager.py | 2 +- .../common/grid/unit_tests/test_gridfile.py | 78 ++++++++++++++++++- .../common/grid/unit_tests/test_horizontal.py | 1 + .../tests/common/grid/unit_tests/test_icon.py | 2 +- .../common/grid/unit_tests/test_refinement.py | 16 ++-- .../common/grid/unit_tests/test_topography.py | 2 + .../common/grid/unit_tests/test_vertical.py | 3 +- 14 files changed, 143 insertions(+), 53 deletions(-) delete mode 100644 model/common/tests/common/grid/unit_tests/conftest.py diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index b4deb1dc4f..cdf30707e1 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -16,7 +16,7 @@ import gt4py.next.backend as gtx_backend import numpy as np -from icon4py.model.common import dimension as dims, exceptions, type_alias as ta, utils +from icon4py.model.common import dimension as dims, type_alias as ta from icon4py.model.common.decomposition import definitions as decomposition, halo from icon4py.model.common.grid import ( base, @@ -96,9 +96,12 @@ def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is FileNotFoundError: raise FileNotFoundError(f"gridfile {self._file_name} not found, aborting") - - - def __call__(self, backend: gtx_backend.Backend | None, keep_skip_values: bool, decomposer:Callable[[np.ndarray, int], np.ndarray]): + def __call__( + self, + backend: gtx_backend.Backend | None, + keep_skip_values: bool, + decomposer: Callable[[np.ndarray, int], np.ndarray] = halo.SingleNodeDecomposer(), + ): if not self._reader: self.open() @@ -306,7 +309,10 @@ def coordinates(self) -> CoordinateDict: return self._coordinates def _construct_grid( - self, backend: gtx_backend.Backend | None, with_skip_values: bool, decomposer:Callable[[np.ndarray, int], np.ndarray] = None + self, + backend: gtx_backend.Backend | None, + with_skip_values: bool, + decomposer: Callable[[np.ndarray, int], np.ndarray] = None, ) -> icon.IconGrid: """Construct the grid topology from the icon grid file. diff --git a/model/common/src/icon4py/model/common/grid/gridfile.py b/model/common/src/icon4py/model/common/grid/gridfile.py index 0c8cc78199..2f1e88dcf2 100644 --- a/model/common/src/icon4py/model/common/grid/gridfile.py +++ b/model/common/src/icon4py/model/common/grid/gridfile.py @@ -268,7 +268,7 @@ class GridFile: def __init__(self, file_name: str, transformation: IndexTransformation): self._filename = file_name - self.transformation = transformation + self._offset = transformation self._dataset = None def dimension(self, name: DimensionName) -> int: @@ -305,7 +305,7 @@ def int_variable( ) variable = self.variable(name, indices, transpose=transpose, dtype=gtx.int32) if apply_transformation: - variable = variable + self.transformation(indices) + return variable + self._offset(variable) return variable def variable( @@ -320,15 +320,21 @@ def variable( If a index array is given it only reads the values at those positions. Args: name: name of the field to read - indices: indices to read + indices: indices to read if requesting a restricted set of indices. We assume this be a 1d array it will be applied to the 1. dimension (after transposition) transpose: flag indicateing whether the array needs to be transposed to match icon4py dimension ordering, defaults to False dtype: datatype of the field """ + + assert indices is None or indices.ndim == 1, "indices must be 1 dimensional" + try: variable = self._dataset.variables[name] + slicer = [slice(None) for _ in range(variable.ndim)] + if indices is not None and indices.size > 0: + slicer[(1 if transpose else 0)] = indices _log.debug(f"reading {name}: transposing = {transpose}") - data = variable[:] if indices is None else variable[indices] + data = variable[tuple(slicer)] data = np.array(data, dtype=dtype) return np.transpose(data) if transpose else data except KeyError as err: @@ -337,6 +343,13 @@ def variable( _log.debug(f"Error: {err}") raise exceptions.IconGridError(msg) from err + def __enter__(self): + self.open() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + def close(self): self._dataset.close() diff --git a/model/common/tests/common/decomposition/mpi_tests/test_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_halo.py index 991b8661d3..c22cd94d07 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_halo.py @@ -49,7 +49,9 @@ .joinpath("icon_grid_0013_R02B04_R_ugrid.nc") ) GRID_FILE = ( - test_defs.grids_path().joinpath(test_defs.Grids.R02B04_GLOBAL.name).joinpath("icon_grid_0012_R02B04_G.nc") + test_defs.grids_path() + .joinpath(test_defs.Grids.R02B04_GLOBAL.name) + .joinpath("icon_grid_0012_R02B04_G.nc") ) backend = None @@ -289,7 +291,7 @@ def test_distributed_fields(processor_props): # F811 # fixture global_vertex_lon = grid_manager.coordinates[dims.VertexDim]["lon"] labels = decompose(single_node_grid, processor_props) - neighbor_tables = {k:v.ndarray for k, v in single_node_grid.connectivities.items()} + neighbor_tables = {k: v.ndarray for k, v in single_node_grid.connectivities.items()} halo_generator = halo.IconLikeHaloConstructor( connectivities=neighbor_tables, run_properties=processor_props, @@ -349,9 +351,11 @@ def assert_gathered_field_against_global( sorted_[gathered_global_indices] = gathered_field assert helpers.dallclose(sorted_, global_reference_field) + # TODO add test including halo access: # Will uses geofac_div and geofac_n2s + @pytest.mark.xfail @pytest.mark.mpi def test_halo_neighbor_access_c2e(): diff --git a/model/common/tests/common/grid/fixtures.py b/model/common/tests/common/grid/fixtures.py index 3e550a25f3..b7959f0733 100644 --- a/model/common/tests/common/grid/fixtures.py +++ b/model/common/tests/common/grid/fixtures.py @@ -7,9 +7,24 @@ # SPDX-License-Identifier: BSD-3-Clause from icon4py.model.testing.fixtures.datatest import ( - processor_props, - grid_savepoint, + backend, + damping_height, + flat_height, data_provider, download_ser_data, + experiment, + grid_savepoint, + icon_grid, + processor_props, + model_top_height, + stretch_factor, + htop_moist_proc, + metrics_savepoint, ranked_data_path, + interpolation_savepoint, + lowest_layer_thickness, + maximal_layer_thickness, + topography_savepoint, + top_height_limit_for_maximal_layer_thickness, + backend, ) diff --git a/model/common/tests/common/grid/integration_tests/test_horizontal.py b/model/common/tests/common/grid/integration_tests/test_horizontal.py index 55bd6647c9..7a7ea6a11b 100644 --- a/model/common/tests/common/grid/integration_tests/test_horizontal.py +++ b/model/common/tests/common/grid/integration_tests/test_horizontal.py @@ -12,7 +12,7 @@ from icon4py.model.common.grid import horizontal as h_grid from icon4py.model.testing import datatest_utils as dt_utils from .. import utils -from ...fixtures import * # noqa: F401, F403 +from ..fixtures import * # noqa: F401, F403 @pytest.mark.datatest diff --git a/model/common/tests/common/grid/unit_tests/conftest.py b/model/common/tests/common/grid/unit_tests/conftest.py deleted file mode 100644 index 0ad54a5edf..0000000000 --- a/model/common/tests/common/grid/unit_tests/conftest.py +++ /dev/null @@ -1,22 +0,0 @@ -from icon4py.model.testing.fixtures import ( - backend, - damping_height, - flat_height, - data_provider, - download_ser_data, - experiment, - grid_savepoint, - icon_grid, - processor_props, - model_top_height, - stretch_factor, - htop_moist_proc, - metrics_savepoint, - ranked_data_path, - interpolation_savepoint, - lowest_layer_thickness, - maximal_layer_thickness, - topography_savepoint, - top_height_limit_for_maximal_layer_thickness, -) - diff --git a/model/common/tests/common/grid/unit_tests/test_geometry.py b/model/common/tests/common/grid/unit_tests/test_geometry.py index 9298885985..6dd3c9e622 100644 --- a/model/common/tests/common/grid/unit_tests/test_geometry.py +++ b/model/common/tests/common/grid/unit_tests/test_geometry.py @@ -21,7 +21,7 @@ from icon4py.model.common.math import helpers as math_helpers from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import datatest_utils as dt_utils, grid_utils, test_utils - +from ..fixtures import * # noqa: F401, F403 def test_geometry_raises_for_unknown_field(backend): @@ -411,7 +411,7 @@ def test_sparse_fields_creator(): (dt_utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT), ], ) -def test_create_auxiliary_orientation_coordinates(backend, grid_savepoint, grid_file): +def test_create_auxiliary_orientation_coordinates(backend, grid_savepoint, grid_file, experiment): gm = grid_utils.get_grid_manager_from_identifier( grid_file_identifier=grid_file, num_levels=1, diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index 947372f9fa..66840f1e3c 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -38,7 +38,7 @@ from .. import utils -from ..fixtures import * +from ..fixtures import * # noqa: F401, F403 MCH_CH_RO4B09_GLOBAL_NUM_CELLS = 83886080 diff --git a/model/common/tests/common/grid/unit_tests/test_gridfile.py b/model/common/tests/common/grid/unit_tests/test_gridfile.py index b44bfab5ac..70065a2c90 100644 --- a/model/common/tests/common/grid/unit_tests/test_gridfile.py +++ b/model/common/tests/common/grid/unit_tests/test_gridfile.py @@ -5,15 +5,21 @@ # # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause - +import numpy as np import pytest +from typing import Iterable from icon4py.model.common import dimension as dims from icon4py.model.common.grid import gridfile -from icon4py.model.testing import datatest_utils as dt_utils, grid_utils as gridtest_utils +from icon4py.model.testing import ( + datatest_utils as dt_utils, + grid_utils as gridtest_utils, + definitions as test_defs, +) from .. import utils +from ..fixtures import * # noqa: F401, F403 @pytest.mark.with_netcdf @@ -38,8 +44,8 @@ def test_grid_file_dimension(): @pytest.mark.parametrize( "grid_file, experiment", [ - (dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), - (dt_utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT), + (test_defs.Grids.MCH_CH_R04B09_DSL.name, test_defs.Experiments.MCH_CH_R04B09.name), + (test_defs.Grids.R02B04_GLOBAL.name, test_defs.Experiments.EXCLAIM_APE.name), ], ) def test_grid_file_vertex_cell_edge_dimensions(grid_savepoint, grid_file): @@ -60,3 +66,67 @@ def test_grid_file_vertex_cell_edge_dimensions(grid_savepoint, grid_file): pytest.fail(f"reading of dimension from netcdf failed: {error}") finally: parser.close() + + +@pytest.mark.parametrize("filename", (test_defs.Grids.R02B04_GLOBAL.name,)) +@pytest.mark.parametrize("apply_transformation", (True, False)) +def test_int_variable(filename, apply_transformation): + file = gridtest_utils.resolve_full_grid_file_name(filename) + with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: + edge_dim = parser.dimension(gridfile.DimensionName.EDGE_NAME) + # use a test field that does not contain Pentagons + test_field = parser.int_variable( + gridfile.ConnectivityName.C2E, apply_transformation=apply_transformation + ) + min_value = 0 if apply_transformation else 1 + max_value = edge_dim - 1 if apply_transformation else edge_dim + assert min_value == np.min(test_field) + assert max_value == np.max(test_field) + + +def index_selection() -> Iterable[list[int]]: + return ( + x + for x in [ + [0, 1, 2, 3, 4, 5], + [], + [0, 2, 4, 6, 7, 8, 24, 57], + [1, 2, 12, 13, 23, 24, 2306], + ] + ) + + +@pytest.mark.parametrize( + "selection", + index_selection(), +) +@pytest.mark.parametrize("filename", (test_defs.Grids.R02B04_GLOBAL.name,)) +def test_index_read_for_1d_fields(filename, selection): + file = gridtest_utils.resolve_full_grid_file_name(filename) + with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: + selection = np.asarray(selection) if len(selection) > 0 else None + full_field = parser.variable(gridfile.CoordinateName.CELL_LATITUDE) + selective_field = parser.variable(gridfile.CoordinateName.CELL_LATITUDE, indices=selection) + assert np.allclose(full_field[selection], selective_field) + + +@pytest.mark.parametrize( + "selection", + index_selection(), +) +@pytest.mark.parametrize("filename", (test_defs.Grids.R02B04_GLOBAL.name,)) +@pytest.mark.parametrize( + "field", + (gridfile.ConnectivityName.V2E, gridfile.ConnectivityName.V2C, gridfile.ConnectivityName.E2V), +) +@pytest.mark.parametrize("apply_offset", (True, False)) +def test_index_read_for_2d_connectivity(filename, selection, field, apply_offset): + file = gridtest_utils.resolve_full_grid_file_name(filename) + with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: + selection = np.asarray(selection) if len(selection) > 0 else None + # TODO(halungge): grid_file.ConnectivityName.V2E:P 2 D fields + full_field = parser.int_variable(field, transpose=True, apply_transformation=apply_offset) + selective_field = parser.int_variable( + field, indices=selection, transpose=True, apply_transformation=apply_offset + ) + assert np.allclose(full_field[selection], selective_field) diff --git a/model/common/tests/common/grid/unit_tests/test_horizontal.py b/model/common/tests/common/grid/unit_tests/test_horizontal.py index c288202237..69d4fe3748 100644 --- a/model/common/tests/common/grid/unit_tests/test_horizontal.py +++ b/model/common/tests/common/grid/unit_tests/test_horizontal.py @@ -11,6 +11,7 @@ import icon4py.model.common.grid.horizontal as h_grid from .. import utils +from ..fixtures import * # noqa: F401, F403 log = logging.getLogger(__name__) diff --git a/model/common/tests/common/grid/unit_tests/test_icon.py b/model/common/tests/common/grid/unit_tests/test_icon.py index f67a2ad967..67c6b42326 100644 --- a/model/common/tests/common/grid/unit_tests/test_icon.py +++ b/model/common/tests/common/grid/unit_tests/test_icon.py @@ -19,7 +19,7 @@ icon, ) from icon4py.model.testing import datatest_utils as dt_utils, grid_utils as gridtest_utils - +from ..fixtures import * # noqa: F401, F403 from .. import utils diff --git a/model/common/tests/common/grid/unit_tests/test_refinement.py b/model/common/tests/common/grid/unit_tests/test_refinement.py index efa6012254..f33f0a7057 100644 --- a/model/common/tests/common/grid/unit_tests/test_refinement.py +++ b/model/common/tests/common/grid/unit_tests/test_refinement.py @@ -12,11 +12,10 @@ from icon4py.model.common.grid import refinement as refin, horizontal as h_grid from icon4py.model.common import dimension as dims from icon4py.model.common.utils import data_allocation as data_alloc, device_utils -from icon4py.model.testing import datatest_utils as dt_utils, grid_utils -from icon4py.model.testing.fixtures import backend +from icon4py.model.testing import datatest_utils as dt_utils, grid_utils, definitions as test_defs from .. import utils - +from ..fixtures import * # noqa: F401, F403 def out_of_range(dim: gtx.Dimension): @@ -66,7 +65,8 @@ def test_valid_refinement_values(dim): @pytest.mark.parametrize("dim", utils.main_horizontal_dims()) @pytest.mark.parametrize( - "grid_file, expected", [(dt_utils.R02B04_GLOBAL, False), (dt_utils.REGIONAL_EXPERIMENT, True)] + "grid_file, expected", + [(test_defs.Grids.R02B04_GLOBAL.name, False), (test_defs.Grids.MCH_CH_R04B09_DSL.name, True)], ) def test_is_local_area_grid_for_grid_files(grid_file, expected, dim, backend): grid = grid_utils.get_grid_manager_from_identifier(grid_file, 1, True, backend).grid @@ -86,13 +86,15 @@ def start_indices(grid_savepoint) -> dict: } +@pytest.mark.xfail @pytest.mark.parametrize("dim", (dims.CellDim, dims.EdgeDim, dims.VertexDim)) @pytest.mark.parametrize( - "grid_file, experiment", [(dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT)] + "grid_file, experiment", + [(test_defs.Grids.MCH_CH_R04B09_DSL.name, test_defs.Experiments.MCH_CH_R04B09.name)], ) -def test_compute_start_index(dim, grid_file, experiment, start_indices): +def test_compute_start_index(dim, grid_file, start_indices, experiment): reference_start = start_indices.get(dim) - grid = grid_utils.get_grid_manager( + grid = grid_utils.get_grid_manager_from_identifier( grid_file, num_levels=1, keep_skip_values=True, backend=None ).grid refinement_control_field = grid.refinement_control[dim] diff --git a/model/common/tests/common/grid/unit_tests/test_topography.py b/model/common/tests/common/grid/unit_tests/test_topography.py index 99568b4d56..c66f2aee7e 100644 --- a/model/common/tests/common/grid/unit_tests/test_topography.py +++ b/model/common/tests/common/grid/unit_tests/test_topography.py @@ -12,6 +12,8 @@ from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import datatest_utils as dt_utils, test_utils +from ..fixtures import * # noqa: F401, F403 + @pytest.mark.embedded_remap_error @pytest.mark.datatest diff --git a/model/common/tests/common/grid/unit_tests/test_vertical.py b/model/common/tests/common/grid/unit_tests/test_vertical.py index 0b43bcfaaa..9f07bd76d1 100644 --- a/model/common/tests/common/grid/unit_tests/test_vertical.py +++ b/model/common/tests/common/grid/unit_tests/test_vertical.py @@ -15,8 +15,7 @@ from icon4py.model.common.grid import vertical as v_grid from icon4py.model.common.utils import data_allocation as data_alloc, device_utils from icon4py.model.testing import datatest_utils as dt_utils, grid_utils, test_utils - - +from ..fixtures import * # noqa: F401, F403 NUM_LEVELS = grid_utils.MCH_CH_R04B09_LEVELS From 14f408ee9db4ab6ccf64caa8c01316dc4f214e0e Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 27 Aug 2025 10:08:12 +0200 Subject: [PATCH 081/240] test:distribute and gather simple fields (WIP) --- .../icon4py/model/common/grid/grid_manager.py | 2 -- .../decomposition/mpi_tests/test_halo.py | 32 ++++++++++++------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index cdf30707e1..5dba41f9cc 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -64,13 +64,11 @@ def __init__( transformation: gridfile.IndexTransformation, grid_file: pathlib.Path | str, config: v_grid.VerticalGridConfig, # TODO(@halungge): remove: - separate vertical from horizontal grid - decomposer: Callable[[np.ndarray, int], np.ndarray] = _single_node_decomposer, run_properties: decomposition.ProcessProperties = _single_node_properties, ): self._run_properties = run_properties self._transformation = transformation self._file_name = str(grid_file) - self._decompose = decomposer self._halo_constructor = None self._vertical_config = config self._grid: icon.IconGrid | None = None diff --git a/model/common/tests/common/decomposition/mpi_tests/test_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_halo.py index c22cd94d07..71105c0c4d 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_halo.py @@ -36,7 +36,7 @@ from icon4py.model.common.grid import ( base as base_grid, grid_manager as gm, - gridfile as grid_file, + gridfile, simple, vertical as v_grid, ) @@ -65,16 +65,26 @@ def simple_neighbor_tables(): return neighbor_tables -def single_node_gridfile_manager(file: pathlib.Path) -> gm.GridManager: +def run_gridmananger_for_single_node(file: pathlib.Path, vertical_config:v_grid.VerticalGridConfig) -> gm.GridManager: + manager = _grid_manager(file, vertical_config) + manager(keep_skip_values=True) + return manager + + +def _grid_manager(file:pathlib.Path,vertical_config:v_grid.VerticalGridConfig): manager = gm.GridManager( - icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation(), + gridfile.ToZeroBasedIndexTransformation(), str(file), - v_grid.VerticalGridConfig(num_levels=1), + vertical_config, ) - manager(keep_skip_values=True) return manager +def run_grid_manager_for_multinode(gm:pathlib.Path, verical_config)->gm.GridManager: + manager = _grid_manager(file, vertical_config) + manager(keep_skip_values=True, run_properties=) + + @pytest.mark.mpi(min_size=4) def test_halo_constructor_owned_cells(processor_props, simple_neighbor_tables): # F811 # fixture halo_generator = halo.IconLikeHaloConstructor( @@ -279,17 +289,16 @@ def decompose(grid: base_grid.Grid, processor_props): # F811 # fixture return labels -@pytest.mark.xfail + @pytest.mark.mpi def test_distributed_fields(processor_props): # F811 # fixture - grid_manager = single_node_gridfile_manager(GRID_FILE) - + grid_manager = run_gridmananger_for_single_node(GRID_FILE) single_node_grid = grid_manager.grid - - global_cell_area = grid_manager.geometry[grid_file.GeometryName.CELL_AREA] + global_cell_area = grid_manager.geometry[gridfile.GeometryName.CELL_AREA] global_edge_lat = grid_manager.coordinates[dims.EdgeDim]["lat"] global_vertex_lon = grid_manager.coordinates[dims.VertexDim]["lon"] + labels = decompose(single_node_grid, processor_props) neighbor_tables = {k: v.ndarray for k, v in single_node_grid.connectivities.items()} halo_generator = halo.IconLikeHaloConstructor( @@ -298,10 +307,11 @@ def test_distributed_fields(processor_props): # F811 # fixture num_levels=1, ) decomposition_info = halo_generator(labels) + multi_node_grid_manager = gm.GridManager(gridfile.ToZeroBased) # distributed read: read one field per dimension ## TODO why is this local?? - local_cell_area = grid_manager.geometry[grid_file.GeometryName.CELL_AREA] + local_cell_area = grid_manager.geometry[gridfile.GeometryName.CELL_AREA] local_edge_lat = grid_manager.coordinates[dims.EdgeDim]["lat"] local_vertex_lon = grid_manager.coordinates[dims.VertexDim]["lon"] print( From eef1ea8891878febb7b0d79562b49a9b7546d8ba Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 29 Aug 2025 15:17:48 +0200 Subject: [PATCH 082/240] simple comparisison global grid manager distributed fields --- .../model/common/decomposition/definitions.py | 24 ++- .../common/decomposition/mpi_decomposition.py | 23 ++- .../icon4py/model/common/grid/grid_manager.py | 133 ++++++++++--- .../decomposition/mpi_tests/test_halo.py | 160 ++------------- .../mpi_tests/test_mpi_decomposition.py | 31 +-- .../decomposition/two_ranks_distribution.py | 17 +- .../unit_tests/test_definitions.py | 17 +- .../tests/common/decomposition/utils.py | 16 +- .../mpi_tests/test_parallel_grid_manager.py | 183 ++++++++++++++++-- .../grid/mpi_tests/test_parallel_icon.py | 12 +- 10 files changed, 370 insertions(+), 246 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 471374ca99..8f051f5aff 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -141,14 +141,22 @@ def global_to_local( sorter = np.argsort(global_indices) mask = np.isin(indices_to_translate, global_indices) - # Find the positions of `values_to_find` in the sorted `global_indices`. - # The `sorter` argument tells searchsorted to work with the sorted version - # of `global_indices` without creating an explicit sorted copy. - local_neighbors = np.where( - mask, - sorter[np.searchsorted(global_indices, indices_to_translate, sorter=sorter)], - gridfile.GridFile.INVALID_INDEX, - ) + positions = np.searchsorted(global_indices, indices_to_translate, sorter=sorter) + local_neighbors = np.full_like(indices_to_translate, gridfile.GridFile.INVALID_INDEX) + local_neighbors[mask] = sorter[positions[mask]] + return local_neighbors + + # TODO (halungge): use for test reference? in test_definitions.py + def global_to_local_ref( + self, dim: gtx.Dimension, indices_to_translate: data_alloc.NDArray + ) -> data_alloc.NDArray: + global_indices = self.global_index(dim) + local_neighbors = np.full_like(indices_to_translate, gridfile.GridFile.INVALID_INDEX) + for i in range(indices_to_translate.shape[0]): + for j in range(indices_to_translate.shape[1]): + if np.isin(indices_to_translate[i, j], global_indices): + pos = np.where(indices_to_translate[i, j] == global_indices)[0] + local_neighbors[i, j] = pos return local_neighbors def owner_mask(self, dim: gtx.Dimension) -> data_alloc.NDArray: diff --git a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py index 1702af6929..402ef26c6a 100644 --- a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py +++ b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py @@ -111,7 +111,6 @@ def get_multinode_properties( return _get_processor_properties(with_mpi=True, comm_id=comm_id) -# TODO (@halungge): changed for dev/testing set back to frozen @dataclass(frozen=False) class MPICommProcessProperties(definitions.ProcessProperties): comm: mpi4py.MPI.Comm = None @@ -146,6 +145,10 @@ def __init__( dim: self._create_domain_descriptor(dim) for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values() } + self._field_size: dict[gtx.Dimension : int] = { + dim: self._decomposition_info.global_index[dim].shape[0] + for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values() + } log.info(f"domain descriptors for dimensions {self._domain_descriptors.keys()} initialized") self._patterns = { dim: self._create_pattern(dim) for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values() @@ -208,15 +211,17 @@ def _create_pattern(self, horizontal_dim: gtx.Dimension): def _slice_field_based_on_dim(self, field: gtx.Field, dim: gtx.Dimension) -> data_alloc.NDArray: """ Slices the field based on the dimension passed in. + + This is a helper function needed for the granule - Fortran integration. As the Fortran fields have nproma + size but the global_index fields used to initialize GHEX exchanges have only local num_edge, + num_cell_num_vertex size. """ - if dim == dims.VertexDim: - return field.ndarray[: self._decomposition_info.num_vertices, :] - elif dim == dims.EdgeDim: - return field.ndarray[: self._decomposition_info.num_edges, :] - elif dim == dims.CellDim: - return field.ndarray[: self._decomposition_info.num_cells, :] - else: - raise ValueError(f"Unknown dimension {dim}") + try: + assert field.ndarray.ndim == 2 + trim_length = self._field_size[dim] + return field.ndarray[:trim_length, :] + except KeyError: + log.warn(f"Trying to trim field of invalid dimension {dim} for exchange. Not trimming.") def exchange(self, dim: gtx.Dimension, *fields: Sequence[gtx.Field]): """ diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 5dba41f9cc..b638c5ba2c 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -5,7 +5,6 @@ # # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause -import functools import logging import pathlib from collections.abc import Callable @@ -18,6 +17,7 @@ from icon4py.model.common import dimension as dims, type_alias as ta from icon4py.model.common.decomposition import definitions as decomposition, halo +from icon4py.model.common.exceptions import InvalidConfigError from icon4py.model.common.grid import ( base, gridfile, @@ -64,9 +64,7 @@ def __init__( transformation: gridfile.IndexTransformation, grid_file: pathlib.Path | str, config: v_grid.VerticalGridConfig, # TODO(@halungge): remove: - separate vertical from horizontal grid - run_properties: decomposition.ProcessProperties = _single_node_properties, ): - self._run_properties = run_properties self._transformation = transformation self._file_name = str(grid_file) self._halo_constructor = None @@ -98,28 +96,45 @@ def __call__( self, backend: gtx_backend.Backend | None, keep_skip_values: bool, - decomposer: Callable[[np.ndarray, int], np.ndarray] = halo.SingleNodeDecomposer(), + run_properties: decomposition.ProcessProperties = _single_node_properties, + decomposer: Callable[[np.ndarray, int], np.ndarray] = _single_node_decomposer, ): + self._run_properties = run_properties + if not run_properties.single_node() and isinstance(decomposer, halo.SingleNodeDecomposer): + raise InvalidConfigError( + f"Need a Decomposer for multi node run: running on: {run_properties.comm_size}" + ) + if not self._reader: self.open() - self._grid = self._construct_grid(backend=backend, with_skip_values=keep_skip_values) + self._grid = self._construct_grid( + backend=backend, with_skip_values=keep_skip_values, decomposer=decomposer + ) self._coordinates = self._read_coordinates(backend) self._geometry = self._read_geometry_fields(backend) + self.close() def _read_coordinates(self, backend: gtx_backend.Backend | None) -> CoordinateDict: + my_cell_indices = self._decomposition_info.global_index(dims.CellDim) + my_edge_indices = self._decomposition_info.global_index(dims.EdgeDim) + my_vertex_indices = self._decomposition_info.global_index(dims.VertexDim) return { dims.CellDim: { "lat": gtx.as_field( (dims.CellDim,), - self._reader.variable(gridfile.CoordinateName.CELL_LATITUDE), + self._reader.variable( + gridfile.CoordinateName.CELL_LATITUDE, indices=my_cell_indices + ), dtype=ta.wpfloat, allocator=backend, ), "lon": gtx.as_field( (dims.CellDim,), - self._reader.variable(gridfile.CoordinateName.CELL_LONGITUDE), + self._reader.variable( + gridfile.CoordinateName.CELL_LONGITUDE, indices=my_cell_indices + ), dtype=ta.wpfloat, allocator=backend, ), @@ -127,13 +142,17 @@ def _read_coordinates(self, backend: gtx_backend.Backend | None) -> CoordinateDi dims.EdgeDim: { "lat": gtx.as_field( (dims.EdgeDim,), - self._reader.variable(gridfile.CoordinateName.EDGE_LATITUDE), + self._reader.variable( + gridfile.CoordinateName.EDGE_LATITUDE, indices=my_edge_indices + ), dtype=ta.wpfloat, allocator=backend, ), "lon": gtx.as_field( (dims.EdgeDim,), - self._reader.variable(gridfile.CoordinateName.EDGE_LONGITUDE), + self._reader.variable( + gridfile.CoordinateName.EDGE_LONGITUDE, indices=my_edge_indices + ), dtype=ta.wpfloat, allocator=backend, ), @@ -141,13 +160,17 @@ def _read_coordinates(self, backend: gtx_backend.Backend | None) -> CoordinateDi dims.VertexDim: { "lat": gtx.as_field( (dims.VertexDim,), - self._reader.variable(gridfile.CoordinateName.VERTEX_LATITUDE), + self._reader.variable( + gridfile.CoordinateName.VERTEX_LATITUDE, indices=my_vertex_indices + ), allocator=backend, dtype=ta.wpfloat, ), "lon": gtx.as_field( (dims.VertexDim,), - self._reader.variable(gridfile.CoordinateName.VERTEX_LONGITUDE), + self._reader.variable( + gridfile.CoordinateName.VERTEX_LONGITUDE, indices=my_vertex_indices + ), allocator=backend, dtype=ta.wpfloat, ), @@ -155,12 +178,15 @@ def _read_coordinates(self, backend: gtx_backend.Backend | None) -> CoordinateDi } def _read_geometry_fields(self, backend: gtx_backend.Backend | None): + my_cell_indices = self._decomposition_info.global_index(dims.CellDim) + my_edge_indices = self._decomposition_info.global_index(dims.EdgeDim) + my_vertex_indices = self._decomposition_info.global_index(dims.VertexDim) return { # TODO(halungge): still needs to ported, values from "our" grid files contains (wrong) values: # based on bug in generator fixed with this [PR40](https://gitlab.dkrz.de/dwd-sw/dwd_icon_tools/-/merge_requests/40) . gridfile.GeometryName.CELL_AREA.value: gtx.as_field( (dims.CellDim,), - self._reader.variable(gridfile.GeometryName.CELL_AREA), + self._reader.variable(gridfile.GeometryName.CELL_AREA, indices=my_cell_indices), allocator=backend, ), # TODO(halungge): easily computed from a neighbor_sum V2C over the cell areas? @@ -171,18 +197,28 @@ def _read_geometry_fields(self, backend: gtx_backend.Backend | None): ), gridfile.GeometryName.EDGE_CELL_DISTANCE.value: gtx.as_field( (dims.EdgeDim, dims.E2CDim), - self._reader.variable(gridfile.GeometryName.EDGE_CELL_DISTANCE, transpose=True), + self._reader.variable( + gridfile.GeometryName.EDGE_CELL_DISTANCE, + transpose=True, + indices=my_edge_indices, + ), allocator=backend, ), gridfile.GeometryName.EDGE_VERTEX_DISTANCE.value: gtx.as_field( (dims.EdgeDim, dims.E2VDim), - self._reader.variable(gridfile.GeometryName.EDGE_VERTEX_DISTANCE, transpose=True), + self._reader.variable( + gridfile.GeometryName.EDGE_VERTEX_DISTANCE, + transpose=True, + indices=my_edge_indices, + ), allocator=backend, ), # TODO(halungge): recompute from coordinates? field in gridfile contains NaN on boundary edges gridfile.GeometryName.TANGENT_ORIENTATION.value: gtx.as_field( (dims.EdgeDim,), - self._reader.variable(gridfile.GeometryName.TANGENT_ORIENTATION), + self._reader.variable( + gridfile.GeometryName.TANGENT_ORIENTATION, indices=my_edge_indices + ), allocator=backend, ), gridfile.GeometryName.CELL_NORMAL_ORIENTATION.value: gtx.as_field( @@ -191,6 +227,7 @@ def _read_geometry_fields(self, backend: gtx_backend.Backend | None): gridfile.GeometryName.CELL_NORMAL_ORIENTATION, transpose=True, apply_transformation=False, + indices=my_cell_indices, ), allocator=backend, ), @@ -200,6 +237,7 @@ def _read_geometry_fields(self, backend: gtx_backend.Backend | None): gridfile.GeometryName.EDGE_ORIENTATION_ON_VERTEX, transpose=True, apply_transformation=False, + indices=my_vertex_indices, ), allocator=backend, ), @@ -207,7 +245,6 @@ def _read_geometry_fields(self, backend: gtx_backend.Backend | None): def _read_grid_refinement_fields( self, - decomposition_info: decomposition.DecompositionInfo | None = None, backend: gtx_backend.Backend | None = None, ) -> dict[gtx.Dimension, gtx.Field]: """ @@ -217,7 +254,6 @@ def _read_grid_refinement_fields( see [refinement.py](refinement.py) Args: - decomposition_info: Optional decomposition information, if not provided the grid is assumed to be a single node run. backend: Optional backend to use for reading the fields, if not provided the default backend is used. Returns: dict[gtx.Dimension, gtx.Field]: A dictionary containing the refinement control fields for each dimension. @@ -231,7 +267,10 @@ def _read_grid_refinement_fields( dim: gtx.as_field( (dim,), self._reader.int_variable( - name, decomposition_info, transpose=False, apply_transformation=False + name, + indices=self._decomposition_info.global_index(dim), + transpose=False, + apply_transformation=False, ), allocator=backend, ) @@ -306,11 +345,15 @@ def geometry(self) -> GeometryDict: def coordinates(self) -> CoordinateDict: return self._coordinates + @property + def decomposition_info(self) -> decomposition.DecompositionInfo: + return self._decomposition_info + def _construct_grid( self, backend: gtx_backend.Backend | None, with_skip_values: bool, - decomposer: Callable[[np.ndarray, int], np.ndarray] = None, + decomposer: Callable[[np.ndarray, int], np.ndarray] | None = None, ) -> icon.IconGrid: """Construct the grid topology from the icon grid file. @@ -325,9 +368,9 @@ def _construct_grid( grid_root = self._reader.attribute(gridfile.MandatoryPropertyName.ROOT) grid_level = self._reader.attribute(gridfile.MandatoryPropertyName.LEVEL) global_params = icon.GlobalGridParams(root=grid_root, level=grid_level) - _determine_limited_area = functools.partial(refinement.is_limited_area_grid, array_ns=xp) - refinement_fields = functools.partial(self._read_grid_refinement_fields, backend=backend)() - limited_area = _determine_limited_area(refinement_fields[dims.CellDim].ndarray) + cell_refinment = self._reader.variable(gridfile.GridRefinementName.CONTROL_CELLS) + limited_area = refinement.is_limited_area_grid(cell_refinment, array_ns=xp) + grid_config = base.GridConfig( horizontal_size=global_grid_size, vertical_size=self._vertical_config.num_levels, @@ -349,21 +392,20 @@ def _construct_grid( if decomposer is not None: # DECOMPOSITION - cells_to_rank_mapping = self._decompose( + cells_to_rank_mapping = decomposer( cell_to_cell_neighbors, self._run_properties.comm_size, ) # HALO CONSTRUCTION # TODO(halungge): reduce the set of neighbor tables used in the halo construction - # TODO (halungge): figure out where to do the host to device copies (xp.asarray...) + # TODO(halungge): figure out where to do the host to device copies (xp.asarray...) neighbor_tables_for_halo_construction = neighbor_tables # halo_constructor - creates the decomposition info, which can then be used to generate the local patches on each rank halo_constructor = self._initialize_halo_constructor( global_grid_size, neighbor_tables_for_halo_construction, backend ) - decomposition_info = halo_constructor(cells_to_rank_mapping) - self._decomposition_info = decomposition_info + self._decomposition_info = halo_constructor(cells_to_rank_mapping) ## TODO(halungge): do local reads (and halo exchanges!!) FIX: my_cells etc are in 0 base python coding - reading from file fails... ## @@ -371,20 +413,19 @@ def _construct_grid( # TODO(halungge): run this only for distrbuted grids otherwise to nothing internally neighbor_tables = { - k: decomposition_info.global_to_local( - k.source, v[decomposition_info.global_index(k.target[0])] + k: self._decomposition_info.global_to_local( + k.source, v[self._decomposition_info.global_index(k.target[0])] ) for k, v in neighbor_tables_for_halo_construction.items() } # COMPUTE remaining derived connectivities else: - # TODO no decomposition - ... + self._decomposition_info = _single_node_decomposition_info(grid_config, xp) # JOINT functionality neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) - + refinement_fields = self._read_grid_refinement_fields(backend) # TODO(halungge): needs to be moved out of the joint section start, end, _ = self._read_start_end_indices() start_indices = { @@ -717,3 +758,33 @@ def construct_local_connectivity( indices = sorted_index_of_global_idx[positions] local_connectivity[i, valid_neighbor_mask] = indices return local_connectivity + + +def _single_node_decomposition_info( + grid_config: base.GridConfig, xp: ModuleType +) -> decomposition.DecompositionInfo: + cell_size = (grid_config.num_cells,) + edge_size = (grid_config.num_edges,) + vertex_size = (grid_config.num_vertices,) + info = ( + decomposition.DecompositionInfo(grid_config.num_levels) + .set_dimension( + dims.CellDim, + xp.arange(cell_size[0], dtype=gtx.int32), + xp.ones(cell_size, dtype=bool), + xp.zeros(cell_size), + ) + .set_dimension( + dims.EdgeDim, + xp.arange(edge_size[0], dtype=gtx.int32), + xp.ones(edge_size, dtype=bool), + xp.zeros(edge_size), + ) + .set_dimension( + dims.VertexDim, + xp.arange(vertex_size[0], dtype=gtx.int32), + xp.ones(vertex_size, dtype=bool), + xp.zeros(vertex_size), + ) + ) + return info diff --git a/model/common/tests/common/decomposition/mpi_tests/test_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_halo.py index 71105c0c4d..beeb433d5d 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_halo.py @@ -6,25 +6,22 @@ # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause -import pathlib - import gt4py.next as gtx import numpy as np import pytest -import icon4py.model.common.grid.gridfile -from ..fixtures import * # noqa: F403 - import icon4py.model.common.dimension as dims from icon4py.model.common import exceptions -from icon4py.model.common.decomposition import definitions as defs, mpi_decomposition - +from icon4py.model.common.decomposition import definitions as defs +from icon4py.model.testing.fixtures import processor_props +from icon4py.model.testing import parallel_helpers from .. import utils -from ..utils import assert_same_entries + try: import mpi4py # import mpi4py to check for optional mpi dependency import mpi4py.MPI + from icon4py.model.common.decomposition import mpi_decomposition mpi_decomposition.init_mpi() except ImportError: @@ -35,10 +32,7 @@ from icon4py.model.common.decomposition import halo from icon4py.model.common.grid import ( base as base_grid, - grid_manager as gm, - gridfile, simple, - vertical as v_grid, ) from icon4py.model.testing import datatest_utils as dt_utils, definitions as test_defs @@ -48,11 +42,6 @@ .joinpath(dt_utils.R02B04_GLOBAL) .joinpath("icon_grid_0013_R02B04_R_ugrid.nc") ) -GRID_FILE = ( - test_defs.grids_path() - .joinpath(test_defs.Grids.R02B04_GLOBAL.name) - .joinpath("icon_grid_0012_R02B04_G.nc") -) backend = None @@ -65,26 +54,6 @@ def simple_neighbor_tables(): return neighbor_tables -def run_gridmananger_for_single_node(file: pathlib.Path, vertical_config:v_grid.VerticalGridConfig) -> gm.GridManager: - manager = _grid_manager(file, vertical_config) - manager(keep_skip_values=True) - return manager - - -def _grid_manager(file:pathlib.Path,vertical_config:v_grid.VerticalGridConfig): - manager = gm.GridManager( - gridfile.ToZeroBasedIndexTransformation(), - str(file), - vertical_config, - ) - return manager - - -def run_grid_manager_for_multinode(gm:pathlib.Path, verical_config)->gm.GridManager: - manager = _grid_manager(file, vertical_config) - manager(keep_skip_values=True, run_properties=) - - @pytest.mark.mpi(min_size=4) def test_halo_constructor_owned_cells(processor_props, simple_neighbor_tables): # F811 # fixture halo_generator = halo.IconLikeHaloConstructor( @@ -136,11 +105,11 @@ def global_indices(dim: gtx.Dimension) -> np.ndarray: @pytest.mark.parametrize("dim", [dims.CellDim, dims.EdgeDim, dims.VertexDim]) @pytest.mark.mpi(min_size=4) +@pytest.mark.parametrize("processor_props", [True], indirect=True) def test_element_ownership_is_unique( dim, processor_props, simple_neighbor_tables ): # F811 # fixture - if processor_props.comm_size != 4: - pytest.skip("This test requires exactly 4 MPI ranks.") + parallel_helpers.check_comm_size(processor_props, sizes=[4]) halo_generator = halo.IconLikeHaloConstructor( connectivities=simple_neighbor_tables, @@ -180,11 +149,14 @@ def test_element_ownership_is_unique( @pytest.mark.mpi(min_size=4) @pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) +@pytest.mark.parametrize("processor_props", [True], indirect=True) def test_halo_constructor_decomposition_info_global_indices( processor_props, simple_neighbor_tables, dim ): # F811 # fixture if processor_props.comm_size != 4: - pytest.skip("This test requires exactly 4 MPI ranks.") + pytest.skip( + f"This test requires exactly 4 MPI ranks, current run has {processor_props.comm_size}" + ) halo_generator = halo.IconLikeHaloConstructor( connectivities=simple_neighbor_tables, run_properties=processor_props, @@ -200,7 +172,7 @@ def test_halo_constructor_decomposition_info_global_indices( ) my_owned = decomp_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED) print(f"rank {processor_props.rank} owns {dim} : {my_owned} ") - assert_same_entries(dim, my_owned, utils.OWNED, processor_props.rank) + utils.assert_same_entries(dim, my_owned, utils.OWNED, processor_props.rank) @pytest.mark.mpi(min_size=4) @@ -236,7 +208,7 @@ def test_halo_constructor_decomposition_info_halo_levels( first_halo_line_global_index = decomp_info.global_index( dim, defs.DecompositionInfo.EntryType.ALL )[first_halo_line_local_index] - assert_same_entries( + utils.assert_same_entries( dim, first_halo_line_global_index, utils.FIRST_HALO_LINE, processor_props.rank ) second_halo_line_local_index = np.where( @@ -245,42 +217,11 @@ def test_halo_constructor_decomposition_info_halo_levels( second_halo_line_global_index = decomp_info.global_index( dim, defs.DecompositionInfo.EntryType.ALL )[second_halo_line_local_index] - assert_same_entries( + utils.assert_same_entries( dim, second_halo_line_global_index, utils.SECOND_HALO_LINE, processor_props.rank ) -# TODO unused - remove or fix and use? -def icon_distribution( - props: defs.ProcessProperties, decomposition_info: defs.DecompositionInfo -) -> np.ndarray: - cell_index = decomposition_info.global_index( - dims.CellDim, defs.DecompositionInfo.EntryType.OWNED - ) - comm = props.comm - local_sizes, recv_buffer = gather_field(cell_index, comm) - distribution = np.empty((sum(local_sizes)), dtype=int) - if comm.rank == 0: - start_index = 0 - for s in comm.size: - end_index = local_sizes[s] - distribution[recv_buffer[start_index:end_index]] = s - start_index = end_index - - comm.Bcast(distribution, root=0) - return distribution - - -def gather_field(field: np.ndarray, comm: mpi4py.MPI.Comm) -> tuple: - local_sizes = np.array(comm.gather(field.size, root=0)) - if comm.rank == 0: - recv_buffer = np.empty(sum(local_sizes), dtype=field.dtype) - else: - recv_buffer = None - comm.Gatherv(sendbuf=field, recvbuf=(recv_buffer, local_sizes), root=0) - return local_sizes, recv_buffer - - def decompose(grid: base_grid.Grid, processor_props): # F811 # fixture partitioner = halo.SimpleMetisDecomposer() labels = partitioner( @@ -289,79 +230,6 @@ def decompose(grid: base_grid.Grid, processor_props): # F811 # fixture return labels - -@pytest.mark.mpi -def test_distributed_fields(processor_props): # F811 # fixture - grid_manager = run_gridmananger_for_single_node(GRID_FILE) - single_node_grid = grid_manager.grid - global_cell_area = grid_manager.geometry[gridfile.GeometryName.CELL_AREA] - global_edge_lat = grid_manager.coordinates[dims.EdgeDim]["lat"] - global_vertex_lon = grid_manager.coordinates[dims.VertexDim]["lon"] - - - labels = decompose(single_node_grid, processor_props) - neighbor_tables = {k: v.ndarray for k, v in single_node_grid.connectivities.items()} - halo_generator = halo.IconLikeHaloConstructor( - connectivities=neighbor_tables, - run_properties=processor_props, - num_levels=1, - ) - decomposition_info = halo_generator(labels) - multi_node_grid_manager = gm.GridManager(gridfile.ToZeroBased) - # distributed read: read one field per dimension - - ## TODO why is this local?? - local_cell_area = grid_manager.geometry[gridfile.GeometryName.CELL_AREA] - local_edge_lat = grid_manager.coordinates[dims.EdgeDim]["lat"] - local_vertex_lon = grid_manager.coordinates[dims.VertexDim]["lon"] - print( - f"rank = {processor_props.rank} has size(cell_area): {local_cell_area.ndarray.shape}, " - f"has size(edge_length): {local_edge_lat.ndarray.shape}, has size(vertex_length): {local_vertex_lon.ndarray.shape}" - ) - # the local number of cells must be at most the global number of cells (analytically computed) - assert ( - local_cell_area.asnumpy().shape[0] <= single_node_grid.global_properties.num_cells - ), "local field is larger than global field" - # global read: read the same (global fields) - - assert_gathered_field_against_global( - decomposition_info, processor_props, dims.CellDim, global_cell_area, local_cell_area - ) - - assert_gathered_field_against_global( - decomposition_info, processor_props, dims.EdgeDim, global_edge_lat, local_edge_lat - ) - assert_gathered_field_against_global( - decomposition_info, processor_props, dims.VertexDim, global_vertex_lon, local_vertex_lon - ) - - -def assert_gathered_field_against_global( - decomposition_info: defs.DecompositionInfo, - processor_props: defs.ProcessProperties, # F811 # fixture - dim: gtx.Dimension, - global_reference_field: np.ndarray, - local_field: np.ndarray, -): - assert ( - local_field.size - == decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.ALL).size - ) - owned_entries = local_field[ - decomposition_info.local_index(dim, defs.DecompositionInfo.EntryType.OWNED) - ] - gathered_sizes, gathered_field = gather_field(owned_entries, processor_props.comm) - global_index_sizes, gathered_global_indices = gather_field( - decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED), - processor_props.comm, - ) - if processor_props.rank == 0: - assert np.all(gathered_sizes == global_index_sizes) - sorted_ = np.zeros(global_reference_field.shape, dtype=gtx.float64) - sorted_[gathered_global_indices] = gathered_field - assert helpers.dallclose(sorted_, global_reference_field) - - # TODO add test including halo access: # Will uses geofac_div and geofac_n2s diff --git a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py index 19b1cf94e5..c16fb0dfed 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py @@ -8,23 +8,32 @@ import numpy as np import pytest +import logging + +from icon4py.model.common import dimension as dims +from icon4py.model.common.decomposition import definitions from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import parallel_helpers -from ..fixtures import * # noqa: F403 +from icon4py.model.testing.fixtures.datatest import ( + experiment, + backend, + download_ser_data, + ranked_data_path, + data_provider, + processor_props, + decomposition_info, +) try: import mpi4py # import mpi4py to check for optional mpi dependency + from icon4py.model.common.decomposition import mpi_decomposition + + mpi_decomposition.init_mpi() except ImportError: pytest.skip("Skipping parallel on single node installation", allow_module_level=True) -import logging - -from icon4py.model.common import dimension as dims - -from icon4py.model.common.decomposition import definitions, mpi_decomposition - _log = logging.getLogger(__name__) @@ -43,6 +52,7 @@ @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_props(processor_props): assert processor_props.comm + assert processor_props.comm_size > 1 @pytest.mark.mpi(min_size=2) @@ -112,12 +122,12 @@ def test_decomposition_info_local_index( decomposition_info, processor_props, ): - parallel_helpers.check_comm_size(processor_props, sizes=[2]) + caplog.set_level("info") + parallel_helpers.check_comm_size(processor_props, sizes=(2,)) my_rank = processor_props.rank all_indices = decomposition_info.local_index(dim, definitions.DecompositionInfo.EntryType.ALL) my_total = total[my_rank] my_owned = owned[my_rank] - assert all_indices.shape[0] == my_total assert np.array_equal(all_indices, np.arange(0, my_total)) halo_indices = decomposition_info.local_index(dim, definitions.DecompositionInfo.EntryType.HALO) @@ -211,7 +221,6 @@ def test_exchange_on_dummy_data( processor_props, decomposition_info, grid_savepoint, - metrics_savepoint, dimension, caplog, ): @@ -234,7 +243,7 @@ def test_exchange_on_dummy_data( dimension, definitions.DecompositionInfo.EntryType.OWNED ) assert np.all(input_field == number) - exchange.exchange_and_wait(dimension, input_field) + exchange.exchange_and_wait(dimension, (input_field,)) result = input_field.asnumpy() _log.info(f"rank={processor_props.rank} - num of halo points ={halo_points.shape}") _log.info( diff --git a/model/common/tests/common/decomposition/two_ranks_distribution.py b/model/common/tests/common/decomposition/two_ranks_distribution.py index 3585a75603..ccd6c6709b 100644 --- a/model/common/tests/common/decomposition/two_ranks_distribution.py +++ b/model/common/tests/common/decomposition/two_ranks_distribution.py @@ -5,12 +5,27 @@ # # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause +from typing import Final import numpy as np TWO_RANKS_DISTRIBUTION: np.ndarray = np.ones(10) -TWO_RANKS_DISTRIBUTION[5:7, 10] = 0 +TWO_RANKS_DISTRIBUTION[5, 6, 10] = 0 # TODO define all the rest +CELL_OWN: Final[dict[int, list[int]]] = { + 0: [6, 7, 10], + 1: [0, 1, 2, 3, 4, 5, 8, 9, 11, 12, 13, 14, 15, 16, 17], +} +EDGE_OWN: Final[dict[int, list[int]]] = { + 0: [13, 14], + 1: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26], +} +VERTEX_OWN: Final[dict[int, list[int]]] = {0: [], 1: [0, 1, 2, 3, 4, 5, 6, 7, 8]} + +CELL_FIRST_HALO_LINE = {0: [3, 4, 11, 13, 9], 1: [6, 7, 14]} +CELL_SECOND_HALO_LINE = {0: [0, 1, 5, 8, 14, 17, 16], 1: []} +EDGE_FIRST_HALO_LINE = {0: [9, 12, 17, 21, 10], 1: []} +EDGE_SECOND_HALO_LINE = {0: [1, 2, 5, 4, 15, 16, 24, 25, 26, 22, 23, 18, 11], 1: [14, 13]} diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index 5fc47e06c2..43d660aacc 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -5,7 +5,6 @@ # # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause -import dataclasses import numpy as np import pytest @@ -18,7 +17,8 @@ from .. import utils -from ..fixtures import * # noqa: F403 +from ..utils import dummy_four_ranks +from icon4py.model.testing.fixtures import processor_props @pytest.mark.parametrize("processor_props", [False], indirect=True) @@ -31,19 +31,6 @@ def test_create_single_node_runtime_without_mpi(processor_props): # fixture assert isinstance(exchange, definitions.SingleNodeExchange) -@dataclasses.dataclass(frozen=True) -class DummyProps(definitions.ProcessProperties): - def __init__(self, rank: int): - object.__setattr__(self, "rank", rank % 4) - object.__setattr__(self, "comm", None) - object.__setattr__(self, "comm_name", "dummy on 4") - object.__setattr__(self, "comm_size", 4) - - -def dummy_four_ranks(rank) -> definitions.ProcessProperties: - return DummyProps(rank=rank) - - def get_neighbor_tables_for_simple_grid() -> dict[str, data_alloc.NDArray]: grid = simple.simple_grid() neighbor_tables = { diff --git a/model/common/tests/common/decomposition/utils.py b/model/common/tests/common/decomposition/utils.py index 41fba0917b..cd85ede0ac 100644 --- a/model/common/tests/common/decomposition/utils.py +++ b/model/common/tests/common/decomposition/utils.py @@ -5,12 +5,13 @@ # # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause +import dataclasses import numpy as np from gt4py import next as gtx from icon4py.model.common import dimension as dims - +from icon4py.model.common.decomposition import definitions """ TESTDATA using the [SimpleGrid](../../../src/icon4py/model/common/grid/simple.py) @@ -154,3 +155,16 @@ def assert_same_entries( ): assert my_owned.size == len(reference[dim][rank]) assert np.setdiff1d(my_owned, reference[dim][rank], assume_unique=True).size == 0 + + +def dummy_four_ranks(rank) -> definitions.ProcessProperties: + return DummyProps(rank=rank) + + +@dataclasses.dataclass(frozen=True) +class DummyProps(definitions.ProcessProperties): + def __init__(self, rank: int): + object.__setattr__(self, "rank", rank % 4) + object.__setattr__(self, "comm", None) + object.__setattr__(self, "comm_name", "dummy on 4") + object.__setattr__(self, "comm_size", 4) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 52907fa977..e5acf4d8f9 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -7,17 +7,26 @@ # SPDX-License-Identifier: BSD-3-Clause import logging +import pathlib +import numpy as np import pytest +from gt4py import next as gtx import icon4py.model.common.grid.gridfile import icon4py.model.testing.grid_utils as grid_utils -from icon4py.model.common import exceptions -from icon4py.model.common.decomposition import definitions, halo, mpi_decomposition -from icon4py.model.common.grid import grid_manager as gm, vertical as v_grid -from icon4py.model.testing import datatest_utils as dt_utils +from icon4py.model.common import exceptions, dimension as dims +from icon4py.model.common.decomposition import halo, mpi_decomposition, definitions as defs +from icon4py.model.common.grid import grid_manager as gm, vertical as v_grid, gridfile +from icon4py.model.testing import ( + datatest_utils as dt_utils, + test_utils as test_helpers, + definitions as test_defs, +) from .. import utils +from ..fixtures import * +from ...decomposition import utils as decomp_utils try: @@ -27,16 +36,45 @@ except ImportError: pytest.skip("Skipping parallel on single node installation", allow_module_level=True) +log = logging.getLogger(__file__) +vertical_config = v_grid.VerticalGridConfig(num_levels=1) -@pytest.mark.mpi(min_size=2) -@pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_props(caplog, processor_props): # fixture - caplog.set_level(logging.DEBUG) - """dummy test to check setup""" - assert processor_props.comm_size > 1 + +def run_gridmananger_for_multinode( + file: pathlib.Path, + vertical_config: v_grid.VerticalGridConfig, + run_properties: defs.ProcessProperties, + decomposer: halo.Decomposer, +) -> gm.GridManager: + manager = _grid_manager(file, vertical_config) + manager( + keep_skip_values=True, backend=None, run_properties=run_properties, decomposer=decomposer + ) + return manager + + +def _grid_manager(file: pathlib.Path, vertical_config: v_grid.VerticalGridConfig): + manager = gm.GridManager( + gridfile.ToZeroBasedIndexTransformation(), + str(file), + vertical_config, + ) + return manager + + +def run_grid_manager_for_singlenode( + file: pathlib.Path, vertical_config: v_grid.VerticalGridConfig +) -> gm.GridManager: + manager = _grid_manager(file, vertical_config) + manager( + keep_skip_values=True, + run_properties=defs.SingleNodeProcessProperties(), + decomposer=None, + backend=None, + ) + return manager -# TODO FIXME @pytest.mark.xfail @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @@ -65,7 +103,7 @@ def test_start_end_index( icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation(), file, v_grid.VerticalGridConfig(1), - run_properties=definitions.get_processor_properties(definitions.SingleNodeRun()), + run_properties=defs.get_processor_properties(defs.SingleNodeRun()), ).grid with manager.set_decomposer(partitioner) as manage: manage(backend=backend, keep_skip_values=True) @@ -83,14 +121,123 @@ def test_start_end_index( @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi(min_size=2) def test_grid_manager_validate_decomposer(processor_props): - file = grid_utils.resolve_full_grid_file_name(dt_utils.R02B04_GLOBAL) + file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL.name) manager = gm.GridManager( - icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation(), + gridfile.ToZeroBasedIndexTransformation(), file, - v_grid.VerticalGridConfig(1), - run_properties=processor_props, + vertical_config, ) with pytest.raises(exceptions.InvalidConfigError) as e: - manager.set_decomposer(halo.SingleNodeDecomposer()) + manager( + keep_skip_values=True, + backend=None, + run_properties=processor_props, + decomposer=halo.SingleNodeDecomposer(), + ) + + assert "Need a Decomposer for multi" in e.value.args[0] + - assert "Need a Decomposer for for multi" in e.value.args[0] +@pytest.mark.mpi +@pytest.mark.parametrize("processor_props", [True], indirect=True) +def test_distributed_fields(processor_props, caplog): + caplog.set_level(logging.INFO) + print(f"myrank - {processor_props.rank}: running with processor_props = {processor_props}") + file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL.name) + grid_manager = run_grid_manager_for_singlenode(file, vertical_config) + single_node_grid = grid_manager.grid + global_cell_area = grid_manager.geometry[gridfile.GeometryName.CELL_AREA] + global_edge_lat = grid_manager.coordinates[dims.EdgeDim]["lat"] + global_vertex_lon = grid_manager.coordinates[dims.VertexDim]["lon"] + + multinode = run_gridmananger_for_multinode( + file, + vertical_config, + run_properties=processor_props, + decomposer=halo.SimpleMetisDecomposer(), + ) + decomposition_info = multinode.decomposition_info + + local_cell_area = multinode.geometry[gridfile.GeometryName.CELL_AREA] + local_edge_lat = multinode.coordinates[dims.EdgeDim]["lat"] + local_vertex_lon = multinode.coordinates[dims.VertexDim]["lon"] + print( + f"rank = {processor_props.rank} has size(cell_area): {local_cell_area.ndarray.shape}, " + f"has size(edge_length): {local_edge_lat.ndarray.shape}, has size(vertex_length): {local_vertex_lon.ndarray.shape}" + ) + # the local number of cells must be at most the global number of cells (analytically computed) + assert ( + local_cell_area.asnumpy().shape[0] <= single_node_grid.global_properties.num_cells + ), "local field is larger than global field" + # global read: read the same (global fields) + + assert_gathered_field_against_global( + decomposition_info, + processor_props, + dims.CellDim, + global_cell_area.asnumpy(), + local_cell_area.asnumpy(), + ) + + assert_gathered_field_against_global( + decomposition_info, + processor_props, + dims.EdgeDim, + global_edge_lat.asnumpy(), + local_edge_lat.asnumpy(), + ) + assert_gathered_field_against_global( + decomposition_info, + processor_props, + dims.VertexDim, + global_vertex_lon.asnumpy(), + local_vertex_lon.asnumpy(), + ) + + +def gather_field(field: np.ndarray, comm: mpi4py.MPI.Comm) -> tuple: + local_sizes = np.array(comm.gather(field.size, root=0)) + if comm.rank == 0: + recv_buffer = np.empty(sum(local_sizes), dtype=field.dtype) + log.debug( + f"rank: {comm.rank} - setup receive buffer with size {sum(local_sizes)} on rank 0" + ) + else: + recv_buffer = None + comm.Gatherv(sendbuf=field, recvbuf=(recv_buffer, local_sizes), root=0) + if comm.rank == 0: + log.debug(f"fields gathered:") + log.debug(f"field sizes {local_sizes}") + + return local_sizes, recv_buffer + + +def assert_gathered_field_against_global( + decomposition_info: defs.DecompositionInfo, + processor_props: defs.ProcessProperties, # F811 # fixture + dim: gtx.Dimension, + global_reference_field: np.ndarray, + local_field: np.ndarray, +): + assert ( + local_field.shape + == decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.ALL).shape + ) + owned_entries = local_field[ + decomposition_info.local_index(dim, defs.DecompositionInfo.EntryType.OWNED) + ] + gathered_sizes, gathered_field = gather_field(owned_entries, processor_props.comm) + global_index_sizes, gathered_global_indices = gather_field( + decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED), + processor_props.comm, + ) + if processor_props.rank == 0: + print(f"rank = {processor_props.rank}: asserting gathered fields") + assert np.all( + gathered_sizes == global_index_sizes + ), f"gathered field sizes do not match {gathered_sizes}" + sorted_ = np.zeros(global_reference_field.shape, dtype=gtx.float64) + sorted_[gathered_global_indices] = gathered_field + assert test_helpers.dallclose( + sorted_, global_reference_field + ), f"Gathered field values do not match for dim {dim}.- " diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py index 3f96bd1513..a81891a8de 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py @@ -12,13 +12,11 @@ import icon4py.model.common.dimension as dims import icon4py.model.common.grid.horizontal as h_grid -from icon4py.model.testing.parallel_helpers import ( - check_comm_size, -) -from icon4py.model.testing import parallel_helpers +from icon4py.model.testing import parallel_helpers +from icon4py.model.common.decomposition import definitions as defs, mpi_decomposition +from ..fixtures import * from .. import utils -from .. import fixtures # noqa F403 try: @@ -64,6 +62,9 @@ def test_distributed_local(processor_props, dim, icon_grid, caplog): caplog.set_level(logging.INFO) parallel_helpers.check_comm_size(processor_props) domain = h_grid.domain(dim)(h_grid.Zone.LOCAL) + print( + f"rank {processor_props.rank}/{processor_props.comm_size} dim = {dim} : {icon_grid.size[dim]}" + ) # local still runs entire field: assert icon_grid.start_index(domain) == 0 print( @@ -120,7 +121,6 @@ def test_distributed_local(processor_props, dim, icon_grid, caplog): @pytest.mark.parametrize("marker", [h_grid.Zone.HALO, h_grid.Zone.HALO_LEVEL_2]) def test_distributed_halo(processor_props, dim, marker, icon_grid): parallel_helpers.check_comm_size(processor_props) - num = int(next(iter(re.findall(r"\d+", marker.value)))) domain = h_grid.domain(dim)(marker) start_index = icon_grid.start_index(domain) end_index = icon_grid.end_index(domain) From e2cdc6dfdfcb9be2057ad71ce4b9868b3a5afe71 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 29 Aug 2025 18:37:46 +0200 Subject: [PATCH 083/240] clean up interfaces --- .../model/common/decomposition/definitions.py | 2 +- .../model/common/decomposition/halo.py | 62 +++++-- .../icon4py/model/common/grid/grid_manager.py | 151 ++++++++---------- .../mpi_tests/test_parallel_grid_manager.py | 23 +-- .../grid/unit_tests/test_grid_manager.py | 2 +- .../src/icon4py/model/testing/grid_utils.py | 2 +- .../icon4py/tools/py2fgen/wrappers/common.py | 5 +- 7 files changed, 124 insertions(+), 123 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 8225009d2b..bbdf46368d 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -78,7 +78,7 @@ def __call__(self): self._counter = self._counter + 1 return next_id - +#TODO(halungge): halo_levels.. can we get rid of them? class DecompositionInfo: class EntryType(IntEnum): ALL = 0 diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 5941d16f43..9f26190a9b 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -84,7 +84,6 @@ def __init__( self._xp = data_alloc.import_array_ns(backend) self._num_levels = num_levels self._props = run_properties - self._connectivities = {_value(k): v for k, v in connectivities.items()} self._assert_all_neighbor_tables() @@ -379,13 +378,11 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: return decomp_info -# TODO(halungge): refine type hints: adjacency_matrix should be a connectivity matrix of C2E2C and -# the return value an array of shape (n_cells,) - - @runtime_checkable class Decomposer(Protocol): - def __call__(self, adjacency_matrix: data_alloc.NDArray, n_part: int) -> data_alloc.NDArray: + def __call__( + self, adjacency_matrix: data_alloc.NDArray, num_partitions: int + ) -> data_alloc.NDArray: """ Call the decomposition. @@ -400,14 +397,17 @@ class SimpleMetisDecomposer(Decomposer): """ A simple decomposer using METIS for partitioning a grid topology. - We use the simple pythonic interface to pymetis: just passing the adjacency matrix + We use the simple pythonic interface to pymetis: just passing the adjacency matrix, which for ICON is + the full grid C2E2C neigbhor table. if more control is needed (for example by using weights we need to switch to the C like interface) https://documen.tician.de/pymetis/functionality.html """ - def __call__(self, adjacency_matrix: data_alloc.NDArray, n_part: int) -> data_alloc.NDArray: + def __call__( + self, adjacency_matrix: data_alloc.NDArray, num_partitions: int + ) -> data_alloc.NDArray: """ - Generate partition labesl for this grid topology using METIS: + Generate partition labels for this grid topology using METIS: https://github.com/KarypisLab/METIS This method utilizes the pymetis Python bindings: @@ -415,16 +415,56 @@ def __call__(self, adjacency_matrix: data_alloc.NDArray, n_part: int) -> data_al Args: n_part: int, number of partitions to create + adjacency_matrix: nd array: neighbor table describing of the main dimension object to be distributed: for example cell -> cell neighbors Returns: np.ndarray: array with partition label (int, rank number) for each cell """ import pymetis - _, partition_index = pymetis.part_graph(nparts=n_part, adjacency=adjacency_matrix) + _, partition_index = pymetis.part_graph(nparts=num_partitions, adjacency=adjacency_matrix) return np.array(partition_index) class SingleNodeDecomposer(Decomposer): - def __call__(self, adjacency_matrix: data_alloc.NDArray, n_part: int) -> data_alloc.NDArray: + def __call__( + self, adjacency_matrix: data_alloc.NDArray, num_partitions=1 + ) -> data_alloc.NDArray: """Dummy decomposer for single node: assigns all cells to rank = 0""" return np.zeros(adjacency_matrix.shape[0], dtype=gtx.int32) + + +def halo_constructor( + run_properties: defs.ProcessProperties, + grid_config: base.GridConfig, + connectivities: dict[gtx.FieldOffset, data_alloc.NDArray], + backend=gtx_backend.Backend | None, +) -> HaloConstructor: + """ + Factory method to create the halo constructor. We need some input data from the global grid and from + Run parameters, hence this method is called during grid construction. + + Currently there is only one halo type (except for single node dummy). + If in the future we want to experiment with different halo types we should add an extra selection + parameter + Args: + processor_props: + grid_config: + connectivities: + backend: + + Returns: a HaloConstructor suitable for the run_properties + + """ + if run_properties.single_node(): + return NoHalos( + num_levels=grid_config.num_levels, + horizontal_size=grid_config.horizontal_size, + backend=backend, + ) + else: + return IconLikeHaloConstructor( + num_levels=grid_config.num_levels, + run_properties=run_properties, + connectivities=connectivities, + backend=backend, + ) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 479925ccd6..b61d21b07a 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -7,7 +7,6 @@ # SPDX-License-Identifier: BSD-3-Clause import logging import pathlib -from collections.abc import Callable from types import ModuleType from typing import Literal, TypeAlias @@ -17,6 +16,7 @@ from icon4py.model.common import dimension as dims, type_alias as ta from icon4py.model.common.decomposition import definitions as decomposition, halo +from icon4py.model.common.decomposition.halo import SingleNodeDecomposer from icon4py.model.common.exceptions import InvalidConfigError from icon4py.model.common.grid import ( base, @@ -30,8 +30,9 @@ _log = logging.getLogger(__name__) -_single_node_properties = decomposition.SingleNodeProcessProperties() _single_node_decomposer = halo.SingleNodeDecomposer() +_single_process_props = decomposition.SingleNodeProcessProperties() +_fortan_to_python_transformer = gridfile.ToZeroBasedIndexTransformation() class IconGridError(RuntimeError): @@ -54,27 +55,28 @@ class GridManager: """ - def open(self): - """Open the gridfile resource for reading.""" - self._reader = gridfile.GridFile(self._file_name, self._transformation) - self._reader.open() - def __init__( self, - transformation: gridfile.IndexTransformation, grid_file: pathlib.Path | str, - config: v_grid.VerticalGridConfig, # TODO(@halungge): remove: - separate vertical from horizontal grid + config: v_grid.VerticalGridConfig, + transformation: gridfile.IndexTransformation = _fortan_to_python_transformer, ): self._transformation = transformation self._file_name = str(grid_file) - self._halo_constructor = None self._vertical_config = config + self._halo_constructor: halo.HaloConstructor | None = None + # Output self._grid: icon.IconGrid | None = None self._decomposition_info: decomposition.DecompositionInfo | None = None self._geometry: GeometryDict = {} self._coordinates: CoordinateDict = {} self._reader = None + def open(self): + """Open the gridfile resource for reading.""" + self._reader = gridfile.GridFile(self._file_name, self._transformation) + self._reader.open() + def close(self): """close the gridfile resource.""" self._reader.close() @@ -96,20 +98,20 @@ def __call__( self, backend: gtx_backend.Backend | None, keep_skip_values: bool, - run_properties: decomposition.ProcessProperties = _single_node_properties, - decomposer: Callable[[np.ndarray, int], np.ndarray] = _single_node_decomposer, + decomposer: halo.Decomposer = _single_node_decomposer, + run_properties=_single_process_props, ): - self._run_properties = run_properties - if not run_properties.single_node() and isinstance(decomposer, halo.SingleNodeDecomposer): - raise InvalidConfigError( - f"Need a Decomposer for multi node run: running on: {run_properties.comm_size}" - ) + if not run_properties.single_node() and isinstance(decomposer, SingleNodeDecomposer): + raise InvalidConfigError("Need a Decomposer for multi node run") if not self._reader: self.open() - self._grid = self._construct_grid( - backend=backend, with_skip_values=keep_skip_values, decomposer=decomposer + self._construct_decomposed_grid( + backend=backend, + with_skip_values=keep_skip_values, + decomposer=decomposer, + run_properties=run_properties, ) self._coordinates = self._read_coordinates(backend) self._geometry = self._read_geometry_fields(backend) @@ -348,12 +350,13 @@ def coordinates(self) -> CoordinateDict: def decomposition_info(self) -> decomposition.DecompositionInfo: return self._decomposition_info - def _construct_grid( + def _construct_decomposed_grid( self, backend: gtx_backend.Backend | None, with_skip_values: bool, - decomposer: Callable[[np.ndarray, int], np.ndarray] | None = None, - ) -> icon.IconGrid: + decomposer: halo.Decomposer, + run_properties: decomposition.ProcessProperties, + ) -> None: """Construct the grid topology from the icon grid file. Reads connectivity fields from the grid file and constructs derived connectivities needed in @@ -362,20 +365,8 @@ def _construct_grid( """ xp = data_alloc.import_array_ns(backend) ## FULL GRID PROPERTIES - uuid_ = self._reader.attribute(gridfile.MandatoryPropertyName.GRID_UUID) - global_grid_size = self._read_full_grid_size() - grid_root = self._reader.attribute(gridfile.MandatoryPropertyName.ROOT) - grid_level = self._reader.attribute(gridfile.MandatoryPropertyName.LEVEL) - global_params = icon.GlobalGridParams(root=grid_root, level=grid_level) - cell_refinment = self._reader.variable(gridfile.GridRefinementName.CONTROL_CELLS) - limited_area = refinement.is_limited_area_grid(cell_refinment, array_ns=xp) + global_params, grid_config = self._construct_config(with_skip_values, xp) - grid_config = base.GridConfig( - horizontal_size=global_grid_size, - vertical_size=self._vertical_config.num_levels, - limited_area=limited_area, - keep_skip_values=with_skip_values, - ) cell_to_cell_neighbors = self._get_index_field(gridfile.ConnectivityName.C2E2C) neighbor_tables = { dims.C2E2C: cell_to_cell_neighbors, @@ -388,41 +379,30 @@ def _construct_grid( dims.E2V: self._get_index_field(gridfile.ConnectivityName.E2V), } - if decomposer is not None: - # DECOMPOSITION - - cells_to_rank_mapping = decomposer( - cell_to_cell_neighbors, - self._run_properties.comm_size, - ) - # HALO CONSTRUCTION - # TODO(halungge): reduce the set of neighbor tables used in the halo construction - # TODO(halungge): figure out where to do the host to device copies (xp.asarray...) - neighbor_tables_for_halo_construction = neighbor_tables - # halo_constructor - creates the decomposition info, which can then be used to generate the local patches on each rank - halo_constructor = self._initialize_halo_constructor( - global_grid_size, neighbor_tables_for_halo_construction, backend - ) - - self._decomposition_info = halo_constructor(cells_to_rank_mapping) - - ## TODO(halungge): do local reads (and halo exchanges!!) FIX: my_cells etc are in 0 base python coding - reading from file fails... - ## - # CONSTRUCT LOCAL PATCH + cells_to_rank_mapping = decomposer(cell_to_cell_neighbors, run_properties.comm_size) + # HALO CONSTRUCTION + # TODO(halungge): reduce the set of neighbor tables used in the halo construction + # TODO(halungge): figure out where to do the host to device copies (xp.asarray...) + neighbor_tables_for_halo_construction = neighbor_tables + halo_constructor = halo.halo_constructor( + run_properties=run_properties, + grid_config=grid_config, + connectivities=neighbor_tables_for_halo_construction, + backend=backend, + ) - # TODO(halungge): run this only for distrbuted grids otherwise to nothing internally - neighbor_tables = { - k: self._decomposition_info.global_to_local( - k.source, v[self._decomposition_info.global_index(k.target[0])] - ) - for k, v in neighbor_tables_for_halo_construction.items() - } + self._decomposition_info = halo_constructor(cells_to_rank_mapping) - # COMPUTE remaining derived connectivities - else: - self._decomposition_info = _single_node_decomposition_info(grid_config, xp) + # TODO(halungge): run this only for distrbuted grids otherwise to nothing internally + neighbor_tables = { + k: self._decomposition_info.global_to_local( + k.source, v[self._decomposition_info.global_index(k.target[0])] + ) + for k, v in neighbor_tables_for_halo_construction.items() + } # JOINT functionality + # COMPUTE remaining derived connectivities neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) start, end = self._read_start_end_indices() @@ -436,9 +416,10 @@ def _construct_grid( for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values() for k, v in h_grid.map_icon_domain_bounds(dim, end[dim]).items() } + refinement_fields = self._read_grid_refinement_fields(backend) grid = icon.icon_grid( - uuid_, + self._reader.attribute(gridfile.MandatoryPropertyName.GRID_UUID), allocator=backend, config=grid_config, neighbor_tables=neighbor_tables, @@ -447,7 +428,21 @@ def _construct_grid( global_properties=global_params, refinement_control=refinement_fields, ) - return grid + self._grid = grid + + def _construct_config(self, with_skip_values, xp): + grid_root = self._reader.attribute(gridfile.MandatoryPropertyName.ROOT) + grid_level = self._reader.attribute(gridfile.MandatoryPropertyName.LEVEL) + global_params = icon.GlobalGridParams(root=grid_root, level=grid_level) + cell_refinement = self._reader.variable(gridfile.GridRefinementName.CONTROL_CELLS) + limited_area = refinement.is_limited_area_grid(cell_refinement, array_ns=xp) + grid_config = base.GridConfig( + horizontal_size=self._read_full_grid_size(), + vertical_size=self._vertical_config.num_levels, + limited_area=limited_area, + keep_skip_values=with_skip_values, + ) + return global_params, grid_config def _get_index_field( self, @@ -475,26 +470,6 @@ def _read_full_grid_size(self) -> base.HorizontalGridSize: ) return full_grid_size - def _initialize_halo_constructor( - self, - grid_size: base.HorizontalGridSize, - connectivities: dict[gtx.FieldOffset, data_alloc.NDArray], - backend=gtx_backend.Backend | None, - ) -> halo.HaloConstructor: - if self._run_properties.single_node(): - return halo.NoHalos( - num_levels=self._vertical_config.num_levels, - horizontal_size=grid_size, - backend=backend, - ) - else: - return halo.IconLikeHaloConstructor( - run_properties=self._run_properties, - connectivities=connectivities, - num_levels=self._vertical_config.num_levels, - backend=backend, - ) - def _get_derived_connectivities( neighbor_tables: dict[gtx.FieldOffset, data_alloc.NDArray], array_ns: ModuleType = np diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index e5acf4d8f9..614424eb35 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -54,11 +54,7 @@ def run_gridmananger_for_multinode( def _grid_manager(file: pathlib.Path, vertical_config: v_grid.VerticalGridConfig): - manager = gm.GridManager( - gridfile.ToZeroBasedIndexTransformation(), - str(file), - vertical_config, - ) + manager = gm.GridManager(str(file), vertical_config) return manager @@ -69,7 +65,7 @@ def run_grid_manager_for_singlenode( manager( keep_skip_values=True, run_properties=defs.SingleNodeProcessProperties(), - decomposer=None, + decomposer=halo.SingleNodeDecomposer(), backend=None, ) return manager @@ -94,20 +90,15 @@ def test_start_end_index( partitioner = halo.SimpleMetisDecomposer() manager = gm.GridManager( - icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation(), file, v_grid.VerticalGridConfig(1), - run_properties=processor_props, + icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation(), ) single_node_grid = gm.GridManager( - icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation(), file, v_grid.VerticalGridConfig(1), - run_properties=defs.get_processor_properties(defs.SingleNodeRun()), + icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation(), ).grid - with manager.set_decomposer(partitioner) as manage: - manage(backend=backend, keep_skip_values=True) - grid = manage.grid for domain in utils.global_grid_domains(dim): assert grid.start_index(domain) == single_node_grid.start_index( @@ -122,11 +113,7 @@ def test_start_end_index( @pytest.mark.mpi(min_size=2) def test_grid_manager_validate_decomposer(processor_props): file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL.name) - manager = gm.GridManager( - gridfile.ToZeroBasedIndexTransformation(), - file, - vertical_config, - ) + manager = gm.GridManager(file, vertical_config, gridfile.ToZeroBasedIndexTransformation()) with pytest.raises(exceptions.InvalidConfigError) as e: manager( keep_skip_values=True, diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index 66840f1e3c..10c7326c98 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -365,9 +365,9 @@ def test_gridmanager_given_file_not_found_then_abort(): fname = "./unknown_grid.nc" with pytest.raises(FileNotFoundError) as error: manager = gm.GridManager( - icon4py.model.common.grid.gridfile.NoTransformation(), fname, v_grid.VerticalGridConfig(num_levels=80), + icon4py.model.common.grid.gridfile.NoTransformation(), ) manager(backend=None, keep_skip_values=True) assert error.value == 1 diff --git a/model/testing/src/icon4py/model/testing/grid_utils.py b/model/testing/src/icon4py/model/testing/grid_utils.py index 613ce50a3d..feb29f5c2c 100644 --- a/model/testing/src/icon4py/model/testing/grid_utils.py +++ b/model/testing/src/icon4py/model/testing/grid_utils.py @@ -87,9 +87,9 @@ def get_grid_manager( backend: the gt4py Backend we are running on """ manager = gm.GridManager( - gridfile.ToZeroBasedIndexTransformation(), filename, v_grid.VerticalGridConfig(num_levels=num_levels), + gridfile.ToZeroBasedIndexTransformation(), ) manager(backend=backend, keep_skip_values=keep_skip_values) return manager diff --git a/tools/src/icon4py/tools/py2fgen/wrappers/common.py b/tools/src/icon4py/tools/py2fgen/wrappers/common.py index 445f86c52c..d0ed970b5d 100644 --- a/tools/src/icon4py/tools/py2fgen/wrappers/common.py +++ b/tools/src/icon4py/tools/py2fgen/wrappers/common.py @@ -267,10 +267,9 @@ def construct_decomposition( e_owner_mask = e_owner_mask[:num_edges] v_owner_mask = v_owner_mask[:num_vertices] + decomposition_info = ( - definitions.DecompositionInfo( - klevels=num_levels, num_cells=num_cells, num_edges=num_edges, num_vertices=num_vertices - ) + definitions.DecompositionInfo(klevels=num_levels) .set_dimension(dims.CellDim, c_glb_index, c_owner_mask) .set_dimension(dims.EdgeDim, e_glb_index, e_owner_mask) .set_dimension(dims.VertexDim, v_glb_index, v_owner_mask) From 39c093da931fa69ebcc808ee34eba53a2dce1440 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Mon, 1 Sep 2025 15:18:46 +0200 Subject: [PATCH 084/240] fix start_index, end_index datatests --- .../model/common/decomposition/definitions.py | 11 ++++++----- .../common/decomposition/mpi_decomposition.py | 7 ++++--- .../mpi_tests/test_mpi_decomposition.py | 4 +++- .../common/grid/mpi_tests/test_parallel_icon.py | 15 ++++++++------- .../src/icon4py/tools/py2fgen/wrappers/common.py | 1 - 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index bbdf46368d..f158a990f3 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -78,7 +78,8 @@ def __call__(self): self._counter = self._counter + 1 return next_id -#TODO(halungge): halo_levels.. can we get rid of them? + +# TODO(halungge): halo_levels.. can we get rid of them? class DecompositionInfo: class EntryType(IntEnum): ALL = 0 @@ -192,9 +193,9 @@ def is_ready(self) -> bool: ... @runtime_checkable class ExchangeRuntime(Protocol): - def exchange(self, dim: gtx.Dimension, *fields: tuple) -> ExchangeResult: ... + def exchange(self, dim: gtx.Dimension, *fields: Sequence[gtx.Field]) -> ExchangeResult: ... - def exchange_and_wait(self, dim: gtx.Dimension, *fields: tuple) -> None: ... + def exchange_and_wait(self, dim: gtx.Dimension, *fields: Sequence[gtx.Field]) -> None: ... def get_size(self) -> int: ... @@ -203,10 +204,10 @@ def my_rank(self) -> int: ... @dataclass class SingleNodeExchange: - def exchange(self, dim: gtx.Dimension, *fields: tuple) -> ExchangeResult: + def exchange(self, dim: gtx.Dimension, fields: Sequence[gtx.Field]) -> ExchangeResult: return SingleNodeResult() - def exchange_and_wait(self, dim: gtx.Dimension, *fields: tuple) -> None: + def exchange_and_wait(self, dim: gtx.Dimension, fields: Sequence[gtx.Field]) -> None: return def my_rank(self) -> int: diff --git a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py index 71520b21ef..6b641f93c9 100644 --- a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py +++ b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py @@ -146,7 +146,7 @@ def __init__( for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values() } self._field_size: dict[gtx.Dimension : int] = { - dim: self._decomposition_info.global_index[dim].shape[0] + dim: self._decomposition_info.global_index(dim).shape[0] for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values() } log.info(f"domain descriptors for dimensions {self._domain_descriptors.keys()} initialized") @@ -216,6 +216,7 @@ def _slice_field_based_on_dim(self, field: gtx.Field, dim: gtx.Dimension) -> dat size but the global_index fields used to initialize GHEX exchanges have only local num_edge, num_cell_num_vertex size. """ + print(f" field = {field}, {type(field)}") try: assert field.ndarray.ndim == 2 trim_length = self._field_size[dim] @@ -223,7 +224,7 @@ def _slice_field_based_on_dim(self, field: gtx.Field, dim: gtx.Dimension) -> dat except KeyError: log.warn(f"Trying to trim field of invalid dimension {dim} for exchange. Not trimming.") - def exchange(self, dim: gtx.Dimension, *fields: Sequence[gtx.Field]): + def exchange(self, dim: gtx.Dimension, fields: Sequence[gtx.Field]): """ Exchange method that slices the fields based on the dimension and then performs halo exchange. @@ -258,7 +259,7 @@ def exchange(self, dim: gtx.Dimension, *fields: Sequence[gtx.Field]): log.debug(f"exchange for {len(fields)} fields of dimension ='{dim.value}' initiated.") return MultiNodeResult(handle, applied_patterns) - def exchange_and_wait(self, dim: gtx.Dimension, *fields: tuple): + def exchange_and_wait(self, dim: gtx.Dimension, *fields: Sequence[gtx.Field]): res = self.exchange(dim, *fields) res.wait() log.debug(f"exchange for {len(fields)} fields of dimension ='{dim.value}' done.") diff --git a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py index c16fb0dfed..b15ee924bd 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py @@ -24,6 +24,8 @@ data_provider, processor_props, decomposition_info, + grid_savepoint, + icon_grid, ) try: @@ -122,7 +124,7 @@ def test_decomposition_info_local_index( decomposition_info, processor_props, ): - caplog.set_level("info") + caplog.set_level(logging.INFO) parallel_helpers.check_comm_size(processor_props, sizes=(2,)) my_rank = processor_props.rank all_indices = decomposition_info.local_index(dim, definitions.DecompositionInfo.EntryType.ALL) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py index a81891a8de..ba26dcb7ac 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py @@ -118,16 +118,17 @@ def test_distributed_local(processor_props, dim, icon_grid, caplog): @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi @pytest.mark.parametrize("dim", utils.main_horizontal_dims()) -@pytest.mark.parametrize("marker", [h_grid.Zone.HALO, h_grid.Zone.HALO_LEVEL_2]) -def test_distributed_halo(processor_props, dim, marker, icon_grid): +@pytest.mark.parametrize("zone, level", [(h_grid.Zone.HALO, 1), (h_grid.Zone.HALO_LEVEL_2, 2)]) +def test_distributed_halo(processor_props, dim, zone, level, icon_grid): parallel_helpers.check_comm_size(processor_props) - domain = h_grid.domain(dim)(marker) + domain = h_grid.domain(dim)(zone) start_index = icon_grid.start_index(domain) end_index = icon_grid.end_index(domain) rank = processor_props.rank print( - f"rank {rank}/{processor_props.comm_size} dim = {dim} {marker} : ({start_index}, {end_index})" + f"rank {rank}/{processor_props.comm_size} dim = {dim} {zone} : ({start_index}, {end_index})" ) - - assert start_index == HALO_IDX[processor_props.comm_size][dim][rank][num - 1] - assert end_index == HALO_IDX[processor_props.comm_size][dim][rank][num] + expected = HALO_IDX[processor_props.comm_size][dim][rank][level - 1] + assert start_index == expected, f"expected start index {expected}, but was {start_index}" + expected = HALO_IDX[processor_props.comm_size][dim][rank][level] + assert end_index == expected, f"expected start index {1}, but was {start_index}" diff --git a/tools/src/icon4py/tools/py2fgen/wrappers/common.py b/tools/src/icon4py/tools/py2fgen/wrappers/common.py index d0ed970b5d..a50a989f22 100644 --- a/tools/src/icon4py/tools/py2fgen/wrappers/common.py +++ b/tools/src/icon4py/tools/py2fgen/wrappers/common.py @@ -267,7 +267,6 @@ def construct_decomposition( e_owner_mask = e_owner_mask[:num_edges] v_owner_mask = v_owner_mask[:num_vertices] - decomposition_info = ( definitions.DecompositionInfo(klevels=num_levels) .set_dimension(dims.CellDim, c_glb_index, c_owner_mask) From 222dee68ca093024d0f1917802851b20f142bc5b Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 2 Sep 2025 09:36:41 +0200 Subject: [PATCH 085/240] add test skeletion for testing neighbor access --- .../decomposition/mpi_tests/test_halo.py | 13 +---------- .../mpi_tests/test_parallel_grid_manager.py | 22 ++++++++++++++++--- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/model/common/tests/common/decomposition/mpi_tests/test_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_halo.py index beeb433d5d..908ecad109 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_halo.py @@ -234,18 +234,7 @@ def decompose(grid: base_grid.Grid, processor_props): # F811 # fixture # Will uses geofac_div and geofac_n2s -@pytest.mark.xfail -@pytest.mark.mpi -def test_halo_neighbor_access_c2e(): - pytest.fail("TODO implement") - # geofac_div = primal_edge_length(C2E) * edge_orientation / area - - # 1. read grid and distribue - GridManager - - # 2. get geometry fields (from GridManger) primal_edge_length, edge_orientation, area (local read) - # 3. compute geofac_div = primal_edge_length * edge_orientation / area - # 4. gather geofac_div - # 5 compare (possible reorder + def test_no_halo(): diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 614424eb35..67284df6a9 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -127,7 +127,7 @@ def test_grid_manager_validate_decomposer(processor_props): @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_distributed_fields(processor_props, caplog): +def test_fields_distribute_and_gather(processor_props, caplog): caplog.set_level(logging.INFO) print(f"myrank - {processor_props.rank}: running with processor_props = {processor_props}") file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL.name) @@ -138,8 +138,8 @@ def test_distributed_fields(processor_props, caplog): global_vertex_lon = grid_manager.coordinates[dims.VertexDim]["lon"] multinode = run_gridmananger_for_multinode( - file, - vertical_config, + file=file, + vertical_config=vertical_config, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), ) @@ -228,3 +228,19 @@ def assert_gathered_field_against_global( assert test_helpers.dallclose( sorted_, global_reference_field ), f"Gathered field values do not match for dim {dim}.- " + + +@pytest.mark.datatest +@pytest.mark.parametrize("processor_props", [True], indirect=True) +@pytest.mark.mpi +def test_halo_neighbor_access_c2e(grid_savepoint): + pytest.fail("TODO implement") + # geofac_div = primal_edge_length(C2E) * edge_orientation / area + + # 1. read grid and distribue - GridManager + + # 2. get geometry fields (from GridManger) primal_edge_length, edge_orientation, area (local read) + # 3. compute geofac_div = primal_edge_length * edge_orientation / area + # 4. gather geofac_div + # 5 compare (possible reorder + ... From ce0869dba417c7a14a32c1cca99f628f1abb84c9 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 3 Sep 2025 11:36:03 +0200 Subject: [PATCH 086/240] setup test for c2e access --- .../decomposition/mpi_tests/test_halo.py | 6 --- .../mpi_tests/test_parallel_grid_manager.py | 42 +++++++++++++++---- .../model/testing/fixtures/datatest.py | 20 ++++----- 3 files changed, 44 insertions(+), 24 deletions(-) diff --git a/model/common/tests/common/decomposition/mpi_tests/test_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_halo.py index 908ecad109..8ee30ee6a2 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_halo.py @@ -230,12 +230,6 @@ def decompose(grid: base_grid.Grid, processor_props): # F811 # fixture return labels -# TODO add test including halo access: -# Will uses geofac_div and geofac_n2s - - - - def test_no_halo(): grid_size = base_grid.HorizontalGridSize(num_cells=9, num_edges=14, num_vertices=6) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 67284df6a9..b547876016 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -14,10 +14,12 @@ from gt4py import next as gtx import icon4py.model.common.grid.gridfile +from icon4py.model.common.utils import data_allocation as data_alloc import icon4py.model.testing.grid_utils as grid_utils from icon4py.model.common import exceptions, dimension as dims from icon4py.model.common.decomposition import halo, mpi_decomposition, definitions as defs -from icon4py.model.common.grid import grid_manager as gm, vertical as v_grid, gridfile +from icon4py.model.common.grid import grid_manager as gm, vertical as v_grid, gridfile, geometry, geometry_attributes +from icon4py.model.common.interpolation import interpolation_fields from icon4py.model.testing import ( datatest_utils as dt_utils, test_utils as test_helpers, @@ -131,11 +133,11 @@ def test_fields_distribute_and_gather(processor_props, caplog): caplog.set_level(logging.INFO) print(f"myrank - {processor_props.rank}: running with processor_props = {processor_props}") file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL.name) - grid_manager = run_grid_manager_for_singlenode(file, vertical_config) - single_node_grid = grid_manager.grid - global_cell_area = grid_manager.geometry[gridfile.GeometryName.CELL_AREA] - global_edge_lat = grid_manager.coordinates[dims.EdgeDim]["lat"] - global_vertex_lon = grid_manager.coordinates[dims.VertexDim]["lon"] + single_node = run_grid_manager_for_singlenode(file, vertical_config) + single_node_grid = single_node.grid + global_cell_area = single_node.geometry[gridfile.GeometryName.CELL_AREA] + global_edge_lat = single_node.coordinates[dims.EdgeDim]["lat"] + global_vertex_lon = single_node.coordinates[dims.VertexDim]["lon"] multinode = run_gridmananger_for_multinode( file=file, @@ -230,11 +232,35 @@ def assert_gathered_field_against_global( ), f"Gathered field values do not match for dim {dim}.- " +# TODO add test including halo access: +# Will uses geofac_div and geofac_n2s + @pytest.mark.datatest @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi -def test_halo_neighbor_access_c2e(grid_savepoint): - pytest.fail("TODO implement") +def test_halo_neighbor_access_c2e(processor_props): + file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL.name) + single_node_grid = run_grid_manager_for_singlenode(file, vertical_config).grid + + + multinode_grid_manager = run_gridmananger_for_multinode( + file=file, + vertical_config=vertical_config, + run_properties=processor_props, + decomposer=halo.SimpleMetisDecomposer(), + ) + distributed_grid = multinode_grid_manager.grid + extra_geometry_fields = multinode_grid_manager.geometry + decomposition_info = multinode_grid_manager.decomposition_info + coordinates = multinode_grid_manager.coordinates + distributed_geometry = geometry.GridGeometry(backend=None, grid=distributed_grid, coordinates=coordinates, decomposition_info=decomposition_info, extra_fields=extra_geometry_fields, + metadata=geometry_attributes.attrs) + edge_length = distributed_geometry.get(geometry_attributes.EDGE_LENGTH) + cell_area = distributed_geometry.get(geometry_attributes.CELL_AREA) + edge_orientation = distributed_geometry.get(geometry_attributes.VERTEX_EDGE_ORIENTATION) + + geofac_div = data_alloc.zero_field(distributed_grid, dims.CellDim, dims.C2EDim) + interpolation_fields.compute_geofac_div(primal_edge_length = edge_length, area = cell_area, edge_orientation = edge_orientation, out = geofac_div, offset_provider={"C2E": distributed_grid.get_connectivity("C2E")}) # geofac_div = primal_edge_length(C2E) * edge_orientation / area # 1. read grid and distribue - GridManager diff --git a/model/testing/src/icon4py/model/testing/fixtures/datatest.py b/model/testing/src/icon4py/model/testing/fixtures/datatest.py index e0fecbe681..c63068fc0a 100644 --- a/model/testing/src/icon4py/model/testing/fixtures/datatest.py +++ b/model/testing/src/icon4py/model/testing/fixtures/datatest.py @@ -61,15 +61,15 @@ def backend(request: pytest.FixtureRequest) -> gtx_backend.Backend: @pytest.fixture -def experiment(): +def experiment()->str: return dt_utils.REGIONAL_EXPERIMENT @pytest.fixture(scope="session", params=[False]) -def processor_props(request): +def processor_props(request)-> decomposition.ProcessProperties: with_mpi = request.param runtype = decomposition.get_runtype(with_mpi=with_mpi) - yield decomposition.get_processor_properties(runtype) + return decomposition.get_processor_properties(runtype) @pytest.fixture(scope="session") @@ -83,7 +83,7 @@ def _download_ser_data( comm_size: int, _ranked_data_path: pathlib.Path, _experiment: str, -): +)->None: # not a fixture to be able to use this function outside of pytest try: destination_path = dt_utils.get_datapath_for_experiment(_ranked_data_path, _experiment) @@ -126,7 +126,7 @@ def download_ser_data( ranked_data_path: pathlib.Path, experiment: str, pytestconfig, -): +)->None: """ Get the binary ICON data from a remote server. @@ -156,7 +156,7 @@ def grid_savepoint( return data_provider.from_savepoint_grid(grid_id, root, level) -def is_regional(experiment_name): +def is_regional(experiment_name)->bool: return experiment_name == dt_utils.REGIONAL_EXPERIMENT @@ -250,9 +250,9 @@ def metrics_savepoint(data_provider): # F811 @pytest.fixture -def metrics_nonhydro_savepoint(data_provider): # F811 +def metrics_nonhydro_savepoint(data_provider)->serialbox.MetricSavepoint: # F811 """Load data from ICON metric state nonhydro savepoint.""" - return data_provider.from_metrics_nonhydro_savepoint() + return data_provider.from_metrics_savepoint() @pytest.fixture @@ -262,7 +262,7 @@ def topography_savepoint(data_provider): # F811 @pytest.fixture -def savepoint_velocity_init(data_provider, step_date_init, istep_init, substep_init): # F811 +def savepoint_velocity_init(data_provider, step_date_init, istep_init, substep_init)->serialbox.IconVelocityInitSavepoint: # F811 """ Load data from ICON savepoint at start of subroutine velocity_tendencies in mo_velocity_advection.f90. @@ -297,7 +297,7 @@ def savepoint_nonhydro_init( @pytest.fixture -def savepoint_dycore_30_to_38_init(data_provider, istep_init, step_date_init, substep_init): +def savepoint_dycore_30_to_38_init(data_provider, istep_init, step_date_init, substep_init)->serialbox.IconDycoreInit30To38Savepoint: """ Load data from ICON savepoint directly before the first stencil in stencils 30 to 38 in mo_solve_nonhydro.f90 of solve_nonhydro module. From 04c765aa13ec8bb83cf5bbde552782a9a6a14104 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 3 Sep 2025 18:09:26 +0200 Subject: [PATCH 087/240] remove connectivity workaround --- .../icon4py/model/common/states/factory.py | 31 ++++--------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/model/common/src/icon4py/model/common/states/factory.py b/model/common/src/icon4py/model/common/states/factory.py index 9d8412ee4a..8850aac52a 100644 --- a/model/common/src/icon4py/model/common/states/factory.py +++ b/model/common/src/icon4py/model/common/states/factory.py @@ -450,28 +450,9 @@ def _map_dim(dim: gtx.Dimension) -> gtx.Dimension: field_domain = {_map_dim(dim): (0, _map_size(dim, grid)) for dim in self._dims} return {k: allocate(field_domain, dtype=dtype[k]) for k in self._fields} - # TODO(halungge): this can be simplified when completely disentangling vertical and horizontal grid. - # the IconGrid should then only contain horizontal connectivities and no longer any Koff which should be moved to the VerticalGrid - def _get_offset_providers(self, grid: icon_grid.IconGrid) -> dict[str, gtx.FieldOffset]: - offset_providers = {} - for dim in self._compute_domain: - if dim.kind == gtx.DimensionKind.HORIZONTAL: - horizontal_offsets = { - k: v - for k, v in grid.connectivities.items() - # TODO(halungge): review this workaround, as the fix should be available in the gt4py baseline - if isinstance(v, gtx.Connectivity) - and v.domain.dims[0].kind == gtx.DimensionKind.HORIZONTAL - } - offset_providers.update(horizontal_offsets) - if dim.kind == gtx.DimensionKind.VERTICAL: - vertical_offsets = { - k: v - for k, v in grid.connectivities.items() - if isinstance(v, gtx.Dimension) and v.kind == gtx.DimensionKind.VERTICAL - } - offset_providers.update(vertical_offsets) - return offset_providers + def _grid_connectivities(self, grid: icon_grid.IconGrid) -> dict[str, gtx.Connectivity | gtx.Dimension]: + return grid.connectivities + def _domain_args( self, grid: icon_grid.IconGrid, vertical_grid: v_grid.VerticalGrid @@ -524,9 +505,9 @@ def _compute( deps = {k: factory.get(v) for k, v in self._dependencies.items()} deps.update(self._params) deps.update({k: self._fields[v] for k, v in self._output.items()}) - dims = self._domain_args(grid_provider.grid, grid_provider.vertical_grid) - offset_providers = self._get_offset_providers(grid_provider.grid) - deps.update(dims) + domain_bounds = self._domain_args(grid_provider.grid, grid_provider.vertical_grid) + deps.update(domain_bounds) + offset_providers = self._grid_connectivities(grid_provider.grid) self._func.with_backend(backend)(**deps, offset_provider=offset_providers) @property From 7353c3cc12a21a1f82dfc0b07255bc4e3c5b5949 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 3 Sep 2025 18:10:32 +0200 Subject: [PATCH 088/240] add test skeleton (WIP) --- .../model/common/decomposition/definitions.py | 2 +- .../icon4py/model/common/grid/grid_manager.py | 7 +- .../decomposition/mpi_tests/test_halo.py | 1 - .../mpi_tests/test_parallel_grid_manager.py | 28 ++++++-- .../common/grid/unit_tests/test_geometry.py | 5 +- .../grid/unit_tests/test_geometry_stencils.py | 70 +++++++++++++++++++ .../grid/unit_tests/test_grid_manager.py | 20 ++++++ .../model/testing/fixtures/datatest.py | 18 +++-- .../src/icon4py/model/testing/grid_utils.py | 11 +-- .../src/icon4py/model/testing/serialbox.py | 2 +- 10 files changed, 135 insertions(+), 29 deletions(-) create mode 100644 model/common/tests/common/grid/unit_tests/test_geometry_stencils.py diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index f158a990f3..22613cbd96 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -79,7 +79,7 @@ def __call__(self): return next_id -# TODO(halungge): halo_levels.. can we get rid of them? +# TODO(halungge): halo_levels.. they are the decomp_domain in ICON class DecompositionInfo: class EntryType(IntEnum): ALL = 0 diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index b61d21b07a..47cb200458 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -291,7 +291,7 @@ def _read_start_end_indices( This should be used for a single node run. In the case of a multi node distributed run the start and end indices need to be reconstructed from the decomposed grid. """ - _CHILD_DOM = 0 + _child_dom = 0 grid_refinement_dimensions = { dims.CellDim: gridfile.DimensionName.CELL_GRF, dims.EdgeDim: gridfile.DimensionName.EDGE_GRF, @@ -307,7 +307,7 @@ def _read_start_end_indices( } start_indices = { - dim: self._get_index_field(name, transpose=False, apply_offset=True)[_CHILD_DOM] + dim: self._get_index_field(name, transpose=False, apply_offset=True)[_child_dom] for dim, name in start_index_names.items() } for dim in grid_refinement_dimensions: @@ -321,7 +321,7 @@ def _read_start_end_indices( dims.VertexDim: gridfile.GridRefinementName.END_INDEX_VERTICES, } end_indices = { - dim: self._get_index_field(name, transpose=False, apply_offset=False)[_CHILD_DOM] + dim: self._get_index_field(name, transpose=False, apply_offset=False)[_child_dom] for dim, name in end_index_names.items() } for dim in grid_refinement_dimensions: @@ -403,6 +403,7 @@ def _construct_decomposed_grid( # JOINT functionality # COMPUTE remaining derived connectivities + neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) start, end = self._read_start_end_indices() diff --git a/model/common/tests/common/decomposition/mpi_tests/test_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_halo.py index 8ee30ee6a2..570ba12bd5 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_halo.py @@ -230,7 +230,6 @@ def decompose(grid: base_grid.Grid, processor_props): # F811 # fixture return labels - def test_no_halo(): grid_size = base_grid.HorizontalGridSize(num_cells=9, num_edges=14, num_vertices=6) halo_generator = halo.NoHalos(horizontal_size=grid_size, num_levels=10, backend=None) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index b547876016..7a2647523a 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -18,7 +18,13 @@ import icon4py.model.testing.grid_utils as grid_utils from icon4py.model.common import exceptions, dimension as dims from icon4py.model.common.decomposition import halo, mpi_decomposition, definitions as defs -from icon4py.model.common.grid import grid_manager as gm, vertical as v_grid, gridfile, geometry, geometry_attributes +from icon4py.model.common.grid import ( + grid_manager as gm, + vertical as v_grid, + gridfile, + geometry, + geometry_attributes, +) from icon4py.model.common.interpolation import interpolation_fields from icon4py.model.testing import ( datatest_utils as dt_utils, @@ -235,6 +241,7 @@ def assert_gathered_field_against_global( # TODO add test including halo access: # Will uses geofac_div and geofac_n2s + @pytest.mark.datatest @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi @@ -242,7 +249,6 @@ def test_halo_neighbor_access_c2e(processor_props): file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL.name) single_node_grid = run_grid_manager_for_singlenode(file, vertical_config).grid - multinode_grid_manager = run_gridmananger_for_multinode( file=file, vertical_config=vertical_config, @@ -253,14 +259,26 @@ def test_halo_neighbor_access_c2e(processor_props): extra_geometry_fields = multinode_grid_manager.geometry decomposition_info = multinode_grid_manager.decomposition_info coordinates = multinode_grid_manager.coordinates - distributed_geometry = geometry.GridGeometry(backend=None, grid=distributed_grid, coordinates=coordinates, decomposition_info=decomposition_info, extra_fields=extra_geometry_fields, - metadata=geometry_attributes.attrs) + distributed_geometry = geometry.GridGeometry( + backend=None, + grid=distributed_grid, + coordinates=coordinates, + decomposition_info=decomposition_info, + extra_fields=extra_geometry_fields, + metadata=geometry_attributes.attrs, + ) edge_length = distributed_geometry.get(geometry_attributes.EDGE_LENGTH) cell_area = distributed_geometry.get(geometry_attributes.CELL_AREA) edge_orientation = distributed_geometry.get(geometry_attributes.VERTEX_EDGE_ORIENTATION) geofac_div = data_alloc.zero_field(distributed_grid, dims.CellDim, dims.C2EDim) - interpolation_fields.compute_geofac_div(primal_edge_length = edge_length, area = cell_area, edge_orientation = edge_orientation, out = geofac_div, offset_provider={"C2E": distributed_grid.get_connectivity("C2E")}) + interpolation_fields.compute_geofac_div( + primal_edge_length=edge_length, + area=cell_area, + edge_orientation=edge_orientation, + out=geofac_div, + offset_provider={"C2E": distributed_grid.get_connectivity("C2E")}, + ) # geofac_div = primal_edge_length(C2E) * edge_orientation / area # 1. read grid and distribue - GridManager diff --git a/model/common/tests/common/grid/unit_tests/test_geometry.py b/model/common/tests/common/grid/unit_tests/test_geometry.py index 6dd3c9e622..18a6d1e4ca 100644 --- a/model/common/tests/common/grid/unit_tests/test_geometry.py +++ b/model/common/tests/common/grid/unit_tests/test_geometry.py @@ -18,9 +18,10 @@ simple as simple, ) from icon4py.model.common.grid.geometry import as_sparse_field + from icon4py.model.common.math import helpers as math_helpers from icon4py.model.common.utils import data_allocation as data_alloc -from icon4py.model.testing import datatest_utils as dt_utils, grid_utils, test_utils +from icon4py.model.testing import datatest_utils as dt_utils, grid_utils, test_utils, definitions as test_defs from ..fixtures import * # noqa: F401, F403 @@ -99,7 +100,7 @@ def test_compute_inverse_edge_length(backend, grid_savepoint, grid_file, experim @pytest.mark.parametrize( "grid_file, experiment, rtol", [ - (dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT, 1e-7), + (test_defs.Grids.MCH_CH_R04B09_DSL.name, dt_utils.REGIONAL_EXPERIMENT, 1e-7), (dt_utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT, 1e-11), ], ) diff --git a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py new file mode 100644 index 0000000000..3ddb470d4d --- /dev/null +++ b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py @@ -0,0 +1,70 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +import numpy as np +import pytest +from icon4py.model.testing import definitions +from icon4py.model.common.utils import data_allocation as data_alloc +from icon4py.model.testing import grid_utils +from icon4py.model.common import dimension as dims +from icon4py.model.common import constants +from icon4py.model.common.grid.geometry_stencils import compute_edge_length, compute_cell_center_arc_distance +from ..fixtures import * + +@pytest.mark.level("unit") +@pytest.mark.datatest +@pytest.mark.parametrize( + "experiment, grid_file", + ((definitions.Experiments.MCH_CH_R04B09.name, definitions.Grids.MCH_CH_R04B09_DSL.name), (definitions.Experiments.EXCLAIM_APE.name, definitions.Grids.R02B04_GLOBAL.name)), +) +def test_edge_length(experiment, grid_file, grid_savepoint, backend): + keep = True + #grid = grid_savepoint.construct_icon_grid(backend, keep_skip_values=keep) + + gm = grid_utils.get_grid_manager_from_identifier(grid_file, keep_skip_values=keep, num_levels=1, + backend=backend) + grid = gm.grid + coordinates = gm.coordinates[dims.VertexDim] + lat = coordinates["lat"] + lon = coordinates["lon"] + #lat = grid_savepoint.lat(dims.VertexDim) + #lon = grid_savepoint.lon(dims.VertexDim) + length = data_alloc.zero_field(grid, dims.EdgeDim) + compute_edge_length.with_backend(backend)( + vertex_lat=lat, + vertex_lon=lon, + radius=constants.EARTH_RADIUS, + length=length, + horizontal_start=0, + horizontal_end=grid.size[dims.EdgeDim], + offset_provider={"E2V": grid.get_connectivity(dims.E2V)}, + ) + + assert np.allclose(length.asnumpy(), grid_savepoint.primal_edge_length().asnumpy()) + + +@pytest.mark.level("unit") +@pytest.mark.datatest +@pytest.mark.parametrize( + "experiment, grid_file", + ((definitions.Experiments.MCH_CH_R04B09.name, definitions.Grids.MCH_CH_R04B09_DSL.name), (definitions.Experiments.EXCLAIM_APE.name, definitions.Grids.R02B04_GLOBAL.name)), +) +def test_compute_dual_edge_length(experiment, grid_file, grid_savepoint, backend): + keep = True + # grid = grid_savepoint.construct_icon_grid(backend, keep_skip_values=keep) + + gm = grid_utils.get_grid_manager_from_identifier(grid_file, keep_skip_values=keep, num_levels=1, + backend=backend) + grid = gm.grid + coordinates = gm.coordinates[dims.VertexDim] + lat = coordinates["lat"] + lon = coordinates["lon"] + # lat = grid_savepoint.lat(dims.VertexDim) + # lon = grid_savepoint.lon(dims.VertexDim) + length = data_alloc.zero_field(grid, dims.EdgeDim) + compute_cell_center_arc_distance.with_backend(backend)() diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index 10c7326c98..d32e5e8a7e 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -612,6 +612,26 @@ def test_limited_area_on_grid(grid_file, expected): assert expected == grid.limited_area +@pytest.mark.parametrize( + "grid_file, experiment", + [ + (dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT), + (dt_utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT), + ], +) +@pytest.mark.parametrize("dim", utils.horizontal_dims()) +def test_decomposition_info_single_node(dim, grid_file, experiment, grid_savepoint, backend): + expected = grid_savepoint.construct_decomposition_info() + gm = utils.run_grid_manager(grid_file, keep_skip_values=True, backend=backend) + result = gm.decomposition_info + assert np.all(result.local_index(dim) == expected.local_index(dim)) + assert np.all(result.global_index(dim) == expected.global_index(dim)) + assert np.all(result.owner_mask(dim) == expected.owner_mask(dim)) + assert np.all(result.halo_levels(dim) == expected.halo_levels(dim)) + + + + # TODO move to mpi_tests folder @pytest.mark.mpi @pytest.mark.parametrize( diff --git a/model/testing/src/icon4py/model/testing/fixtures/datatest.py b/model/testing/src/icon4py/model/testing/fixtures/datatest.py index c63068fc0a..efef42a89a 100644 --- a/model/testing/src/icon4py/model/testing/fixtures/datatest.py +++ b/model/testing/src/icon4py/model/testing/fixtures/datatest.py @@ -61,7 +61,7 @@ def backend(request: pytest.FixtureRequest) -> gtx_backend.Backend: @pytest.fixture -def experiment()->str: +def experiment() -> str: return dt_utils.REGIONAL_EXPERIMENT @@ -83,7 +83,7 @@ def _download_ser_data( comm_size: int, _ranked_data_path: pathlib.Path, _experiment: str, -)->None: +) -> None: # not a fixture to be able to use this function outside of pytest try: destination_path = dt_utils.get_datapath_for_experiment(_ranked_data_path, _experiment) @@ -126,7 +126,7 @@ def download_ser_data( ranked_data_path: pathlib.Path, experiment: str, pytestconfig, -)->None: +) -> None: """ Get the binary ICON data from a remote server. @@ -156,7 +156,7 @@ def grid_savepoint( return data_provider.from_savepoint_grid(grid_id, root, level) -def is_regional(experiment_name)->bool: +def is_regional(experiment_name) -> bool: return experiment_name == dt_utils.REGIONAL_EXPERIMENT @@ -250,7 +250,7 @@ def metrics_savepoint(data_provider): # F811 @pytest.fixture -def metrics_nonhydro_savepoint(data_provider)->serialbox.MetricSavepoint: # F811 +def metrics_nonhydro_savepoint(data_provider) -> serialbox.MetricSavepoint: # F811 """Load data from ICON metric state nonhydro savepoint.""" return data_provider.from_metrics_savepoint() @@ -262,7 +262,9 @@ def topography_savepoint(data_provider): # F811 @pytest.fixture -def savepoint_velocity_init(data_provider, step_date_init, istep_init, substep_init)->serialbox.IconVelocityInitSavepoint: # F811 +def savepoint_velocity_init( + data_provider, step_date_init, istep_init, substep_init +) -> serialbox.IconVelocityInitSavepoint: # F811 """ Load data from ICON savepoint at start of subroutine velocity_tendencies in mo_velocity_advection.f90. @@ -297,7 +299,9 @@ def savepoint_nonhydro_init( @pytest.fixture -def savepoint_dycore_30_to_38_init(data_provider, istep_init, step_date_init, substep_init)->serialbox.IconDycoreInit30To38Savepoint: +def savepoint_dycore_30_to_38_init( + data_provider, istep_init, step_date_init, substep_init +) -> serialbox.IconDycoreInit30To38Savepoint: """ Load data from ICON savepoint directly before the first stencil in stencils 30 to 38 in mo_solve_nonhydro.f90 of solve_nonhydro module. diff --git a/model/testing/src/icon4py/model/testing/grid_utils.py b/model/testing/src/icon4py/model/testing/grid_utils.py index feb29f5c2c..52160570e2 100644 --- a/model/testing/src/icon4py/model/testing/grid_utils.py +++ b/model/testing/src/icon4py/model/testing/grid_utils.py @@ -9,7 +9,6 @@ import gt4py.next.backend as gtx_backend -from icon4py.model.common.decomposition import halo from icon4py.model.common.grid import ( geometry, geometry_attributes as geometry_attrs, @@ -17,7 +16,7 @@ gridfile, vertical as v_grid, ) -from icon4py.model.common.utils import data_allocation as data_alloc, device_utils +from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import ( config, data_handling, @@ -148,8 +147,6 @@ def get_num_levels(experiment: str) -> int: def get_grid_geometry( backend: gtx_backend.Backend | None, experiment: str, grid_file: str ) -> geometry.GridGeometry: - on_gpu = device_utils.is_cupy_device(backend) - xp = data_alloc.array_ns(on_gpu) num_levels = get_num_levels(experiment) register_name = "_".join((experiment, data_alloc.backend_name(backend))) @@ -158,12 +155,8 @@ def _construct_grid_geometry() -> geometry.GridGeometry: grid_file, keep_skip_values=True, num_levels=num_levels, backend=backend ) grid = gm.grid - dummy_halo_constructor = halo.NoHalos( - horizontal_size=grid.config.horizontal_size, num_levels=num_levels, backend=backend - ) - decomposition_info = dummy_halo_constructor(xp.zeros((grid.num_levels,), dtype=int)) geometry_source = geometry.GridGeometry( - grid, decomposition_info, backend, gm.coordinates, gm.geometry, geometry_attrs.attrs + grid, gm.decomposition_info, backend, gm.coordinates, gm.geometry, geometry_attrs.attrs ) return geometry_source diff --git a/model/testing/src/icon4py/model/testing/serialbox.py b/model/testing/src/icon4py/model/testing/serialbox.py index ab93a9e6f9..05dcead052 100644 --- a/model/testing/src/icon4py/model/testing/serialbox.py +++ b/model/testing/src/icon4py/model/testing/serialbox.py @@ -482,7 +482,7 @@ def construct_decomposition_info(self): def _get_decomposition_fields(self, dim: gtx.Dimension): global_index = self.global_index(dim) mask = self.owner_mask(dim)[0 : self.num(dim)] - halo_levels = self.decomp_domain(dim) + halo_levels = self.decomp_domain(dim)[0 : self.num(dim)] return dim, global_index, mask, halo_levels def construct_icon_grid( From 34956c6487269c03e42590ae46700da3bb99deda Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 4 Sep 2025 09:01:39 +0200 Subject: [PATCH 089/240] compute local size for config --- .../model/common/decomposition/definitions.py | 14 ++-- .../model/common/decomposition/halo.py | 12 ++-- .../icon4py/model/common/grid/grid_manager.py | 27 ++++---- .../icon4py/model/common/states/factory.py | 5 +- .../unit_tests/test_definitions.py | 26 ++++++++ .../mpi_tests/test_parallel_grid_manager.py | 66 ++++++++++++++----- .../common/grid/unit_tests/test_geometry.py | 7 +- .../grid/unit_tests/test_geometry_stencils.py | 32 ++++++--- .../grid/unit_tests/test_grid_manager.py | 2 - .../unit_tests/test_interpolation_fields.py | 2 +- .../model/testing/fixtures/datatest.py | 2 +- 11 files changed, 140 insertions(+), 55 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 22613cbd96..802e3f7adc 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -19,8 +19,8 @@ import gt4py.next as gtx import numpy as np -from icon4py.model.common import utils -from icon4py.model.common.grid import gridfile +from icon4py.model.common import dimension as dims, utils +from icon4py.model.common.grid import base, gridfile from icon4py.model.common.utils import data_allocation as data_alloc @@ -79,7 +79,6 @@ def __call__(self): return next_id -# TODO(halungge): halo_levels.. they are the decomp_domain in ICON class DecompositionInfo: class EntryType(IntEnum): ALL = 0 @@ -174,13 +173,20 @@ def global_index(self, dim: gtx.Dimension, entry_type: EntryType = EntryType.ALL case _: raise NotImplementedError() + def get_horizontal_size(self): + return base.HorizontalGridSize( + num_cells=self.global_index(dims.CellDim, self.EntryType.ALL).shape[0], + num_edges=self.global_index(dims.EdgeDim, self.EntryType.ALL).shape[0], + num_vertices=self.global_index(dims.VertexDim, self.EntryType.ALL).shape[0], + ) + def halo_levels(self, dim: gtx.Dimension): return self._halo_levels[dim] def halo_level_mask(self, dim: gtx.Dimension, level: DecompositionFlag): return np.where(self._halo_levels[dim] == level, True, False) - # TODO (@halungge): unused - delete + # TODO (@halungge): unused - delete? def is_on_node(self, dim, index: int, entryType: EntryType = EntryType.ALL) -> bool: return np.isin(index, self.global_index(dim, entry_type=entryType)).item() diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 9f26190a9b..f7d48203ac 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -435,7 +435,8 @@ def __call__( def halo_constructor( run_properties: defs.ProcessProperties, - grid_config: base.GridConfig, + num_levels: int, + full_grid_size: base.HorizontalGridSize, connectivities: dict[gtx.FieldOffset, data_alloc.NDArray], backend=gtx_backend.Backend | None, ) -> HaloConstructor: @@ -448,7 +449,8 @@ def halo_constructor( parameter Args: processor_props: - grid_config: + num_levels: + full_grid_size connectivities: backend: @@ -457,13 +459,13 @@ def halo_constructor( """ if run_properties.single_node(): return NoHalos( - num_levels=grid_config.num_levels, - horizontal_size=grid_config.horizontal_size, + num_levels=num_levels, + horizontal_size=full_grid_size, backend=backend, ) else: return IconLikeHaloConstructor( - num_levels=grid_config.num_levels, + num_levels=num_levels, run_properties=run_properties, connectivities=connectivities, backend=backend, diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 47cb200458..05891fb20c 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -365,7 +365,10 @@ def _construct_decomposed_grid( """ xp = data_alloc.import_array_ns(backend) ## FULL GRID PROPERTIES - global_params, grid_config = self._construct_config(with_skip_values, xp) + global_params = self._construct_global_params() + cell_refinement = self._reader.variable(gridfile.GridRefinementName.CONTROL_CELLS) + global_size = self._read_full_grid_size() + limited_area = refinement.is_limited_area_grid(cell_refinement, array_ns=xp) cell_to_cell_neighbors = self._get_index_field(gridfile.ConnectivityName.C2E2C) neighbor_tables = { @@ -386,12 +389,14 @@ def _construct_decomposed_grid( neighbor_tables_for_halo_construction = neighbor_tables halo_constructor = halo.halo_constructor( run_properties=run_properties, - grid_config=grid_config, + num_levels=self._vertical_config.num_levels, + full_grid_size=global_size, connectivities=neighbor_tables_for_halo_construction, backend=backend, ) self._decomposition_info = halo_constructor(cells_to_rank_mapping) + distributed_size = self._decomposition_info.get_horizontal_grid_size() # TODO(halungge): run this only for distrbuted grids otherwise to nothing internally neighbor_tables = { @@ -418,6 +423,12 @@ def _construct_decomposed_grid( for k, v in h_grid.map_icon_domain_bounds(dim, end[dim]).items() } refinement_fields = self._read_grid_refinement_fields(backend) + grid_config = base.GridConfig( + horizontal_size=distributed_size, + vertical_size=self._vertical_config.num_levels, + limited_area=limited_area, + keep_skip_values=with_skip_values, + ) grid = icon.icon_grid( self._reader.attribute(gridfile.MandatoryPropertyName.GRID_UUID), @@ -431,19 +442,11 @@ def _construct_decomposed_grid( ) self._grid = grid - def _construct_config(self, with_skip_values, xp): + def _construct_global_params(self): grid_root = self._reader.attribute(gridfile.MandatoryPropertyName.ROOT) grid_level = self._reader.attribute(gridfile.MandatoryPropertyName.LEVEL) global_params = icon.GlobalGridParams(root=grid_root, level=grid_level) - cell_refinement = self._reader.variable(gridfile.GridRefinementName.CONTROL_CELLS) - limited_area = refinement.is_limited_area_grid(cell_refinement, array_ns=xp) - grid_config = base.GridConfig( - horizontal_size=self._read_full_grid_size(), - vertical_size=self._vertical_config.num_levels, - limited_area=limited_area, - keep_skip_values=with_skip_values, - ) - return global_params, grid_config + return global_params def _get_index_field( self, diff --git a/model/common/src/icon4py/model/common/states/factory.py b/model/common/src/icon4py/model/common/states/factory.py index 8850aac52a..17284a14e8 100644 --- a/model/common/src/icon4py/model/common/states/factory.py +++ b/model/common/src/icon4py/model/common/states/factory.py @@ -450,10 +450,11 @@ def _map_dim(dim: gtx.Dimension) -> gtx.Dimension: field_domain = {_map_dim(dim): (0, _map_size(dim, grid)) for dim in self._dims} return {k: allocate(field_domain, dtype=dtype[k]) for k in self._fields} - def _grid_connectivities(self, grid: icon_grid.IconGrid) -> dict[str, gtx.Connectivity | gtx.Dimension]: + def _grid_connectivities( + self, grid: icon_grid.IconGrid + ) -> dict[str, gtx.Connectivity | gtx.Dimension]: return grid.connectivities - def _domain_args( self, grid: icon_grid.IconGrid, vertical_grid: v_grid.VerticalGrid ) -> dict[str : gtx.int32]: diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index 43d660aacc..a581d3ba7b 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -17,6 +17,7 @@ from .. import utils +from ..mpi_tests.test_halo import simple_neighbor_tables from ..utils import dummy_four_ranks from icon4py.model.testing.fixtures import processor_props @@ -99,3 +100,28 @@ def test_halo_constructor_decomposition_info_global_indices(dim, rank): my_owned = decomp_info.global_index(dim, definitions.DecompositionInfo.EntryType.OWNED) print(f"rank {props.rank} owns {dim} : {my_owned} ") utils.assert_same_entries(dim, my_owned, utils.OWNED, props.rank) + + +@pytest.mark.parametrize("rank", (0, 1, 2, 3)) +def test_horizontal_size(rank): + simple_neighbor_tables = get_neighbor_tables_for_simple_grid() + props = dummy_four_ranks(rank) + halo_generator = halo.IconLikeHaloConstructor( + connectivities=simple_neighbor_tables, + run_properties=props, + num_levels=1, + ) + decomp_info = halo_generator(utils.SIMPLE_DISTRIBUTION) + horizontal_size = decomp_info.get_horizontal_size() + expected_verts = len(utils.OWNED[dims.VertexDim][rank]) + len(utils.HALO[dims.VertexDim][rank]) + assert ( + horizontal_size.num_vertices == expected_verts + ), f"local size mismatch on rank={rank} for {dims.VertexDim}: expected {expected_verts}, but was {horizontal_size.num_vertices}" + expected_edges = len(utils.OWNED[dims.EdgeDim][rank]) + len(utils.HALO[dims.EdgeDim][rank]) + assert ( + horizontal_size.num_edges == expected_edges + ), f"local size mismatch on rank={rank} for {dims.EdgeDim}: expected {expected_edges}, but was {horizontal_size.num_edges}" + expected_cells = len(utils.OWNED[dims.CellDim][rank]) + len(utils.HALO[dims.CellDim][rank]) + assert ( + horizontal_size.num_cells == expected_cells + ), f"local size mismatch on rank={rank} for {dims.CellDim}: expected {expected_cells}, but was {horizontal_size.num_cells}" diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 7a2647523a..39211c5bdc 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -25,7 +25,7 @@ geometry, geometry_attributes, ) -from icon4py.model.common.interpolation import interpolation_fields +from icon4py.model.common.interpolation.interpolation_fields import compute_geofac_div from icon4py.model.testing import ( datatest_utils as dt_utils, test_utils as test_helpers, @@ -109,10 +109,10 @@ def test_start_end_index( ).grid for domain in utils.global_grid_domains(dim): - assert grid.start_index(domain) == single_node_grid.start_index( + assert icon_grid.start_index(domain) == single_node_grid.start_index( domain ), f"start index wrong for domain {domain}" - assert grid.end_index(domain) == single_node_grid.end_index( + assert icon_grid.end_index(domain) == single_node_grid.end_index( domain ), f"end index wrong for domain {domain}" @@ -239,15 +239,40 @@ def assert_gathered_field_against_global( # TODO add test including halo access: -# Will uses geofac_div and geofac_n2s +# Will uses +# - geofac_div +# - geofac_n2s @pytest.mark.datatest @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi -def test_halo_neighbor_access_c2e(processor_props): +def test_halo_neighbor_access_c2e(processor_props, interpolation_savepoint): file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL.name) - single_node_grid = run_grid_manager_for_singlenode(file, vertical_config).grid + backend = None + single_node = run_grid_manager_for_singlenode(file, vertical_config) + single_node_grid = single_node.grid + single_node_geometry = geometry.GridGeometry( + backend=backend, + grid=single_node_grid, + coordinates=single_node.coordinates, + decomposition_info=single_node.decomposition_info, + extra_fields=single_node.geometry, + metadata=geometry_attributes.attrs, + ) + reference = data_alloc.zero_field(single_node_grid, dims.CellDim, dims.C2EDim) + single_node_edge_length = single_node_geometry.get(geometry_attributes.EDGE_LENGTH) + single_node_cell_area = single_node_geometry.get(geometry_attributes.CELL_AREA) + single_node_edge_orientation = single_node_geometry.get( + geometry_attributes.VERTEX_EDGE_ORIENTATION + ) + # compute_geofac_div.with_backend(backend)( + # primal_edge_length=single_node_edge_length, + # area=single_node_cell_area, + # edge_orientation=single_node_edge_orientation, + # out=reference, + # offset_provider={"C2E": single_node_grid.get_connectivity("C2E")}, + # ) multinode_grid_manager = run_gridmananger_for_multinode( file=file, @@ -258,11 +283,11 @@ def test_halo_neighbor_access_c2e(processor_props): distributed_grid = multinode_grid_manager.grid extra_geometry_fields = multinode_grid_manager.geometry decomposition_info = multinode_grid_manager.decomposition_info - coordinates = multinode_grid_manager.coordinates + distributed_coordinates = multinode_grid_manager.coordinates distributed_geometry = geometry.GridGeometry( - backend=None, + backend=backend, grid=distributed_grid, - coordinates=coordinates, + coordinates=distributed_coordinates, decomposition_info=decomposition_info, extra_fields=extra_geometry_fields, metadata=geometry_attributes.attrs, @@ -271,16 +296,23 @@ def test_halo_neighbor_access_c2e(processor_props): cell_area = distributed_geometry.get(geometry_attributes.CELL_AREA) edge_orientation = distributed_geometry.get(geometry_attributes.VERTEX_EDGE_ORIENTATION) - geofac_div = data_alloc.zero_field(distributed_grid, dims.CellDim, dims.C2EDim) - interpolation_fields.compute_geofac_div( - primal_edge_length=edge_length, - area=cell_area, - edge_orientation=edge_orientation, - out=geofac_div, - offset_provider={"C2E": distributed_grid.get_connectivity("C2E")}, - ) # geofac_div = primal_edge_length(C2E) * edge_orientation / area + geofac_div = data_alloc.zero_field(distributed_grid, dims.CellDim, dims.C2EDim) + # compute_geofac_div.with_backend(backend)( + # primal_edge_length=edge_length, + # area=cell_area, + # edge_orientation=edge_orientation, + # out=geofac_div, + # offset_provider={"C2E": distributed_grid.get_connectivity("C2E")}, + # ) + assert_gathered_field_against_global( + decomposition_info, + processor_props, + dims.EdgeDim, + global_reference_field=reference.asnumpy(), + local_field=geofac_div.asnumpy(), + ) # 1. read grid and distribue - GridManager # 2. get geometry fields (from GridManger) primal_edge_length, edge_orientation, area (local read) diff --git a/model/common/tests/common/grid/unit_tests/test_geometry.py b/model/common/tests/common/grid/unit_tests/test_geometry.py index 18a6d1e4ca..1e245e3faa 100644 --- a/model/common/tests/common/grid/unit_tests/test_geometry.py +++ b/model/common/tests/common/grid/unit_tests/test_geometry.py @@ -21,7 +21,12 @@ from icon4py.model.common.math import helpers as math_helpers from icon4py.model.common.utils import data_allocation as data_alloc -from icon4py.model.testing import datatest_utils as dt_utils, grid_utils, test_utils, definitions as test_defs +from icon4py.model.testing import ( + datatest_utils as dt_utils, + grid_utils, + test_utils, + definitions as test_defs, +) from ..fixtures import * # noqa: F401, F403 diff --git a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py index 3ddb470d4d..b455001d46 100644 --- a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py +++ b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py @@ -13,27 +13,35 @@ from icon4py.model.testing import grid_utils from icon4py.model.common import dimension as dims from icon4py.model.common import constants -from icon4py.model.common.grid.geometry_stencils import compute_edge_length, compute_cell_center_arc_distance +from icon4py.model.common.grid.geometry_stencils import ( + compute_edge_length, + compute_cell_center_arc_distance, +) from ..fixtures import * + @pytest.mark.level("unit") @pytest.mark.datatest @pytest.mark.parametrize( "experiment, grid_file", - ((definitions.Experiments.MCH_CH_R04B09.name, definitions.Grids.MCH_CH_R04B09_DSL.name), (definitions.Experiments.EXCLAIM_APE.name, definitions.Grids.R02B04_GLOBAL.name)), + ( + (definitions.Experiments.MCH_CH_R04B09.name, definitions.Grids.MCH_CH_R04B09_DSL.name), + (definitions.Experiments.EXCLAIM_APE.name, definitions.Grids.R02B04_GLOBAL.name), + ), ) def test_edge_length(experiment, grid_file, grid_savepoint, backend): keep = True - #grid = grid_savepoint.construct_icon_grid(backend, keep_skip_values=keep) + # grid = grid_savepoint.construct_icon_grid(backend, keep_skip_values=keep) - gm = grid_utils.get_grid_manager_from_identifier(grid_file, keep_skip_values=keep, num_levels=1, - backend=backend) + gm = grid_utils.get_grid_manager_from_identifier( + grid_file, keep_skip_values=keep, num_levels=1, backend=backend + ) grid = gm.grid coordinates = gm.coordinates[dims.VertexDim] lat = coordinates["lat"] lon = coordinates["lon"] - #lat = grid_savepoint.lat(dims.VertexDim) - #lon = grid_savepoint.lon(dims.VertexDim) + # lat = grid_savepoint.lat(dims.VertexDim) + # lon = grid_savepoint.lon(dims.VertexDim) length = data_alloc.zero_field(grid, dims.EdgeDim) compute_edge_length.with_backend(backend)( vertex_lat=lat, @@ -52,14 +60,18 @@ def test_edge_length(experiment, grid_file, grid_savepoint, backend): @pytest.mark.datatest @pytest.mark.parametrize( "experiment, grid_file", - ((definitions.Experiments.MCH_CH_R04B09.name, definitions.Grids.MCH_CH_R04B09_DSL.name), (definitions.Experiments.EXCLAIM_APE.name, definitions.Grids.R02B04_GLOBAL.name)), + ( + (definitions.Experiments.MCH_CH_R04B09.name, definitions.Grids.MCH_CH_R04B09_DSL.name), + (definitions.Experiments.EXCLAIM_APE.name, definitions.Grids.R02B04_GLOBAL.name), + ), ) def test_compute_dual_edge_length(experiment, grid_file, grid_savepoint, backend): keep = True # grid = grid_savepoint.construct_icon_grid(backend, keep_skip_values=keep) - gm = grid_utils.get_grid_manager_from_identifier(grid_file, keep_skip_values=keep, num_levels=1, - backend=backend) + gm = grid_utils.get_grid_manager_from_identifier( + grid_file, keep_skip_values=keep, num_levels=1, backend=backend + ) grid = gm.grid coordinates = gm.coordinates[dims.VertexDim] lat = coordinates["lat"] diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index d32e5e8a7e..6cc50c26c7 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -630,8 +630,6 @@ def test_decomposition_info_single_node(dim, grid_file, experiment, grid_savepoi assert np.all(result.halo_levels(dim) == expected.halo_levels(dim)) - - # TODO move to mpi_tests folder @pytest.mark.mpi @pytest.mark.parametrize( diff --git a/model/common/tests/common/interpolation/unit_tests/test_interpolation_fields.py b/model/common/tests/common/interpolation/unit_tests/test_interpolation_fields.py index 0e74d797ae..c6b2d86356 100644 --- a/model/common/tests/common/interpolation/unit_tests/test_interpolation_fields.py +++ b/model/common/tests/common/interpolation/unit_tests/test_interpolation_fields.py @@ -90,7 +90,7 @@ def test_compute_geofac_div( area = grid_savepoint.cell_areas() geofac_div_ref = interpolation_savepoint.geofac_div() geofac_div = data_alloc.zero_field(mesh, dims.CellDim, dims.C2EDim) - compute_geofac_div.with_backend(backend)( + compute_geofac_div.with_backend(None)( primal_edge_length=primal_edge_length, edge_orientation=edge_orientation, area=area, diff --git a/model/testing/src/icon4py/model/testing/fixtures/datatest.py b/model/testing/src/icon4py/model/testing/fixtures/datatest.py index efef42a89a..b04dcfb5f9 100644 --- a/model/testing/src/icon4py/model/testing/fixtures/datatest.py +++ b/model/testing/src/icon4py/model/testing/fixtures/datatest.py @@ -66,7 +66,7 @@ def experiment() -> str: @pytest.fixture(scope="session", params=[False]) -def processor_props(request)-> decomposition.ProcessProperties: +def processor_props(request) -> decomposition.ProcessProperties: with_mpi = request.param runtype = decomposition.get_runtype(with_mpi=with_mpi) return decomposition.get_processor_properties(runtype) From 003b70c77439232cc2d487af09687ffc2f32bf42 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 5 Sep 2025 10:09:42 +0200 Subject: [PATCH 090/240] neighbor access test (WIP) --- .../model/common/decomposition/definitions.py | 3 + .../src/icon4py/model/common/grid/base.py | 3 + .../icon4py/model/common/grid/grid_manager.py | 24 ++-- .../src/icon4py/model/common/grid/icon.py | 4 +- .../icon4py/model/common/states/factory.py | 2 + .../mpi_tests/test_parallel_grid_manager.py | 109 +++++++++++------- 6 files changed, 86 insertions(+), 59 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 802e3f7adc..a283bf501c 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -180,6 +180,9 @@ def get_horizontal_size(self): num_vertices=self.global_index(dims.VertexDim, self.EntryType.ALL).shape[0], ) + def get_halo_size(self, dim: gtx.Dimension, flag: DecompositionFlag)->int: + return np.count_nonzero(self.halo_level_mask(dim, flag)) + def halo_levels(self, dim: gtx.Dimension): return self._halo_levels[dim] diff --git a/model/common/src/icon4py/model/common/grid/base.py b/model/common/src/icon4py/model/common/grid/base.py index 9ddd490f57..a4293f5e57 100644 --- a/model/common/src/icon4py/model/common/grid/base.py +++ b/model/common/src/icon4py/model/common/grid/base.py @@ -44,6 +44,9 @@ class HorizontalGridSize: num_edges: int num_cells: int + def __repr__(self): + return f"{self.__class__} (, , bool: value = dimension in CONNECTIVITIES_ON_PENTAGONS or ( limited_area and dimension in CONNECTIVITIES_ON_BOUNDARIES ) - return value + + #return value + return True def _should_replace_skip_values( diff --git a/model/common/src/icon4py/model/common/states/factory.py b/model/common/src/icon4py/model/common/states/factory.py index 17284a14e8..915ae0d43d 100644 --- a/model/common/src/icon4py/model/common/states/factory.py +++ b/model/common/src/icon4py/model/common/states/factory.py @@ -43,6 +43,7 @@ import enum import functools import logging +import pdb import types import typing from collections.abc import Callable, Mapping, MutableMapping, Sequence @@ -61,6 +62,7 @@ icon as icon_grid, vertical as v_grid, ) +from icon4py.model.common.io.writers import processor_properties from icon4py.model.common.states import model, utils as state_utils from icon4py.model.common.utils import data_allocation as data_alloc diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 39211c5bdc..4e1bc7c2ae 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -5,8 +5,9 @@ # # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause - +import functools import logging +import operator import pathlib import numpy as np @@ -25,6 +26,7 @@ geometry, geometry_attributes, ) +from icon4py.model.common.interpolation import interpolation_factory, interpolation_attributes from icon4py.model.common.interpolation.interpolation_fields import compute_geofac_div from icon4py.model.testing import ( datatest_utils as dt_utils, @@ -191,11 +193,16 @@ def test_fields_distribute_and_gather(processor_props, caplog): def gather_field(field: np.ndarray, comm: mpi4py.MPI.Comm) -> tuple: + constant_dims = field.shape[1:] + constant_length = functools.reduce(operator.mul, constant_dims) if len(constant_dims) > 0 else 1 + local_sizes = np.array(comm.gather(field.size, root=0)) + + if comm.rank == 0: - recv_buffer = np.empty(sum(local_sizes), dtype=field.dtype) + recv_buffer = np.empty(np.sum(local_sizes), dtype=field.dtype) log.debug( - f"rank: {comm.rank} - setup receive buffer with size {sum(local_sizes)} on rank 0" + f"rank:{comm} - {comm.rank} - setup receive buffer with size {sum(local_sizes)} on rank 0" ) else: recv_buffer = None @@ -204,9 +211,12 @@ def gather_field(field: np.ndarray, comm: mpi4py.MPI.Comm) -> tuple: log.debug(f"fields gathered:") log.debug(f"field sizes {local_sizes}") - return local_sizes, recv_buffer + local_first_dim = tuple(ls/constant_length for ls in local_sizes) + gathered_field = recv_buffer.reshape((-1 + constant_dims)) + return local_first_dim, gathered_field +#TODO (halungge): deal with 2d fields def assert_gathered_field_against_global( decomposition_info: defs.DecompositionInfo, processor_props: defs.ProcessProperties, # F811 # fixture @@ -214,9 +224,10 @@ def assert_gathered_field_against_global( global_reference_field: np.ndarray, local_field: np.ndarray, ): + assert ( - local_field.shape - == decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.ALL).shape + local_field.shape[0] + == decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.ALL).shape[0] ) owned_entries = local_field[ decomposition_info.local_index(dim, defs.DecompositionInfo.EntryType.OWNED) @@ -244,12 +255,12 @@ def assert_gathered_field_against_global( # - geofac_n2s -@pytest.mark.datatest -@pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi -def test_halo_neighbor_access_c2e(processor_props, interpolation_savepoint): +@pytest.mark.parametrize("processor_props", [True], indirect=True) +def test_halo_neighbor_access_c2e(processor_props): file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL.name) backend = None + print(f"running on {processor_props.comm}") single_node = run_grid_manager_for_singlenode(file, vertical_config) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( @@ -260,63 +271,73 @@ def test_halo_neighbor_access_c2e(processor_props, interpolation_savepoint): extra_fields=single_node.geometry, metadata=geometry_attributes.attrs, ) + + print(f"rank = {processor_props.rank} : single node grid has size {single_node.decomposition_info.get_horizontal_size()!r}") reference = data_alloc.zero_field(single_node_grid, dims.CellDim, dims.C2EDim) single_node_edge_length = single_node_geometry.get(geometry_attributes.EDGE_LENGTH) single_node_cell_area = single_node_geometry.get(geometry_attributes.CELL_AREA) single_node_edge_orientation = single_node_geometry.get( - geometry_attributes.VERTEX_EDGE_ORIENTATION + geometry_attributes.CELL_NORMAL_ORIENTATION + ) + compute_geofac_div.with_backend(backend)( + primal_edge_length=single_node_edge_length, + area=single_node_cell_area, + edge_orientation=single_node_edge_orientation, + out=reference, + offset_provider={"C2E": single_node_grid.get_connectivity("C2E")}, ) - # compute_geofac_div.with_backend(backend)( - # primal_edge_length=single_node_edge_length, - # area=single_node_cell_area, - # edge_orientation=single_node_edge_orientation, - # out=reference, - # offset_provider={"C2E": single_node_grid.get_connectivity("C2E")}, - # ) - + print( + f"rank = {processor_props.rank} : single node computed field reference has size {reference.asnumpy().shape}") + processor_props.comm.barrier() multinode_grid_manager = run_gridmananger_for_multinode( - file=file, - vertical_config=vertical_config, - run_properties=processor_props, - decomposer=halo.SimpleMetisDecomposer(), + file=file, + vertical_config=vertical_config, + run_properties=processor_props, + decomposer=halo.SimpleMetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid extra_geometry_fields = multinode_grid_manager.geometry decomposition_info = multinode_grid_manager.decomposition_info + + print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") + print(f"rank = {processor_props.rank}: halo size for 'CellDim' (1 : {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.SECOND_HALO_LINE)})") distributed_coordinates = multinode_grid_manager.coordinates distributed_geometry = geometry.GridGeometry( - backend=backend, - grid=distributed_grid, - coordinates=distributed_coordinates, - decomposition_info=decomposition_info, - extra_fields=extra_geometry_fields, - metadata=geometry_attributes.attrs, + backend=backend, + grid=distributed_grid, + coordinates=distributed_coordinates, + decomposition_info=decomposition_info, + extra_fields=extra_geometry_fields, + metadata=geometry_attributes.attrs, ) + edge_length = distributed_geometry.get(geometry_attributes.EDGE_LENGTH) cell_area = distributed_geometry.get(geometry_attributes.CELL_AREA) - edge_orientation = distributed_geometry.get(geometry_attributes.VERTEX_EDGE_ORIENTATION) + edge_orientation = distributed_geometry.get(geometry_attributes.CELL_NORMAL_ORIENTATION) - # geofac_div = primal_edge_length(C2E) * edge_orientation / area + # ### geofac_div = primal_edge_length(C2E) * edge_orientation / area geofac_div = data_alloc.zero_field(distributed_grid, dims.CellDim, dims.C2EDim) - # compute_geofac_div.with_backend(backend)( - # primal_edge_length=edge_length, - # area=cell_area, - # edge_orientation=edge_orientation, - # out=geofac_div, - # offset_provider={"C2E": distributed_grid.get_connectivity("C2E")}, - # ) + compute_geofac_div.with_backend(backend)( + primal_edge_length=edge_length, + area=cell_area, + edge_orientation=edge_orientation, + out=geofac_div, + offset_provider={"C2E": distributed_grid.get_connectivity("C2E")}, + ) assert_gathered_field_against_global( - decomposition_info, - processor_props, - dims.EdgeDim, - global_reference_field=reference.asnumpy(), - local_field=geofac_div.asnumpy(), - ) + decomposition_info, + processor_props, + dims.CellDim, + global_reference_field=reference.asnumpy(), + local_field=geofac_div.asnumpy(), + ) + + print(f"rank = {processor_props.rank} - DONE") # 1. read grid and distribue - GridManager # 2. get geometry fields (from GridManger) primal_edge_length, edge_orientation, area (local read) # 3. compute geofac_div = primal_edge_length * edge_orientation / area # 4. gather geofac_div # 5 compare (possible reorder - ... + From d1f35cbac91260f8c3aae578224b2e4e5dc97e66 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 9 Sep 2025 10:00:53 +0200 Subject: [PATCH 091/240] fix field gather assertion for multi dimensional field --- .../model/common/decomposition/definitions.py | 2 +- .../icon4py/model/common/grid/grid_manager.py | 26 ++++-- .../src/icon4py/model/common/grid/icon.py | 3 +- .../icon4py/model/common/states/factory.py | 2 - .../mpi_tests/test_parallel_grid_manager.py | 80 +++++++++++-------- 5 files changed, 68 insertions(+), 45 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index a283bf501c..cb18dd972f 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -180,7 +180,7 @@ def get_horizontal_size(self): num_vertices=self.global_index(dims.VertexDim, self.EntryType.ALL).shape[0], ) - def get_halo_size(self, dim: gtx.Dimension, flag: DecompositionFlag)->int: + def get_halo_size(self, dim: gtx.Dimension, flag: DecompositionFlag) -> int: return np.count_nonzero(self.halo_level_mask(dim, flag)) def halo_levels(self, dim: gtx.Dimension): diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 8471ca2a86..c4a6d5bbcc 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -408,18 +408,28 @@ def _construct_decomposed_grid( # COMPUTE remaining derived connectivities start_indices = {d: 0 for d in h_grid.EDGE_ZONES} - start_indices.update({d:0 for d in h_grid.VERTEX_ZONES}) - start_indices.update({d:0 for d in h_grid.CELL_ZONES}) - end_indices = {h_grid.domain(dims.EdgeDim)(d): distributed_size.num_edges for d in h_grid.EDGE_ZONES} - end_indices.update({h_grid.domain(dims.VertexDim)(d): distributed_size.num_vertices for d in h_grid.VERTEX_ZONES}) - end_indices.update({h_grid.domain(dims.CellDim)(d): distributed_size.num_vertices for d in h_grid.CELL_ZONES}) + start_indices.update({d: 0 for d in h_grid.VERTEX_ZONES}) + start_indices.update({d: 0 for d in h_grid.CELL_ZONES}) + end_indices = { + h_grid.domain(dims.EdgeDim)(d): distributed_size.num_edges for d in h_grid.EDGE_ZONES + } + end_indices.update( + { + h_grid.domain(dims.VertexDim)(d): distributed_size.num_vertices + for d in h_grid.VERTEX_ZONES + } + ) + end_indices.update( + { + h_grid.domain(dims.CellDim)(d): distributed_size.num_vertices + for d in h_grid.CELL_ZONES + } + ) neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) - - refinement_fields = self._read_grid_refinement_fields(backend) - # todo compute start end index + # TODO(halungge): compute start end index grid_config = base.GridConfig( horizontal_size=distributed_size, vertical_size=self._vertical_config.num_levels, diff --git a/model/common/src/icon4py/model/common/grid/icon.py b/model/common/src/icon4py/model/common/grid/icon.py index 8332018d59..142b5acb52 100644 --- a/model/common/src/icon4py/model/common/grid/icon.py +++ b/model/common/src/icon4py/model/common/grid/icon.py @@ -130,8 +130,7 @@ def _has_skip_values(offset: gtx.FieldOffset, limited_area: bool) -> bool: limited_area and dimension in CONNECTIVITIES_ON_BOUNDARIES ) - #return value - return True + return value def _should_replace_skip_values( diff --git a/model/common/src/icon4py/model/common/states/factory.py b/model/common/src/icon4py/model/common/states/factory.py index 915ae0d43d..17284a14e8 100644 --- a/model/common/src/icon4py/model/common/states/factory.py +++ b/model/common/src/icon4py/model/common/states/factory.py @@ -43,7 +43,6 @@ import enum import functools import logging -import pdb import types import typing from collections.abc import Callable, Mapping, MutableMapping, Sequence @@ -62,7 +61,6 @@ icon as icon_grid, vertical as v_grid, ) -from icon4py.model.common.io.writers import processor_properties from icon4py.model.common.states import model, utils as state_utils from icon4py.model.common.utils import data_allocation as data_alloc diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 4e1bc7c2ae..7a2095e711 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -194,11 +194,10 @@ def test_fields_distribute_and_gather(processor_props, caplog): def gather_field(field: np.ndarray, comm: mpi4py.MPI.Comm) -> tuple: constant_dims = field.shape[1:] - constant_length = functools.reduce(operator.mul, constant_dims) if len(constant_dims) > 0 else 1 + constant_length = functools.reduce(operator.mul, constant_dims) if len(constant_dims) > 0 else 1 local_sizes = np.array(comm.gather(field.size, root=0)) - if comm.rank == 0: recv_buffer = np.empty(np.sum(local_sizes), dtype=field.dtype) log.debug( @@ -206,17 +205,20 @@ def gather_field(field: np.ndarray, comm: mpi4py.MPI.Comm) -> tuple: ) else: recv_buffer = None + comm.Gatherv(sendbuf=field, recvbuf=(recv_buffer, local_sizes), root=0) if comm.rank == 0: log.debug(f"fields gathered:") log.debug(f"field sizes {local_sizes}") - - - local_first_dim = tuple(ls/constant_length for ls in local_sizes) - gathered_field = recv_buffer.reshape((-1 + constant_dims)) + local_first_dim = tuple(size / constant_length for size in local_sizes) + gathered_field = recv_buffer.reshape((-1, *constant_dims)) + else: + gathered_field = None + local_first_dim = field.shape[0] return local_first_dim, gathered_field -#TODO (halungge): deal with 2d fields + +# TODO (halungge): deal with 2d fields def assert_gathered_field_against_global( decomposition_info: defs.DecompositionInfo, processor_props: defs.ProcessProperties, # F811 # fixture @@ -224,7 +226,6 @@ def assert_gathered_field_against_global( global_reference_field: np.ndarray, local_field: np.ndarray, ): - assert ( local_field.shape[0] == decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.ALL).shape[0] @@ -238,15 +239,26 @@ def assert_gathered_field_against_global( processor_props.comm, ) if processor_props.rank == 0: - print(f"rank = {processor_props.rank}: asserting gathered fields") + print(f"rank = {processor_props.rank}: asserting gathered fields: ") assert np.all( gathered_sizes == global_index_sizes ), f"gathered field sizes do not match {gathered_sizes}" + print( + f"rank = {processor_props.rank}: Checking field size: --- gathered sizes {gathered_sizes}" + ) + print( + f"rank = {processor_props.rank}: --- gathered field has size {gathered_sizes}" + ) sorted_ = np.zeros(global_reference_field.shape, dtype=gtx.float64) sorted_[gathered_global_indices] = gathered_field assert test_helpers.dallclose( sorted_, global_reference_field ), f"Gathered field values do not match for dim {dim}.- " + print( + f"rank = {processor_props.rank}: comparing fields (samples) " + f"\n -- gathered {sorted_[:6]} " + f"\n -- global ref {global_reference_field[:6]}" + ) # TODO add test including halo access: @@ -272,13 +284,15 @@ def test_halo_neighbor_access_c2e(processor_props): metadata=geometry_attributes.attrs, ) - print(f"rank = {processor_props.rank} : single node grid has size {single_node.decomposition_info.get_horizontal_size()!r}") + print( + f"rank = {processor_props.rank} : single node grid has size {single_node.decomposition_info.get_horizontal_size()!r}" + ) reference = data_alloc.zero_field(single_node_grid, dims.CellDim, dims.C2EDim) single_node_edge_length = single_node_geometry.get(geometry_attributes.EDGE_LENGTH) single_node_cell_area = single_node_geometry.get(geometry_attributes.CELL_AREA) single_node_edge_orientation = single_node_geometry.get( - geometry_attributes.CELL_NORMAL_ORIENTATION - ) + geometry_attributes.CELL_NORMAL_ORIENTATION + ) compute_geofac_div.with_backend(backend)( primal_edge_length=single_node_edge_length, area=single_node_cell_area, @@ -287,28 +301,31 @@ def test_halo_neighbor_access_c2e(processor_props): offset_provider={"C2E": single_node_grid.get_connectivity("C2E")}, ) print( - f"rank = {processor_props.rank} : single node computed field reference has size {reference.asnumpy().shape}") + f"rank = {processor_props.rank} : single node computed field reference has size {reference.asnumpy().shape}" + ) processor_props.comm.barrier() multinode_grid_manager = run_gridmananger_for_multinode( - file=file, - vertical_config=vertical_config, - run_properties=processor_props, - decomposer=halo.SimpleMetisDecomposer(), + file=file, + vertical_config=vertical_config, + run_properties=processor_props, + decomposer=halo.SimpleMetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid extra_geometry_fields = multinode_grid_manager.geometry decomposition_info = multinode_grid_manager.decomposition_info print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") - print(f"rank = {processor_props.rank}: halo size for 'CellDim' (1 : {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.SECOND_HALO_LINE)})") + print( + f"rank = {processor_props.rank}: halo size for 'CellDim' (1 : {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + ) distributed_coordinates = multinode_grid_manager.coordinates distributed_geometry = geometry.GridGeometry( - backend=backend, - grid=distributed_grid, - coordinates=distributed_coordinates, - decomposition_info=decomposition_info, - extra_fields=extra_geometry_fields, - metadata=geometry_attributes.attrs, + backend=backend, + grid=distributed_grid, + coordinates=distributed_coordinates, + decomposition_info=decomposition_info, + extra_fields=extra_geometry_fields, + metadata=geometry_attributes.attrs, ) edge_length = distributed_geometry.get(geometry_attributes.EDGE_LENGTH) @@ -323,15 +340,15 @@ def test_halo_neighbor_access_c2e(processor_props): edge_orientation=edge_orientation, out=geofac_div, offset_provider={"C2E": distributed_grid.get_connectivity("C2E")}, - ) + ) assert_gathered_field_against_global( - decomposition_info, - processor_props, - dims.CellDim, - global_reference_field=reference.asnumpy(), - local_field=geofac_div.asnumpy(), - ) + decomposition_info, + processor_props, + dims.CellDim, + global_reference_field=reference.asnumpy(), + local_field=geofac_div.asnumpy(), + ) print(f"rank = {processor_props.rank} - DONE") # 1. read grid and distribue - GridManager @@ -340,4 +357,3 @@ def test_halo_neighbor_access_c2e(processor_props): # 3. compute geofac_div = primal_edge_length * edge_orientation / area # 4. gather geofac_div # 5 compare (possible reorder - From 29ffd9ef6b215a8669527fedf56bc6330207e5b4 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 9 Sep 2025 15:49:58 +0200 Subject: [PATCH 092/240] fix some tests after merge --- .../icon4py/model/common/grid/grid_manager.py | 5 +++- .../mpi_tests/test_parallel_grid_manager.py | 5 ++-- .../common/grid/unit_tests/test_gridfile.py | 6 ++-- .../tests/common/grid/unit_tests/test_icon.py | 2 +- .../common/grid/unit_tests/test_refinement.py | 2 +- .../common/grid/unit_tests/test_vertical.py | 30 +++++++++---------- 6 files changed, 27 insertions(+), 23 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 2f087fc2c6..67671242d3 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -26,6 +26,7 @@ refinement, vertical as v_grid, ) +from icon4py.model.common.grid.icon import GridSubdivision from icon4py.model.common.utils import data_allocation as data_alloc @@ -451,7 +452,9 @@ def _construct_decomposed_grid( def _construct_global_params(self): grid_root = self._reader.attribute(gridfile.MandatoryPropertyName.ROOT) grid_level = self._reader.attribute(gridfile.MandatoryPropertyName.LEVEL) - global_params = icon.GlobalGridParams(root=grid_root, level=grid_level) + geometry_type = self._reader.try_attribute(gridfile.MPIMPropertyName.GEOMETRY) + shape = icon.GridShape(geometry_type=geometry_type, subdivision=icon.GridSubdivision(root=grid_root, level=grid_level)) + global_params = icon.GlobalGridParams(grid_shape=shape) return global_params def _get_index_field( diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index e9e568750b..6cd501e951 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -86,9 +86,9 @@ def run_grid_manager_for_singlenode( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize( - "grid_file, experiment", + "experiment", [ - (dt_utils.R02B04_GLOBAL, dt_utils.GLOBAL_EXPERIMENT), + (test_defs.Experiments.EXCLAIM_APE), # (dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT) ], ) @@ -97,6 +97,7 @@ def test_start_end_index( caplog, backend, processor_props, grid_file, experiment, dim, icon_grid ): # fixture caplog.set_level(logging.INFO) + grid_file = experiment.grid file = grid_utils.resolve_full_grid_file_name(grid_file) partitioner = halo.SimpleMetisDecomposer() diff --git a/model/common/tests/common/grid/unit_tests/test_gridfile.py b/model/common/tests/common/grid/unit_tests/test_gridfile.py index ca16f08fc4..4ec581639d 100644 --- a/model/common/tests/common/grid/unit_tests/test_gridfile.py +++ b/model/common/tests/common/grid/unit_tests/test_gridfile.py @@ -75,7 +75,7 @@ def test_grid_file_vertex_cell_edge_dimensions( parser.close() -@pytest.mark.parametrize("filename", (test_defs.Grids.R02B04_GLOBAL.name,)) +@pytest.mark.parametrize("filename", (definitions.Grids.R02B04_GLOBAL,)) @pytest.mark.parametrize("apply_transformation", (True, False)) def test_int_variable(filename, apply_transformation): file = gridtest_utils.resolve_full_grid_file_name(filename) @@ -107,7 +107,7 @@ def index_selection() -> Iterable[list[int]]: "selection", index_selection(), ) -@pytest.mark.parametrize("filename", (test_defs.Grids.R02B04_GLOBAL.name,)) +@pytest.mark.parametrize("filename", (definitions.Grids.R02B04_GLOBAL.name,)) def test_index_read_for_1d_fields(filename, selection): file = gridtest_utils.resolve_full_grid_file_name(filename) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: @@ -121,7 +121,7 @@ def test_index_read_for_1d_fields(filename, selection): "selection", index_selection(), ) -@pytest.mark.parametrize("filename", (test_defs.Grids.R02B04_GLOBAL.name,)) +@pytest.mark.parametrize("filename", (definitions.Grids.R02B04_GLOBAL.name,)) @pytest.mark.parametrize( "field", (gridfile.ConnectivityName.V2E, gridfile.ConnectivityName.V2C, gridfile.ConnectivityName.E2V), diff --git a/model/common/tests/common/grid/unit_tests/test_icon.py b/model/common/tests/common/grid/unit_tests/test_icon.py index 189534cd0c..b8a840c8a6 100644 --- a/model/common/tests/common/grid/unit_tests/test_icon.py +++ b/model/common/tests/common/grid/unit_tests/test_icon.py @@ -21,7 +21,7 @@ horizontal as h_grid, icon, ) -from icon4py.model.testing import datatest_utils as dt_utils, grid_utils as gridtest_utils +from icon4py.model.testing import definitions, grid_utils as gridtest_utils from ..fixtures import * # noqa: F401, F403 from .. import utils diff --git a/model/common/tests/common/grid/unit_tests/test_refinement.py b/model/common/tests/common/grid/unit_tests/test_refinement.py index fdbe521690..6efc183a76 100644 --- a/model/common/tests/common/grid/unit_tests/test_refinement.py +++ b/model/common/tests/common/grid/unit_tests/test_refinement.py @@ -66,7 +66,7 @@ def test_valid_refinement_values(dim): @pytest.mark.parametrize("dim", utils.main_horizontal_dims()) @pytest.mark.parametrize( - "grid_file, expected", + "grid_descriptor, expected", [(test_defs.Grids.R02B04_GLOBAL, False), (test_defs.Grids.MCH_CH_R04B09_DSL, True)], ) def test_is_local_area_grid_for_grid_files(grid_descriptor, expected, dim, backend): diff --git a/model/common/tests/common/grid/unit_tests/test_vertical.py b/model/common/tests/common/grid/unit_tests/test_vertical.py index aadf3f26da..24b82dbddc 100644 --- a/model/common/tests/common/grid/unit_tests/test_vertical.py +++ b/model/common/tests/common/grid/unit_tests/test_vertical.py @@ -14,7 +14,7 @@ from icon4py.model.common import dimension as dims, type_alias as ta from icon4py.model.common.grid import vertical as v_grid from icon4py.model.common.utils import data_allocation as data_alloc, device_utils -from icon4py.model.testing import datatest_utils as dt_utils, grid_utils, test_utils +from icon4py.model.testing import definitions, test_utils from ..fixtures import * # noqa: F401, F403 @@ -42,7 +42,7 @@ def test_damping_layer_calculation(max_h, damping_height, delta, flat_height): @pytest.mark.datatest -@pytest.mark.parametrize("experiment", [dt_utils.REGIONAL_EXPERIMENT, dt_utils.GLOBAL_EXPERIMENT]) +@pytest.mark.parametrize("experiment", [definitions.Experiments.MCH_CH_R04B09, definitions.Experiments.EXCLAIM_APE]) def test_damping_layer_calculation_from_icon_input( grid_savepoint, experiment, damping_height, flat_height ): @@ -116,7 +116,7 @@ def configure_vertical_grid(grid_savepoint, top_moist_threshold=22500.0): @pytest.mark.datatest @pytest.mark.parametrize( "experiment, expected_moist_level", - [(dt_utils.REGIONAL_EXPERIMENT, 0), (dt_utils.GLOBAL_EXPERIMENT, 25)], + [(definitions.Experiments.MCH_CH_R04B09, 0), (definitions.Experiments.EXCLAIM_APE, 25)], ) def test_moist_level_calculation(grid_savepoint, experiment, expected_moist_level): threshold = 22500.0 @@ -134,8 +134,8 @@ def test_interface_physical_height(grid_savepoint): @pytest.mark.datatest -@pytest.mark.parametrize("experiment", [dt_utils.REGIONAL_EXPERIMENT, dt_utils.GLOBAL_EXPERIMENT]) -def test_flat_level_calculation(grid_savepoint, experiment, flat_height): +@pytest.mark.parametrize("experiment", [definitions.Experiments.MCH_CH_R04B09, definitions.Experiments.EXCLAIM_APE]) +def test_flat_level_calculation(grid_savepoint, experiment): vertical_grid = configure_vertical_grid(grid_savepoint) assert grid_savepoint.nflatlev() == vertical_grid.nflatlev @@ -171,7 +171,7 @@ def test_grid_index_top(grid_savepoint, dim, offset): @pytest.mark.datatest -@pytest.mark.parametrize("experiment, levels", [(dt_utils.GLOBAL_EXPERIMENT, 60)]) +@pytest.mark.parametrize("experiment, levels", [(definitions.Experiments.EXCLAIM_APE, 60)]) @pytest.mark.parametrize("dim", [dims.KDim, dims.KHalfDim]) @pytest.mark.parametrize("offset", offsets()) def test_grid_index_damping(grid_savepoint, experiment, levels, dim, offset): @@ -186,7 +186,7 @@ def test_grid_index_damping(grid_savepoint, experiment, levels, dim, offset): @pytest.mark.datatest -@pytest.mark.parametrize("experiment, levels", [(dt_utils.GLOBAL_EXPERIMENT, 60)]) +@pytest.mark.parametrize("experiment, levels", [(definitions.Experiments.EXCLAIM_APE, 60)]) @pytest.mark.parametrize("dim", [dims.KDim, dims.KHalfDim]) @pytest.mark.parametrize("offset", offsets()) def test_grid_index_moist(grid_savepoint, experiment, levels, dim, offset): @@ -201,7 +201,7 @@ def test_grid_index_moist(grid_savepoint, experiment, levels, dim, offset): @pytest.mark.datatest -@pytest.mark.parametrize("experiment, levels", [(dt_utils.GLOBAL_EXPERIMENT, 60)]) +@pytest.mark.parametrize("experiment, levels", [(definitions.Experiments.EXCLAIM_APE, 60)]) @pytest.mark.parametrize("dim", [dims.KDim, dims.KHalfDim]) @pytest.mark.parametrize("offset", offsets()) def test_grid_index_flat(grid_savepoint, experiment, levels, dim, offset): @@ -231,7 +231,7 @@ def test_grid_index_bottom(grid_savepoint, experiment, dim, offset): @pytest.mark.datatest -@pytest.mark.parametrize("experiment, levels", [(dt_utils.GLOBAL_EXPERIMENT, 60)]) +@pytest.mark.parametrize("experiment, levels", [(definitions.Experiments.EXCLAIM_APE, 60)]) @pytest.mark.parametrize("zone", vertical_zones()) @pytest.mark.parametrize("dim", [dims.KDim, dims.KHalfDim]) @pytest.mark.parametrize("offset", offsets()) @@ -247,7 +247,7 @@ def test_grid_index_raises_if_index_above_num_levels( @pytest.mark.datatest -@pytest.mark.parametrize("experiment, levels", [(dt_utils.GLOBAL_EXPERIMENT, 60)]) +@pytest.mark.parametrize("experiment, levels", [(definitions.Experiments.EXCLAIM_APE, 60)]) @pytest.mark.parametrize("zone", vertical_zones()) @pytest.mark.parametrize("dim", [dims.KDim, dims.KHalfDim]) @pytest.mark.parametrize("offset", offsets()) @@ -263,7 +263,7 @@ def test_grid_index_raises_if_index_below_zero( @pytest.mark.datatest -@pytest.mark.parametrize("experiment", (dt_utils.REGIONAL_EXPERIMENT, dt_utils.GLOBAL_EXPERIMENT)) +@pytest.mark.parametrize("experiment", (definitions.Experiments.MCH_CH_R04B09, definitions.Experiments.EXCLAIM_APE)) def test_vct_a_vct_b_calculation_from_icon_input( grid_savepoint, experiment, @@ -298,7 +298,7 @@ def test_vct_a_vct_b_calculation_from_icon_input( @pytest.mark.datatest @pytest.mark.parametrize( "experiment", - [dt_utils.REGIONAL_EXPERIMENT, dt_utils.GAUSS3D_EXPERIMENT, dt_utils.GLOBAL_EXPERIMENT], + [definitions.Experiments.MCH_CH_R04B09, definitions.Experiments.GAUSS3D, definitions.Experiments.EXCLAIM_APE], ) def test_compute_vertical_coordinate( grid_savepoint, @@ -317,7 +317,7 @@ def test_compute_vertical_coordinate( specific_values = ( {"rayleigh_damping_height": 12500.0, "stretch_factor": 0.65, "lowest_layer_thickness": 20.0} - if experiment == dt_utils.REGIONAL_EXPERIMENT + if experiment == definitions.Experiments.MCH_CH_R04B09 else { "rayleigh_damping_height": 45000.0, "stretch_factor": 1.0, @@ -340,9 +340,9 @@ def test_compute_vertical_coordinate( ) assert vertical_geometry.nflatlev == grid_savepoint.nflatlev() - if experiment in (dt_utils.REGIONAL_EXPERIMENT, dt_utils.GAUSS3D_EXPERIMENT): + if experiment in (definitions.Experiments.MCH_CH_R04B09, definitions.Experiments.GAUSS3D): topography = topography_savepoint.topo_c() - elif experiment == dt_utils.GLOBAL_EXPERIMENT: + elif experiment == definitions.Experiments.EXCLAIM_APE: topography = data_alloc.zero_field( icon_grid, dims.CellDim, backend=backend, dtype=ta.wpfloat ) From dd0fc140f4108efdaaa94dd18681b60ffd43052e Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 9 Sep 2025 18:42:16 +0200 Subject: [PATCH 093/240] fix some tests after merge --- model/common/src/icon4py/model/common/grid/grid_manager.py | 1 + model/common/tests/common/grid/unit_tests/test_gridfile.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 67671242d3..689e30b92e 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -435,6 +435,7 @@ def _construct_decomposed_grid( horizontal_size=distributed_size, vertical_size=self._vertical_config.num_levels, limited_area=limited_area, + keep_skip_values=with_skip_values ) grid = icon.icon_grid( diff --git a/model/common/tests/common/grid/unit_tests/test_gridfile.py b/model/common/tests/common/grid/unit_tests/test_gridfile.py index 4ec581639d..9f3ebe12da 100644 --- a/model/common/tests/common/grid/unit_tests/test_gridfile.py +++ b/model/common/tests/common/grid/unit_tests/test_gridfile.py @@ -15,7 +15,6 @@ from icon4py.model.common import dimension as dims from icon4py.model.common.grid import gridfile from icon4py.model.testing import ( - datatest_utils as dt_utils, grid_utils as gridtest_utils, definitions, ) @@ -31,7 +30,7 @@ def test_grid_file_dimension(): grid_descriptor = definitions.Grids.R02B04_GLOBAL global_grid_file = str(gridtest_utils.resolve_full_grid_file_name(grid_descriptor)) - parser = gridfile.GridFile(global_grid_file) + parser = gridfile.GridFile(global_grid_file, transformation=gridfile.NoTransformation()) try: parser.open() assert parser.dimension(gridfile.DimensionName.CELL_NAME) == grid_descriptor.sizes["cell"] @@ -107,7 +106,7 @@ def index_selection() -> Iterable[list[int]]: "selection", index_selection(), ) -@pytest.mark.parametrize("filename", (definitions.Grids.R02B04_GLOBAL.name,)) +@pytest.mark.parametrize("filename", (definitions.Grids.R02B04_GLOBAL,)) def test_index_read_for_1d_fields(filename, selection): file = gridtest_utils.resolve_full_grid_file_name(filename) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: @@ -121,7 +120,7 @@ def test_index_read_for_1d_fields(filename, selection): "selection", index_selection(), ) -@pytest.mark.parametrize("filename", (definitions.Grids.R02B04_GLOBAL.name,)) +@pytest.mark.parametrize("filename", (definitions.Grids.R02B04_GLOBAL,)) @pytest.mark.parametrize( "field", (gridfile.ConnectivityName.V2E, gridfile.ConnectivityName.V2C, gridfile.ConnectivityName.E2V), From ed9a36c382375e733dba768ecc9aa987296af6a7 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 18 Sep 2025 11:12:38 +0200 Subject: [PATCH 094/240] pre-commit --- .../icon4py/model/common/grid/grid_manager.py | 8 ++++-- .../decomposition/mpi_tests/test_halo.py | 9 +++--- .../mpi_tests/test_mpi_decomposition.py | 12 ++++---- .../unit_tests/test_definitions.py | 3 +- .../tests/common/decomposition/utils.py | 11 ++++---- model/common/tests/common/grid/fixtures.py | 17 ++++++----- .../mpi_tests/test_parallel_grid_manager.py | 26 ++++++++--------- .../grid/mpi_tests/test_parallel_icon.py | 6 ++-- .../grid/unit_tests/test_geometry_stencils.py | 28 +++++++++++-------- .../grid/unit_tests/test_grid_manager.py | 2 +- .../common/grid/unit_tests/test_gridfile.py | 2 ++ .../common/grid/unit_tests/test_horizontal.py | 2 +- .../common/grid/unit_tests/test_refinement.py | 6 ++-- 13 files changed, 69 insertions(+), 63 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 689e30b92e..8db9ebfaaf 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -26,7 +26,6 @@ refinement, vertical as v_grid, ) -from icon4py.model.common.grid.icon import GridSubdivision from icon4py.model.common.utils import data_allocation as data_alloc @@ -435,7 +434,7 @@ def _construct_decomposed_grid( horizontal_size=distributed_size, vertical_size=self._vertical_config.num_levels, limited_area=limited_area, - keep_skip_values=with_skip_values + keep_skip_values=with_skip_values, ) grid = icon.icon_grid( @@ -454,7 +453,10 @@ def _construct_global_params(self): grid_root = self._reader.attribute(gridfile.MandatoryPropertyName.ROOT) grid_level = self._reader.attribute(gridfile.MandatoryPropertyName.LEVEL) geometry_type = self._reader.try_attribute(gridfile.MPIMPropertyName.GEOMETRY) - shape = icon.GridShape(geometry_type=geometry_type, subdivision=icon.GridSubdivision(root=grid_root, level=grid_level)) + shape = icon.GridShape( + geometry_type=geometry_type, + subdivision=icon.GridSubdivision(root=grid_root, level=grid_level), + ) global_params = icon.GlobalGridParams(grid_shape=shape) return global_params diff --git a/model/common/tests/common/decomposition/mpi_tests/test_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_halo.py index 570ba12bd5..93539b14d9 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_halo.py @@ -13,14 +13,16 @@ import icon4py.model.common.dimension as dims from icon4py.model.common import exceptions from icon4py.model.common.decomposition import definitions as defs -from icon4py.model.testing.fixtures import processor_props from icon4py.model.testing import parallel_helpers +from icon4py.model.testing.fixtures import processor_props + from .. import utils try: import mpi4py # import mpi4py to check for optional mpi dependency import mpi4py.MPI + from icon4py.model.common.decomposition import mpi_decomposition mpi_decomposition.init_mpi() @@ -30,10 +32,7 @@ from gt4py.next import common as gtx_common from icon4py.model.common.decomposition import halo -from icon4py.model.common.grid import ( - base as base_grid, - simple, -) +from icon4py.model.common.grid import base as base_grid, simple from icon4py.model.testing import datatest_utils as dt_utils, definitions as test_defs diff --git a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py index 42a63db2ec..99648988ae 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py @@ -6,33 +6,31 @@ # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause +import logging + import numpy as np import pytest -import logging from icon4py.model.common import dimension as dims - from icon4py.model.common.decomposition import definitions from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import parallel_helpers - from icon4py.model.testing.fixtures.datatest import ( - experiment, backend, - download_ser_data, - ranked_data_path, data_provider, - processor_props, decomposition_info, download_ser_data, experiment, grid_savepoint, icon_grid, + processor_props, ranked_data_path, ) + try: import mpi4py # import mpi4py to check for optional mpi dependency + from icon4py.model.common.decomposition import mpi_decomposition mpi_decomposition.init_mpi() diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index a581d3ba7b..91415351bf 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -14,12 +14,11 @@ import icon4py.model.common.utils.data_allocation as data_alloc from icon4py.model.common.decomposition import definitions, halo from icon4py.model.common.grid import simple - +from icon4py.model.testing.fixtures import processor_props from .. import utils from ..mpi_tests.test_halo import simple_neighbor_tables from ..utils import dummy_four_ranks -from icon4py.model.testing.fixtures import processor_props @pytest.mark.parametrize("processor_props", [False], indirect=True) diff --git a/model/common/tests/common/decomposition/utils.py b/model/common/tests/common/decomposition/utils.py index cd85ede0ac..deb6a0e535 100644 --- a/model/common/tests/common/decomposition/utils.py +++ b/model/common/tests/common/decomposition/utils.py @@ -13,6 +13,7 @@ from icon4py.model.common import dimension as dims from icon4py.model.common.decomposition import definitions + """ TESTDATA using the [SimpleGrid](../../../src/icon4py/model/common/grid/simple.py) The distribution maps all of the 18 cells of the simple grid to ranks 0..3 @@ -152,15 +153,11 @@ def assert_same_entries( dim: gtx.Dimension, my_owned: np.ndarray, reference: dict[gtx.Dimension, dict], rank: int -): +) -> None: assert my_owned.size == len(reference[dim][rank]) assert np.setdiff1d(my_owned, reference[dim][rank], assume_unique=True).size == 0 -def dummy_four_ranks(rank) -> definitions.ProcessProperties: - return DummyProps(rank=rank) - - @dataclasses.dataclass(frozen=True) class DummyProps(definitions.ProcessProperties): def __init__(self, rank: int): @@ -168,3 +165,7 @@ def __init__(self, rank: int): object.__setattr__(self, "comm", None) object.__setattr__(self, "comm_name", "dummy on 4") object.__setattr__(self, "comm_size", 4) + + +def dummy_four_ranks(rank: int) -> definitions.ProcessProperties: + return DummyProps(rank=rank) diff --git a/model/common/tests/common/grid/fixtures.py b/model/common/tests/common/grid/fixtures.py index b7959f0733..bf1ce2b54f 100644 --- a/model/common/tests/common/grid/fixtures.py +++ b/model/common/tests/common/grid/fixtures.py @@ -9,22 +9,21 @@ from icon4py.model.testing.fixtures.datatest import ( backend, damping_height, - flat_height, data_provider, download_ser_data, experiment, + flat_height, grid_savepoint, - icon_grid, - processor_props, - model_top_height, - stretch_factor, htop_moist_proc, - metrics_savepoint, - ranked_data_path, + icon_grid, interpolation_savepoint, lowest_layer_thickness, maximal_layer_thickness, - topography_savepoint, + metrics_savepoint, + model_top_height, + processor_props, + ranked_data_path, + stretch_factor, top_height_limit_for_maximal_layer_thickness, - backend, + topography_savepoint, ) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 6cd501e951..ac2f6060ac 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -9,39 +9,39 @@ import logging import operator import pathlib -from typing import Iterator +from collections.abc import Iterator import numpy as np import pytest from gt4py import next as gtx import icon4py.model.common.grid.gridfile -from icon4py.model.common.utils import data_allocation as data_alloc -import icon4py.model.testing.grid_utils as grid_utils -from icon4py.model.common import exceptions, dimension as dims -from icon4py.model.common.decomposition import halo, mpi_decomposition, definitions as defs +from icon4py.model.common import dimension as dims, exceptions +from icon4py.model.common.decomposition import definitions as defs, halo, mpi_decomposition from icon4py.model.common.grid import ( - grid_manager as gm, - vertical as v_grid, - gridfile, geometry, geometry_attributes, + grid_manager as gm, + gridfile, + vertical as v_grid, ) -from icon4py.model.common.interpolation import interpolation_factory, interpolation_attributes +from icon4py.model.common.interpolation import interpolation_attributes, interpolation_factory from icon4py.model.common.interpolation.interpolation_fields import compute_geofac_div +from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import ( datatest_utils as dt_utils, - test_utils as test_helpers, definitions as test_defs, + grid_utils, + test_utils as test_helpers, ) +from ...decomposition import utils as decomp_utils from .. import utils from ..fixtures import * -from ...decomposition import utils as decomp_utils try: - import mpi4py # noqa F401: import mpi4py to check for optional mpi dependency + import mpi4py mpi_decomposition.init_mpi() except ImportError: @@ -210,7 +210,7 @@ def gather_field(field: np.ndarray, comm: mpi4py.MPI.Comm) -> tuple: comm.Gatherv(sendbuf=field, recvbuf=(recv_buffer, local_sizes), root=0) if comm.rank == 0: - log.debug(f"fields gathered:") + log.debug("fields gathered:") log.debug(f"field sizes {local_sizes}") local_first_dim = tuple(size / constant_length for size in local_sizes) gathered_field = recv_buffer.reshape((-1, *constant_dims)) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py index a9474c2c3d..9b6538e572 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py @@ -15,11 +15,11 @@ import icon4py.model.common.dimension as dims import icon4py.model.common.grid.horizontal as h_grid - -from icon4py.model.testing import parallel_helpers from icon4py.model.common.decomposition import definitions as defs, mpi_decomposition -from ..fixtures import * +from icon4py.model.testing import parallel_helpers + from .. import utils +from ..fixtures import * if TYPE_CHECKING: diff --git a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py index b455001d46..d8cfbaec66 100644 --- a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py +++ b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py @@ -8,15 +8,15 @@ import numpy as np import pytest -from icon4py.model.testing import definitions -from icon4py.model.common.utils import data_allocation as data_alloc -from icon4py.model.testing import grid_utils -from icon4py.model.common import dimension as dims -from icon4py.model.common import constants + +from icon4py.model.common import constants, dimension as dims from icon4py.model.common.grid.geometry_stencils import ( - compute_edge_length, compute_cell_center_arc_distance, + compute_edge_length, ) +from icon4py.model.common.utils import data_allocation as data_alloc +from icon4py.model.testing import definitions, grid_utils + from ..fixtures import * @@ -74,9 +74,15 @@ def test_compute_dual_edge_length(experiment, grid_file, grid_savepoint, backend ) grid = gm.grid coordinates = gm.coordinates[dims.VertexDim] - lat = coordinates["lat"] - lon = coordinates["lon"] - # lat = grid_savepoint.lat(dims.VertexDim) - # lon = grid_savepoint.lon(dims.VertexDim) + # lat = coordinates["lat"] + # lon = coordinates["lon"] + lat_vertex = grid_savepoint.lat(dims.VertexDim) + lon_vertex = grid_savepoint.lon(dims.VertexDim) + lat_edges = grid_savepoint.lat(dims.EdgeDim) + lon_edges = grid_savepoint.lon(dims.EdgeDim) + radius = constants.EARTH_RADIUS + length = data_alloc.zero_field(grid, dims.EdgeDim) - compute_cell_center_arc_distance.with_backend(backend)() + compute_cell_center_arc_distance.with_backend(backend)( + lat_edges, lon_edges, lat_vertex, lon_vertex, radius, length + ) diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index 5abefa31d9..66bb74d4ef 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -9,7 +9,7 @@ import logging import typing -from typing import Iterator +from collections.abc import Iterator import gt4py.next as gtx import gt4py.next.typing as gtx_typing diff --git a/model/common/tests/common/grid/unit_tests/test_gridfile.py b/model/common/tests/common/grid/unit_tests/test_gridfile.py index d4fdc412c3..dc164ff075 100644 --- a/model/common/tests/common/grid/unit_tests/test_gridfile.py +++ b/model/common/tests/common/grid/unit_tests/test_gridfile.py @@ -7,8 +7,10 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations +from collections.abc import Iterable from typing import TYPE_CHECKING +import numpy as np import pytest from icon4py.model.common import dimension as dims diff --git a/model/common/tests/common/grid/unit_tests/test_horizontal.py b/model/common/tests/common/grid/unit_tests/test_horizontal.py index 9a84a11737..1df2e9cec3 100644 --- a/model/common/tests/common/grid/unit_tests/test_horizontal.py +++ b/model/common/tests/common/grid/unit_tests/test_horizontal.py @@ -17,7 +17,7 @@ import icon4py.model.common.grid.horizontal as h_grid from .. import utils -from ..fixtures import * # noqa: F401, F403 +from ..fixtures import * # noqa: F403 if TYPE_CHECKING: diff --git a/model/common/tests/common/grid/unit_tests/test_refinement.py b/model/common/tests/common/grid/unit_tests/test_refinement.py index 9dba19f77a..784fe2b56a 100644 --- a/model/common/tests/common/grid/unit_tests/test_refinement.py +++ b/model/common/tests/common/grid/unit_tests/test_refinement.py @@ -12,14 +12,14 @@ import gt4py.next as gtx import pytest -from icon4py.model.common.grid import refinement as refin, horizontal as h_grid from icon4py.model.common import dimension as dims +from icon4py.model.common.grid import horizontal as h_grid, refinement as refin from icon4py.model.common.utils import data_allocation as data_alloc, device_utils from icon4py.model.testing import definitions as test_defs, grid_utils from icon4py.model.testing.fixtures import backend from .. import utils -from ..fixtures import * # noqa: F401, F403 +from ..fixtures import * # noqa: F403 if TYPE_CHECKING: @@ -78,7 +78,7 @@ def test_valid_refinement_values(dim: gtx.Dimension) -> None: [(test_defs.Grids.R02B04_GLOBAL, False), (test_defs.Grids.MCH_CH_R04B09_DSL, True)], ) def test_is_local_area_grid_for_grid_files( - grid_descriptor: definitions.GridDescription, + grid_descriptor: test_defs.GridDescription, expected: bool, dim: gtx.Dimension, backend: gtx_typing.Backend, From 271d70b10eddd5ff2638656167574a86c325b618 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 18 Sep 2025 11:34:47 +0200 Subject: [PATCH 095/240] fix some typting in test_parallel_grid_manager.py --- .../mpi_tests/test_parallel_grid_manager.py | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index ac2f6060ac..710059df1a 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -10,6 +10,7 @@ import operator import pathlib from collections.abc import Iterator +from typing import Any import numpy as np import pytest @@ -23,17 +24,16 @@ geometry_attributes, grid_manager as gm, gridfile, - vertical as v_grid, + vertical as v_grid, base, ) -from icon4py.model.common.interpolation import interpolation_attributes, interpolation_factory from icon4py.model.common.interpolation.interpolation_fields import compute_geofac_div from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import ( - datatest_utils as dt_utils, definitions as test_defs, grid_utils, - test_utils as test_helpers, + test_utils as test_helpers, definitions, ) +from gt4py.next import typing as gtx_typing from ...decomposition import utils as decomp_utils from .. import utils @@ -64,7 +64,7 @@ def run_gridmananger_for_multinode( return manager -def _grid_manager(file: pathlib.Path, vertical_config: v_grid.VerticalGridConfig): +def _grid_manager(file: pathlib.Path, vertical_config: v_grid.VerticalGridConfig)->gm.GridManager: manager = gm.GridManager(str(file), vertical_config) return manager @@ -94,8 +94,13 @@ def run_grid_manager_for_singlenode( ) @pytest.mark.parametrize("dim", utils.horizontal_dims()) def test_start_end_index( - caplog, backend, processor_props, grid_file, experiment, dim, icon_grid -): # fixture + caplog:Any, + backend:gtx_typing.Backend|None, + processor_props:defs.ProcessProperties, + experiment:definitions.Experiment, + dim:gtx.Dimension, + icon_grid:base.Grid +)->None: # fixture caplog.set_level(logging.INFO) grid_file = experiment.grid file = grid_utils.resolve_full_grid_file_name(grid_file) @@ -123,7 +128,7 @@ def test_start_end_index( @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi(min_size=2) -def test_grid_manager_validate_decomposer(processor_props): +def test_grid_manager_validate_decomposer(processor_props:defs.ProcessProperties)->None: file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL.name) manager = gm.GridManager(file, vertical_config, gridfile.ToZeroBasedIndexTransformation()) with pytest.raises(exceptions.InvalidConfigError) as e: @@ -139,10 +144,10 @@ def test_grid_manager_validate_decomposer(processor_props): @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_fields_distribute_and_gather(processor_props, caplog): +def test_fields_distribute_and_gather(processor_props:defs.ProcessProperties, caplog:Any)->None: caplog.set_level(logging.INFO) print(f"myrank - {processor_props.rank}: running with processor_props = {processor_props}") - file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL.name) + file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) single_node = run_grid_manager_for_singlenode(file, vertical_config) single_node_grid = single_node.grid global_cell_area = single_node.geometry[gridfile.GeometryName.CELL_AREA] @@ -227,7 +232,7 @@ def assert_gathered_field_against_global( dim: gtx.Dimension, global_reference_field: np.ndarray, local_field: np.ndarray, -): +)->None: assert ( local_field.shape[0] == decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.ALL).shape[0] @@ -271,7 +276,7 @@ def assert_gathered_field_against_global( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_halo_neighbor_access_c2e(processor_props): +def test_halo_neighbor_access_c2e(processor_props:defs.ProcessProperties): file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL.name) backend = None print(f"running on {processor_props.comm}") From eca2a8f113a9097042257acde02e13b401dd5a49 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 19 Sep 2025 09:30:23 +0200 Subject: [PATCH 096/240] make test run again --- .../src/icon4py/model/common/grid/base.py | 3 ++ .../icon4py/model/common/grid/grid_manager.py | 3 +- .../mpi_tests/test_parallel_grid_manager.py | 52 ++++++++++--------- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/base.py b/model/common/src/icon4py/model/common/grid/base.py index a4293f5e57..23b267574f 100644 --- a/model/common/src/icon4py/model/common/grid/base.py +++ b/model/common/src/icon4py/model/common/grid/base.py @@ -167,6 +167,9 @@ def get_connectivity(self, offset: str | gtx.FieldOffset) -> gtx_common.Neighbor assert gtx_common.is_neighbor_table(connectivity) return connectivity + def get_neighbor_tables(self): + return {k:v.ndarray for k, v in self.connectivities.items() if gtx_common.is_neighbor_connectivity(v)} + def start_index(self, domain: h_grid.Domain) -> gtx.int32: """ Use to specify lower end of domains of a field for field_operators. diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 8db9ebfaaf..c47134ba6e 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -225,10 +225,9 @@ def _read_geometry_fields(self, backend: gtx_typing.Backend | None): ), gridfile.GeometryName.CELL_NORMAL_ORIENTATION.value: gtx.as_field( (dims.CellDim, dims.C2EDim), - self._reader.int_variable( + self._reader.variable( gridfile.GeometryName.CELL_NORMAL_ORIENTATION, transpose=True, - apply_transformation=False, indices=my_cell_indices, ), allocator=backend, diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 710059df1a..bdc36c94fe 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -15,25 +15,27 @@ import numpy as np import pytest from gt4py import next as gtx +from gt4py.next import typing as gtx_typing import icon4py.model.common.grid.gridfile from icon4py.model.common import dimension as dims, exceptions from icon4py.model.common.decomposition import definitions as defs, halo, mpi_decomposition from icon4py.model.common.grid import ( + base, geometry, geometry_attributes, grid_manager as gm, gridfile, - vertical as v_grid, base, + vertical as v_grid, ) from icon4py.model.common.interpolation.interpolation_fields import compute_geofac_div from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import ( + definitions, definitions as test_defs, grid_utils, - test_utils as test_helpers, definitions, + test_utils as test_helpers, ) -from gt4py.next import typing as gtx_typing from ...decomposition import utils as decomp_utils from .. import utils @@ -64,7 +66,7 @@ def run_gridmananger_for_multinode( return manager -def _grid_manager(file: pathlib.Path, vertical_config: v_grid.VerticalGridConfig)->gm.GridManager: +def _grid_manager(file: pathlib.Path, vertical_config: v_grid.VerticalGridConfig) -> gm.GridManager: manager = gm.GridManager(str(file), vertical_config) return manager @@ -94,13 +96,13 @@ def run_grid_manager_for_singlenode( ) @pytest.mark.parametrize("dim", utils.horizontal_dims()) def test_start_end_index( - caplog:Any, - backend:gtx_typing.Backend|None, - processor_props:defs.ProcessProperties, - experiment:definitions.Experiment, - dim:gtx.Dimension, - icon_grid:base.Grid -)->None: # fixture + caplog: Any, + backend: gtx_typing.Backend | None, + processor_props: defs.ProcessProperties, + experiment: definitions.Experiment, + dim: gtx.Dimension, + icon_grid: base.Grid, +) -> None: # fixture caplog.set_level(logging.INFO) grid_file = experiment.grid file = grid_utils.resolve_full_grid_file_name(grid_file) @@ -128,8 +130,8 @@ def test_start_end_index( @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi(min_size=2) -def test_grid_manager_validate_decomposer(processor_props:defs.ProcessProperties)->None: - file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL.name) +def test_grid_manager_validate_decomposer(processor_props: defs.ProcessProperties) -> None: + file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) manager = gm.GridManager(file, vertical_config, gridfile.ToZeroBasedIndexTransformation()) with pytest.raises(exceptions.InvalidConfigError) as e: manager( @@ -144,7 +146,7 @@ def test_grid_manager_validate_decomposer(processor_props:defs.ProcessProperties @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_fields_distribute_and_gather(processor_props:defs.ProcessProperties, caplog:Any)->None: +def test_fields_distribute_and_gather(processor_props: defs.ProcessProperties, caplog: Any) -> None: caplog.set_level(logging.INFO) print(f"myrank - {processor_props.rank}: running with processor_props = {processor_props}") file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) @@ -232,7 +234,7 @@ def assert_gathered_field_against_global( dim: gtx.Dimension, global_reference_field: np.ndarray, local_field: np.ndarray, -)->None: +) -> None: assert ( local_field.shape[0] == decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.ALL).shape[0] @@ -273,12 +275,12 @@ def assert_gathered_field_against_global( # - geofac_div # - geofac_n2s - +# TODO (halungge): fix non contiguous dimension for embedded @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_halo_neighbor_access_c2e(processor_props:defs.ProcessProperties): - file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL.name) - backend = None +def test_halo_neighbor_access_c2e(processor_props: defs.ProcessProperties, backend:gtx_typing.Backend|None): +# processor_props = decomp_utils.DummyProps(rank = 1) + file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) print(f"running on {processor_props.comm}") single_node = run_grid_manager_for_singlenode(file, vertical_config) single_node_grid = single_node.grid @@ -300,7 +302,7 @@ def test_halo_neighbor_access_c2e(processor_props:defs.ProcessProperties): single_node_edge_orientation = single_node_geometry.get( geometry_attributes.CELL_NORMAL_ORIENTATION ) - compute_geofac_div.with_backend(backend)( + compute_geofac_div.with_backend(None)( primal_edge_length=single_node_edge_length, area=single_node_cell_area, edge_orientation=single_node_edge_orientation, @@ -310,7 +312,7 @@ def test_halo_neighbor_access_c2e(processor_props:defs.ProcessProperties): print( f"rank = {processor_props.rank} : single node computed field reference has size {reference.asnumpy().shape}" ) - processor_props.comm.barrier() + #processor_props.comm.barrier() multinode_grid_manager = run_gridmananger_for_multinode( file=file, vertical_config=vertical_config, @@ -341,7 +343,7 @@ def test_halo_neighbor_access_c2e(processor_props:defs.ProcessProperties): # ### geofac_div = primal_edge_length(C2E) * edge_orientation / area geofac_div = data_alloc.zero_field(distributed_grid, dims.CellDim, dims.C2EDim) - compute_geofac_div.with_backend(backend)( + compute_geofac_div.with_backend(None)( primal_edge_length=edge_length, area=cell_area, edge_orientation=edge_orientation, @@ -382,8 +384,8 @@ def test_local_connectivities( test_defs.Grids.R02B04_GLOBAL, keep_skip_values=True, backend=None ).grid partitioner = halo.SimpleMetisDecomposer() - face_face_connectivity = grid.connectivities[dims.C2E2CDim.value].ndarray # type: ignore[union-attr] - neighbor_tables = {k: v.ndarray for k, v in grid.connectivities.items()} + face_face_connectivity = grid.get_connectivity(dims.C2E2C).ndarray # type: ignore[union-attr] + neighbor_tables = grid.get_neighbor_tables() labels = partitioner(face_face_connectivity, num_partitions=processor_props.comm_size) halo_generator = halo.IconLikeHaloConstructor( connectivities=neighbor_tables, @@ -394,7 +396,7 @@ def test_local_connectivities( decomposition_info = halo_generator(labels) connectivity = gm.construct_local_connectivity( - field_offset, decomposition_info, connectivity=grid.connectivities[field_offset].ndarray + field_offset, decomposition_info, connectivity=grid.get_connectivity(field_offset).ndarray ) # there is an neighbor list for each index of the target dimension on the node assert ( From 3698b9a940b89274ad76f099bd3feaee5a60ca55 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Sat, 20 Sep 2025 22:51:53 +0200 Subject: [PATCH 097/240] fixes from merge --- .../icon4py/model/common/grid/grid_manager.py | 28 ++++----------- .../mpi_tests/test_parallel_grid_manager.py | 28 +++++---------- .../grid/mpi_tests/test_parallel_icon.py | 1 + .../grid/unit_tests/test_geometry_stencils.py | 36 ------------------- 4 files changed, 17 insertions(+), 76 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index d4bccabfae..52c4b6013d 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -5,6 +5,7 @@ # # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause +import functools import logging import pathlib from types import ModuleType @@ -22,7 +23,6 @@ base, grid_refinement as refinement, gridfile, - horizontal as h_grid, icon, vertical as v_grid, ) @@ -406,29 +406,15 @@ def _construct_decomposed_grid( } # COMPUTE remaining derived connectivities - start_indices = {d: 0 for d in h_grid.EDGE_ZONES} - start_indices.update({d: 0 for d in h_grid.VERTEX_ZONES}) - start_indices.update({d: 0 for d in h_grid.CELL_ZONES}) - end_indices = { - h_grid.domain(dims.EdgeDim)(d): distributed_size.num_edges for d in h_grid.EDGE_ZONES - } - end_indices.update( - { - h_grid.domain(dims.VertexDim)(d): distributed_size.num_vertices - for d in h_grid.VERTEX_ZONES - } - ) - end_indices.update( - { - h_grid.domain(dims.CellDim)(d): distributed_size.num_vertices - for d in h_grid.CELL_ZONES - } - ) - neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) refinement_fields = self._read_grid_refinement_fields(backend) - # TODO(halungge): compute start end index + + domain_bounds_constructor = functools.partial( + refinement.compute_domain_bounds, refinement_fields=refinement_fields, array_ns=xp + ) + start_index, end_index = icon.get_start_and_end_index(domain_bounds_constructor) + grid_config = base.GridConfig( horizontal_size=distributed_size, vertical_size=self._vertical_config.num_levels, diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 857349ec47..c32b583e0c 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -26,6 +26,7 @@ geometry_attributes, grid_manager as gm, gridfile, + horizontal as h_grid, vertical as v_grid, ) from icon4py.model.common.interpolation.interpolation_fields import compute_geofac_div @@ -84,14 +85,14 @@ def run_grid_manager_for_singlenode( return manager -@pytest.mark.xfail +@pytest.mark.xfail("fix test, add data for APE") @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize( "experiment", [ (test_defs.Experiments.EXCLAIM_APE), - # (dt_utils.REGIONAL_EXPERIMENT, dt_utils.REGIONAL_EXPERIMENT) + # (test_defs.Experiments.MCH_CH_R04B09) ], ) @pytest.mark.parametrize("dim", utils.horizontal_dims()) @@ -113,17 +114,15 @@ def test_start_end_index( v_grid.VerticalGridConfig(1), icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation(), ) - single_node_grid = gm.GridManager( - file, - v_grid.VerticalGridConfig(1), - icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation(), - ).grid + manager(backend, keep_skip_values=True, decomposer=partitioner, run_properties=processor_props) + grid = manager.grid - for domain in utils.global_grid_domains(dim): - assert icon_grid.start_index(domain) == single_node_grid.start_index( + domains = (h_grid.domain(dim)(z) for z in h_grid.VERTEX_AND_CELL_ZONES) + for domain in domains: + assert icon_grid.start_index(domain) == grid.start_index( domain ), f"start index wrong for domain {domain}" - assert icon_grid.end_index(domain) == single_node_grid.end_index( + assert icon_grid.end_index(domain) == grid.end_index( domain ), f"end index wrong for domain {domain}" @@ -315,7 +314,6 @@ def test_halo_neighbor_access_c2e( print( f"rank = {processor_props.rank} : single node computed field reference has size {reference.asnumpy().shape}" ) - # processor_props.comm.barrier() multinode_grid_manager = run_gridmananger_for_multinode( file=file, vertical_config=vertical_config, @@ -344,7 +342,6 @@ def test_halo_neighbor_access_c2e( cell_area = distributed_geometry.get(geometry_attributes.CELL_AREA) edge_orientation = distributed_geometry.get(geometry_attributes.CELL_NORMAL_ORIENTATION) - # ### geofac_div = primal_edge_length(C2E) * edge_orientation / area geofac_div = data_alloc.zero_field(distributed_grid, dims.CellDim, dims.C2EDim) compute_geofac_div.with_backend(None)( primal_edge_length=edge_length, @@ -363,15 +360,8 @@ def test_halo_neighbor_access_c2e( ) print(f"rank = {processor_props.rank} - DONE") - # 1. read grid and distribue - GridManager - - # 2. get geometry fields (from GridManger) primal_edge_length, edge_orientation, area (local read) - # 3. compute geofac_div = primal_edge_length * edge_orientation / area - # 4. gather geofac_div - # 5 compare (possible reorder -# TODO move to mpi_tests folder @pytest.mark.mpi @pytest.mark.parametrize( "field_offset", diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py index 9b6538e572..d49b92a6ff 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py @@ -135,6 +135,7 @@ def test_distributed_halo( dim: gtx.Dimension, zone: h_grid.Zone, icon_grid: base_grid.Grid, + level: int, ) -> None: parallel_helpers.check_comm_size(processor_props) domain = h_grid.domain(dim)(zone) diff --git a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py index d8cfbaec66..8827ecdb93 100644 --- a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py +++ b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py @@ -31,8 +31,6 @@ ) def test_edge_length(experiment, grid_file, grid_savepoint, backend): keep = True - # grid = grid_savepoint.construct_icon_grid(backend, keep_skip_values=keep) - gm = grid_utils.get_grid_manager_from_identifier( grid_file, keep_skip_values=keep, num_levels=1, backend=backend ) @@ -40,8 +38,6 @@ def test_edge_length(experiment, grid_file, grid_savepoint, backend): coordinates = gm.coordinates[dims.VertexDim] lat = coordinates["lat"] lon = coordinates["lon"] - # lat = grid_savepoint.lat(dims.VertexDim) - # lon = grid_savepoint.lon(dims.VertexDim) length = data_alloc.zero_field(grid, dims.EdgeDim) compute_edge_length.with_backend(backend)( vertex_lat=lat, @@ -54,35 +50,3 @@ def test_edge_length(experiment, grid_file, grid_savepoint, backend): ) assert np.allclose(length.asnumpy(), grid_savepoint.primal_edge_length().asnumpy()) - - -@pytest.mark.level("unit") -@pytest.mark.datatest -@pytest.mark.parametrize( - "experiment, grid_file", - ( - (definitions.Experiments.MCH_CH_R04B09.name, definitions.Grids.MCH_CH_R04B09_DSL.name), - (definitions.Experiments.EXCLAIM_APE.name, definitions.Grids.R02B04_GLOBAL.name), - ), -) -def test_compute_dual_edge_length(experiment, grid_file, grid_savepoint, backend): - keep = True - # grid = grid_savepoint.construct_icon_grid(backend, keep_skip_values=keep) - - gm = grid_utils.get_grid_manager_from_identifier( - grid_file, keep_skip_values=keep, num_levels=1, backend=backend - ) - grid = gm.grid - coordinates = gm.coordinates[dims.VertexDim] - # lat = coordinates["lat"] - # lon = coordinates["lon"] - lat_vertex = grid_savepoint.lat(dims.VertexDim) - lon_vertex = grid_savepoint.lon(dims.VertexDim) - lat_edges = grid_savepoint.lat(dims.EdgeDim) - lon_edges = grid_savepoint.lon(dims.EdgeDim) - radius = constants.EARTH_RADIUS - - length = data_alloc.zero_field(grid, dims.EdgeDim) - compute_cell_center_arc_distance.with_backend(backend)( - lat_edges, lon_edges, lat_vertex, lon_vertex, radius, length - ) From c9be1d09d2ae3de9aa0d3d7f4e606b9284670e83 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 7 Oct 2025 18:04:38 +0200 Subject: [PATCH 098/240] fix merge: global grid params --- .../model/common/decomposition/halo.py | 2 +- .../icon4py/model/common/grid/grid_manager.py | 57 ++++++++++++++++--- .../mpi_tests/test_parallel_grid_manager.py | 10 ++-- .../grid/unit_tests/test_geometry_stencils.py | 17 ++---- .../grid/unit_tests/test_grid_manager.py | 9 +-- .../common/grid/unit_tests/test_gridfile.py | 13 +++-- .../tests/common/grid/unit_tests/test_icon.py | 2 +- .../src/icon4py/model/testing/grid_utils.py | 9 ++- 8 files changed, 78 insertions(+), 41 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index f7d48203ac..409a41faf5 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -140,7 +140,7 @@ def _assert_all_neighbor_tables(self): ] for d in relevant_dimension: assert ( - d.value in self._connectivities.keys() + d.value in self._connectivities ), f"Table for {d} is missing from the neighbor table array." def _connectivity(self, offset: gtx.FieldOffset | str) -> data_alloc.NDArray: diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 828d1e422a..befbaff763 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -9,7 +9,7 @@ import logging import pathlib from types import ModuleType -from typing import Literal, Protocol, TypeAlias +from typing import Literal, TypeAlias import gt4py.next as gtx import gt4py.next.typing as gtx_typing @@ -101,14 +101,21 @@ def __call__( decomposer: halo.Decomposer = _single_node_decomposer, run_properties=_single_process_props, ): - if not run_properties.single_node() and isinstance(decomposer, SingleNodeDecomposer): + if not run_properties.single_node() and isinstance(decomposer, halo.SingleNodeDecomposer): raise InvalidConfigError("Need a Decomposer for multi node run") if not self._reader: self.open() - self._grid = self._construct_grid(backend=backend, with_skip_values=keep_skip_values) + + self._construct_decomposed_grid( + backend=backend, + with_skip_values=keep_skip_values, + decomposer=decomposer, + run_properties=run_properties, + ) self._coordinates = self._read_coordinates(backend) self._geometry = self._read_geometry_fields(backend) + self._geometry = self._read_geometry_fields(backend) self.close() @@ -368,9 +375,10 @@ def _construct_decomposed_grid( """ xp = data_alloc.import_array_ns(backend) ## FULL GRID PROPERTIES - global_params = self._construct_global_params() + cell_refinement = self._reader.variable(gridfile.GridRefinementName.CONTROL_CELLS) global_size = self._read_full_grid_size() + global_params = self._construct_global_params(backend, global_size) limited_area = refinement.is_limited_area_grid(cell_refinement, array_ns=xp) cell_to_cell_neighbors = self._get_index_field(gridfile.ConnectivityName.C2E2C) @@ -438,15 +446,50 @@ def _construct_decomposed_grid( ) self._grid = grid - def _construct_global_params(self): + def _construct_global_params( + self, backend: gtx_typing.Backend, global_size: base.HorizontalGridSize + ): grid_root = self._reader.attribute(gridfile.MandatoryPropertyName.ROOT) grid_level = self._reader.attribute(gridfile.MandatoryPropertyName.LEVEL) - geometry_type = self._reader.try_attribute(gridfile.MPIMPropertyName.GEOMETRY) + if geometry_type := self._reader.try_attribute(gridfile.MPIMPropertyName.GEOMETRY): + geometry_type = base.GeometryType(geometry_type) + sphere_radius = self._reader.try_attribute(gridfile.MPIMPropertyName.SPHERE_RADIUS) + domain_length = self._reader.try_attribute(gridfile.MPIMPropertyName.DOMAIN_LENGTH) + domain_height = self._reader.try_attribute(gridfile.MPIMPropertyName.DOMAIN_HEIGHT) + mean_edge_length = self._reader.try_attribute(gridfile.MPIMPropertyName.MEAN_EDGE_LENGTH) + mean_dual_edge_length = self._reader.try_attribute( + gridfile.MPIMPropertyName.MEAN_DUAL_EDGE_LENGTH + ) + mean_cell_area = self._reader.try_attribute(gridfile.MPIMPropertyName.MEAN_CELL_AREA) + mean_dual_cell_area = self._reader.try_attribute( + gridfile.MPIMPropertyName.MEAN_DUAL_CELL_AREA + ) + # TODO (@halungge): Fix this: reads the global fields... + edge_lengths = self._reader.variable(gridfile.GeometryName.EDGE_LENGTH) + dual_edge_lengths = self._reader.variable(gridfile.GeometryName.DUAL_EDGE_LENGTH) + cell_areas = self._reader.variable(gridfile.GeometryName.CELL_AREA.value) + dual_cell_areas = self._reader.variable(gridfile.GeometryName.DUAL_AREA) shape = icon.GridShape( geometry_type=geometry_type, subdivision=icon.GridSubdivision(root=grid_root, level=grid_level), ) - global_params = icon.GlobalGridParams(grid_shape=shape) + global_params = icon.GlobalGridParams.from_fields( + backend=backend, + grid_shape=shape, + radius=sphere_radius, + domain_length=domain_length, + domain_height=domain_height, + num_cells=global_size.num_cells, + mean_edge_length=mean_edge_length, + mean_dual_edge_length=mean_dual_edge_length, + mean_cell_area=mean_cell_area, + mean_dual_cell_area=mean_dual_cell_area, + edge_lengths=edge_lengths, + dual_edge_lengths=dual_edge_lengths, + cell_areas=cell_areas, + dual_cell_areas=dual_cell_areas, + ) + return global_params def _get_index_field( diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index c32b583e0c..677824c538 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -151,7 +151,7 @@ def test_fields_distribute_and_gather(processor_props: defs.ProcessProperties, c file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) single_node = run_grid_manager_for_singlenode(file, vertical_config) single_node_grid = single_node.grid - global_cell_area = single_node.geometry[gridfile.GeometryName.CELL_AREA] + global_cell_area = single_node.geometry_fields[gridfile.GeometryName.CELL_AREA] global_edge_lat = single_node.coordinates[dims.EdgeDim]["lat"] global_vertex_lon = single_node.coordinates[dims.VertexDim]["lon"] @@ -163,7 +163,7 @@ def test_fields_distribute_and_gather(processor_props: defs.ProcessProperties, c ) decomposition_info = multinode.decomposition_info - local_cell_area = multinode.geometry[gridfile.GeometryName.CELL_AREA] + local_cell_area = multinode.geometry_fields[gridfile.GeometryName.CELL_AREA] local_edge_lat = multinode.coordinates[dims.EdgeDim]["lat"] local_vertex_lon = multinode.coordinates[dims.VertexDim]["lon"] print( @@ -291,7 +291,7 @@ def test_halo_neighbor_access_c2e( grid=single_node_grid, coordinates=single_node.coordinates, decomposition_info=single_node.decomposition_info, - extra_fields=single_node.geometry, + extra_fields=single_node.geometry_fields, metadata=geometry_attributes.attrs, ) @@ -321,7 +321,7 @@ def test_halo_neighbor_access_c2e( decomposer=halo.SimpleMetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid - extra_geometry_fields = multinode_grid_manager.geometry + extra_geometry_fields = multinode_grid_manager.geometry_fields decomposition_info = multinode_grid_manager.decomposition_info print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") @@ -377,7 +377,7 @@ def test_local_connectivities( test_defs.Grids.R02B04_GLOBAL, keep_skip_values=True, backend=None ).grid partitioner = halo.SimpleMetisDecomposer() - face_face_connectivity = grid.get_connectivity(dims.C2E2C).ndarray # type: ignore[union-attr] + face_face_connectivity = grid.get_connectivity(dims.C2E2C).ndarray neighbor_tables = grid.get_neighbor_tables() labels = partitioner(face_face_connectivity, num_partitions=processor_props.comm_size) halo_generator = halo.IconLikeHaloConstructor( diff --git a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py index 8827ecdb93..f48fcd0d19 100644 --- a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py +++ b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py @@ -10,27 +10,18 @@ import pytest from icon4py.model.common import constants, dimension as dims -from icon4py.model.common.grid.geometry_stencils import ( - compute_cell_center_arc_distance, - compute_edge_length, -) +from icon4py.model.common.grid.geometry_stencils import compute_edge_length from icon4py.model.common.utils import data_allocation as data_alloc -from icon4py.model.testing import definitions, grid_utils +from icon4py.model.testing import grid_utils from ..fixtures import * @pytest.mark.level("unit") @pytest.mark.datatest -@pytest.mark.parametrize( - "experiment, grid_file", - ( - (definitions.Experiments.MCH_CH_R04B09.name, definitions.Grids.MCH_CH_R04B09_DSL.name), - (definitions.Experiments.EXCLAIM_APE.name, definitions.Grids.R02B04_GLOBAL.name), - ), -) -def test_edge_length(experiment, grid_file, grid_savepoint, backend): +def test_edge_length(experiment, grid_savepoint, backend) -> None: keep = True + grid_file = experiment.grid gm = grid_utils.get_grid_manager_from_identifier( grid_file, keep_skip_values=keep, num_levels=1, backend=backend ) diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index 1d1f50fe27..b02b9b084f 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -547,22 +547,15 @@ def test_limited_area_on_grid(grid_descriptor: definitions.GridDescription, expe assert expected == grid.limited_area -@pytest.mark.parametrize( - "grid_file", - [ - (definitions.Grids.MCH_CH_R04B09_DSL), - (definitions.Grids.R02B04_GLOBAL), - ], -) @pytest.mark.parametrize("dim", utils.horizontal_dims()) def test_decomposition_info_single_node( dim: gtx.Dimension, - grid_file: definitions.GridDescription, experiment: definitions.Experiment, grid_savepoint: serialbox.IconGridSavepoint, backend: gtx_typing.Backend, ) -> None: expected = grid_savepoint.construct_decomposition_info() + grid_file = experiment.grid gm = utils.run_grid_manager(grid_file, keep_skip_values=True, backend=backend) result = gm.decomposition_info assert np.all(result.local_index(dim) == expected.local_index(dim)) diff --git a/model/common/tests/common/grid/unit_tests/test_gridfile.py b/model/common/tests/common/grid/unit_tests/test_gridfile.py index dc164ff075..1f6e9c0f2a 100644 --- a/model/common/tests/common/grid/unit_tests/test_gridfile.py +++ b/model/common/tests/common/grid/unit_tests/test_gridfile.py @@ -55,7 +55,7 @@ def test_grid_file_vertex_cell_edge_dimensions( experiment: definitions.Experiment, grid_savepoint: serialbox.IconGridSavepoint ) -> None: file = gridtest_utils.resolve_full_grid_file_name(experiment.grid) - parser = gridfile.GridFile(str(file)) + parser = gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) try: parser.open() assert parser.dimension(gridfile.DimensionName.CELL_NAME) == grid_savepoint.num( @@ -75,7 +75,7 @@ def test_grid_file_vertex_cell_edge_dimensions( @pytest.mark.parametrize("filename", (definitions.Grids.R02B04_GLOBAL,)) @pytest.mark.parametrize("apply_transformation", (True, False)) -def test_int_variable(filename, apply_transformation): +def test_int_variable(filename, apply_transformation) -> None: file = gridtest_utils.resolve_full_grid_file_name(filename) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: edge_dim = parser.dimension(gridfile.DimensionName.EDGE_NAME) @@ -106,7 +106,7 @@ def index_selection() -> Iterable[list[int]]: index_selection(), ) @pytest.mark.parametrize("filename", (definitions.Grids.R02B04_GLOBAL,)) -def test_index_read_for_1d_fields(filename, selection): +def test_index_read_for_1d_fields(filename: definitions.GridDescription, selection: list) -> None: file = gridtest_utils.resolve_full_grid_file_name(filename) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: selection = np.asarray(selection) if len(selection) > 0 else None @@ -125,7 +125,12 @@ def test_index_read_for_1d_fields(filename, selection): (gridfile.ConnectivityName.V2E, gridfile.ConnectivityName.V2C, gridfile.ConnectivityName.E2V), ) @pytest.mark.parametrize("apply_offset", (True, False)) -def test_index_read_for_2d_connectivity(filename, selection, field, apply_offset): +def test_index_read_for_2d_connectivity( + filename: definitions.GridDescription, + selection: list, + field: gridfile.FieldName, + apply_offset: bool, +) -> None: file = gridtest_utils.resolve_full_grid_file_name(filename) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: selection = np.asarray(selection) if len(selection) > 0 else None diff --git a/model/common/tests/common/grid/unit_tests/test_icon.py b/model/common/tests/common/grid/unit_tests/test_icon.py index 2cfef6f978..b8706fab92 100644 --- a/model/common/tests/common/grid/unit_tests/test_icon.py +++ b/model/common/tests/common/grid/unit_tests/test_icon.py @@ -524,7 +524,7 @@ def test_global_grid_params_from_grid_manager( grid_descriptor: definitions.GridDescription, backend: gtx_typing.Backend, geometry_type: base.GeometryType, - subdivision: base.GridSubdivision, + subdivision: icon.GridSubdivision, radius: float, domain_length: float, domain_height: float, diff --git a/model/testing/src/icon4py/model/testing/grid_utils.py b/model/testing/src/icon4py/model/testing/grid_utils.py index 27329a3ff4..86383c6c71 100644 --- a/model/testing/src/icon4py/model/testing/grid_utils.py +++ b/model/testing/src/icon4py/model/testing/grid_utils.py @@ -13,7 +13,7 @@ geometry, geometry_attributes as geometry_attrs, grid_manager as gm, - icon, + gridfile, vertical as v_grid, ) from icon4py.model.common.utils import data_allocation as data_alloc @@ -112,7 +112,12 @@ def _construct_grid_geometry() -> geometry.GridGeometry: ) grid = gm.grid geometry_source = geometry.GridGeometry( - grid, gm.decomposition_info, backend, gm.coordinates, gm.geometry_fields, geometry_attrs.attrs + grid, + gm.decomposition_info, + backend, + gm.coordinates, + gm.geometry_fields, + geometry_attrs.attrs, ) return geometry_source From 0a2d12675bd3cb29d12bb2d486fbaf50ef78948d Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 8 Oct 2025 10:44:42 +0200 Subject: [PATCH 099/240] fix typings --- .../model/common/decomposition/definitions.py | 5 +++ .../model/common/decomposition/halo.py | 5 +-- .../icon4py/model/common/grid/grid_manager.py | 3 +- .../decomposition/two_ranks_distribution.py | 2 +- .../tests/common/decomposition/utils.py | 10 ++++- .../mpi_tests/test_parallel_grid_manager.py | 16 ++++---- .../grid/mpi_tests/test_parallel_icon.py | 4 +- .../grid/unit_tests/test_geometry_stencils.py | 11 ++++-- .../common/grid/unit_tests/test_gridfile.py | 38 +++++++++++-------- pyproject.toml | 3 +- uv.lock | 4 ++ 11 files changed, 64 insertions(+), 37 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index cb18dd972f..b247ee91a6 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -52,6 +52,11 @@ def __str__(self): @dataclass(frozen=True, init=False) class SingleNodeProcessProperties(ProcessProperties): + comm: Any + comm_name: str + comm_size: int + rank: int + def __init__(self): object.__setattr__(self, "comm", None) object.__setattr__(self, "rank", 0) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 409a41faf5..b174a4f06c 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -168,10 +168,7 @@ def next_halo_line(self, cells: data_alloc.NDArray, depot: data_alloc.NDArray | ), "input should be 1d array" # TODO(halungge): otherwise reshape instead cell_neighbors = self._find_cell_neighbors(cells) - if depot is not None: - cells_so_far = self._xp.hstack((depot, cells)) - else: - cells_so_far = cells + cells_so_far = self._xp.hstack((depot, cells)) if depot is not None else cells next_halo_cells = self._xp.setdiff1d( self._xp.unique(cell_neighbors), cells_so_far, assume_unique=True diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index befbaff763..40d322a64b 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -11,6 +11,7 @@ from types import ModuleType from typing import Literal, TypeAlias +import gt4py._core.definitions as gtx_core_defs import gt4py.next as gtx import gt4py.next.typing as gtx_typing import numpy as np @@ -735,7 +736,7 @@ def _patch_with_dummy_lastline(ar, array_ns: ModuleType = np): def construct_local_connectivity( field_offset: gtx.FieldOffset, decomposition_info: decomposition.DecompositionInfo, - connectivity: np.ndarray, + connectivity: np.ndarray | gtx_core_defs.NDArrayObject, ) -> np.ndarray: """ Construct a connectivity table for use on a given rank: it maps from source to target dimension in _local_ indices. diff --git a/model/common/tests/common/decomposition/two_ranks_distribution.py b/model/common/tests/common/decomposition/two_ranks_distribution.py index ccd6c6709b..44097f6e88 100644 --- a/model/common/tests/common/decomposition/two_ranks_distribution.py +++ b/model/common/tests/common/decomposition/two_ranks_distribution.py @@ -14,7 +14,7 @@ TWO_RANKS_DISTRIBUTION[5, 6, 10] = 0 -# TODO define all the rest +# TODO (@halungge): define all the rest or delete CELL_OWN: Final[dict[int, list[int]]] = { 0: [6, 7, 10], 1: [0, 1, 2, 3, 4, 5, 8, 9, 11, 12, 13, 14, 15, 16, 17], diff --git a/model/common/tests/common/decomposition/utils.py b/model/common/tests/common/decomposition/utils.py index deb6a0e535..cf249f9725 100644 --- a/model/common/tests/common/decomposition/utils.py +++ b/model/common/tests/common/decomposition/utils.py @@ -6,6 +6,7 @@ # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause import dataclasses +from typing import Any import numpy as np from gt4py import next as gtx @@ -158,8 +159,13 @@ def assert_same_entries( assert np.setdiff1d(my_owned, reference[dim][rank], assume_unique=True).size == 0 -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass(frozen=True, init=False) class DummyProps(definitions.ProcessProperties): + comm: Any + comm_name: str + comm_size: int + rank: int + def __init__(self, rank: int): object.__setattr__(self, "rank", rank % 4) object.__setattr__(self, "comm", None) @@ -168,4 +174,4 @@ def __init__(self, rank: int): def dummy_four_ranks(rank: int) -> definitions.ProcessProperties: - return DummyProps(rank=rank) + return DummyProps(rank) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 677824c538..b90ecb8952 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -40,7 +40,7 @@ from ...decomposition import utils as decomp_utils from .. import utils -from ..fixtures import * +from ..fixtures import backend, experiment, grid_savepoint, icon_grid, processor_props try: @@ -170,9 +170,11 @@ def test_fields_distribute_and_gather(processor_props: defs.ProcessProperties, c f"rank = {processor_props.rank} has size(cell_area): {local_cell_area.ndarray.shape}, " f"has size(edge_length): {local_edge_lat.ndarray.shape}, has size(vertex_length): {local_vertex_lon.ndarray.shape}" ) + global_num_cells = single_node_grid.config.num_cells + # the local number of cells must be at most the global number of cells (analytically computed) assert ( - local_cell_area.asnumpy().shape[0] <= single_node_grid.global_properties.num_cells + local_cell_area.asnumpy().shape[0] <= global_num_cells ), "local field is larger than global field" # global read: read the same (global fields) @@ -219,10 +221,10 @@ def gather_field(field: np.ndarray, comm: mpi4py.MPI.Comm) -> tuple: log.debug("fields gathered:") log.debug(f"field sizes {local_sizes}") local_first_dim = tuple(size / constant_length for size in local_sizes) - gathered_field = recv_buffer.reshape((-1, *constant_dims)) + gathered_field = recv_buffer.reshape((-1, *constant_dims)) # type: ignore [union-attr] else: gathered_field = None - local_first_dim = field.shape[0] + local_first_dim = field.shape return local_first_dim, gathered_field @@ -257,7 +259,7 @@ def assert_gathered_field_against_global( print( f"rank = {processor_props.rank}: --- gathered field has size {gathered_sizes}" ) - sorted_ = np.zeros(global_reference_field.shape, dtype=gtx.float64) + sorted_ = np.zeros(global_reference_field.shape, dtype=gtx.float64) # type: ignore [attr-defined] sorted_[gathered_global_indices] = gathered_field assert test_helpers.dallclose( sorted_, global_reference_field @@ -280,8 +282,8 @@ def assert_gathered_field_against_global( @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_halo_neighbor_access_c2e( processor_props: defs.ProcessProperties, backend: gtx_typing.Backend | None -): - # processor_props = decomp_utils.DummyProps(rank = 1) +) -> None: + # processor_props = decomp_utils.DummyProps(1) file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) print(f"running on {processor_props.comm}") single_node = run_grid_manager_for_singlenode(file, vertical_config) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py index d49b92a6ff..c996a520fb 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py @@ -19,7 +19,7 @@ from icon4py.model.testing import parallel_helpers from .. import utils -from ..fixtures import * +from ..fixtures import icon_grid, processor_props if TYPE_CHECKING: @@ -30,7 +30,7 @@ try: - import mpi4py # type: ignore[import-not-found] # F401: import mpi4py to check for optional mpi dependency + import mpi4py except ImportError: pytest.skip("Skipping parallel on single node installation", allow_module_level=True) diff --git a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py index f48fcd0d19..8bd6b0af70 100644 --- a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py +++ b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py @@ -8,18 +8,23 @@ import numpy as np import pytest +from gt4py.next import typing as gtx_typing from icon4py.model.common import constants, dimension as dims from icon4py.model.common.grid.geometry_stencils import compute_edge_length from icon4py.model.common.utils import data_allocation as data_alloc -from icon4py.model.testing import grid_utils +from icon4py.model.testing import definitions, grid_utils, serialbox -from ..fixtures import * +from ..fixtures import backend, experiment, grid_savepoint @pytest.mark.level("unit") @pytest.mark.datatest -def test_edge_length(experiment, grid_savepoint, backend) -> None: +def test_edge_length( + experiment: definitions.Experiment, + grid_savepoint: serialbox.IconGridSavepoint, + backend: gtx_typing.Backend, +) -> None: keep = True grid_file = experiment.grid gm = grid_utils.get_grid_manager_from_identifier( diff --git a/model/common/tests/common/grid/unit_tests/test_gridfile.py b/model/common/tests/common/grid/unit_tests/test_gridfile.py index 1f6e9c0f2a..030cb52ee8 100644 --- a/model/common/tests/common/grid/unit_tests/test_gridfile.py +++ b/model/common/tests/common/grid/unit_tests/test_gridfile.py @@ -73,10 +73,12 @@ def test_grid_file_vertex_cell_edge_dimensions( parser.close() -@pytest.mark.parametrize("filename", (definitions.Grids.R02B04_GLOBAL,)) +@pytest.mark.parametrize("grid_descriptor", (definitions.Grids.R02B04_GLOBAL,)) @pytest.mark.parametrize("apply_transformation", (True, False)) -def test_int_variable(filename, apply_transformation) -> None: - file = gridtest_utils.resolve_full_grid_file_name(filename) +def test_int_variable( + grid_descriptor: definitions.GridDescription, apply_transformation: bool +) -> None: + file = gridtest_utils.resolve_full_grid_file_name(grid_descriptor) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: edge_dim = parser.dimension(gridfile.DimensionName.EDGE_NAME) # use a test field that does not contain Pentagons @@ -105,38 +107,42 @@ def index_selection() -> Iterable[list[int]]: "selection", index_selection(), ) -@pytest.mark.parametrize("filename", (definitions.Grids.R02B04_GLOBAL,)) -def test_index_read_for_1d_fields(filename: definitions.GridDescription, selection: list) -> None: - file = gridtest_utils.resolve_full_grid_file_name(filename) +@pytest.mark.parametrize("grid_descriptor", (definitions.Grids.R02B04_GLOBAL,)) +def test_index_read_for_1d_fields( + grid_descriptor: definitions.GridDescription, selection: list[int] +) -> None: + file = gridtest_utils.resolve_full_grid_file_name(grid_descriptor) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: - selection = np.asarray(selection) if len(selection) > 0 else None + indices_to_read = np.asarray(selection) if len(selection) > 0 else None full_field = parser.variable(gridfile.CoordinateName.CELL_LATITUDE) - selective_field = parser.variable(gridfile.CoordinateName.CELL_LATITUDE, indices=selection) - assert np.allclose(full_field[selection], selective_field) + selective_field = parser.variable( + gridfile.CoordinateName.CELL_LATITUDE, indices=indices_to_read + ) + assert np.allclose(full_field[indices_to_read], selective_field) @pytest.mark.parametrize( "selection", index_selection(), ) -@pytest.mark.parametrize("filename", (definitions.Grids.R02B04_GLOBAL,)) +@pytest.mark.parametrize("grid_descriptor", (definitions.Grids.R02B04_GLOBAL,)) @pytest.mark.parametrize( "field", (gridfile.ConnectivityName.V2E, gridfile.ConnectivityName.V2C, gridfile.ConnectivityName.E2V), ) @pytest.mark.parametrize("apply_offset", (True, False)) def test_index_read_for_2d_connectivity( - filename: definitions.GridDescription, - selection: list, + grid_descriptor: definitions.GridDescription, + selection: list[int], field: gridfile.FieldName, apply_offset: bool, ) -> None: - file = gridtest_utils.resolve_full_grid_file_name(filename) + file = gridtest_utils.resolve_full_grid_file_name(grid_descriptor) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: - selection = np.asarray(selection) if len(selection) > 0 else None + indices_to_read = np.asarray(selection) if len(selection) > 0 else None # TODO(halungge): grid_file.ConnectivityName.V2E:P 2 D fields full_field = parser.int_variable(field, transpose=True, apply_transformation=apply_offset) selective_field = parser.int_variable( - field, indices=selection, transpose=True, apply_transformation=apply_offset + field, indices=indices_to_read, transpose=True, apply_transformation=apply_offset ) - assert np.allclose(full_field[selection], selective_field) + assert np.allclose(full_field[indices_to_read], selective_field) diff --git a/pyproject.toml b/pyproject.toml index 062b3dbda1..b1ac478fad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,8 @@ test = [ typing = [ "mypy[faster-cache]>=1.13.0", "typing-extensions>=4.11.0", - "types-cffi>=1.16.0" + "types-cffi>=1.16.0", + "mpi4py>=4.0.0" ] # -- Standard project description options (PEP 621) -- diff --git a/uv.lock b/uv.lock index c5103e1f9c..4ff102217e 100644 --- a/uv.lock +++ b/uv.lock @@ -1626,6 +1626,7 @@ dev = [ { name = "coverage", extra = ["toml"] }, { name = "esbonio" }, { name = "icon4py-testing" }, + { name = "mpi4py" }, { name = "mypy", extra = ["faster-cache"] }, { name = "myst-parser" }, { name = "nox" }, @@ -1688,6 +1689,7 @@ test = [ { name = "pytest-xdist", extra = ["psutil"] }, ] typing = [ + { name = "mpi4py" }, { name = "mypy", extra = ["faster-cache"] }, { name = "types-cffi" }, { name = "typing-extensions" }, @@ -1726,6 +1728,7 @@ dev = [ { name = "coverage", extras = ["toml"], specifier = ">=7.5.0" }, { name = "esbonio", specifier = ">=0.16.0" }, { name = "icon4py-testing", editable = "model/testing" }, + { name = "mpi4py", specifier = ">=4.0.0" }, { name = "mypy", extras = ["faster-cache"], specifier = ">=1.13.0" }, { name = "myst-parser", specifier = ">=4.0.0" }, { name = "nox", git = "https://github.com/wntrblm/nox.git?rev=aa09595437608dfe21eb776d8a4bcc0bd5f9916b" }, @@ -1788,6 +1791,7 @@ test = [ { name = "pytest-xdist", extras = ["psutil"], specifier = ">=3.5.0" }, ] typing = [ + { name = "mpi4py", specifier = ">=4.0.0" }, { name = "mypy", extras = ["faster-cache"], specifier = ">=1.13.0" }, { name = "types-cffi", specifier = ">=1.16.0" }, { name = "typing-extensions", specifier = ">=4.11.0" }, From 785f7d1500f32efed33e0ec7615be7367aef1891 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 8 Oct 2025 10:52:14 +0200 Subject: [PATCH 100/240] remove obsolete TODO --- .../common/grid/mpi_tests/test_parallel_grid_manager.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index b90ecb8952..4729beadf8 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -228,7 +228,6 @@ def gather_field(field: np.ndarray, comm: mpi4py.MPI.Comm) -> tuple: return local_first_dim, gathered_field -# TODO (halungge): deal with 2d fields def assert_gathered_field_against_global( decomposition_info: defs.DecompositionInfo, processor_props: defs.ProcessProperties, # F811 # fixture @@ -277,7 +276,7 @@ def assert_gathered_field_against_global( # - geofac_n2s -# TODO (halungge): fix non contiguous dimension for embedded +# TODO (halungge): fix non contiguous dimension for embedded in gt4py @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_halo_neighbor_access_c2e( @@ -373,7 +372,7 @@ def test_local_connectivities( processor_props: defs.ProcessProperties, caplog: Iterator, field_offset: gtx.FieldOffset, -) -> None: # fixture +) -> None: caplog.set_level(logging.INFO) # type: ignore [attr-defined] grid = utils.run_grid_manager( test_defs.Grids.R02B04_GLOBAL, keep_skip_values=True, backend=None From c5ad92bfdc4959dd5c88d2413b385d4f20fd6e1c Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 10 Oct 2025 15:56:38 +0200 Subject: [PATCH 101/240] fix parallel diffusion tests --- .../mpi_tests/test_parallel_diffusion.py | 127 +++++++++++------- .../mpi_tests/test_parallel_solve_nonhydro.py | 10 +- .../model/common/decomposition/definitions.py | 8 +- .../common/decomposition/mpi_decomposition.py | 4 +- .../src/icon4py/model/testing/definitions.py | 6 +- 5 files changed, 90 insertions(+), 65 deletions(-) diff --git a/model/atmosphere/diffusion/tests/diffusion/mpi_tests/test_parallel_diffusion.py b/model/atmosphere/diffusion/tests/diffusion/mpi_tests/test_parallel_diffusion.py index ad23ccd009..8867fd659f 100644 --- a/model/atmosphere/diffusion/tests/diffusion/mpi_tests/test_parallel_diffusion.py +++ b/model/atmosphere/diffusion/tests/diffusion/mpi_tests/test_parallel_diffusion.py @@ -6,45 +6,58 @@ # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause +from typing import Any + import pytest +from gt4py.next import typing as gtx_typing from icon4py.model.atmosphere.diffusion import diffusion as diffusion_, diffusion_states from icon4py.model.common import dimension as dims -from icon4py.model.common.decomposition import definitions as decompositon -from icon4py.model.common.grid import vertical as v_grid -from icon4py.model.testing import definitions, parallel_helpers, test_utils +from icon4py.model.common.decomposition import definitions as decomposition, mpi_decomposition +from icon4py.model.common.grid import icon, vertical as v_grid +from icon4py.model.testing import definitions, parallel_helpers, serialbox, test_utils from .. import utils from ..fixtures import * # noqa: F403 @pytest.mark.mpi -@pytest.mark.parametrize("experiment", [definitions.Experiments.MCH_CH_R04B09]) +@pytest.mark.parametrize( + "experiment, step_date_init, step_date_exit", + [ + ( + definitions.Experiments.MCH_CH_R04B09, + "2021-06-20T12:00:10.000", + "2021-06-20T12:00:10.000", + ), + (definitions.Experiments.EXCLAIM_APE, "2000-01-01T00:00:02.000", "2000-01-01T00:00:02.000"), + ], +) @pytest.mark.parametrize("ndyn_substeps", [2]) -@pytest.mark.parametrize("linit", [True, False]) -@pytest.mark.parametrize("orchestration", [False, True]) -@pytest.mark.parametrize("processor_props", [True, False], indirect=True) +@pytest.mark.parametrize("orchestration", [False]) +@pytest.mark.parametrize("processor_props", [True], indirect=True) def test_parallel_diffusion( - experiment, - step_date_init, - linit, - ndyn_substeps, - processor_props, - decomposition_info, - icon_grid, - savepoint_diffusion_init, - savepoint_diffusion_exit, - grid_savepoint, + experiment: definitions.Experiment, + step_date_init: str, + step_date_exit: str, + linit: bool, + ndyn_substeps: int, + processor_props: decomposition.ProcessProperties, + decomposition_info: decomposition.DecompositionInfo, + icon_grid: icon.IconGrid, + savepoint_diffusion_init: serialbox.IconDiffusionInitSavepoint, + savepoint_diffusion_exit: serialbox.IconDiffusionExitSavepoint, + grid_savepoint: serialbox.IconGridSavepoint, metric_state: diffusion_states.DiffusionMetricState, interpolation_state: diffusion_states.DiffusionInterpolationState, - lowest_layer_thickness, - model_top_height, - stretch_factor, - damping_height, - caplog, - backend, - orchestration, -): + lowest_layer_thickness: float, + model_top_height: int, + stretch_factor: float, + damping_height: int, + caplog: Any, + backend: gtx_typing.Backend, + orchestration: bool, +) -> None: if orchestration and not test_utils.is_dace(backend): raise pytest.skip("This test is only executed for `dace` backends.") caplog.set_level("INFO") @@ -54,9 +67,9 @@ def test_parallel_diffusion( ) print( f"rank={processor_props.rank}/{processor_props.comm_size}: decomposition info : klevels = {decomposition_info.klevels}, " - f"local cells = {decomposition_info.global_index(dims.CellDim, decompositon.DecompositionInfo.EntryType.ALL).shape} " - f"local edges = {decomposition_info.global_index(dims.EdgeDim, decompositon.DecompositionInfo.EntryType.ALL).shape} " - f"local vertices = {decomposition_info.global_index(dims.VertexDim, decompositon.DecompositionInfo.EntryType.ALL).shape}" + f"local cells = {decomposition_info.global_index(dims.CellDim, decomposition.DecompositionInfo.EntryType.ALL).shape} " + f"local edges = {decomposition_info.global_index(dims.EdgeDim, decomposition.DecompositionInfo.EntryType.ALL).shape} " + f"local vertices = {decomposition_info.global_index(dims.VertexDim, decomposition.DecompositionInfo.EntryType.ALL).shape}" ) print( f"rank={processor_props.rank}/{processor_props.comm_size}: GHEX context setup: from {processor_props.comm_name} with {processor_props.comm_size} nodes" @@ -81,7 +94,7 @@ def test_parallel_diffusion( diffusion_params = diffusion_.DiffusionParams(config) cell_geometry = grid_savepoint.construct_cell_geometry() edge_geometry = grid_savepoint.construct_edge_geometry() - exchange = decompositon.create_exchange(processor_props, decomposition_info) + exchange = decomposition.create_exchange(processor_props, decomposition_info) diffusion = diffusion_.Diffusion( grid=icon_grid, config=config, @@ -136,29 +149,39 @@ def test_parallel_diffusion( @pytest.mark.mpi -@pytest.mark.parametrize("experiment", [definitions.Experiments.MCH_CH_R04B09]) +@pytest.mark.parametrize( + "experiment, step_date_init, step_date_exit", + [ + ( + definitions.Experiments.MCH_CH_R04B09, + "2021-06-20T12:00:10.000", + "2021-06-20T12:00:10.000", + ), + (definitions.Experiments.EXCLAIM_APE, "2000-01-01T00:00:02.000", "2000-01-01T00:00:02.000"), + ], +) @pytest.mark.parametrize("ndyn_substeps", [2]) -@pytest.mark.parametrize("linit", [True]) @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_parallel_diffusion_multiple_steps( - experiment, - step_date_init, - linit, - ndyn_substeps, - processor_props, - decomposition_info, - icon_grid, - savepoint_diffusion_init, - savepoint_diffusion_exit, - grid_savepoint, + experiment: definitions.Experiment, + step_date_init: str, + step_date_exit: str, + linit: bool, + ndyn_substeps: int, + processor_props: decomposition.ProcessProperties, + decomposition_info: decomposition.DecompositionInfo, + icon_grid: icon.IconGrid, + savepoint_diffusion_init: serialbox.IconDiffusionInitSavepoint, + savepoint_diffusion_exit: serialbox.IconDiffusionExitSavepoint, + grid_savepoint: serialbox.IconGridSavepoint, metric_state: diffusion_states.DiffusionMetricState, interpolation_state: diffusion_states.DiffusionInterpolationState, - lowest_layer_thickness, - model_top_height, - stretch_factor, - damping_height, - caplog, - backend, + lowest_layer_thickness: float, + model_top_height: float, + stretch_factor: float, + damping_height: float, + caplog: Any, + backend: gtx_typing.Backend, ): if not test_utils.is_dace(backend): raise pytest.skip("This test is only executed for `dace backends.") @@ -172,9 +195,9 @@ def test_parallel_diffusion_multiple_steps( ) print( f"rank={processor_props.rank}/{processor_props.comm_size}: decomposition info : klevels = {decomposition_info.klevels}, " - f"local cells = {decomposition_info.global_index(dims.CellDim, decompositon.DecompositionInfo.EntryType.ALL).shape} " - f"local edges = {decomposition_info.global_index(dims.EdgeDim, decompositon.DecompositionInfo.EntryType.ALL).shape} " - f"local vertices = {decomposition_info.global_index(dims.VertexDim, decompositon.DecompositionInfo.EntryType.ALL).shape}" + f"local cells = {decomposition_info.global_index(dims.CellDim, decomposition.DecompositionInfo.EntryType.ALL).shape} " + f"local edges = {decomposition_info.global_index(dims.EdgeDim, decomposition.DecompositionInfo.EntryType.ALL).shape} " + f"local vertices = {decomposition_info.global_index(dims.VertexDim, decomposition.DecompositionInfo.EntryType.ALL).shape}" ) print( f"rank={processor_props.rank}/{processor_props.comm_size}: GHEX context setup: from {processor_props.comm_name} with {processor_props.comm_size} nodes" @@ -199,7 +222,7 @@ def test_parallel_diffusion_multiple_steps( print( f"rank={processor_props.rank}/{processor_props.comm_size}: setup: using {processor_props.comm_name} with {processor_props.comm_size} nodes" ) - exchange = decompositon.create_exchange(processor_props, decomposition_info) + exchange = decomposition.create_exchange(processor_props, decomposition_info) ###################################################################### # DaCe NON-Orchestrated Backend @@ -256,7 +279,7 @@ def test_parallel_diffusion_multiple_steps( # DaCe Orchestrated Backend ###################################################################### - exchange = decompositon.create_exchange(processor_props, decomposition_info) + exchange = decomposition.create_exchange(processor_props, decomposition_info) diffusion = diffusion_.Diffusion( grid=icon_grid, config=config, diff --git a/model/atmosphere/dycore/tests/dycore/mpi_tests/test_parallel_solve_nonhydro.py b/model/atmosphere/dycore/tests/dycore/mpi_tests/test_parallel_solve_nonhydro.py index f08d77028b..6c09ec7ae3 100644 --- a/model/atmosphere/dycore/tests/dycore/mpi_tests/test_parallel_solve_nonhydro.py +++ b/model/atmosphere/dycore/tests/dycore/mpi_tests/test_parallel_solve_nonhydro.py @@ -11,12 +11,13 @@ from icon4py.model.atmosphere.dycore import dycore_states, solve_nonhydro as nh from icon4py.model.common import dimension as dims, utils as common_utils -from icon4py.model.common.decomposition import definitions +from icon4py.model.common.decomposition import definitions, mpi_decomposition from icon4py.model.common.grid import states as grid_states, vertical as v_grid from icon4py.model.common.utils import data_allocation as data_alloc -from icon4py.model.testing import parallel_helpers, test_utils +from icon4py.model.testing import definitions as test_defs, parallel_helpers, test_utils from .. import utils +from ..fixtures import * # noqa: F403 @pytest.mark.parametrize("processor_props", [True], indirect=True) @@ -75,7 +76,7 @@ def test_run_solve_nonhydro_single_step( f"rank={processor_props.rank}/{processor_props.comm_size}: number of halo cells {np.count_nonzero(np.invert(owned_cells))}" ) - config = definitions.construct_nonhydrostatic_config(experiment) + config = test_defs.construct_nonhydrostatic_config(experiment) sp = savepoint_nonhydro_init sp_step_exit = savepoint_nonhydro_step_final nonhydro_params = nh.NonHydrostaticParams(config) @@ -90,12 +91,10 @@ def test_run_solve_nonhydro_single_step( config=vertical_config, vct_a=grid_savepoint.vct_a(), vct_b=grid_savepoint.vct_b(), - _min_index_flat_horizontal_grad_pressure=grid_savepoint.nflat_gradp(), ) sp_v = savepoint_velocity_init dtime = sp_v.get_metadata("dtime").get("dtime") lprep_adv = sp_v.get_metadata("prep_adv").get("prep_adv") - # clean_mflx = sp_v.get_metadata("clean_mflx").get("clean_mflx") # noqa: ERA001 [commented-out-code] prep_adv = dycore_states.PrepAdvection( vn_traj=sp.vn_traj(), mass_flx_me=sp.mass_flx_me(), @@ -106,7 +105,6 @@ def test_run_solve_nonhydro_single_step( ) recompute = sp_v.get_metadata("recompute").get("recompute") - # linit = sp_v.get_metadata("linit").get("linit") # noqa: ERA001 [commented-out-code] diagnostic_state_nh = dycore_states.DiagnosticStateNonHydro( max_vertical_cfl=0.0, diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index b247ee91a6..0935fab6d9 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -207,9 +207,9 @@ def is_ready(self) -> bool: ... @runtime_checkable class ExchangeRuntime(Protocol): - def exchange(self, dim: gtx.Dimension, *fields: Sequence[gtx.Field]) -> ExchangeResult: ... + def exchange(self, dim: gtx.Dimension, *fields: tuple[gtx.Field, ...]) -> ExchangeResult: ... - def exchange_and_wait(self, dim: gtx.Dimension, *fields: Sequence[gtx.Field]) -> None: ... + def exchange_and_wait(self, dim: gtx.Dimension, *fields: tuple[gtx.Field, ...]) -> None: ... def get_size(self) -> int: ... @@ -218,10 +218,10 @@ def my_rank(self) -> int: ... @dataclass class SingleNodeExchange: - def exchange(self, dim: gtx.Dimension, fields: Sequence[gtx.Field]) -> ExchangeResult: + def exchange(self, dim: gtx.Dimension, *fields: tuple[gtx.Field, ...]) -> ExchangeResult: return SingleNodeResult() - def exchange_and_wait(self, dim: gtx.Dimension, fields: Sequence[gtx.Field]) -> None: + def exchange_and_wait(self, dim: gtx.Dimension, *fields: tuple[gtx.Field, ...]) -> None: return def my_rank(self) -> int: diff --git a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py index 6b641f93c9..5f59119c0f 100644 --- a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py +++ b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py @@ -224,7 +224,7 @@ def _slice_field_based_on_dim(self, field: gtx.Field, dim: gtx.Dimension) -> dat except KeyError: log.warn(f"Trying to trim field of invalid dimension {dim} for exchange. Not trimming.") - def exchange(self, dim: gtx.Dimension, fields: Sequence[gtx.Field]): + def exchange(self, dim: gtx.Dimension, *fields: tuple[gtx.Field, ...]): """ Exchange method that slices the fields based on the dimension and then performs halo exchange. @@ -259,7 +259,7 @@ def exchange(self, dim: gtx.Dimension, fields: Sequence[gtx.Field]): log.debug(f"exchange for {len(fields)} fields of dimension ='{dim.value}' initiated.") return MultiNodeResult(handle, applied_patterns) - def exchange_and_wait(self, dim: gtx.Dimension, *fields: Sequence[gtx.Field]): + def exchange_and_wait(self, dim: gtx.Dimension, *fields: tuple[gtx.Field, ...]): res = self.exchange(dim, *fields) res.wait() log.debug(f"exchange for {len(fields)} fields of dimension ='{dim.value}' done.") diff --git a/model/testing/src/icon4py/model/testing/definitions.py b/model/testing/src/icon4py/model/testing/definitions.py index 29159b1a60..e64410906c 100644 --- a/model/testing/src/icon4py/model/testing/definitions.py +++ b/model/testing/src/icon4py/model/testing/definitions.py @@ -143,7 +143,11 @@ class Experiments: description="EXCLAIM Aquaplanet experiment", grid=Grids.R02B04_GLOBAL, num_levels=60, - partitioned_data={1: "https://polybox.ethz.ch/index.php/s/2n2WpTgZFlTCTHu/download"}, + partitioned_data={ + 1: "https://polybox.ethz.ch/index.php/s/2n2WpTgZFlTCTHu/download", + 2: "https://polybox.ethz.ch/index.php/s/GQNcLtp4CN7ERbi/download", + 4: "https://polybox.ethz.ch/index.php/s/NfES3j9no15A0aX/download", + }, ) MCH_CH_R04B09: Final = Experiment( name="mch_ch_r04b09_dsl", From 8791be096ee6816d0773df175e764df3afeec74e Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 10 Oct 2025 17:52:55 +0200 Subject: [PATCH 102/240] fix parallel non hydro test --- .../pytest_benchmark_results_3.10.json | 23870 ++++++++++++++++ .../atmosphere/diffusion/config/__init__.py | 0 .../unit_tests/diffusion_default.yaml | 21 + .../mpi_tests/test_parallel_solve_nonhydro.py | 87 +- pytest | 0 pytest_benchmark_results_3.10.json | 0 6 files changed, 23919 insertions(+), 59 deletions(-) create mode 100644 model/atmosphere/diffusion/pytest_benchmark_results_3.10.json create mode 100644 model/atmosphere/diffusion/src/icon4py/model/atmosphere/diffusion/config/__init__.py create mode 100644 model/atmosphere/diffusion/tests/diffusion/unit_tests/diffusion_default.yaml create mode 100644 pytest create mode 100644 pytest_benchmark_results_3.10.json diff --git a/model/atmosphere/diffusion/pytest_benchmark_results_3.10.json b/model/atmosphere/diffusion/pytest_benchmark_results_3.10.json new file mode 100644 index 0000000000..0b27906f0f --- /dev/null +++ b/model/atmosphere/diffusion/pytest_benchmark_results_3.10.json @@ -0,0 +1,23870 @@ +{ + "machine_info": { + "node": "luzm", + "processor": "x86_64", + "machine": "x86_64", + "python_compiler": "Clang 20.1.4 ", + "python_implementation": "CPython", + "python_implementation_version": "3.10.18", + "python_version": "3.10.18", + "python_build": [ + "main", + "Jul 11 2025 22:43:29" + ], + "release": "6.14.0-27-generic", + "system": "Linux", + "cpu": { + "python_version": "3.10.18.final.0 (64 bit)", + "cpuinfo_version": [ + 9, + 0, + 0 + ], + "cpuinfo_version_string": "9.0.0", + "arch": "X86_64", + "bits": 64, + "count": 8, + "arch_string_raw": "x86_64", + "vendor_id_raw": "GenuineIntel", + "brand_raw": "11th Gen Intel(R) Core(TM) i7-1165G7 @ 2.80GHz", + "hz_advertised_friendly": "2.8000 GHz", + "hz_actual_friendly": "3.6964 GHz", + "hz_advertised": [ + 2800000000, + 0 + ], + "hz_actual": [ + 3696436000, + 0 + ], + "stepping": 1, + "model": 140, + "family": 6, + "flags": [ + "3dnowprefetch", + "abm", + "acpi", + "adx", + "aes", + "aperfmperf", + "apic", + "arat", + "arch_capabilities", + "arch_perfmon", + "art", + "avx", + "avx2", + "avx512_bitalg", + "avx512_vbmi2", + "avx512_vnni", + "avx512_vp2intersect", + "avx512_vpopcntdq", + "avx512bw", + "avx512cd", + "avx512dq", + "avx512f", + "avx512ifma", + "avx512vbmi", + "avx512vl", + "bmi1", + "bmi2", + "bts", + "cat_l2", + "cdp_l2", + "clflush", + "clflushopt", + "clwb", + "cmov", + "constant_tsc", + "cpuid", + "cpuid_fault", + "cx16", + "cx8", + "de", + "ds_cpl", + "dtes64", + "dtherm", + "dts", + "epb", + "ept", + "ept_ad", + "erms", + "est", + "f16c", + "flexpriority", + "flush_l1d", + "fma", + "fpu", + "fsgsbase", + "fsrm", + "fxsr", + "gfni", + "ht", + "hwp", + "hwp_act_window", + "hwp_epp", + "hwp_notify", + "hwp_pkg_req", + "ibpb", + "ibrs", + "ibrs_enhanced", + "ibt", + "ida", + "intel_pt", + "invpcid", + "lahf_lm", + "lm", + "mca", + "mce", + "md_clear", + "mmx", + "monitor", + "movbe", + "movdir64b", + "movdiri", + "msr", + "mtrr", + "nonstop_tsc", + "nopl", + "nx", + "ospke", + "pae", + "pat", + "pbe", + "pcid", + "pclmulqdq", + "pdcm", + "pdpe1gb", + "pebs", + "pge", + "pku", + "pln", + "pni", + "popcnt", + "pse", + "pse36", + "pts", + "rdpid", + "rdrand", + "rdseed", + "rdt_a", + "rdtscp", + "rep_good", + "sdbg", + "sep", + "sha_ni", + "smap", + "smep", + "split_lock_detect", + "ss", + "ssbd", + "sse", + "sse2", + "sse4_1", + "sse4_2", + "ssse3", + "stibp", + "syscall", + "tm", + "tm2", + "tpr_shadow", + "tsc", + "tsc_adjust", + "tsc_deadline_timer", + "tsc_known_freq", + "umip", + "user_shstk", + "vaes", + "vme", + "vmx", + "vnmi", + "vpclmulqdq", + "vpid", + "x2apic", + "xgetbv1", + "xsave", + "xsavec", + "xsaveopt", + "xsaves", + "xtopology", + "xtpr" + ], + "l3_cache_size": 12582912, + "l2_cache_size": 5242880, + "l1_data_cache_size": 196608, + "l1_instruction_cache_size": 131072 + } + }, + "commit_info": { + "id": "4701cc492a34ba6f88f368973e3338ba63e2ebde", + "time": "2025-08-15T18:27:55+02:00", + "author_time": "2025-08-15T18:27:55+02:00", + "dirty": true, + "project": "icon4py", + "branch": "main" + }, + "benchmarks": [ + { + "group": null, + "name": "test_stencil", + "fullname": "TestApplyDiffusionToVn", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.0006441649966291152, + "max": 0.0189960389980115, + "mean": 0.003602255105346275, + "stddev": 0.004880199014392282, + "rounds": 1236, + "median": 0.000828790998639306, + "iqr": 0.0031777689982845914, + "q1": 0.0006916990023455583, + "q3": 0.0038694680006301496, + "iqr_outliers": 213, + "stddev_outliers": 214, + "outliers": "214;213", + "ld15iqr": 0.0006441649966291152, + "hd15iqr": 0.00882603400532389, + "ops": 277.60388166730706, + "total": 4.452387310207996, + "data": [ + 0.0017910800015670247, + 0.001038121001329273, + 0.0019165579942637123, + 0.0010132710012840107, + 0.0009261520026484504, + 0.004629479000868741, + 0.0008771790016908199, + 0.0012659669955610298, + 0.005838680997840129, + 0.0009167329990305007, + 0.0017449129954911768, + 0.005366599005355965, + 0.006050699004845228, + 0.003211258001101669, + 0.004396117001306266, + 0.00084640400018543, + 0.0024797090009087697, + 0.0016823099940665998, + 0.0008128440022119321, + 0.0008048430026974529, + 0.0007994300030986778, + 0.0014144599990686402, + 0.003872152003168594, + 0.0008281319969682954, + 0.001688773998466786, + 0.0017891370007419027, + 0.000797337997937575, + 0.0008002229951671325, + 0.005781596999440808, + 0.0008682740008225664, + 0.000802238006144762, + 0.0007971419981913641, + 0.0007916649992694147, + 0.002750313993601594, + 0.0009287129942094907, + 0.0008001629976206459, + 0.0007929630010039546, + 0.0037936029984848574, + 0.0008390020011574961, + 0.0008065389993134886, + 0.0007898670010035858, + 0.0007860760015319102, + 0.003697043997817673, + 0.0012517420036601834, + 0.0007957960042404011, + 0.0007862189959269017, + 0.0008000800007721409, + 0.005445961003715638, + 0.0008278699970105663, + 0.0007592789988848381, + 0.0007590489985886961, + 0.0007524009997723624, + 0.005092090999823995, + 0.0008520120027242228, + 0.0007779300067340955, + 0.0007659469993086532, + 0.0007740900036878884, + 0.0007893439978943206, + 0.0008224100020015612, + 0.000788523000665009, + 0.0007705399984843098, + 0.0072099870012607425, + 0.0015989450039342046, + 0.0008028480006032623, + 0.0007706379983574152, + 0.0007822869956726208, + 0.005300292999891099, + 0.001071673999831546, + 0.0007753850004519336, + 0.0007566089989268221, + 0.012242778997460846, + 0.01343844900111435, + 0.010967854999762494, + 0.014991911004472058, + 0.015992868000466842, + 0.014988950999395456, + 0.015996636997442693, + 0.013996709996717982, + 0.01500271300028544, + 0.013999702998262364, + 0.011992769002972636, + 0.013994649998494424, + 0.012100498002837412, + 0.014886088996718172, + 0.014009182996232994, + 0.012987723996047862, + 0.014995753997936845, + 0.013992221996886656, + 0.0149990389982122, + 0.014988329996413086, + 0.016006876998289954, + 0.013994688000821043, + 0.01097399299760582, + 0.012134317999880295, + 0.01583950199710671, + 0.01399741999921389, + 0.015000230996520258, + 0.011995387001661584, + 0.015997483998944517, + 0.018990815995493904, + 0.013995096000144258, + 0.014473850998911075, + 0.014523929996357765, + 0.011994951004453469, + 0.010994398006005213, + 0.016998277998936828, + 0.014996942998550367, + 0.014010519000294153, + 0.014747959001397248, + 0.014230581997253466, + 0.01399532100185752, + 0.013998589005495887, + 0.014007051002408843, + 0.01298610599769745, + 0.0051392789973760955, + 0.0009202680012094788, + 0.0007703189985477366, + 0.0030886509994161315, + 0.0008120610000332817, + 0.0007868709944887087, + 0.0007098870046320371, + 0.0006741730030626059, + 0.0006771789994672872, + 0.0034376510011497885, + 0.0010097769991261885, + 0.0013776490013697185, + 0.0007482189976144582, + 0.0006907060014782473, + 0.0006935190031072125, + 0.0006853170052636415, + 0.0006840689966338687, + 0.0017324999935226515, + 0.0009010850044433028, + 0.0006875019971630536, + 0.0006748980013071559, + 0.0006843349983682856, + 0.0006826409953646362, + 0.004676359996665269, + 0.0009134450010606088, + 0.0008405240005231462, + 0.0020881609962088987, + 0.0007717850021435879, + 0.000826235998829361, + 0.0006955919961910695, + 0.0006876129991724156, + 0.004158816002018284, + 0.0008792500011622906, + 0.002929597001639195, + 0.000708334002410993, + 0.0006838389963377267, + 0.0006767490049242042, + 0.003451850003330037, + 0.000850248005008325, + 0.0006967220033402555, + 0.0006935470009921119, + 0.0007926910038804635, + 0.0021651139977620915, + 0.0007912029977887869, + 0.0006857030020910315, + 0.0006827429970144294, + 0.001303429999097716, + 0.0026995899970643222, + 0.0007826500004739501, + 0.0007142590038711205, + 0.0006970189933781512, + 0.0008166030020220205, + 0.0010894769948208705, + 0.0015481800000998192, + 0.0007005610023043118, + 0.0006860220019007102, + 0.0006968589950702153, + 0.0009087079961318523, + 0.0007494289966416545, + 0.000702844001352787, + 0.001902452000649646, + 0.0007230479968711734, + 0.0006832869985373691, + 0.0006783950011595152, + 0.00070327399589587, + 0.005190780997509137, + 0.0008920629989006557, + 0.0007052569999359548, + 0.0034616859993548132, + 0.0009027089981827885, + 0.0022201649990165606, + 0.0007084669996402226, + 0.0007376709982054308, + 0.0006867730044177733, + 0.0006797189998906106, + 0.005230715003563091, + 0.0009112719999393448, + 0.000815441002487205, + 0.0006780999974580482, + 0.0006866279945825227, + 0.0006723890037392266, + 0.005616223003016785, + 0.0008412009992753156, + 0.0007267809996847063, + 0.0006852669976069592, + 0.0006903789981151931, + 0.006091538998589385, + 0.0009737299988046288, + 0.0007210430048871785, + 0.0006902410023030825, + 0.0006961269973544404, + 0.005290292996505741, + 0.0008242100011557341, + 0.0008306529998662882, + 0.0006930720046511851, + 0.0006762459961464629, + 0.0006921829990460537, + 0.0006861619986011647, + 0.0006841010035714135, + 0.0006810150007368065, + 0.0006723330006934702, + 0.0006782819982618093, + 0.0006768980019842274, + 0.006596188999537844, + 0.000919655998586677, + 0.0007246750028571114, + 0.0006889689975650981, + 0.0006798319955123588, + 0.0006720399978803471, + 0.005219725004280917, + 0.0008396979974349961, + 0.0007689699996262789, + 0.0006852070000604726, + 0.0006828020050306804, + 0.0006788080063415691, + 0.004055716999573633, + 0.000846512004500255, + 0.0016434919962193817, + 0.0006885229959152639, + 0.0006782359996577725, + 0.0006871969962958246, + 0.0029133729985915124, + 0.0008294500003103167, + 0.001203034000354819, + 0.0007630440013599582, + 0.000751017993025016, + 0.0006791370033170097, + 0.0006889659998705611, + 0.0006898530045873486, + 0.004213459003949538, + 0.0010181969992117956, + 0.0008593070015194826, + 0.0006856960026198067, + 0.0006748019950464368, + 0.0007484379966626875, + 0.0007289580025826581, + 0.000738310998713132, + 0.0034082240017596632, + 0.0008494829962728545, + 0.000776590000896249, + 0.001259662996744737, + 0.0006905920017743483, + 0.0012289579972275533, + 0.000685144004819449, + 0.002149034000467509, + 0.0007475829988834448, + 0.0006964900021557696, + 0.0006933309996384196, + 0.0006761490003555082, + 0.0048670210017007776, + 0.0008394500036956742, + 0.0007035880043986253, + 0.0006794599976274185, + 0.0006767599988961592, + 0.006080268001824152, + 0.0009304570048698224, + 0.0006987810047576204, + 0.0006869270000606775, + 0.0006958629965083674, + 0.005236438999418169, + 0.0008326259994646534, + 0.0007064410019665956, + 0.0006727880027028732, + 0.0006837069959146902, + 0.005553811002755538, + 0.0020070940008736216, + 0.0007699159978074022, + 0.0007866730011301115, + 0.0006747890001861379, + 0.0035857870025211014, + 0.0008065969959716313, + 0.0007472510042134672, + 0.0006813910003984347, + 0.0006746689978172071, + 0.003838793003524188, + 0.0008327440009452403, + 0.0007404409989248961, + 0.0007590080058434978, + 0.0006722549951518886, + 0.0032885819964576513, + 0.0008321939967572689, + 0.0007692299986956641, + 0.0006855060055386275, + 0.0006880729997646995, + 0.0006753459965693764, + 0.013277963000291493, + 0.014996954996604472, + 0.01400588000251446, + 0.013997317997564096, + 0.014990302996011451, + 0.0119932469970081, + 0.015031267997983377, + 0.014960679000068922, + 0.014997362995927688, + 0.014998681006545667, + 0.009994018000725191, + 0.0139999489983893, + 0.014027012002770789, + 0.01402186400082428, + 0.014922634996764828, + 0.011998909998510499, + 0.015992255001037847, + 0.017012759002682287, + 0.015979450006852858, + 0.015995595000276808, + 0.01399729099648539, + 0.015997946997231338, + 0.00816186799784191, + 0.013839431005180813, + 0.011990033999609295, + 0.01399113599472912, + 0.015995378002116922, + 0.014997542995843105, + 0.012995884993870277, + 0.012996395998925436, + 0.014997926999058109, + 0.016995944002701435, + 0.012996407000173349, + 0.012142337996920105, + 0.0010720740028773434, + 0.0007739220018265769, + 0.00921738900069613, + 0.0010049550037365407, + 0.00835946699953638, + 0.001091660000383854, + 0.008205612997699063, + 0.00101242600067053, + 0.007878041004005354, + 0.001033660002576653, + 0.008108693997201044, + 0.0011257290025241673, + 0.0070560599997406825, + 0.0010155300042242743, + 0.0008834160034894012, + 0.007685147997108288, + 0.0009773570054676384, + 0.0073984570044558495, + 0.0010255140005028807, + 0.0008422340033575892, + 0.007080301998939831, + 0.0010176259966101497, + 0.0009207919938489795, + 0.003618520997406449, + 0.0008373159944312647, + 0.0006624589950661175, + 0.0006661289953626692, + 0.0006601030036108568, + 0.0053736840054625645, + 0.0009387800018885173, + 0.0008009119992493652, + 0.0006687670029350556, + 0.0006622039945796132, + 0.0006656810001004487, + 0.0006609379997826181, + 0.0006610670025111176, + 0.003530932001012843, + 0.0010325059993192554, + 0.000679892000334803, + 0.000677563002682291, + 0.002730662999965716, + 0.0008024830021895468, + 0.0006642020016442984, + 0.0006685269981971942, + 0.002612967000459321, + 0.0008180069999070838, + 0.0007221660052891821, + 0.0037159029961912893, + 0.0008379869977943599, + 0.0007986510026967153, + 0.0022608359940932132, + 0.0007860870027798228, + 0.000665643994580023, + 0.0006730829991283827, + 0.0006669259964837693, + 0.0006608140029129572, + 0.006249673999263905, + 0.000901301005796995, + 0.0007788880029693246, + 0.0006685859989374876, + 0.0006551970000145957, + 0.005126595999172423, + 0.0014320670015877113, + 0.0006652900046901777, + 0.0006679729995084926, + 0.0006613799996557645, + 0.0006604529989999719, + 0.0006652240044786595, + 0.000651909998850897, + 0.00351586800388759, + 0.0014690399984829128, + 0.0007013659997028299, + 0.0007391670005745254, + 0.0006547949960804544, + 0.0006645640023634769, + 0.0006578989996341988, + 0.000658456003293395, + 0.0027067019982496276, + 0.000782657996751368, + 0.0031238439987646416, + 0.000808352000603918, + 0.0006871520017739385, + 0.0006570369951077737, + 0.004035136000311468, + 0.0008187990024453029, + 0.0007005279985605739, + 0.0006751819964847527, + 0.0006659170030616224, + 0.00629470800049603, + 0.0008239150047302246, + 0.0007601379984407686, + 0.0006842779985163361, + 0.0006582579953828827, + 0.0006632799995713867, + 0.006981098995311186, + 0.0008957399986684322, + 0.0007338850045925938, + 0.000655679999908898, + 0.00065670700132614, + 0.005755550999310799, + 0.0008629369986010715, + 0.0007685009986744262, + 0.0006624219968216494, + 0.0006584259972441941, + 0.0006555549989570864, + 0.00433205500303302, + 0.0019576179984142072, + 0.0009757139996509068, + 0.001187513000331819, + 0.0006658679994870909, + 0.0012016670007142238, + 0.0006727420040988363, + 0.0017837729974417016, + 0.0006780079929740168, + 0.0006643629967584275, + 0.0006728230000589974, + 0.0006652550000580959, + 0.0006663370004389435, + 0.002592982003989164, + 0.0008373349992325529, + 0.0026158489999943413, + 0.0023612910008523613, + 0.0007161040048231371, + 0.00067721900268225, + 0.0006851099969935603, + 0.005526471002667677, + 0.0007175790015025996, + 0.0006839790003141388, + 0.001467125999624841, + 0.0020381970025482588, + 0.0010490800050320104, + 0.001983059002668597, + 0.0010545699988142587, + 0.000693579000653699, + 0.0006772519991500303, + 0.0006753790003131144, + 0.0006721170066157356, + 0.002581227003247477, + 0.0007170859971665777, + 0.0006907110000611283, + 0.002507149998564273, + 0.0006870400029583834, + 0.0019304599991301075, + 0.0006960670070839114, + 0.0006807949976064265, + 0.0006779269970138557, + 0.0006863909948151559, + 0.0011748649994842708, + 0.002878939005313441, + 0.0007106350021786056, + 0.0006847620024927892, + 0.0006700459998683073, + 0.0013088090054225177, + 0.0020193320015096106, + 0.0007081650037434883, + 0.0018973309997818433, + 0.0009361350021208636, + 0.0017246229981537908, + 0.0007085549950716086, + 0.0006753330017090775, + 0.0006766779988538474, + 0.0006891470038681291, + 0.0006870720026199706, + 0.005214709999563638, + 0.0007031560016912408, + 0.000689591994159855, + 0.0014511680055875331, + 0.0007158640000852756, + 0.0006776890004402958, + 0.0006778859969926998, + 0.000671159999910742, + 0.002394701004959643, + 0.0013668770043295808, + 0.0017691259999992326, + 0.0007009969995124266, + 0.0006871260047773831, + 0.002504557000065688, + 0.0010904869996011257, + 0.0006928770017111674, + 0.0006915250050951727, + 0.001246539999556262, + 0.0010909169941442087, + 0.000688129999616649, + 0.0022335239991662093, + 0.0006951329996809363, + 0.00223937799455598, + 0.0007098260030033998, + 0.0006957570003578439, + 0.0007053109948174097, + 0.0006876989937154576, + 0.002564862006693147, + 0.0006882289962959476, + 0.001398373999109026, + 0.0006695260017295368, + 0.0006807380050304346, + 0.0014034650012035854, + 0.0008949980037868954, + 0.0006754249989171512, + 0.0006810500053688884, + 0.001405041002726648, + 0.002864344001864083, + 0.0007108950012479909, + 0.0006898870051372796, + 0.000678028998663649, + 0.001494257005106192, + 0.0006964130006963387, + 0.0007005590014159679, + 0.0006871120058349334, + 0.0015778640008647926, + 0.00221269199391827, + 0.0006781219999538735, + 0.0006867629999760538, + 0.0027849890029756352, + 0.0007597730000270531, + 0.0006855060055386275, + 0.0007122509996406734, + 0.0020545329971355386, + 0.0011135120003018528, + 0.0006825899981777184, + 0.0006785109944758005, + 0.0006781219999538735, + 0.0031887820005067624, + 0.0007630219988641329, + 0.0007112840030458756, + 0.003957031003665179, + 0.000749092003388796, + 0.0006988620007177815, + 0.0006892059973324649, + 0.004495975997997448, + 0.0007085629986249842, + 0.001034615001117345, + 0.0017626250046305358, + 0.0006837410037405789, + 0.0006739859964000061, + 0.0006863849994260818, + 0.0037853359972359613, + 0.0009548950038151816, + 0.0006877539999550208, + 0.0006787080055801198, + 0.0006861049987492152, + 0.0022048329992685467, + 0.0007115670014172792, + 0.002972616995975841, + 0.0007278300035977736, + 0.0006941309984540567, + 0.0006766689984942786, + 0.004057909005496185, + 0.0007205699948826805, + 0.0006775490037398413, + 0.0006850369973108172, + 0.001393377999193035, + 0.00070643299841322, + 0.000680536002619192, + 0.0006866859985166229, + 0.0006690630034427159, + 0.0057137350013363175, + 0.0006828359983046539, + 0.000672313995892182, + 0.0038154860012582503, + 0.000689309999870602, + 0.0006728130028932355, + 0.0007057580005493946, + 0.0018902369993156753, + 0.0006944979977561161, + 0.0006762630000594072, + 0.0006833129955339245, + 0.003848602995276451, + 0.0006970559988985769, + 0.0006809239930589683, + 0.0006803099968237802, + 0.0008943220018409193, + 0.0033171359973493963, + 0.000700976001098752, + 0.000823524002043996, + 0.0006887830022606067, + 0.0006752499975846149, + 0.002297744002135005, + 0.0006994849973125383, + 0.0028659050003625453, + 0.0007681380011490546, + 0.003481259998807218, + 0.000715172995114699, + 0.0006863460002932698, + 0.000680360994010698, + 0.0011495290018501692, + 0.0006909040021128021, + 0.0030735059990547597, + 0.0006914670011610724, + 0.0006749940002919175, + 0.0006723510014126077, + 0.0009121270049945451, + 0.0019783509997068904, + 0.0007132620012271218, + 0.0006877160049043596, + 0.0006825110031059012, + 0.0006736649957019836, + 0.0031842519965721294, + 0.001590730003954377, + 0.0006900720036355779, + 0.000681030003761407, + 0.0006787180027458817, + 0.005444091999379452, + 0.0007174739948823117, + 0.0006827769975643605, + 0.000681227000313811, + 0.0006828810001024976, + 0.003960244001063984, + 0.0006969610039959662, + 0.0006755590002285317, + 0.0016811459936434403, + 0.0007047300023259595, + 0.0024264460007543676, + 0.001438552004401572, + 0.0007114920008461922, + 0.0007034739974187687, + 0.0006976500008022413, + 0.0010882770002353936, + 0.0010688300026231445, + 0.0006971140028326772, + 0.0006982889972277917, + 0.0007098669957485981, + 0.005251219998172019, + 0.0007156339997891337, + 0.0006982740014791489, + 0.0029132030031178147, + 0.0007328459978452884, + 0.0022250499969231896, + 0.0007178769956226461, + 0.0014682149994769134, + 0.0010176200012210757, + 0.0007034320005914196, + 0.0006974119969527237, + 0.0027817829977720976, + 0.002993792004417628, + 0.0007249809932545759, + 0.0013100010037305765, + 0.0017852689998107962, + 0.0007438800021191128, + 0.0006969719979679212, + 0.0007084440003382042, + 0.002715000999160111, + 0.0019542000009096228, + 0.0007005899969954044, + 0.0007002620041021146, + 0.0012344019996817224, + 0.0035043480020249262, + 0.0007569090012111701, + 0.0007075620014802553, + 0.0030635959992650896, + 0.0007067620026646182, + 0.0006918280050740577, + 0.0016197620061575435, + 0.002282294000906404, + 0.0014180979997036047, + 0.0007271309950738214, + 0.0025243920026696287, + 0.0007452229983755387, + 0.0007058269984554499, + 0.0006961140024941415, + 0.0006890700024086982, + 0.0018224250015919097, + 0.0031207190040731803, + 0.0007449479962815531, + 0.0008608069983893074, + 0.0020296660004532896, + 0.0007447489988408051, + 0.0006992649941821583, + 0.0006991369955358095, + 0.0032927349966485053, + 0.0007206199952634051, + 0.0016941339999902993, + 0.0013757519991486333, + 0.0008682800034875982, + 0.0007359400042332709, + 0.0007009140026639216, + 0.000693074005539529, + 0.003865333004796412, + 0.001095166000595782, + 0.0007157270010793582, + 0.0014535279988194816, + 0.0018351059989072382, + 0.000734460998501163, + 0.0007044180019875057, + 0.0006894530015415512, + 0.0030038519980735146, + 0.0013576779965660535, + 0.0007181300024967641, + 0.0007024599981377833, + 0.0012384410001686774, + 0.0022200050007086247, + 0.0007282540027517825, + 0.00417627199931303, + 0.0007483500012313016, + 0.0007085430042934604, + 0.005535593998502009, + 0.0007870209956308827, + 0.0011025339990737848, + 0.0017045849963324144, + 0.0034600380022311583, + 0.002302426000824198, + 0.0007047610051813535, + 0.0006993639981374145, + 0.0032406310056103393, + 0.001342467003269121, + 0.0053512369995587505, + 0.0011755170053220354, + 0.0007462610010406934, + 0.0007037510004010983, + 0.0012827310056309216, + 0.0014206860068952665, + 0.0007302849990082905, + 0.0007054640009300783, + 0.000703749006788712, + 0.00156170800619293, + 0.0019955909956479445, + 0.0007022179997875355, + 0.0007006819942034781, + 0.0009914190013660118, + 0.003422150999540463, + 0.000715813999704551, + 0.0006959899983485229, + 0.000692592999257613, + 0.00334922899492085, + 0.0007189200041466393, + 0.0007022279969532974, + 0.0006999600009294227, + 0.0011550870040082373, + 0.0016998030041577294, + 0.0006975960059207864, + 0.0007036309980321676, + 0.0027305390030960552, + 0.0007340619995375164, + 0.0018107630021404475, + 0.0007106850025593303, + 0.0007229940019897185, + 0.0014076110019232146, + 0.00201169300271431, + 0.0007083230011630803, + 0.0007144949995563366, + 0.0007097619964042678, + 0.0006997029995545745, + 0.0036892650023219176, + 0.0025281889975303784, + 0.0007388489975710399, + 0.0014548940016538836, + 0.0013216720035416074, + 0.0007065530007821508, + 0.0006958200028748251, + 0.0007154269987950101, + 0.003230203001294285, + 0.0015059009965625592, + 0.0010338199936086312, + 0.0006942510008229874, + 0.0006971030015847646, + 0.0025428749941056594, + 0.0007087889971444383, + 0.0006974169955356047, + 0.0006982269987929612, + 0.0015545970018138178, + 0.0014501300029223785, + 0.000706057995557785, + 0.0006919500010553747, + 0.0006922200045664795, + 0.004539338005997706, + 0.0007425009971484542, + 0.0007006849991739728, + 0.0024432509962935, + 0.0036128540014033206, + 0.0007056879985611886, + 0.0037446940041263588, + 0.0033494610033812933, + 0.0007768559953547083, + 0.0007169829987105913, + 0.0007041470016702078, + 0.0009051700035342947, + 0.003812253999058157, + 0.0020949229947291315, + 0.0007240609993459657, + 0.0007075110042933375, + 0.0007015649971435778, + 0.0007105870026862249, + 0.0032772599952295423, + 0.0007912270011729561, + 0.0007335129994316958, + 0.0007059319977997802, + 0.0006990180045249872, + 0.006193378001626115, + 0.0008627930001239292, + 0.0007589070009998977, + 0.0006915699996170588, + 0.0006897889979882166, + 0.003330450002977159, + 0.000906525005120784, + 0.000781043003371451, + 0.0006982650011195801, + 0.000694044996635057, + 0.0006898470019223168, + 0.004675897995184641, + 0.009329692999017425, + 0.015998779999790713, + 0.013996175002830569, + 0.017996565002249554, + 0.014997022000898141, + 0.013002581996261142, + 0.01399139500426827, + 0.017008114999043755, + 0.013989232000312768, + 0.015991132997442037, + 0.014999168000940699, + 0.012994498996704351, + 0.013993189000757411, + 0.013999613001942635, + 0.014055856001505163, + 0.014932019999832846, + 0.013996579000377096, + 0.010168239998165518, + 0.00882603400532389, + 0.0189960389980115, + 0.014001965995703358, + 0.015009688002464827, + 0.0008891190009308048, + 0.000827873001981061, + 0.0006866819967399351, + 0.0006826600001659244, + 0.0006736900031683035, + 0.0018607809979585, + 0.0007085880060913041, + 0.004029736999655142, + 0.0008575549945817329, + 0.0007179370004450902, + 0.0006744559941580519, + 0.0006819109985372052, + 0.004156554001383483, + 0.0020543280043057166, + 0.0006963719933992252, + 0.000680156001180876, + 0.0006794840010115877, + 0.0006738670053891838, + 0.000667729000269901, + 0.0026178709958912805, + 0.0008472560002701357, + 0.0006976689983275719, + 0.0006826259996159934, + 0.0006797639944124967, + 0.0057821380032692105, + 0.0008597550040576607, + 0.0007626489968970418, + 0.0022597330025746487, + 0.0007829230016795918, + 0.0007823560008546337, + 0.0006848449993412942, + 0.0006732479960191995, + 0.0006814540029154159, + 0.01721084000018891, + 0.013990293999086134, + 0.015001831998233683, + 0.014985650996095501, + 0.012996694997127634, + 0.014998753998952452, + 0.01199634500517277, + 0.013995281005918514, + 0.008998073004477192, + 0.01000490099977469, + 0.01698785299959127, + 0.013086255996313412, + 0.012908202996186446, + 0.010967248999804724, + 0.010027083000750281, + 0.016007022000849247, + 0.014995539997471496, + 0.012101795000489801, + 0.013976592999824788, + 0.013912536996940617, + 0.015983441000571474, + 0.011993097999948077, + 0.008997261997137684, + 0.010995410004397854, + 0.013998877999256365, + 0.010957386002701242, + 0.013034584997512866, + 0.01299632099835435, + 0.01399909100291552, + 0.017210156001965515, + 0.011780458000430372, + 0.015006030000222381, + 0.008204630998079665, + 0.013781535002635792, + 0.015449698999873362, + 0.008544001000700518, + 0.010570547005045228, + 0.011469051998574287, + 0.007524071996158455, + 0.007418832996336278, + 0.009999041001719888, + 0.0129947710011038, + 0.01499587600119412, + 0.01299915999697987, + 0.014997711004980374, + 0.013997406997077633, + 0.013463531999150291, + 0.010525315999984741, + 0.01399249200039776, + 0.014995074001490138, + 0.01299564799410291, + 0.014000293005665299, + 0.009335827999166213, + 0.014654366998001933, + 0.013995138004247565, + 0.01202050700521795, + 0.011971794003329705, + 0.01098796000587754, + 0.01317414699587971, + 0.011819847000879236, + 0.014992199998232536, + 0.015996132999134716, + 0.015996570997231174, + 0.00990859200101113, + 0.01108338100311812, + 0.013997317000757903, + 0.015999062998162117, + 0.012997612000617664, + 0.010843969997949898, + 0.011147671000799164, + 0.010996580997016281, + 0.01499785600026371, + 0.011997271001746412, + 0.014997639998910017, + 0.014997935999417678, + 0.011010803005774505, + 0.013004902000830043, + 0.016166642999451142, + 0.008060742002271581, + 0.015736809000372887, + 0.013995386005262844, + 0.0159973620029632, + 0.011679576004098635, + 0.008315186998515856, + 0.012249552004504949, + 0.009710174002975691, + 0.010061015003884677, + 0.008966413995949551, + 0.01699818899942329, + 0.010994519994710572, + 0.013997187998029403, + 0.013000286002352368, + 0.009993918996769935, + 0.01399671500257682, + 0.013998062000609934, + 0.012997527002880815, + 0.013992230000440031, + 0.012705887995252851, + 0.012662977998843417, + 0.011630150002019946, + 0.011756139996577986, + 0.000875347999681253, + 0.0006930349991307594, + 0.0006804830045439303, + 0.0006741949982824735, + 0.007205429006717168, + 0.0009332019981229678, + 0.0008181290031643584, + 0.0006819479967816733, + 0.006743777004885487, + 0.0008641840031486936, + 0.0007692069993936457, + 0.0006661229999735951, + 0.0006698919969494455, + 0.0006707959983032197, + 0.00899465899419738, + 0.0008899239983293228, + 0.0007529919967055321, + 0.0006819629998062737, + 0.0006668620044365525, + 0.007933629996841773, + 0.0008920649997889996, + 0.0007031060013105161, + 0.0006765350044588558, + 0.000675780996971298, + 0.0035401640052441508, + 0.000847084003908094, + 0.0007730670040473342, + 0.0006847350014140829, + 0.0006722700054524466, + 0.0006733939953846857, + 0.005794488999526948, + 0.0014429130023927428, + 0.0006988960012677126, + 0.0006843029987066984, + 0.0006741430042893626, + 0.0006623540029977448, + 0.0006736630020895973, + 0.004057046004163567, + 0.0007998669971129857, + 0.0006957369987503625, + 0.0006773969944333658, + 0.0006963589985389262, + 0.0049242930035688914, + 0.0008505189980496652, + 0.0007333890025620349, + 0.0006777140006306581, + 0.0006943820044398308, + 0.0056434940051985905, + 0.0008966950044850819, + 0.0008442089965683408, + 0.0006760719988960773, + 0.004272825004591141, + 0.0013183129995013587, + 0.0006868929995107464, + 0.0006716029965900816, + 0.0006728500011377037, + 0.0006724479972035624, + 0.0034345449967077, + 0.0008453020054730587, + 0.0007633400018676184, + 0.0006711539972457103, + 0.0006811790008214302, + 0.0006779220057069324, + 0.005257974000414833, + 0.0008752130015636794, + 0.0007913080044090748, + 0.0006692220049444586, + 0.0006719960001646541, + 0.0006756380025763065, + 0.005556652999075595, + 0.0008274159990833141, + 0.000739420996978879, + 0.0006862189984531142, + 0.0006738439988112077, + 0.0006695960037177429, + 0.005007834995922167, + 0.0009048799984157085, + 0.0008181640005204827, + 0.0006763519995729439, + 0.003470529001788236, + 0.0008065370057011023, + 0.0007425760049954988, + 0.0006795509980292991, + 0.0006654489989159629, + 0.014282055999501608, + 0.014998681996075902, + 0.013997670997923706, + 0.014995279998402111, + 0.013994829001603648, + 0.015999165996618103, + 0.013994423003168777, + 0.014996181002061348, + 0.013996458001201972, + 0.013998007001646329, + 0.014002175004861783, + 0.015000366998719983, + 0.014989848001278006, + 0.01301359900389798, + 0.013992136002343614, + 0.01115384699369315, + 0.007882895995862782, + 0.0009154349972959608, + 0.0007981029993970878, + 0.000671084999339655, + 0.0006660069993813522, + 0.0006613890000153333, + 0.0017708480008877814, + 0.0006780450057704002, + 0.0006609499978367239, + 0.0006655770048382692, + 0.0007110980004654266, + 0.0008024959970498458, + 0.0006791150008211844, + 0.0006587739990209229, + 0.0006562670023413375, + 0.0008436389980488457, + 0.0007920400021248497, + 0.0007625349971931428, + 0.0008127940018312074, + 0.0007400390022667125, + 0.0006790990009903908, + 0.0006680110018351115, + 0.0007353190012509003, + 0.0006654529934166931, + 0.000694153000949882, + 0.0033360279994667508, + 0.0007754809994366951, + 0.0007598259981023148, + 0.0006938399965292774, + 0.000667038002575282, + 0.0006820150010753423, + 0.0031000509989098646, + 0.0007829739988665096, + 0.000700103999406565, + 0.000666849002300296, + 0.004227871002512984, + 0.0008113330040941946, + 0.0008093090000329539, + 0.0036166319987387396, + 0.0008273060011561029, + 0.0007655360022909008, + 0.0007472560027963482, + 0.0006787840029573999, + 0.0006730309978593141, + 0.0006718459990224801, + 0.002792012004647404, + 0.001506660002632998, + 0.0007012249989202246, + 0.001143160006904509, + 0.0008142529986798763, + 0.0006782769996789284, + 0.0006730219974997453, + 0.0006832569997641258, + 0.0006713780021527782, + 0.000667309999698773, + 0.0008310639968840405, + 0.0006789430044591427, + 0.000673335998726543, + 0.0006651549992966466, + 0.0006756209986633621, + 0.009154687002592254, + 0.0008588060009060428, + 0.0006862579975859262, + 0.0006687319983029738, + 0.0006766899969079532, + 0.007199638006568421, + 0.00090012900182046, + 0.000712305998604279, + 0.000679506003507413, + 0.0006677390047116205, + 0.006892980003613047, + 0.0008698439996805973, + 0.000689045999024529, + 0.0006791509949835017, + 0.0006694670009892434, + 0.007896093004092108, + 0.0009740610039443709, + 0.0007165659990278073, + 0.0006809689948568121, + 0.000689997999870684, + 0.007670699997106567, + 0.0009882910017040558, + 0.0007867530002840795, + 0.000679896998917684, + 0.005662076000589877, + 0.002210350001405459, + 0.000745679993997328, + 0.0006926639980520122, + 0.0006751970067853108, + 0.0006753020061296411, + 0.0006773080021957867, + 0.005479168001329526, + 0.0008371859948965721, + 0.0007276300020748749, + 0.0006576129962923005, + 0.0006620429994654842, + 0.0006503039985545911, + 0.006897346000187099, + 0.0008572070000809617, + 0.0007476390019292012, + 0.0006576960004167631, + 0.0006486919955932535, + 0.0049372170033166185, + 0.0010415380020276643, + 0.0007120220034266822, + 0.0006639359999098815, + 0.0006542799965245649, + 0.0006517350047943182, + 0.00294785900041461, + 0.0008692920018802397, + 0.000693622998369392, + 0.0006526090000988916, + 0.0006565809962921776, + 0.0006441649966291152, + 0.006381900006090291, + 0.0008673100019223057, + 0.0007240010017994791, + 0.0006595669983653352, + 0.0006546989970956929, + 0.0006510179955512285, + 0.005161802997463383, + 0.0008239210001192987, + 0.0007591289977426641, + 0.0006559510002261959, + 0.0006503399999928661, + 0.0006628459959756583, + 0.006093924996093847, + 0.0008499609975842759, + 0.0006861099973320961, + 0.0006622689979849383, + 0.0006520099996123463, + 0.004953345000103582, + 0.0008464999991701916, + 0.0006593030047952197, + 0.0006519840026157908, + 0.0006503330005216412, + 0.0048150570000871085, + 0.0008382349988096394, + 0.008344993002538104, + 0.0008942290005506948, + 0.0007426170050166547, + 0.0006680410006083548, + 0.0006727979998686351, + 0.006251193000935018, + 0.0010079470011987723, + 0.0007276050018845126, + 0.0006617379985982552, + 0.0006751940018148161, + 0.003866783998091705 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil", + "fullname": "TestApplyDiffusionToWAndComputeHorizontalGradientsForTurbulence", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.0006457909985329024, + "max": 0.04000352900038706, + "mean": 0.00400752713244851, + "stddev": 0.008077417551711983, + "rounds": 1314, + "median": 0.0008514215005561709, + "iqr": 0.0011078840034315363, + "q1": 0.0007288289998541586, + "q3": 0.0018367130032856949, + "iqr_outliers": 231, + "stddev_outliers": 127, + "outliers": "127;231", + "ld15iqr": 0.0006457909985329024, + "hd15iqr": 0.0035382470014155842, + "ops": 249.5304378361182, + "total": 5.265890652037342, + "data": [ + 0.00077442799374694, + 0.0018532749963924289, + 0.000916031000087969, + 0.0014507240048260428, + 0.0007920850039226934, + 0.0009337419978692196, + 0.0007719889981672168, + 0.0007630020045326091, + 0.0017018709986587055, + 0.0007858419994590804, + 0.0007685279997531325, + 0.0007649699982721359, + 0.002698370997677557, + 0.0008242500043706968, + 0.0008363980014109984, + 0.000772204999520909, + 0.0011583710002014413, + 0.0008893339982023463, + 0.0011884140039910562, + 0.0008578499982832, + 0.0007884509977884591, + 0.0007802230029483326, + 0.0010829520033439621, + 0.0008356420003110543, + 0.0007971919985720888, + 0.002005868998821825, + 0.0007452229983755387, + 0.0007695200038142502, + 0.0018337970032007433, + 0.0007786970018059947, + 0.0007794650009600446, + 0.0007438609973178245, + 0.004853819998970721, + 0.0008506919984938577, + 0.0007837379962438717, + 0.0009050289954757318, + 0.0021129099986865185, + 0.0008086169982561842, + 0.0007708449993515387, + 0.0008593879974796437, + 0.0007667430036235601, + 0.0010544500037212856, + 0.0007558780052931979, + 0.0019881429980159737, + 0.0028000740057905205, + 0.0008560700007365085, + 0.0007617170049343258, + 0.0007421849950333126, + 0.0007415680011035874, + 0.0008956780002336018, + 0.0008545570017304271, + 0.001191261995700188, + 0.0007476600003428757, + 0.0009343130004708655, + 0.0016764739993959665, + 0.0008163669990608469, + 0.0009733849947224371, + 0.0007512420052080415, + 0.0007441779962391593, + 0.0007492419972550124, + 0.0008181570010492578, + 0.0008682680054334924, + 0.0009551260009175166, + 0.0009213390003424138, + 0.002339077000215184, + 0.0007688530022278428, + 0.0007785319976392202, + 0.0018367130032856949, + 0.0008795390021987259, + 0.0008219679948524572, + 0.0008295689985970967, + 0.0007565719934063964, + 0.0007433790015056729, + 0.0007411580008920282, + 0.0008488100065733306, + 0.0007455790037056431, + 0.0007494259989471175, + 0.0007503329979954287, + 0.0007426129959640093, + 0.0008702639970579185, + 0.0009202859946526587, + 0.0009092010004678741, + 0.000760117000027094, + 0.0007511299991165288, + 0.0007392469997284934, + 0.0007466070019290783, + 0.000848846000735648, + 0.0015564259956590831, + 0.0015150720064411871, + 0.0011640529992291704, + 0.000757408000936266, + 0.0007987050048541278, + 0.000750163002521731, + 0.0008558799963793717, + 0.0008651620009914041, + 0.0007520660001318902, + 0.0007478029947378673, + 0.0007411379992845468, + 0.0008190419976017438, + 0.0019483899959595874, + 0.0007787079957779497, + 0.002816832005919423, + 0.0019585240006563254, + 0.0008283209972432815, + 0.0007485649985028431, + 0.0008089760012808256, + 0.0008789220009930432, + 0.0009139450048678555, + 0.0011081039992859587, + 0.000762791998567991, + 0.0007479010018869303, + 0.0010326370029360987, + 0.0008040210013859905, + 0.000754628999857232, + 0.00093315100093605, + 0.000756075001845602, + 0.0007589550004922785, + 0.000741356001526583, + 0.0007411319966195151, + 0.0015166270022746176, + 0.0016136970007210039, + 0.002023695000389125, + 0.000800528003310319, + 0.0008215549969463609, + 0.0007185770009527914, + 0.0007120410009520128, + 0.0007149239972932264, + 0.0007175350037869066, + 0.0008745779996388592, + 0.000908551002794411, + 0.0007289029963430949, + 0.0007185400027083233, + 0.0008399680009461008, + 0.0009604419974493794, + 0.0008145789979607798, + 0.0007191789991338737, + 0.0007259370031533763, + 0.0007298319978872314, + 0.001548826003272552, + 0.0007808060036040843, + 0.0007403460040222853, + 0.0015896939949016087, + 0.0009177940009976737, + 0.002340906001336407, + 0.000743151998904068, + 0.0007209320028778166, + 0.0015914079995127395, + 0.0011069760003010742, + 0.0007263990046340041, + 0.0007231089984998107, + 0.000729876002878882, + 0.0007269080015248619, + 0.02465823999955319, + 0.03199898800085066, + 0.03202362199954223, + 0.02895602199714631, + 0.033997705999354366, + 0.029978872000356205, + 0.030989227001555264, + 0.028998112997214776, + 0.027002154994988814, + 0.03201166300277691, + 0.025977030003559776, + 0.030009289002919104, + 0.02198553700145567, + 0.02998564000154147, + 0.030999986003735103, + 0.031989271999918856, + 0.026994284002284985, + 0.03000758800044423, + 0.02898513300169725, + 0.028993122999963816, + 0.03200163899600739, + 0.029867540994018782, + 0.02811903799738502, + 0.02899829300440615, + 0.027995643999020103, + 0.02699330799805466, + 0.027995220996672288, + 0.029006511998886708, + 0.025989943998865783, + 0.02799480400426546, + 0.024036424998485018, + 0.027961279003648087, + 0.027989342997898348, + 0.028983084004721604, + 0.02800291400490096, + 0.027210609005123843, + 0.0239983739957097, + 0.02876254199509276, + 0.02793400800146628, + 0.027989852002065163, + 0.028001939004752785, + 0.01637267199839698, + 0.01802093100559432, + 0.030583466003008652, + 0.014996969999629073, + 0.026431767997564748, + 0.011537070000485983, + 0.027014577004592866, + 0.029001667993725277, + 0.031002421004814096, + 0.027971392999461386, + 0.02702135399886174, + 0.03154648100462509, + 0.03741034400445642, + 0.022994439001195133, + 0.04000352900038706, + 0.028225814996403642, + 0.03277729499677662, + 0.031003878997580614, + 0.03098092800064478, + 0.029967089998535812, + 0.031014061998575926, + 0.0310600919983699, + 0.02893962000234751, + 0.027963341999566182, + 0.02598815400415333, + 0.035991705997730605, + 0.028016821001074277, + 0.02896342799795093, + 0.027974069002084434, + 0.023002527996140998, + 0.023997733995201997, + 0.02398909500334412, + 0.025983822997659445, + 0.03004683699691668, + 0.030911196998204105, + 0.026990464997652452, + 0.02799783699447289, + 0.02698978199623525, + 0.02399986799719045, + 0.02399632900051074, + 0.0310335159956594, + 0.026946876998408698, + 0.029993367003044114, + 0.027999666999676265, + 0.026987857003405225, + 0.029017818000284024, + 0.028984150994801894, + 0.021976326002913993, + 0.02523916499922052, + 0.026750123004603665, + 0.015059836994623765, + 0.026929134997772053, + 0.027021095003874507, + 0.029976065998198465, + 0.02905906399973901, + 0.03192309199948795, + 0.030002460996911395, + 0.03205992400035029, + 0.0319276149966754, + 0.026703584000642877, + 0.03028732500388287, + 0.029992390002007596, + 0.030026241001905873, + 0.025957375000871252, + 0.03501569700165419, + 0.03096974900108762, + 0.027004753006622195, + 0.02698286299710162, + 0.026005093001003843, + 0.031033238003146835, + 0.035936677995778155, + 0.027973737996944692, + 0.026988376994268037, + 0.02620010799728334, + 0.03180055000120774, + 0.027470395994896535, + 0.025503179000224918, + 0.03500026700203307, + 0.031003217001853045, + 0.028974433997063898, + 0.029994186996191274, + 0.012385759000608232, + 0.0008500879994244315, + 0.0008262920018751174, + 0.000653845003398601, + 0.009072995999304112, + 0.00096381099865539, + 0.0007404780044453219, + 0.0006625099995289929, + 0.0006492940010502934, + 0.004101951002667192, + 0.0031682310000178404, + 0.0007739549982943572, + 0.0006719270022585988, + 0.0006531309991260059, + 0.0006457909985329024, + 0.0006527290024678223, + 0.0034462940020603128, + 0.0038571520053665154, + 0.0007666390010854229, + 0.004747505001432728, + 0.0007945039978949353, + 0.0006991639966145158, + 0.0006507280049845576, + 0.007521864004957024, + 0.00898761699500028, + 0.0011325230007059872, + 0.0006494389963336289, + 0.0037696390063501894, + 0.000824949995148927, + 0.0007568680011900142, + 0.0006689430010737851, + 0.0006701919992337935, + 0.005615424997813534, + 0.0008278160021291114, + 0.0020008289939141832, + 0.0007314149988815188, + 0.0007151530007831752, + 0.0030612359987571836, + 0.0007810360039002262, + 0.0007147600044845603, + 0.000674789996992331, + 0.000677301999530755, + 0.0006658189959125593, + 0.0029439330028253607, + 0.004803543000889476, + 0.0008768300031078979, + 0.000770033999287989, + 0.003814395000517834, + 0.0008620890002930537, + 0.00114027300151065, + 0.0006717399955959991, + 0.007097511996107642, + 0.0008315469967783429, + 0.009022151003591716, + 0.0008516849993611686, + 0.0007419820030918345, + 0.00979068799642846, + 0.0016720400017220527, + 0.004217012996377889, + 0.0009371859996463172, + 0.0036119430005783215, + 0.0008552729996154085, + 0.0007387130026472732, + 0.004114515002584085, + 0.002768260004813783, + 0.0008039930035010912, + 0.0007049949999782257, + 0.0008558030021958984, + 0.0006719719967804849, + 0.0006676409975625575, + 0.0025564250026945956, + 0.0008054219943005592, + 0.0032483109971508384, + 0.000822693997179158, + 0.0006907380011398345, + 0.0010807979997480288, + 0.0007646259982720949, + 0.0007137079956009984, + 0.0006826210010331124, + 0.0028439150046324357, + 0.0020209409995004535, + 0.0012242160009918734, + 0.0007009690016275272, + 0.0011031760004698299, + 0.0007243139989441261, + 0.0006771490006940439, + 0.0019919910046155564, + 0.0014774009978282265, + 0.0023540250040241517, + 0.0007795460041961633, + 0.0007646770027349703, + 0.0006833860024926253, + 0.0006997589953243732, + 0.0015452770021511242, + 0.0006883440000819974, + 0.0006649679999100044, + 0.00132683999981964, + 0.0029916959974798374, + 0.0008137109980452806, + 0.0009060409938683733, + 0.0027108759968541563, + 0.0008711340051377192, + 0.0006964610001887195, + 0.0007139139997889288, + 0.0018121770044672303, + 0.0012501399978646077, + 0.0006702359969494864, + 0.0007048719999147579, + 0.0006855030005681328, + 0.003468365001026541, + 0.0035382470014155842, + 0.0015514459955738857, + 0.0009069780062418431, + 0.000699462994816713, + 0.006473078996350523, + 0.0008490850013913587, + 0.0006795580047764815, + 0.0009018310011015274, + 0.007294235001609195, + 0.0008552720028092153, + 0.0007149770026444457, + 0.0006824739975854754, + 0.0007185420035966672, + 0.00671289800084196, + 0.0008571310027036816, + 0.0008019009983399883, + 0.0006720510064042173, + 0.0006756680013495497, + 0.00672603900602553, + 0.0008501609991071746, + 0.0008109460031846538, + 0.0010050950004369952, + 0.0007310869987122715, + 0.007393046995275654, + 0.0008725840016268194, + 0.0009661719959694892, + 0.0007280279969563708, + 0.0007310590008273721, + 0.006418038996343967, + 0.0008431449969066307, + 0.0007222629938041791, + 0.000673677001032047, + 0.0007347430000663735, + 0.0028638779986067675, + 0.0010597999935271218, + 0.001825813997129444, + 0.0008024890048545785, + 0.0007628160019521601, + 0.0021887519978918135, + 0.0007098460046108812, + 0.0007186380025814287, + 0.000673255002766382, + 0.0008141409998643212, + 0.0008022689944482408, + 0.0006926049973117188, + 0.0008435440031462349, + 0.0006710410016239621, + 0.0008000599991646595, + 0.0006897940038470551, + 0.0006844609961262904, + 0.0008024870039662346, + 0.0009236240002792329, + 0.0006943309999769554, + 0.0007156130013754591, + 0.0008733740032766946, + 0.0010337730054743588, + 0.0008166190018528141, + 0.0007355100024142303, + 0.0006769120009266771, + 0.000672187001327984, + 0.0006738290030625649, + 0.0006668099958915263, + 0.0006905459958943538, + 0.0006693209943477996, + 0.0007606759972986765, + 0.0007038240000838414, + 0.0014060169996810146, + 0.0008548030018573627, + 0.0008396789999096654, + 0.0007900049968156964, + 0.0006796320012654178, + 0.0006777779999538325, + 0.0006762160046491772, + 0.0006783749995520338, + 0.0006674169999314472, + 0.0006986759981373325, + 0.0006776590016670525, + 0.0008410810041823424, + 0.0007876339950598776, + 0.000722486998711247, + 0.0008444339982816018, + 0.0007917429975350387, + 0.0008771369975875132, + 0.0008271270053228363, + 0.0007604120037285611, + 0.0009025730032590218, + 0.0008444600025541149, + 0.0006691429953207262, + 0.0007202179986052215, + 0.0006945170025574043, + 0.0009141010013991036, + 0.0006722469988744706, + 0.0007094789980328642, + 0.0007085079996613786, + 0.000723860997823067, + 0.0006811779967392795, + 0.000666086001729127, + 0.0006798519971198402, + 0.0006757869996363297, + 0.0008435880008619279, + 0.000854636004078202, + 0.0008094639997580089, + 0.0008401279992540367, + 0.0007263980005518533, + 0.0007571899986942299, + 0.0007306740008061752, + 0.0007674869993934408, + 0.0008590919969719835, + 0.0007872499991208315, + 0.0020851259978371672, + 0.0007723400049144402, + 0.0007143900002120063, + 0.0014457810029853135, + 0.0009239110004273243, + 0.0009351949993288144, + 0.0017922349943546578, + 0.0008108339970931411, + 0.0009817879981710576, + 0.0008635880003566854, + 0.0008276080043287948, + 0.0010922590008703992, + 0.0012835169982281514, + 0.0007018690012046136, + 0.0006865629984531552, + 0.0006784240031265654, + 0.0028202160028740764, + 0.0009408549958607182, + 0.0008795690009719692, + 0.0006913940014783293, + 0.0006876369952806272, + 0.0013750749931205064, + 0.0006966190048842691, + 0.0007255360033013858, + 0.0007194299978436902, + 0.0006833149964222685, + 0.0008737300013308413, + 0.0009257659985451028, + 0.001368223995086737, + 0.0011142470029881224, + 0.0007002010024734773, + 0.0007906019964138977, + 0.0016154829936567694, + 0.0007204919966170564, + 0.0006856440013507381, + 0.004561460002150852, + 0.013781686000584159, + 0.006798492999223527, + 0.0008153369999490678, + 0.0007171539982664399, + 0.0006935890050954185, + 0.000684015998558607, + 0.005086684999696445, + 0.0008439770026598126, + 0.0007211839983938262, + 0.004094493997399695, + 0.000810331002867315, + 0.0007231420022435486, + 0.0028128020057920367, + 0.0007290110006579198, + 0.0038398079996113665, + 0.0007193860001279972, + 0.0008246270008385181, + 0.0038395849987864494, + 0.0007859469988034107, + 0.0009002890001283959, + 0.0026076520007336512, + 0.0014405779947992414, + 0.0007291340007213876, + 0.0007158829976106063, + 0.0006858979977550916, + 0.0018843819998437539, + 0.000990131993603427, + 0.0007168149968492799, + 0.000708637002389878, + 0.001376186999550555, + 0.0009516240024822764, + 0.0008474549977108836, + 0.001867973005573731, + 0.0009449829958612099, + 0.002326143003301695, + 0.0009362880009575747, + 0.0007374449996859767, + 0.0006842020011390559, + 0.0006877340056234971, + 0.0027068490016972646, + 0.0010856180015252903, + 0.0006958269950700924, + 0.0006932780015631579, + 0.0006900900043547153, + 0.0008438389995717444, + 0.0031610180012648925, + 0.0009981539988075383, + 0.0007709999990765937, + 0.0006939559971215203, + 0.0025323770023533143, + 0.0013072370056761429, + 0.0007238690013764426, + 0.0007042420038487762, + 0.0006864210008643568, + 0.0016603839976596646, + 0.0014686229988001287, + 0.0007098470014170744, + 0.0006920489977346733, + 0.0006939879967831075, + 0.0006908030045451596, + 0.0008484209974994883, + 0.0012906410047435202, + 0.0006874230020912364, + 0.0013672280038008466, + 0.0017948060049093328, + 0.0009365480000269599, + 0.0007239839978865348, + 0.0006984729989198968, + 0.0006940950042917393, + 0.0008328269977937452, + 0.0008339770065504126, + 0.002748899001744576, + 0.0008119760022964329, + 0.0006859909990453161, + 0.0006903399989823811, + 0.0014484089988400228, + 0.0008739609984331764, + 0.0014540809934260324, + 0.0012846740064560436, + 0.0008976910030469298, + 0.0026787699971464463, + 0.0007852300041122362, + 0.0007056539980112575, + 0.0006903489993419498, + 0.00121280099847354, + 0.0020452690005186014, + 0.000763513999118004, + 0.0008428700020886026, + 0.001010626001516357, + 0.0010719700003392063, + 0.0006866679977974854, + 0.0006867499978397973, + 0.0006951989998924546, + 0.0008368959970539436, + 0.0010836000001290813, + 0.0008347240000148304, + 0.0029221660006442107, + 0.0007767370043438859, + 0.0008350030038855039, + 0.0007767999995849095, + 0.0009147050004685298, + 0.0012513439942267723, + 0.0006867799966130406, + 0.00073648699617479, + 0.0010246259989799, + 0.0007003790015005507, + 0.004955730997608043, + 0.0009369319959660061, + 0.0008965160013758577, + 0.0007231119961943477, + 0.0007554409967269748, + 0.0007499610001104884, + 0.0010528450002311729, + 0.0010982360035995953, + 0.0008331499993801117, + 0.0008911289987736382, + 0.0010921459979726933, + 0.0007857089949538931, + 0.0007179769963840954, + 0.0017078879973269068, + 0.0012627809992409311, + 0.0009425539974472485, + 0.0026538279998931102, + 0.0008153600065270439, + 0.0007083079981384799, + 0.0007018589967628941, + 0.0006856550025986508, + 0.0006928710063220933, + 0.0008732490023248829, + 0.0012303570038056932, + 0.0009897370036924258, + 0.0014850289953756146, + 0.0007515539982705377, + 0.0006990319961914793, + 0.0006959399979677983, + 0.0008493630011798814, + 0.0018904239987023175, + 0.0017279859966947697, + 0.000731403000827413, + 0.0007479980049538426, + 0.0006919820007169619, + 0.0011058119998779148, + 0.0009460810033488087, + 0.0008534839944331907, + 0.000720758005627431, + 0.0008698729943716899, + 0.0009306759966420941, + 0.00225118800153723, + 0.0007155179991968907, + 0.0008288859971798956, + 0.0009812549978960305, + 0.0009355279980809428, + 0.0007898780022514984, + 0.0014333790022647008, + 0.0012921329980599694, + 0.0007262750004883856, + 0.0006972940027480945, + 0.0006972990013309754, + 0.0006917930004419759, + 0.0008204069963539951, + 0.0009127270022872835, + 0.0010387719958089292, + 0.0008496439986629412, + 0.0008050209944485687, + 0.002613420998386573, + 0.000713255001755897, + 0.0006884899994474836, + 0.0007342009994317777, + 0.0006923940018168651, + 0.0009317950025433674, + 0.0011943700010306202, + 0.0014227559950086288, + 0.000999190000584349, + 0.0007358579969150014, + 0.0012508159998105839, + 0.0007091739971656352, + 0.000971011002548039, + 0.0009018759956234135, + 0.0007267350010806695, + 0.0006817870016675442, + 0.000693118003255222, + 0.0007336970011238009, + 0.0008555949971196242, + 0.0009587859967723489, + 0.0008819470021990128, + 0.0008208860017475672, + 0.0015267910057445988, + 0.0008511580017511733, + 0.001031143001455348, + 0.0008268640012829565, + 0.0028379119976307265, + 0.0007245969973155297, + 0.0006852219958091155, + 0.0006873759994050488, + 0.0008686579967616126, + 0.0010540109942667186, + 0.0009378350005135871, + 0.0008122709987219423, + 0.0007022979989415035, + 0.0006966010041651316, + 0.0018847360042855144, + 0.000757858004362788, + 0.0008463679987471551, + 0.0007202030028565787, + 0.0010253689979435876, + 0.0008838909998303279, + 0.0009541539984638803, + 0.0017858409992186353, + 0.0013422349948086776, + 0.0012120129977120087, + 0.0009018429991556332, + 0.0013131340019754134, + 0.0007698830013396218, + 0.0006992150010773912, + 0.0006890700024086982, + 0.0012072299941792153, + 0.002002828005061019, + 0.0027390320028644055, + 0.0007650589977856725, + 0.00069922400143696, + 0.0025331659999210387, + 0.0018625539960339665, + 0.004182821998256259, + 0.0009857879995252006, + 0.0007201670014183037, + 0.0024559160010539927, + 0.0018555550050223246, + 0.0029957959995954297, + 0.0008103500003926456, + 0.0021406689993455075, + 0.0008594309983891435, + 0.0008087320020422339, + 0.0007160960012697615, + 0.0009191700009978376, + 0.0009682090021669865, + 0.0009652820008341223, + 0.0007935820030979812, + 0.0014979449988459237, + 0.0007325350015889853, + 0.0013049869958194904, + 0.0009682180025265552, + 0.0009240430008503608, + 0.0008910149990697391, + 0.0007233770011225715, + 0.0006961059989407659, + 0.0022790820003137924, + 0.000955151001107879, + 0.0007199660030892119, + 0.0007129820005502552, + 0.0007082939991960302, + 0.0007007849999354221, + 0.0010026429954450577, + 0.0035543319972930476, + 0.0010167639993596822, + 0.0007266659958986565, + 0.0007024760052445345, + 0.004081062004843261, + 0.000750287996197585, + 0.0017955129951587878, + 0.0009352570050396025, + 0.0007838229939807206, + 0.0007494590026908554, + 0.0007539319994975813, + 0.0009214160018018447, + 0.0021715159964514896, + 0.0007616349976160564, + 0.000715157002559863, + 0.0013081129945931025, + 0.001422964000084903, + 0.0009595129959052429, + 0.0009597570024197921, + 0.0008363839951925911, + 0.0007820810060366057, + 0.0007257550023496151, + 0.000711242995748762, + 0.0007894219961599447, + 0.0011505540023790672, + 0.0026476869970792904, + 0.0007328680003411137, + 0.0007209010000224225, + 0.0007031269997241907, + 0.0010612100013531744, + 0.0009605270024621859, + 0.0011899770033778623, + 0.0007333910034503788, + 0.0007484629968530498, + 0.0007203490022220649, + 0.0008196670023608021, + 0.0028265880027902313, + 0.0013895209995098412, + 0.0007324030011659488, + 0.0007170160024543293, + 0.0009334560018032789, + 0.001224457002535928, + 0.000776851004047785, + 0.0007301859950530343, + 0.0007032720022834837, + 0.0006997129967203364, + 0.0010891139972954988, + 0.0007254359952639788, + 0.0013274039956741035, + 0.002023815999564249, + 0.0007196370061137713, + 0.0015744839984108694, + 0.0007671170023968443, + 0.0007099529975675978, + 0.0011917689989786595, + 0.002952928000013344, + 0.000726792000932619, + 0.000709904998075217, + 0.0007173600024543703, + 0.0007037200048216619, + 0.004075222997926176, + 0.0011324750012136064, + 0.0007852990020182915, + 0.0011959359981119633, + 0.0007363049953710288, + 0.0007138489963836037, + 0.0007150950041250326, + 0.0007395919965347275, + 0.002008028001000639, + 0.0009501569948042743, + 0.0007663050055271015, + 0.0007254079973790795, + 0.0017345290034427308, + 0.0007211819975054823, + 0.0007209670002339408, + 0.0007065329991746694, + 0.0016835360002005473, + 0.0010446649976074696, + 0.001939629000844434, + 0.0007429730030708015, + 0.0007120000009308569, + 0.0008760869968682528, + 0.002441486998577602, + 0.0008032389960135333, + 0.0007194569989223965, + 0.0016674310027156025, + 0.0009984669959521852, + 0.000914972995815333, + 0.0007619160023750737, + 0.0007156790015869774, + 0.0007030519991531037, + 0.0006978829987929203, + 0.001249893000931479, + 0.0009818339967750944, + 0.0020837310003116727, + 0.0007234599979710765, + 0.0014628580029238947, + 0.0008923749992391095, + 0.0016695019949111156, + 0.0007661070048925467, + 0.0007314920003409497, + 0.0007084179960656911, + 0.0006977720040595159, + 0.0009548320012982003, + 0.0016550010041100904, + 0.0007370419989456423, + 0.0007148220029193908, + 0.001192542003991548, + 0.0008448830049019307, + 0.00073781399987638, + 0.0009209499985445291, + 0.0008498960014549084, + 0.0007226460002129897, + 0.0007002040001680143, + 0.0007461959976353683, + 0.0007139989975257777, + 0.0013083339945296757, + 0.0015681599979870953, + 0.0007680650014663115, + 0.000704593003320042, + 0.0008995249954750761, + 0.0009445569958188571, + 0.0012940730011905544, + 0.0007306269981199875, + 0.0007406750009977259, + 0.0007475359961972572, + 0.0007140430025174282, + 0.0009759489985299297, + 0.0009635839960537851, + 0.0009918539944919758, + 0.00110855299863033, + 0.0007317039999179542, + 0.0007158180014812388, + 0.0007032209978206083, + 0.0007337619972531684, + 0.000946087995544076, + 0.0008984290034277365, + 0.0007397540030069649, + 0.0027004120056517422, + 0.0012103650005883537, + 0.0018697160048759542, + 0.0007528269998147152, + 0.0007046890023048036, + 0.0007042200013529509, + 0.0008692520059412345, + 0.0009569889953127131, + 0.001015663001453504, + 0.0018405089940642938, + 0.0007621070035384037, + 0.0007878870019339956, + 0.00284192700200947, + 0.0007438510001520626, + 0.0007103480020305142, + 0.0010369130031904206, + 0.0009872129958239384, + 0.0009021730002132244, + 0.000980615004664287, + 0.0013278760015964508, + 0.0007523320018663071, + 0.0007658329996047541, + 0.0011902039987035096, + 0.0007308750064112246, + 0.001090128003852442, + 0.0009551379989716224, + 0.0007381980030913837, + 0.000938074997975491, + 0.0016550500004086643, + 0.0007390620012301952, + 0.0007611359978909604, + 0.0007093960011843592, + 0.0006962289990042336, + 0.0011938060051761568, + 0.001708595002128277, + 0.0007463759975507855, + 0.002191325998865068, + 0.0007341759992414154, + 0.0015523440015385859, + 0.0010155330019188114, + 0.0007288289998541586, + 0.0007086320038069971, + 0.0007266469983733259, + 0.0007881639976403676, + 0.004725930004497059, + 0.0007581209938507527, + 0.0007310419969144277, + 0.0007233340002130717, + 0.0011255139979766682, + 0.0010312919985153712, + 0.001136614999268204, + 0.0007356919959420338, + 0.0007069189960020594, + 0.000726419995771721, + 0.0008621430024504662, + 0.002753319997282233, + 0.0043371779975132085, + 0.0020292170011089183, + 0.0009098270020331256, + 0.0007424799987347797, + 0.000719082003342919, + 0.0009697240020614117, + 0.0025126799955614842, + 0.0010146970016648993, + 0.0007396500004688278, + 0.0007163289992604405, + 0.0007373370026471093, + 0.0009146060037892312, + 0.0015452820007340051, + 0.0007314489994314499, + 0.0010039439948741347, + 0.0008856659987941384, + 0.0007408769961330108, + 0.0014770810012123547, + 0.0008462889963993803, + 0.0007267159962793812, + 0.0007224910004879348, + 0.0007045449965517037, + 0.0009769419993972406, + 0.0009709189980640076, + 0.0009658449998823926, + 0.0007893430010881275, + 0.0007731769946985878, + 0.0007336179987760261, + 0.0007040529962978326, + 0.000937798002269119, + 0.0010175770003115758, + 0.0008500769981765188, + 0.000736481997591909, + 0.0007108940044417977, + 0.0006980640027904883, + 0.0014032760009285994, + 0.0009397760004503652, + 0.0007189300013124011, + 0.000978535994363483, + 0.0007544499967480078, + 0.001520047997473739, + 0.0007710510035394691, + 0.0007176130020525306, + 0.0007193969940999523, + 0.0008509119943482801, + 0.0009534839991829358, + 0.0009565010041114874, + 0.0009404849988641217, + 0.0007692129947827198, + 0.0007110369988367893, + 0.0009449639983358793, + 0.0016987679991871119, + 0.0007698569970671088, + 0.0013660870026797056, + 0.0007358020011452027, + 0.000767431003623642, + 0.0007910140047897585, + 0.0014746029992238618, + 0.0011054179994971491, + 0.0010499339987291023, + 0.0007905829988885671, + 0.0007107130004442297, + 0.0007080909999785945, + 0.002844568000000436, + 0.0007420779948006384, + 0.0007026409948593937, + 0.0007137760039768182, + 0.0007035050002741627, + 0.0007027100000414066, + 0.0009607050014892593, + 0.0008150880021275952, + 0.0007212069976958446, + 0.0008832200037431903, + 0.0010293689992977306, + 0.0009662260054028593, + 0.000742768999771215, + 0.0007103360039764084, + 0.005325322003045585, + 0.0008368019989575259, + 0.0010197529991273768, + 0.0020825850006076507, + 0.0007975609987624921, + 0.0007265570020535961, + 0.0008784770034253597, + 0.0013190209938329645, + 0.0007568220025859773, + 0.0007387850055238232, + 0.000724282996088732, + 0.0007049690029816702, + 0.001118161999329459, + 0.0009849910056800582, + 0.0009624550002627075, + 0.0009785730071598664, + 0.0008037009974941611, + 0.0007199550018412992, + 0.0007083589953253977, + 0.0006983409984968603, + 0.0012524720004876144, + 0.0007273649971466511, + 0.0007063700031721964, + 0.0008276180014945567, + 0.0009551059993100353, + 0.003471120995527599, + 0.005744520996813662, + 0.0008001320020412095, + 0.0007213979988591745, + 0.0008749190019443631, + 0.0007067830010782927, + 0.004123031998460647, + 0.0009197540057357401, + 0.0007578220029245131, + 0.0007228610047604889, + 0.0007540670048911124, + 0.0007079089991748333, + 0.005746051007008646, + 0.0007969220023369417, + 0.0008396929988521151, + 0.000716768998245243, + 0.0007608239975525066, + 0.005804171996715013, + 0.0007185639988165349, + 0.0006922820030013099, + 0.0006890939985169098, + 0.000698533003742341, + 0.0006800560004194267, + 0.0014689190065837465, + 0.003763412998523563, + 0.0009747400035848841, + 0.0006970799950067885, + 0.0006883969981572591, + 0.0055556890001753345, + 0.000879202998476103, + 0.0007026450039120391, + 0.0007566129934275523, + 0.00073654100560816, + 0.005306906998157501, + 0.0007336859998758882, + 0.0006917030041222461, + 0.0006936830031918362, + 0.0033833939960459247, + 0.000731480002286844, + 0.0006902399982209317, + 0.0006871220030006953, + 0.0007323480022023432, + 0.005878959003894124, + 0.0007563910039607435, + 0.0006928719958523288, + 0.0006915200065122917, + 0.0006842279981356114, + 0.0034819569991668686, + 0.0030541000014636666, + 0.0007018269971013069, + 0.0006877510022604838, + 0.0006855859974166378, + 0.0006799699986004271, + 0.0028819619983551092, + 0.003449570998782292, + 0.0007265359963639639, + 0.0006906790003995411, + 0.0006885559996590018, + 0.0006902439999976195, + 0.002728530998865608, + 0.0007454150036210194, + 0.0006941219980944879, + 0.0006929070004844107, + 0.0006915120029589161, + 0.00637327699951129, + 0.0007433789942297153, + 0.0007000170007813722, + 0.0006876889965496957, + 0.004923783002595883, + 0.0009263720057788305, + 0.0007018850010354072, + 0.0006889170035719872, + 0.000686109000525903, + 0.006429372995626181, + 0.0007313009991776198, + 0.0006926360001671128, + 0.0006917630016687326, + 0.0006874799946672283, + 0.0006852070000604726, + 0.0059736280018114485, + 0.0007316760020330548, + 0.0006957360019441694, + 0.0007219870021799579, + 0.0007029339976725169, + 0.006657090001681354, + 0.0007367370053543709, + 0.0006928029979462735, + 0.0007440459958161227, + 0.000696929004334379, + 0.0006809390033595264, + 0.008102243999019265, + 0.0007441569978254847, + 0.000695834998623468, + 0.0006965040010982193, + 0.0006819249974796548, + 0.003707506002683658, + 0.000767213998187799, + 0.0006956360011827201, + 0.0006936360005056486, + 0.0007550829977844842, + 0.0014138240003376268, + 0.0018184130021836609, + 0.0037673580009141006, + 0.0022450140022556297, + 0.002177049005695153, + 0.0010458250035298988, + 0.004621825995855033, + 0.0012424730011844076, + 0.0006899430009070784, + 0.0007234290023916401, + 0.0006819560003350489, + 0.012981266001588665, + 0.002339573999051936, + 0.0007116339984349906, + 0.004486628997256048, + 0.00858919500024058, + 0.002322679996723309, + 0.0007608349988004193, + 0.007961650997458491, + 0.001167192000139039, + 0.0006953979973332025, + 0.000682111996866297, + 0.015986179001629353, + 0.0041653549997136, + 0.007580409001093358, + 0.008830095997836906, + 0.0009391209969180636, + 0.006154263995995279, + 0.009526153997285292, + 0.007486882001103368, + 0.017018168997310568, + 0.0030574339980375953, + 0.006662430001597386, + 0.006317311999737285, + 0.002048975002253428, + 0.008564965006371494, + 0.004712835994723719, + 0.0018059979993267916, + 0.006183233002957422, + 0.0019254340004408732, + 0.004615721001755446, + 0.0017689859960228205, + 0.008569698002247605, + 0.000973596004769206, + 0.00622433700482361, + 0.02857370799756609, + 0.007461766996129882, + 0.0009095959976548329, + 0.0007384540003840812, + 0.005284945997118484, + 0.008932430995628238, + 0.004051603005791549, + 0.0007870100016589276, + 0.009412742001586594, + 0.001859782998508308, + 0.00145579400123097, + 0.005841094003699254, + 0.0008164259998011403, + 0.0007443069989676587, + 0.0007366549980361015, + 0.0007333940011449158, + 0.004903162996924948, + 0.0007809620001353323, + 0.0007291849979083054, + 0.0008379010032513179, + 0.0007863099963287823, + 0.004202891002933029, + 0.0009033629976329394, + 0.0007520520011894405, + 0.0007308330023079179, + 0.000724999001249671, + 0.005226975001278333, + 0.0016959460044745356, + 0.0007365350029431283, + 0.0007808149966876954, + 0.004738130002806429, + 0.0009204720045090653, + 0.0007457940009771846, + 0.000730882995412685, + 0.0007841959959478118, + 0.006482313001470175, + 0.0008167580017470755, + 0.000742279997211881, + 0.0007331350061576813, + 0.0006982259947108105, + 0.005489553994266316, + 0.0007790460003889166, + 0.0007190809992607683, + 0.0007137089996831492, + 0.0007133629987947643, + 0.005623031000141054, + 0.000781968999945093, + 0.000720691001333762, + 0.0007113289975677617, + 0.0007051150023471564, + 0.0034177320048911497, + 0.0007502709995605983, + 0.0007181580003816634, + 0.000842934001411777, + 0.000749785998777952, + 0.000711082000634633, + 0.005801979001262225, + 0.0011773939986596815, + 0.0007171789984568022, + 0.0007077879999997094, + 0.004271816003893036, + 0.0008932850032579154, + 0.005218831000092905, + 0.0008347619950654916, + 0.0007291469955816865, + 0.00732767899899045, + 0.0009357360031572171, + 0.0007425579970004037, + 0.005867056002898607, + 0.0023440669974661432, + 0.0008063249988481402, + 0.004294343001674861, + 0.004775016997882631, + 0.0007641699994564988, + 0.003430463999393396, + 0.005678002002241556, + 0.002249310004117433, + 0.00896358000318287, + 0.0025734869996085763, + 0.00622986099915579, + 0.008280789996206295, + 0.0008486010046908632, + 0.004929226997774094, + 0.0011874650008394383, + 0.006726987005094998, + 0.0011522309941938147, + 0.010131912000360899, + 0.011571277005714364, + 0.008810767001705244, + 0.00112060499668587, + 0.0062385150013142265, + 0.01173161299811909 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil", + "fullname": "TestApplyNabla2AndNabla4GlobalToVn", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.00039931200444698334, + "max": 0.019986325001809746, + "mean": 0.004566150372730821, + "stddev": 0.0033528803643517098, + "rounds": 2058, + "median": 0.005793093998363474, + "iqr": 0.006479188006778713, + "q1": 0.0005192009994061664, + "q3": 0.00699838900618488, + "iqr_outliers": 3, + "stddev_outliers": 1051, + "outliers": "1051;3", + "ld15iqr": 0.00039931200444698334, + "hd15iqr": 0.017988031002460048, + "ops": 219.00286201085893, + "total": 9.397137467080029, + "data": [ + 0.000849110001581721, + 0.0004843339993385598, + 0.0005138319975230843, + 0.00048041399713838473, + 0.0017207330020028166, + 0.0010303910021320917, + 0.0004963859973940998, + 0.0004971070011379197, + 0.0004960939986631274, + 0.00048297100147465244, + 0.0004886249953415245, + 0.0004909499984933063, + 0.0008418560028076172, + 0.0007908600018708967, + 0.0019391400055610575, + 0.0005233300034888089, + 0.0005507500027306378, + 0.0005142179943504743, + 0.0009753519989317283, + 0.0011250469979131594, + 0.0007332269960897975, + 0.0005424120026873425, + 0.0005133610029588453, + 0.0005126199976075441, + 0.0004992079993826337, + 0.0019838879961753264, + 0.0011630610024440102, + 0.0005152550002094358, + 0.0005082709976704791, + 0.0004990419984096661, + 0.0004722500016214326, + 0.0006321869950625114, + 0.0006683980027446523, + 0.0006922629982000217, + 0.00047247399925254285, + 0.0004763039978570305, + 0.0004927670015604235, + 0.0005604959951597266, + 0.0008943430002545938, + 0.0012978629965800792, + 0.000633553005172871, + 0.0004975740012014285, + 0.00047916000039549544, + 0.0008542229988961481, + 0.0006197859984240495, + 0.0005009369997424074, + 0.0004831510013900697, + 0.0004732410016003996, + 0.0005303249999997206, + 0.0004737690032925457, + 0.0005360009963624179, + 0.0004778139991685748, + 0.0004908580012852326, + 0.00047292200179072097, + 0.0004949380017933436, + 0.0005751360004069284, + 0.0006288899967330508, + 0.004128188003960531, + 0.0007412940030917525, + 0.0005539179983315989, + 0.0004914169985568151, + 0.0004834479987039231, + 0.0004816750006284565, + 0.0004788859951077029, + 0.007905243001005147, + 0.0006642090011155233, + 0.0005215659984969534, + 0.0004791920000570826, + 0.004186444995866623, + 0.0006161860001157038, + 0.0005121889989823103, + 0.004363692001788877, + 0.0005438579973997548, + 0.0005546430038521066, + 0.0005111370046506636, + 0.014544705001753755, + 0.0055251070007216185, + 0.010039668006356806, + 0.007494762001442723, + 0.008136989999911748, + 0.007109959995432291, + 0.010497048999241088, + 0.004138379001233261, + 0.007838704994355794, + 0.006994101997406688, + 0.007000021003477741, + 0.006990991998463869, + 0.005061910997028463, + 0.0059250259946566075, + 0.006003172995406203, + 0.006993337003223132, + 0.006995110998104792, + 0.005996217005304061, + 0.007994069004780613, + 0.00901813600648893, + 0.006978566001635045, + 0.0069876540001132526, + 0.006995332005317323, + 0.006996043004619423, + 0.0070079020006232895, + 0.007984349002072122, + 0.005998301996442024, + 0.007993671999429353, + 0.007995751002454199, + 0.0039924489974509925, + 0.005000384997401852, + 0.006999994002399035, + 0.004171204003796447, + 0.0038706579944118857, + 0.005943084004684351, + 0.00799853199714562, + 0.00599272800172912, + 0.006994673996814527, + 0.0069982669956516474, + 0.006994473995291628, + 0.0069957500018063, + 0.007995581996510737, + 0.007995464002306107, + 0.007995959997060709, + 0.003995327999291476, + 0.003993915001046844, + 0.00700024000252597, + 0.006996423006057739, + 0.007996641004865523, + 0.003992311998445075, + 0.006367202004184946, + 0.006630110998230521, + 0.005993984996166546, + 0.005991933001496363, + 0.01100147599936463, + 0.007995038002263755, + 0.007994608000444714, + 0.007996314998308662, + 0.007997417000296991, + 0.006997008000325877, + 0.007993692001036834, + 0.0069984760048100725, + 0.0076225510056247, + 0.006368378002662212, + 0.007045712001854554, + 0.007947363999846857, + 0.006992440001340583, + 0.006995649004238658, + 0.005992831000185106, + 0.008000699002877809, + 0.006994244002271444, + 0.007002917998761404, + 0.006016166000335943, + 0.007977039000252262, + 0.006988918001297861, + 0.0059931209980277345, + 0.004998654992959928, + 0.002992122004798148, + 0.0060016589995939285, + 0.007994334999239072, + 0.007999852998182178, + 0.007993121995241381, + 0.00799574200209463, + 0.007998831999429967, + 0.008029780998185743, + 0.007949246995849535, + 0.010004628995375242, + 0.006989313995291013, + 0.009995230000640731, + 0.007000244004302658, + 0.008991954993689433, + 0.007999133995326702, + 0.005992869999317918, + 0.008010204001038801, + 0.007985943004314322, + 0.006001389003358781, + 0.0059903650035266764, + 0.005995153995172586, + 0.005996899002639111, + 0.005995612998958677, + 0.007996061001904309, + 0.0059984199979226105, + 0.0069968909956514835, + 0.006993101000261959, + 0.007998752000276, + 0.006993580005655531, + 0.005994475002808031, + 0.00799333599570673, + 0.008997926001029555, + 0.006996628995693754, + 0.004995969000447076, + 0.006157789001008496, + 0.009842084000410978, + 0.001992356999835465, + 0.007050539999909233, + 0.007944697004859336, + 0.006428233995393384, + 0.006559198001923505, + 0.007998148998012766, + 0.006986050997511484, + 0.007993491999513935, + 0.006995100004132837, + 0.005994996005028952, + 0.006994859999394976, + 0.007996029002242722, + 0.004993978000129573, + 0.00599740900361212, + 0.006994413000938948, + 0.006999065000854898, + 0.007998916997166816, + 0.007991482998477295, + 0.009996452004997991, + 0.007996117004950065, + 0.006994544994086027, + 0.006006459996569902, + 0.007983970994246192, + 0.0060007489955751225, + 0.005994241997541394, + 0.00799658599862596, + 0.005002117999538314, + 0.005989424003928434, + 0.005997920001391321, + 0.010002351002185605, + 0.009992906998377293, + 0.00799847100279294, + 0.00809000699518947, + 0.006895548998727463, + 0.007999069996003527, + 0.007003170998359565, + 0.005988755998259876, + 0.0043344689984223805, + 0.003653756997664459, + 0.007083378994138911, + 0.007914240995887667, + 0.006190227999468334, + 0.006803352996939793, + 0.010993101997883059, + 0.011003597996023018, + 0.007991014994331636, + 0.009993595995183568, + 0.0033077410043915734, + 0.008035061000555288, + 0.008659335995616857, + 0.005984299001283944, + 0.005994604995066766, + 0.009999816997151356, + 0.009995153995987494, + 0.0067661950015462935, + 0.0032316400029230863, + 0.00399047699465882, + 0.005999396998959128, + 0.006995741001446731, + 0.006998247001320124, + 0.005995874998916406, + 0.012998354999581352, + 0.0069955500002834015, + 0.003998569998657331, + 0.003993348000221886, + 0.00800205000268761, + 0.007007412001257762, + 0.008983622996311169, + 0.003997029998572543, + 0.006994578005105723, + 0.006994708004640415, + 0.006998430006206036, + 0.00400463400001172, + 0.004028556999401189, + 0.006953383002837654, + 0.005996742998831905, + 0.005993999999191146, + 0.00699676999647636, + 0.006997648000833578, + 0.005996239997330122, + 0.007997742002771702, + 0.006992087000980973, + 0.008001145994057879, + 0.006993543000135105, + 0.006998764001764357, + 0.00799958199786488, + 0.00899337200098671, + 0.0069958169988240115, + 0.007998237997526303, + 0.007993548002559692, + 0.006997399999818299, + 0.005994172002829146, + 0.00699706200248329, + 0.0069930409954395145, + 0.00699880400497932, + 0.006996211006480735, + 0.006996984993747901, + 0.007582074002129957, + 0.007412163002300076, + 0.00799562699830858, + 0.008994338000775315, + 0.005995347004500218, + 0.006000218003464397, + 0.00799409100000048, + 0.004993611000827514, + 0.004993415997887496, + 0.006998071003181394, + 0.007995231004315428, + 0.010998607998772059, + 0.0069922280017635785, + 0.005997246000333689, + 0.006997203003265895, + 0.006997012998908758, + 0.005996754996886011, + 0.008006190997548401, + 0.005988430995785166, + 0.007996375003131106, + 0.00502524300100049, + 0.00998031900235219, + 0.0059808389996760525, + 0.019006259004527237, + 0.013993726999615319, + 0.012987242997041903, + 0.007078235001245048, + 0.003011445005540736, + 0.00588725799752865, + 0.007995707004738506, + 0.008005035000678618, + 0.0030250070049078204, + 0.006557344997418113, + 0.014878232002956793, + 0.0025042670022230595, + 0.005072002000815701, + 0.0070520319932256825, + 0.002656892997038085, + 0.005198156999540515, + 0.006034837002516724, + 0.00410852899949532, + 0.005006462000892498, + 0.004833617997064721, + 0.003240464997361414, + 0.002934969998023007, + 0.00481246599520091, + 0.010999842001183424, + 0.011995335997198708, + 0.00499272200249834, + 0.006998273995122872, + 0.006994798000960145, + 0.005697642998711672, + 0.008298884997202549, + 0.006996369003900327, + 0.0070067090055090375, + 0.007980879003298469, + 0.010998840996762738, + 0.005238402998656966, + 0.009755150000273716, + 0.004003544003353454, + 0.006952921001357026, + 0.003028459999768529, + 0.003156151004077401, + 0.002834762002748903, + 0.0051348599954508245, + 0.006845190000603907, + 0.007009376997302752, + 0.00900363500113599, + 0.007979055000760127, + 0.005993660000967793, + 0.0069956919978722, + 0.007997740001883358, + 0.006000278997817077, + 0.004996727999241557, + 0.009997993998695165, + 0.0051227250005467795, + 0.006867732998216525, + 0.000999298004899174, + 0.00399354499677429, + 0.006998397999268491, + 0.0069998969993321225, + 0.006995290998020209, + 0.004002330999355763, + 0.005996930005494505, + 0.006994704999669921, + 0.006994491996010765, + 0.006997379998210818, + 0.006995695999648888, + 0.006996919997618534, + 0.006998228003794793, + 0.00699722900026245, + 0.006997010998020414, + 0.008471597997413483, + 0.006521358998725191, + 0.006993751005211379, + 0.00600956600101199, + 0.0039829070010455325, + 0.004995339004381094, + 0.006998734002991114, + 0.00799665300291963, + 0.00699811200320255, + 0.007995075997314416, + 0.00599769100517733, + 0.006997448996116873, + 0.006998123993980698, + 0.00499832299828995, + 0.005997972999466583, + 0.007993769002496265, + 0.006997582997428253, + 0.00699831700330833, + 0.005583744998148177, + 0.0034033330011880025, + 0.007004557002801448, + 0.00800162100495072, + 0.006985244996030815, + 0.004665134998504072, + 0.005327797996869776, + 0.00800625300325919, + 0.007996583997737616, + 0.006991321002715267, + 0.008046481001656502, + 0.005940923001617193, + 0.007998249995580409, + 0.006690690002869815, + 0.00830275600310415, + 0.00699873200210277, + 0.004558861997793429, + 0.005434125996544026, + 0.003997584004537202, + 0.010998576995916665, + 0.0069980679982108995, + 0.006995394003752153, + 0.006998655997449532, + 0.007996963999175932, + 0.010996889999660198, + 0.006995496994932182, + 0.00799782700050855, + 0.005995877007080708, + 0.005311761997290887, + 0.005697638000128791, + 0.007984128998941742, + 0.006005096001899801, + 0.007991130994923878, + 0.007991657003003638, + 0.007998850000149105, + 0.007992773003934417, + 0.006002988004183862, + 0.006989169000007678, + 0.006997039999987464, + 0.007995041996764485, + 0.005997193999064621, + 0.00799779299995862, + 0.007996230997378007, + 0.005997318003210239, + 0.007997136999620125, + 0.007998145993042272, + 0.00600735199986957, + 0.005986074000247754, + 0.003135354992991779, + 0.002858774001651909, + 0.004996328003471717, + 0.0059989090004819445, + 0.007013695998466574, + 0.00698201400518883, + 0.004505798999161925, + 0.0009923360048560426, + 0.007487563998438418, + 0.008182507001038175, + 0.007810644994606264, + 0.006530799000756815, + 0.005464973000925966, + 0.004521216003922746, + 0.004472337997867726, + 0.0070019949998822995, + 0.0049817219987744465, + 0.006999399003689177, + 0.00700159899861319, + 0.006989093999436591, + 0.0089986540042446, + 0.0069929989986121655, + 0.006998262004344724, + 0.007993252998858225, + 0.007002407000982203, + 0.006991760004893877, + 0.007994403000338934, + 0.0040032329998211935, + 0.004396973003167659, + 0.004585763999784831, + 0.007999443005246576, + 0.007994829000381287, + 0.007999647001270205, + 0.008001630994840525, + 0.0079900450000423, + 0.007996207001269795, + 0.00786882499960484, + 0.006124159001046792, + 0.006989768000494223, + 0.011004571999364998, + 0.009058314994035754, + 0.007279208999534603, + 0.0076155650021974, + 0.006998182994720992, + 0.010151077003683895, + 0.008157409996783827, + 0.01067414200224448, + 0.0059916620011790656, + 0.006985524996707682, + 0.006997503005550243, + 0.007993694998731371, + 0.006995217998337466, + 0.006999344004725572, + 0.006989087996771559, + 0.007995387000846677, + 0.01300138799706474, + 0.005989359997329302, + 0.00799544600158697, + 0.006991708003624808, + 0.006994620998739265, + 0.005996689003950451, + 0.007998512999620289, + 0.007994512998266146, + 0.0160583909964771, + 0.006944617998669855, + 0.011011782997229602, + 0.00995819899981143, + 0.008012879996385891, + 0.0059468960025697015, + 0.006988583998463582, + 0.006990066001890227, + 0.007020331999228802, + 0.007662329000595491, + 0.003315163994557224, + 0.007378559996141121, + 0.003568171006918419, + 0.005999906999932136, + 0.005992750004224945, + 0.0059930539937340654, + 0.0049912249960470945, + 0.011001638995367102, + 0.003989103999629151, + 0.0049971210028161295, + 0.007004491999396123, + 0.00944667400472099, + 0.005540552003367338, + 0.016021512004954275, + 0.014986927999416366, + 0.008728100001462735, + 0.013225497998064384, + 0.010965923996991478, + 0.012021991002256982, + 0.004102111000975128, + 0.00296287899982417, + 0.008979072998045012, + 0.014894244995957706, + 0.019986325001809746, + 0.010994137002853677, + 0.017988031002460048, + 0.007578005002869759, + 0.008384647997445427, + 0.005978486005915329, + 0.0059958630008623, + 0.009003740997286513, + 0.006984812003793195, + 0.007101196999428794, + 0.01088699400133919, + 0.004991440997400787, + 0.007999164001375902, + 0.006993982999119908, + 0.007119155001419131, + 0.00787547099753283, + 0.006995650001044851, + 0.005999512002745178, + 0.010995882003044244, + 0.007995729996764567, + 0.006996287003858015, + 0.006998333003139123, + 0.0079996500062407, + 0.007016548006504308, + 0.007968739002535585, + 0.006988465000176802, + 0.008004461997188628, + 0.0069916669963276945, + 0.0019964369930676185, + 0.0019907310052076355, + 0.006132925002020784, + 0.006865878000098746, + 0.006986084998061415, + 0.008007221993466374, + 0.006985741994867567, + 0.007997007996891625, + 0.006997557997237891, + 0.0069975630030967295, + 0.005997010004648473, + 0.00799846200243337, + 0.006996864998654928, + 0.006997518998105079, + 0.007997533000889234, + 0.006996656993578654, + 0.0069957310042809695, + 0.007000087003689259, + 0.006994543000473641, + 0.0056128949945559725, + 0.002379534998908639, + 0.005998155000270344, + 0.00699978099873988, + 0.006999264995101839, + 0.006992369999352377, + 0.00699787699704757, + 0.007994395004061516, + 0.007999542001925875, + 0.00799696800095262, + 0.0059977729979436845, + 0.005992667000100482, + 0.006997305004915688, + 0.008001043999684043, + 0.0069953229976817966, + 0.0059946059991489165, + 0.006992820999585092, + 0.0069985649970476516, + 0.007236802004626952, + 0.00776060300268, + 0.00699898500170093, + 0.00799113199900603, + 0.004464675999770407, + 0.0065262839998467825, + 0.005999735003570095, + 0.00799478300177725, + 0.00899895799375372, + 0.006992424001509789, + 0.005997764994390309, + 0.006994399998802692, + 0.007000029007031117, + 0.0079939569986891, + 0.007170354001573287, + 0.0068210560057195835, + 0.005993826001940761, + 0.006001847003062721, + 0.009352156994282268, + 0.008636631995614152, + 0.006000381996273063, + 0.006989010995312128, + 0.007988474993908312, + 0.007000645993684884, + 0.00699149799766019, + 0.006670856004348025, + 0.0033278889968642034, + 0.006003848000545986, + 0.008989946996734943, + 0.0066554600052768365, + 0.00461040699883597, + 0.005717164000088815, + 0.007996877997356933, + 0.007995935004146304, + 0.005994619001285173, + 0.00599870099540567, + 0.006998612996540032, + 0.0059960500002489425, + 0.006996898002398666, + 0.006996525997237768, + 0.007999663001100998, + 0.00699830300436588, + 0.0059956250042887405, + 0.006995905998337548, + 0.006997295997280162, + 0.012000277994957287, + 0.00699378799617989, + 0.006997292002779432, + 0.0069985279988031834, + 0.006995120995270554, + 0.00699766800244106, + 0.006997113996476401, + 0.006997175994911231, + 0.005998877000820357, + 0.006001415000355337, + 0.00499180600309046, + 0.004997472002287395, + 0.007997863998753019, + 0.006993467002757825, + 0.00699911799893016, + 0.006995545998506714, + 0.00699676700605778, + 0.005985896998026874, + 0.004005695001978893, + 0.006001547000778373, + 0.006045624002581462, + 0.00795081500109518, + 0.006993581002461724, + 0.006997448996116873, + 0.007993568004167173, + 0.007997439002792817, + 0.005999950000841636, + 0.007995212996320333, + 0.008001665002666414, + 0.007994034000148531, + 0.007998692002729513, + 0.0069992169956094585, + 0.006988765999267343, + 0.0059906009992118925, + 0.007992890001332853, + 0.005999806999170687, + 0.00799643299978925, + 0.005997926993586589, + 0.007992050996108446, + 0.005997020998620428, + 0.0049996360030490905, + 0.007994820996827912, + 0.006996096002694685, + 0.006997526004852261, + 0.006996727999649011, + 0.007998065004358068, + 0.0010064899979624897, + 0.006988465996982995, + 0.00799710599676473, + 0.005996089006657712, + 0.005996683001285419, + 0.006999667995842174, + 0.006994783994741738, + 0.007997922002687119, + 0.006999047000135761, + 0.009995561995310709, + 0.0070020700004533865, + 0.007991113001480699, + 0.008010626996110659, + 0.006994212002609856, + 0.006976644006499555, + 0.006994431998464279, + 0.008000269997864962, + 0.007995740001206286, + 0.01099387599970214, + 0.009996741995564662, + 0.006398973004252184, + 0.0075934659980703145, + 0.005997877000481822, + 0.006995805000769906, + 0.007997901004273444, + 0.00300515800336143, + 0.0075731370016001165, + 0.007415344000037294, + 0.006996304000495002, + 0.007000915997195989, + 0.006993075003265403, + 0.006998058997851331, + 0.005996225001581479, + 0.007000254998274613, + 0.008996841999760363, + 0.008002433998626657, + 0.004994455004634801, + 0.0040149069973267615, + 0.005990271994960494, + 0.005980128000373952, + 0.006992953996814322, + 0.006994276998739224, + 0.005999269000312779, + 0.00799670100241201, + 0.005998590000672266, + 0.008000207999430131, + 0.006988047003687825, + 0.005000868004572112, + 0.007001211000897456, + 0.010989888003678061, + 0.003995166996901389, + 0.004690984002081677, + 0.006298350999713875, + 0.009003106999443844, + 0.006991535003180616, + 0.006993310002144426, + 0.006995682997512631, + 0.006994424998993054, + 0.0069999409970478155, + 0.009998093999456614, + 0.00599207900086185, + 0.008000517002074048, + 0.007993205996172037, + 0.0059956049954053015, + 0.0059965519976685755, + 0.005823913001222536, + 0.00616686099965591, + 0.005997504995320924, + 0.007996127002115827, + 0.00899599999684142, + 0.006996028998401016, + 0.008016465995751787, + 0.005850109002494719, + 0.006163383004604839, + 0.0029523960038204677, + 0.006001251000270713, + 0.006992063994402997, + 0.007995102001586929, + 0.00799560800078325, + 0.006996286996582057, + 0.00800099500338547, + 0.006991237998590805, + 0.006995622003159951, + 0.005096474997117184, + 0.004896112994174473, + 0.007019427001068834, + 0.005983653005387168, + 0.007992506994924042, + 0.00600953699904494, + 0.004976432996045332, + 0.009998187000746839, + 0.002983696002047509, + 0.004001575995062012, + 0.011006213004293386, + 0.007790689000103157, + 0.00434215600398602, + 0.007860343001084402, + 0.002965974999824539, + 0.004006602997833397, + 0.0069440419974853285, + 0.009045104998222087, + 0.00799199200264411, + 0.0060795919998781756, + 0.003906420002749655, + 0.007002710000961088, + 0.007995439998921938, + 0.006995712996285874, + 0.007002359998296015, + 0.010989804002747405, + 0.0099961070009158, + 0.0069958299936843105, + 0.008007624994206708, + 0.00698433599609416, + 0.009042127996508498, + 0.005951230996288359, + 0.0049911370006157085, + 0.005994434999593068, + 0.0070011989955673926, + 0.006995007999648806, + 0.00800293300562771, + 0.006989016001170967, + 0.006996035997872241, + 0.008994904994324315, + 0.0069925079951644875, + 0.0079954219982028, + 0.006996291005634703, + 0.0059976150005240925, + 0.004026660004456062, + 0.0029577570021501742, + 0.004006410999863874, + 0.00498894300108077, + 0.004993828006263357, + 0.004237565997755155, + 0.005750780001108069, + 0.008008550998056307, + 0.0019843310001306236, + 0.006999730001552962, + 0.006994766001298558, + 0.007996145999641158, + 0.006992198002990335, + 0.01311893000092823, + 0.005889463005587459, + 0.006000522000249475, + 0.010946695001621265, + 0.015030219998152461, + 0.008986264001578093, + 0.006991851005295757, + 0.00599592000071425, + 0.006996585005254019, + 0.007997216998774093, + 0.006997021002462134, + 0.008000333997188136, + 0.007994419000169728, + 0.002996041002916172, + 0.00699838900618488, + 0.008021446003112942, + 0.006968818997847848, + 0.006995451003604103, + 0.007003694998275023, + 0.007989818004716653, + 0.008996247997856699, + 0.007996633998118341, + 0.00800137199985329, + 0.007993231003638357, + 0.006000498004141264, + 0.006993104005232453, + 0.006994557996222284, + 0.006998891003604513, + 0.006997443000727799, + 0.0010011919948738068, + 0.007991692997165956, + 0.005997667998599354, + 0.00699650299793575, + 0.008007942000404, + 0.00600281500373967, + 0.006975397001951933, + 0.006992488997639157, + 0.006996989002800547, + 0.005078606998722535, + 0.00692008099576924, + 0.008996019001642708, + 0.006995701005507726, + 0.005003480000596028, + 0.007987114004208706, + 0.00599652600067202, + 0.007996610998816323, + 0.006997682998189703, + 0.007997635002539027, + 0.0069966239971108735, + 0.005996754000079818, + 0.008997700999316294, + 0.005198464001296088, + 0.00879780999821378, + 0.00499430200579809, + 0.0059986060005030595, + 0.006856273001176305, + 0.003138876003504265, + 0.00799762700626161, + 0.007996034997631796, + 0.007998027998837642, + 0.005998375003400724, + 0.007996724001714028, + 0.005996733998472337, + 0.007003572005487513, + 0.004993112997908611, + 0.006997090000368189, + 0.006997115000558551, + 0.007998910004971549, + 0.006989679000980686, + 0.00800188300490845, + 0.006992372997046914, + 0.006998573000601027, + 0.007996517997526098, + 0.006997083997703157, + 0.006997637996391859, + 0.007997590000741184, + 0.007010264998825733, + 0.005987825003103353, + 0.007001983998634387, + 0.004643552005290985, + 0.005336542999430094, + 0.007184264999523293, + 0.007170826000219677, + 0.0076351449970388785, + 0.0069994670047890395, + 0.007174590005888604, + 0.006816915003582835, + 0.004997515003196895, + 0.007005103005212732, + 0.003990133001934737, + 0.001621257993974723, + 0.003987235999375116, + 0.006086143999709748, + 0.003184383996995166, + 0.005109142999572214, + 0.005527868001081515, + 0.006305205999524333, + 0.0007609329986735247, + 0.006530760998430196, + 0.007991151003807317, + 0.008867884003848303, + 0.006997504002356436, + 0.005428686003142502, + 0.006566055002622306, + 0.003996697996626608, + 0.004994950999389403, + 0.006028099996910896, + 0.004968867004208732, + 0.005993733000650536, + 0.0049980119947576895, + 0.006995286996243522, + 0.006998299002589192, + 0.005006229002901819, + 0.007987496006535366, + 0.005997565996949561, + 0.005997438995109405, + 0.010999771999195218, + 0.006994604002102278, + 0.007997419997991528, + 0.006997572003456298, + 0.007007680003880523, + 0.006987022003158927, + 0.006996624993917067, + 0.007003272003203165, + 0.0079916390022845, + 0.008997543001896702, + 0.004995098999643233, + 0.005000983001082204, + 0.00799450499471277, + 0.005997635002131574, + 0.007998549997864757, + 0.007003020997217391, + 0.005989298995700665, + 0.006996520001848694, + 0.007997895001608413, + 0.007996758999070153, + 0.007998804998351261, + 0.006012664001900703, + 0.00806956400629133, + 0.005835635005496442, + 0.006998279997787904, + 0.007992983002623077, + 0.0059995999981765635, + 0.00699524900119286, + 0.007997289998456836, + 0.005998704000376165, + 0.0069957390005583875, + 0.005003175996534992, + 0.005997596999804955, + 0.003990768003859557, + 0.006998458004090935, + 0.006996716998401098, + 0.007002518999797758, + 0.006993580005655531, + 0.007997417997103184, + 0.007998358996701427, + 0.0029988559981575236, + 0.00599389300623443, + 0.006999065000854898, + 0.006240414004423656, + 0.006752481000148691, + 0.005996922998747323, + 0.008136125004966743, + 0.00685799300117651, + 0.007027095001831185, + 0.005034443995100446, + 0.0029318230008357204, + 0.004996873998607043, + 0.006008880998706445, + 0.007984422998561058, + 0.007997635002539027, + 0.007999101995665114, + 0.004771117004565895, + 0.00522198300313903, + 0.007998109998879954, + 0.010016232998168562, + 0.007971799997903872, + 0.006996322998020332, + 0.007997495995368809, + 0.006996759002504405, + 0.00799948199710343, + 0.007995406005647965, + 0.008996969998406712, + 0.00918188499781536, + 0.0038121170000522397, + 0.004007718001957983, + 0.004987986998457927, + 0.006997139993472956, + 0.006997248005063739, + 0.00699702299607452, + 0.006997580996539909, + 0.007997168999281712, + 0.006002406997140497, + 0.00699373099632794, + 0.008997284996439703, + 0.006997128999501001, + 0.007997551998414565, + 0.008005275994946714, + 0.005989187004161067, + 0.0079963329990278, + 0.007995423999091145, + 0.005997161999403033, + 0.004995950002921745, + 0.004661090002628043, + 0.0043344630030333064, + 0.005998471999191679, + 0.006996846001129597, + 0.003974986000685021, + 0.00601975300378399, + 0.0059979490033583716, + 0.007994446998054627, + 0.007997500004421454, + 0.006997154996497557, + 0.006007543997839093, + 0.008345663001819048, + 0.006647501002589706, + 0.006989705994783435, + 0.004524753996520303, + 0.007470452997949906, + 0.008993808995001018, + 0.01099551300285384, + 0.005996497995511163, + 0.008019020002393518, + 0.00797613200120395, + 0.009165846000541933, + 0.006835993001004681, + 0.0023255519990925677, + 0.004707434003648814, + 0.007026799001323525, + 0.007472432000213303, + 0.0064538809965597466, + 0.006190501997480169, + 0.0017912080002133735, + 0.0022847359941806644, + 0.0037102040005265735, + 0.004997447002097033, + 0.005484861001605168, + 0.006509115999506321, + 0.005998207001539413, + 0.006997695003519766, + 0.004217189001792576, + 0.005780649997177534, + 0.004994144001102541, + 0.007999164998182096, + 0.007998282999324147, + 0.006993478004005738, + 0.005996182000671979, + 0.004998393997084349, + 0.005111099002533592, + 0.002882534005038906, + 0.0051206990028731525, + 0.007875575000070967, + 0.005185031004657503, + 0.002808917000947986, + 0.005017625000618864, + 0.0019763590025831945, + 0.006000162000418641, + 0.005995731997245457, + 0.0059979649959132075, + 0.00799763599934522, + 0.005997923006361816, + 0.00799491599900648, + 0.003997066000010818, + 0.004997595999157056, + 0.00699700899713207, + 0.006998569006100297, + 0.008995060998131521, + 0.006999946999712847, + 0.007994725005119108, + 0.003997112005890813, + 0.008997734003060032, + 0.00799933000234887, + 0.006995129995630123, + 0.0069985980007913895, + 0.00701109799410915, + 0.0069815269962418824, + 0.005997877000481822, + 0.007002007005212363, + 0.007992620005097706, + 0.011004447005689144, + 0.007991924998350441, + 0.002995554001245182, + 0.005997996006044559, + 0.007999517998541705, + 0.006997441996645648, + 0.007995266001671553, + 0.008004431001609191, + 0.006989950998104177, + 0.006996565993176773, + 0.006997046999458689, + 0.008003224997082725, + 0.004988788001355715, + 0.007003669998084661, + 0.009989840000343975, + 0.0049967879967880435, + 0.007831731003534514, + 0.006161591001728084, + 0.005997672000376042, + 0.004355625998869073, + 0.001635426000575535, + 0.006299520005995873, + 0.010698707003029995, + 0.0060508050009957515, + 0.003684800998598803, + 0.002362560000619851, + 0.006890379001561087, + 0.004996458999812603, + 0.005535131000215188, + 0.006463096993684303, + 0.008094119002635125, + 0.006029448995832354, + 0.008867758006090298, + 0.0059937990008620545, + 0.009006332998978905, + 0.00558929200633429, + 0.0033978040009969845, + 0.004996073999791406, + 0.006992756003455725, + 0.009502616994723212, + 0.005489826995471958, + 0.004997684998670593, + 0.003996344996266998, + 0.007997795997653157, + 0.0060023889964213595, + 0.006992328002525028, + 0.006421691003197338, + 0.008573104998504277, + 0.005394908002926968, + 0.004680578000261448, + 0.009916000002704095, + 0.005350596002244856, + 0.00965098100277828, + 0.008001344001968391, + 0.007988223995198496, + 0.006994960996962618, + 0.005998551998345647, + 0.005996911000693217, + 0.005996986998070497, + 0.007995690997631755, + 0.005892374996619765, + 0.005102301998704206, + 0.011998688001767732, + 0.0069956590014044195, + 0.006997266995313112, + 0.006999773002462462, + 0.005996239997330122, + 0.006997068005148321, + 0.006758914998499677, + 0.009236830002919305, + 0.007993252998858225, + 0.00200333799875807, + 0.005425184004707262, + 0.0075634450040524825, + 0.00799651399574941, + 0.005001883000659291, + 0.006995522999204695, + 0.004994031995011028, + 0.005999439999868628, + 0.008014969003852457, + 0.006986940003116615, + 0.004539980996923987, + 0.0034432149986969307, + 0.006995725001615938, + 0.006881160996272229, + 0.0071130329961306415, + 0.006996335003350396, + 0.007998031003808137, + 0.006997356002102606, + 0.007996499000000767, + 0.006999197001277935, + 0.00299572399671888, + 0.0059989529981976375, + 0.00799689200357534, + 0.003996416999143548, + 0.009013955997943413, + 0.0030562070023734123, + 0.003919406997738406, + 0.007998103996214923, + 0.00599870099540567, + 0.0069890279992250726, + 0.006994482995651197, + 0.005998345004627481, + 0.006996677999268286, + 0.007005678999121301, + 0.006983597006183118, + 0.006997222000791226, + 0.005995880004775245, + 0.006998643002589233, + 0.0070147250007721595, + 0.0079815380013315, + 0.007186357004684396, + 0.0018104409973602742, + 0.003989706005086191, + 0.006991235000896268, + 0.006719748002069537, + 0.006292389996815473, + 0.006770739004423376, + 0.006208081002114341, + 0.0029879819994675927, + 0.006000028995913453, + 0.004996600997401401, + 0.006998210999881849, + 0.008996765995107125, + 0.005995250998239499, + 0.006332153003313579, + 0.0016534419992240146, + 0.006242811999982223, + 0.00575846300489502, + 0.006831651000538841, + 0.006162662997667212, + 0.005993325998133514, + 0.003992596997704823, + 0.0030102529999567196, + 0.003967095006373711, + 0.006528183999762405, + 0.010497154995391611, + 0.005958980000286829, + 0.004994289003661834, + 0.0066656880007940345, + 0.0033136659985757433, + 0.006004141003359109, + 0.005990507001115475, + 0.004718508003861643, + 0.008282165996206459, + 0.0069883499963907525, + 0.004443322999577504, + 0.0035471359951770864, + 0.0069961170011083595, + 0.006995241994445678, + 0.006997853997745551, + 0.007334743997489568, + 0.007652222004253417, + 0.00799389500025427, + 0.007002913000178523, + 0.00599494599737227, + 0.009026473999256268, + 0.007962283001688775, + 0.004602645996783394, + 0.005408981000073254, + 0.004964742998708971, + 0.007566604996100068, + 0.007427676995575894, + 0.00599004200194031, + 0.006999328004894778, + 0.006000444998790044, + 0.007996278000064194, + 0.006999826000537723, + 0.007991462996869814, + 0.005995084997266531, + 0.006985897998674773, + 0.0079932029984775, + 0.006994638002652209, + 0.007011982997937594, + 0.00697993300127564, + 0.005991565005388111, + 0.0026889430009759963, + 0.0053084090031916276, + 0.0069981710039428435, + 0.006990904999838676, + 0.0069976929953554645, + 0.005715606006560847, + 0.004272741003660485, + 0.002109450004354585, + 0.0038834880033391528, + 0.004177262999291997, + 0.005835606003529392, + 0.00698286199622089, + 0.0069926479991409, + 0.006993545997829642, + 0.002807953998853918, + 0.0005893279958399944, + 0.00045897599920863286, + 0.0004213880019960925, + 0.00041237700497731566, + 0.00042335699981777, + 0.0004186550067970529, + 0.00041174699435941875, + 0.005773154000053182, + 0.0005311920031090267, + 0.0004833470011362806, + 0.00044719599827658385, + 0.0004231450002407655, + 0.003905874997144565, + 0.0005994129969622009, + 0.0005052939959568903, + 0.002591693999420386, + 0.0005083590003778227, + 0.002468833001330495, + 0.0005090209961053915, + 0.0004951070004608482, + 0.00046980100159998983, + 0.00041821200284175575, + 0.0039387159995385446, + 0.0009322629994130693, + 0.00051651599642355, + 0.0005038300005253404, + 0.00043583599472185597, + 0.0004144790000282228, + 0.00041921700176317245, + 0.0004099220022908412, + 0.00041240300197387114, + 0.00048096100363181904, + 0.007387561003270093, + 0.0005983299997751601, + 0.0005283080026856624, + 0.001963545997568872, + 0.0005741909990319982, + 0.007141993999539409, + 0.0006277529973885976, + 0.00045301299542188644, + 0.00043289300083415583, + 0.0019188540027244017, + 0.0005726160015910864, + 0.0004732939996756613, + 0.00042412400216562673, + 0.00040811399958329275, + 0.0004625000001396984, + 0.004709800996351987, + 0.0005536259996006265, + 0.0005343070006347261, + 0.00043224999535596, + 0.0004659269980038516, + 0.000408599000365939, + 0.00040659200021764264, + 0.00040993700531544164, + 0.006057279999367893, + 0.0005822349994559772, + 0.0004609039970091544, + 0.00040949799586087465, + 0.0004043419976369478, + 0.00039931200444698334, + 0.0004072720039403066, + 0.0004003180001745932, + 0.006708965003781486, + 0.0006573680002475157, + 0.0004917670012218878, + 0.00041548500303179026, + 0.00046903300244593993, + 0.0004074559983564541, + 0.0004152370020165108, + 0.00040100800106301904, + 0.006694408002658747, + 0.000575268997636158, + 0.00185005600360455, + 0.0029108989983797073, + 0.000552738994883839, + 0.0004911389987682924, + 0.002028867995250039, + 0.0005613219982478768, + 0.0004457079994608648, + 0.0004344279950601049, + 0.002309224000782706, + 0.0005044799981988035, + 0.0004493369997362606, + 0.0003997289968538098, + 0.0017714400019031018, + 0.0004474279994610697, + 0.00041977400542236865, + 0.0009727130018291064, + 0.0013309179994394071, + 0.0035502489990903996, + 0.0006200239949976094, + 0.0005075209992355667, + 0.0016360040026484057, + 0.0005403649993240833, + 0.0005343800003174692, + 0.0004323570028645918, + 0.00040687400178285316, + 0.0004025980015285313, + 0.0005558319971896708, + 0.0009049720028997399, + 0.0004906250032945536, + 0.0004142539983149618, + 0.0004192689957562834, + 0.00041974399937316775, + 0.00041306600178359076, + 0.0004032020005979575, + 0.006082920997869223, + 0.003009138999914285, + 0.005415140003606211, + 0.0005527059984160587, + 0.0005361250005080365, + 0.0004406360021675937, + 0.0004052619988215156, + 0.00040309900214197114, + 0.0004023200017400086, + 0.0004039550040033646, + 0.000408480002079159, + 0.005514217002200894, + 0.0006598239997401834, + 0.0005578930067713372, + 0.00042913299694191664, + 0.0004306230039219372, + 0.0004104819963686168, + 0.0004157679941272363, + 0.0004003670037491247, + 0.002798542001983151, + 0.0005323520017554983, + 0.00046168299741111696, + 0.0004285969989723526, + 0.0004116160052944906, + 0.00040595600148662925, + 0.0004068890048074536, + 0.00039932900108397007, + 0.0003995480001321994, + 0.00490952000109246, + 0.0005246939981589094, + 0.00044496900227386504, + 0.00041309999505756423, + 0.00041541999962646514, + 0.00040013300167629495, + 0.0004841129994019866, + 0.0004067629997734912, + 0.00423031700483989, + 0.0005893959969398566, + 0.00042528899939497933, + 0.0004067390036652796, + 0.0004058439953951165, + 0.00040919399907579646, + 0.0004030510026495904, + 0.0004124789993511513, + 0.00040713699854677543, + 0.00039996900159167126, + 0.004566217998217326, + 0.0021361300023272634, + 0.0005244850035523996, + 0.00043509600072866306, + 0.00042977100383723155, + 0.0004565010021906346, + 0.00040546200034441426, + 0.00042258499888703227, + 0.0005974849991616793, + 0.005589348002104089, + 0.0005876380018889904, + 0.0005339369963621721, + 0.00042372300231363624, + 0.0004199780014459975, + 0.0004294870013836771, + 0.00042015699727926403, + 0.0004175809954176657, + 0.00043020700104534626, + 0.00046261899842647836, + 0.0014627279961132444, + 0.0006034440011717379, + 0.0004575849961838685, + 0.0004265340030542575, + 0.0004136499992455356, + 0.0004167829974903725, + 0.0004192460037302226, + 0.0004908119954052381, + 0.00042356699850643054, + 0.00043177499901503325, + 0.0004836980006075464, + 0.00043944199569523335, + 0.00042740299977594987, + 0.00046874599502189085, + 0.0004710020002676174, + 0.0005300800039549358, + 0.0004579660017043352, + 0.00042324300011387095, + 0.00042522900184849277, + 0.0004357349971542135, + 0.0004092860035598278, + 0.0004111070011276752, + 0.0004155430069658905, + 0.0004346470013842918, + 0.0004123960025026463, + 0.00040875199920265004, + 0.00042596999992383644, + 0.0004170680040260777, + 0.0004282329973648302, + 0.00042309099808335304, + 0.00042325899994466454, + 0.00043371600622776896, + 0.0004238980036461726, + 0.00047582700062775984, + 0.00044597400119528174, + 0.0004477719994611107, + 0.0005402380047598854, + 0.0004628220049198717, + 0.00044023599912179634, + 0.00042534199747024104, + 0.0004449990010471083, + 0.0004429280015756376, + 0.00046746800217079, + 0.00043287099833833054, + 0.00042421800026204437, + 0.00043831099901581183, + 0.0004215750013827346, + 0.0004366450011730194, + 0.00043000800360459834, + 0.0004246130047249608, + 0.0004930419963784516, + 0.0004677909964811988, + 0.0004474520028452389, + 0.0004917740006931126, + 0.00044018599874107167, + 0.0004221680064802058, + 0.00042257699533365667, + 0.00041337999573443085, + 0.0004219599941279739, + 0.0004477719994611107, + 0.0004418989992700517, + 0.00043986899981973693, + 0.00043716800428228453, + 0.00044093500036979094, + 0.0004216570014250465, + 0.0004331460004323162, + 0.00042302600195398554, + 0.000426780003181193, + 0.00044893199810758233, + 0.00046882099559297785, + 0.0004257589971530251, + 0.000429085994255729, + 0.0004194310022285208, + 0.0004827090015169233, + 0.000490660997456871, + 0.00045354800386121497, + 0.00045328000123845413, + 0.00043312699563102797, + 0.00043986999662593007, + 0.00043444300536066294, + 0.000439362003817223, + 0.00042303800000809133, + 0.00042762900557136163, + 0.0004586790018947795, + 0.00042069800110766664, + 0.0004173199995420873, + 0.0005182169988984242, + 0.0004604069981724024, + 0.00045862099796067923, + 0.0004428590036695823, + 0.0005070309998700395, + 0.0004571080062305555, + 0.00041501499799778685, + 0.00040785000601317734, + 0.0004170319953118451, + 0.0004073210002388805, + 0.003108046999841463, + 0.0005600000004051253, + 0.0013571099989349023, + 0.0004399539975565858, + 0.00041789199894992635, + 0.00041015399619936943, + 0.00046636300248792395, + 0.00046564999502152205, + 0.005148782998730894, + 0.0006061519961804152, + 0.0004445410013431683, + 0.0004168310042587109, + 0.00045888699969509616, + 0.00047143400297500193, + 0.0004239950067130849, + 0.006852272003015969, + 0.0007195939979283139, + 0.0004272370060789399, + 0.0004960359947290272, + 0.0004103210012544878, + 0.0004125740015297197, + 0.0004160110038355924, + 0.00041015399619936943, + 0.00041398200119147077, + 0.00040564400114817545, + 0.005503396001586225, + 0.0005785959947388619, + 0.0004816899963770993, + 0.0004449460029718466, + 0.00041417800093768165, + 0.000410963999456726, + 0.00040774299850454554, + 0.000411877001170069, + 0.00041647600301075727, + 0.007862619000661653, + 0.0006997770033194683, + 0.000733927998226136, + 0.000498016997880768, + 0.0010593979968689382, + 0.0004357619982329197, + 0.00042146000487264246, + 0.0004097150012967177, + 0.0019253489954280667, + 0.0004374920026748441, + 0.0004236780005157925, + 0.0004242489958414808, + 0.00042395800119265914, + 0.0008056680017034523, + 0.0031756120006321, + 0.0005129989949637093, + 0.000457558999187313, + 0.00041165600123349577, + 0.001211383001646027, + 0.0005407130011008121, + 0.005232711002463475, + 0.0006116380027378909, + 0.000691451998136472, + 0.0004932489973725751, + 0.00042578799912007526, + 0.00041931799933081493, + 0.004649107999284752, + 0.0005397220011218451, + 0.0011894779963768087, + 0.00041530499584041536, + 0.0013364710030145943, + 0.0005554640010814182, + 0.0004286760013201274, + 0.0005394399995566346, + 0.0015245600006892346, + 0.0005748359981225803, + 0.00045709999540122226, + 0.0017841480002971366, + 0.00047188200551318005, + 0.000500303998705931, + 0.0004452759967534803, + 0.0008081949999905191, + 0.00044076699850847945, + 0.000550598997506313, + 0.0004410559995449148, + 0.0005076040033600293, + 0.0004271970028639771, + 0.0004131560053792782, + 0.000422930002969224, + 0.00041467000119155273, + 0.0004165639984421432, + 0.0016139729996211827, + 0.000792326005466748, + 0.00042574300459818915, + 0.00041465800313744694, + 0.0012701240048045292, + 0.00243062200024724, + 0.0005630469968309626, + 0.0004313159952289425, + 0.0005132180012878962, + 0.0006384670050465502, + 0.00047497400373686105, + 0.00043742600246332586, + 0.00041409899858990684, + 0.00041179900290444493, + 0.0004848489988944493, + 0.0004216999950585887, + 0.00041995300125563517, + 0.00041185799636878073, + 0.004139583994401619, + 0.0005771170035586692, + 0.00043779899715445936, + 0.00042371499876026064, + 0.00041168200550600886, + 0.0004135109993512742, + 0.00041185400186805055, + 0.000410842003475409, + 0.0005442370020318776, + 0.0005098039982840419, + 0.0005975079984636977, + 0.0008429139998042956, + 0.0008573239974793978, + 0.0010511469954508357, + 0.00044152699410915375, + 0.0011690129977068864, + 0.0004389570021885447, + 0.0004228960024192929, + 0.0004189140017842874, + 0.0004119100049138069, + 0.0004766490019392222, + 0.0004474620000110008, + 0.00041926799895009026, + 0.0025811990053625777, + 0.0015877949990681373, + 0.0004263030059519224, + 0.0004859320033574477, + 0.00048793099995236844, + 0.0004360899984021671, + 0.0004131219975533895, + 0.0004081899969605729, + 0.0011239289960940368, + 0.0016837509974720888, + 0.000965792998613324, + 0.0005848669970873743, + 0.0004203689968562685, + 0.00041159099782817066, + 0.00042084400047315285, + 0.0004135269991820678, + 0.00041655100358184427, + 0.00041046199476113543, + 0.00044154100032756105, + 0.00042176999704679474, + 0.000409067994041834, + 0.0008330809941980988, + 0.0019419130039750598, + 0.000836181003251113, + 0.0004225899974699132, + 0.0004155139977228828, + 0.0004099569996469654, + 0.0004184019999229349, + 0.00041122599941445515, + 0.00041153000347549096, + 0.000413905996538233, + 0.0004082430023117922, + 0.0004658920006477274, + 0.00045406699791783467, + 0.0006168530017021112, + 0.00247904600109905, + 0.0007618520030518994, + 0.003098316003161017, + 0.0005031140026403591, + 0.00042403700354043394, + 0.00041875900205923244, + 0.00041210299968952313, + 0.00040988700493471697, + 0.004289761003747117, + 0.0005467480004881509, + 0.0005185249974601902, + 0.0019665469953906722, + 0.0004568610020214692, + 0.0004811890030396171, + 0.0004582210021908395, + 0.0013440179973258637, + 0.005050351996032987, + 0.004520132999459747, + 0.0019623889966169372, + 0.0017506660005892627, + 0.0014627009950345382, + 0.00070852800126886, + 0.001436575003026519, + 0.001302479999139905, + 0.0005258340024738573, + 0.0005060590046923608, + 0.0006119859972386621, + 0.0004202249983791262, + 0.0009955200002877973, + 0.00046713499614270404, + 0.00048732299910625443, + 0.0004473879962461069, + 0.001970242999959737, + 0.0011343219957780093, + 0.0004310020012781024, + 0.0004224009971949272, + 0.0004336580022936687, + 0.00042989700159523636, + 0.005107106000650674, + 0.005266679996566381, + 0.014999858998635318, + 0.0029539069946622476, + 0.0009996759981731884, + 0.0030267940019257367, + 0.0006770089967176318, + 0.0004419860051712021, + 0.00043163999362150207, + 0.00043454099795781076, + 0.0004373539995867759, + 0.00042621100146789104, + 0.005196728998271283, + 0.0018891189974965528, + 0.0005192009994061664, + 0.00043393299711169675, + 0.0005100010021124035, + 0.00042821300303330645, + 0.006168899999465793, + 0.0079924489982659, + 0.007993428996996954, + 0.007995692998520099, + 0.005022416000429075, + 0.0006168350009829737, + 0.001215545999002643, + 0.0004399110039230436, + 0.0004652360003092326, + 0.0007317639974644408, + 0.0007758569990983233, + 0.005462615998112597, + 0.0005935439985478297, + 0.0009405299933860078, + 0.0028292229981161654, + 0.0005396740016294643, + 0.0004423849968588911, + 0.0005223219995968975, + 0.0004360759994597174, + 0.0005175870028324425, + 0.00043631799780996516, + 0.0004322159948060289, + 0.00042726699757622555, + 0.009150302001216915, + 0.004959778001648374, + 0.0008896770013961941, + 0.0011620540026342496, + 0.00494571500166785, + 0.0022475539954029955, + 0.0005130279969307594, + 0.0023446569975931197, + 0.005256322998320684, + 0.000707118000718765, + 0.0004922619991702959, + 0.0004870949996984564, + 0.000437201997556258, + 0.0005126800024299882, + 0.00042497899994486943, + 0.0004316600025049411, + 0.0013469730038195848, + 0.00047297600394813344, + 0.0004373219999251887, + 0.00042451500485185534, + 0.0004248770055710338, + 0.0004241939968778752, + 0.00042387299617985263, + 0.0004257260006852448, + 0.00042174999543931335, + 0.002728508996369783, + 0.0024525349945179187, + 0.0005246069995337166, + 0.0004461719945538789, + 0.0005158869971637614, + 0.0004650850023608655, + 0.0004528130011749454, + 0.0004327349961386062, + 0.004358765996585134, + 0.0005702009948436171, + 0.0005147740012034774, + 0.0004647050009225495, + 0.000429525003710296, + 0.0004256890024407767, + 0.0004269579949323088, + 0.00042387300345581025, + 0.0004279229979147203, + 0.0026965510041918606, + 0.0027819280003313906, + 0.000564293994102627, + 0.0005910079999011941, + 0.0004541550006251782, + 0.00043634000030579045, + 0.00045075200614519417, + 0.0004393560011521913, + 0.0004264289964339696, + 0.005805537999549415, + 0.0005808950008940883, + 0.0005109639969305135, + 0.0004316600025049411, + 0.0005748839976149611, + 0.0017935489959199913, + 0.001604575001692865, + 0.000501141999848187, + 0.0017770000049495138, + 0.00048509699990972877, + 0.0005314759982866235, + 0.00043602000368991867, + 0.00046626199764432386, + 0.0004497539994190447, + 0.000437351998698432, + 0.00042340700019849464, + 0.0012587379969772883, + 0.0018455810059094802, + 0.0012248679995536804, + 0.00043808000191347674, + 0.0005278749958961271, + 0.0012545339995995164, + 0.00044431499554775655, + 0.00047083299432415515, + 0.00046792899956926703, + 0.0004235050000716001, + 0.00043595900206128135, + 0.0015799789980519563, + 0.0032255670012091286, + 0.0005111659993417561, + 0.0006115830037742853, + 0.0004489229977480136, + 0.00045907699677627534, + 0.0004376779979793355, + 0.0004336929996497929, + 0.0004965469997841865, + 0.0007007320018601604, + 0.001592954999068752, + 0.00044435499876271933, + 0.00042419700184836984, + 0.0004236370004946366, + 0.0004242590002832003, + 0.0004256890024407767, + 0.000417760995333083, + 0.00041889099520631135, + 0.0017392859954270534, + 0.0016034910004236735, + 0.0018899360002251342, + 0.0004939849968650378, + 0.0004367609944893047, + 0.0004321079977671616, + 0.00042632899567252025, + 0.0005060929979663342, + 0.00043147899850737303, + 0.006581768000614829, + 0.0005895180001971312, + 0.0005091959974379279, + 0.0004254300001775846, + 0.00042234900320181623, + 0.00042507699981797487, + 0.00042648900125641376, + 0.00041591099579818547, + 0.006594184000277892, + 0.0006180630007293075, + 0.0005730819975724444, + 0.0004413620044942945, + 0.0004597890001605265, + 0.0004380430036690086, + 0.0004287710034986958, + 0.0004818259985768236, + 0.007139991997973993, + 0.0005716890009352937, + 0.0004581850007525645, + 0.00042797799687832594, + 0.0018880520001403056, + 0.0004938209967804141, + 0.0004627670059562661, + 0.0004350059971329756, + 0.0010306309995939955, + 0.0005094929947517812, + 0.00045845900604035705, + 0.0004341280000517145, + 0.0004233590007061139, + 0.00044363200140651315, + 0.0005052990018157288, + 0.00046369300252990797, + 0.004448856998351403, + 0.0011135280001326464, + 0.00042310099524911493, + 0.0004238260007696226, + 0.0004244290030328557, + 0.0004268500051693991, + 0.0004598829982569441, + 0.0004429479959071614, + 0.00042992499948013574, + 0.004180626005108934, + 0.0006696060008835047, + 0.0004779860028065741, + 0.00045028800377622247, + 0.0004427979947649874, + 0.00052683700050693, + 0.0006816519962740131, + 0.0006394559968612157, + 0.0005079769980511628, + 0.0004631420015357435, + 0.0004276670006220229, + 0.0004263130031176843, + 0.000422443998104427, + 0.0019549459975678474, + 0.00046013700193725526, + 0.0004442950012162328, + 0.0004495590037549846, + 0.0004328479990363121, + 0.00042523500451352447, + 0.0019179650043952279, + 0.002973447997646872, + 0.0005370959988795221, + 0.002821308000420686, + 0.0005983769951853901, + 0.0009628959960537031, + 0.0006365369990817271, + 0.0005064509969088249, + 0.00044259899732423946, + 0.007883489997766446, + 0.0007038080002530478, + 0.00043556199671002105, + 0.00041051099833566695, + 0.0004167380029684864, + 0.00041293600224889815, + 0.00042051399941556156, + 0.0004096630000276491, + 0.008301637004478835, + 0.0006101530016167089, + 0.0005058809983893298, + 0.00048303200310328975, + 0.00043103999632876366, + 0.0004183710043434985, + 0.00041793299897108227, + 0.005567845997575205, + 0.0005826709966640919, + 0.0005374090033001266, + 0.0004286519979359582, + 0.0004947119959979318, + 0.00042985400068573654, + 0.0004251869977451861, + 0.006088528003601823, + 0.000671276000502985, + 0.0004980300000170246, + 0.00042566599586280063, + 0.005146460003743414, + 0.0014868569996906444, + 0.005325068996171467, + 0.005994006001856178, + 0.008004033996257931, + 0.00412221800070256, + 0.007868110995332245, + 0.008021148998523131, + 0.008315480001328979, + 0.0076290029974188656, + 0.007966858000145294, + 0.008115434997307602, + 0.004877728002611548, + 0.006960826998692937, + 0.012390636002237443, + 0.009891097004583571, + 0.006405741994967684, + 0.007270049005455803, + 0.007976983004482463, + 0.006981789003475569, + 0.005996892999974079, + 0.004609898998751305, + 0.006953416006581392, + 0.006693666000501253, + 0.0026925159982056357, + 0.006106196000473574, + 0.0007108600038918667, + 0.0005514509975910187, + 0.0004326539929024875, + 0.00044223000440979376, + 0.0006158249962027185, + 0.0005129519995534793, + 0.0004297129999031313, + 0.005739988999266643, + 0.0005379510039347224, + 0.0006950210008653812, + 0.00046390500210691243, + 0.0004238290057401173, + 0.00042304800444981083, + 0.0005066000012448058, + 0.005497856000147294, + 0.0008519760012859479, + 0.0010510209976928309, + 0.000511669997649733, + 0.0005121729991515167 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil", + "fullname": "TestApplyNabla2AndNabla4ToVn", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.0004973409959347919, + "max": 0.058552956994390115, + "mean": 0.004031990086174464, + "stddev": 0.0048726557169137584, + "rounds": 743, + "median": 0.0020961110058124177, + "iqr": 0.0064202395024040015, + "q1": 0.0005695842482964508, + "q3": 0.006989823750700452, + "iqr_outliers": 15, + "stddev_outliers": 56, + "outliers": "56;15", + "ld15iqr": 0.0004973409959347919, + "hd15iqr": 0.016996696998830885, + "ops": 248.0164828353524, + "total": 2.995768634027627, + "data": [ + 0.006994562005274929, + 0.006991027003095951, + 0.005631523999909405, + 0.0063559890040778555, + 0.002378231001785025, + 0.003608762999647297, + 0.006002092995913699, + 0.005542909995710943, + 0.00445655300427461, + 0.007015672999841627, + 0.007958791000419296, + 0.00699589800206013, + 0.007017026997345965, + 0.004994367998733651, + 0.007973642997967545, + 0.007005079998634756, + 0.002881583000998944, + 0.006096122997405473, + 0.0059965159962303005, + 0.00500260699482169, + 0.005992222002532799, + 0.005994234001263976, + 0.007348270999500528, + 0.00664360399969155, + 0.004607791997841559, + 0.004972246999386698, + 0.005358853995858226, + 0.0050204250001115724, + 0.011986496996541973, + 0.006678450001345482, + 0.004364954998891335, + 0.0029140600017854013, + 0.0019970120047219098, + 0.005040560004999861, + 0.007959331000165548, + 0.008996851996926125, + 0.006984968997130636, + 0.00613956899906043, + 0.008656292004161514, + 0.01590454700635746, + 0.007063761004246771, + 0.004177809001703281, + 0.009028174994455185, + 0.005219322003540583, + 0.036127849998592865, + 0.058552956994390115, + 0.008960809995187446, + 0.004985585997928865, + 0.0029832810032530688, + 0.0006678110003122129, + 0.0005811189985251985, + 0.0005671550024999306, + 0.003948980993300211, + 0.0006183750010677613, + 0.0005771709984401241, + 0.000567188995773904, + 0.0005966619937680662, + 0.0005660599999828264, + 0.0066457790017011575, + 0.0006662870000582188, + 0.0006052800017641857, + 0.0005676740038325079, + 0.0005677000008290634, + 0.0005664149939548224, + 0.0005797109988634475, + 0.008043335998081602, + 0.005994635001115967, + 0.006998886994551867, + 0.006059131002984941, + 0.006931538999197073, + 0.0059968519999529235, + 0.0080002320028143, + 0.0039951189974090084, + 0.006998056000156794, + 0.005812134004372638, + 0.0051825179980369285, + 0.011007768000126816, + 0.00798626100004185, + 0.004740664997370914, + 0.004680704005295411, + 0.0075858799973502755, + 0.00797694699576823, + 0.006997620999754872, + 0.006096311997680459, + 0.00795734699931927, + 0.007936092995805666, + 0.00699226300639566, + 0.006988197004829999, + 0.01100306100124726, + 0.005990965000819415, + 0.006876409002870787, + 0.03130758999759564, + 0.017450975996325724, + 0.006330295997031499, + 0.0072219940047943965, + 0.01576943699910771, + 0.007984333002241328, + 0.00500780400034273, + 0.006980900994676631, + 0.007997835004061926, + 0.007999840003321879, + 0.006988398003159091, + 0.006997960001172032, + 0.007996105996426195, + 0.015007540001533926, + 0.007004330000199843, + 0.0069741999977850355, + 0.007993500999873504, + 0.006993415001488756, + 0.008032603000174277, + 0.005945443001110107, + 0.007989359000930563, + 0.006994648996624164, + 0.007137139997212216, + 0.006857950997073203, + 0.00799287900008494, + 0.006998343000304885, + 0.0069964459980838, + 0.0060244589985813946, + 0.007980123999004718, + 0.006968109999434091, + 0.006996915995841846, + 0.007995526997547131, + 0.00658665900118649, + 0.011409502993046772, + 0.007994179999514017, + 0.00800040900503518, + 0.0059939850034425035, + 0.005995601997710764, + 0.006131230002210941, + 0.006865515999379568, + 0.006994287999987137, + 0.007999980000022333, + 0.006993959999817889, + 0.006380777995218523, + 0.002612044001580216, + 0.005008995998650789, + 0.008997354001621716, + 0.007989359997736756, + 0.006991103000473231, + 0.00800332299695583, + 0.007990974001586437, + 0.005995570005325135, + 0.00802095299877692, + 0.0069759410034748726, + 0.006990785004745703, + 0.005993061997287441, + 0.00599350099946605, + 0.010996153003361542, + 0.006990886002313346, + 0.006996061994868796, + 0.006997555996349547, + 0.006999411001743283, + 0.006006217001413461, + 0.0069850560030317865, + 0.005997750995447859, + 0.005997414999001194, + 0.007997170003363863, + 0.0069976489976397716, + 0.006995525996899232, + 0.007998587003385182, + 0.006996823998633772, + 0.006998470998951234, + 0.007995636995474342, + 0.008006392999959644, + 0.00799943000311032, + 0.00800268400053028, + 0.00998302500374848, + 0.006992570997681469, + 0.0066425340046407655, + 0.00535209499503253, + 0.003996389001258649, + 0.0042673719945014454, + 0.0037253039990901016, + 0.007004428000072949, + 0.006990298999880906, + 0.0070012160067562945, + 0.0069918370063533075, + 0.005489588998898398, + 0.005503510001290124, + 0.005997133004711941, + 0.00526943699514959, + 0.007725327006482985, + 0.008000112997251563, + 0.007995791995199397, + 0.006011361998389475, + 0.005982627997582313, + 0.005993833001411986, + 0.006999349003308453, + 0.00799556100537302, + 0.006998526005190797, + 0.007998672997928225, + 0.008005513001990039, + 0.002831731006153859, + 0.00615058599942131, + 0.006996809002885129, + 0.006998087003012188, + 0.0069961970002623275, + 0.005270785994071048, + 0.006732321999152191, + 0.00799919600103749, + 0.008002187001693528, + 0.007976642998983152, + 0.007000614998105448, + 0.00800067499949364, + 0.0059865930015803315, + 0.007996555999852717, + 0.005996566993417218, + 0.007997609995072708, + 0.008006119001947809, + 0.006980546997510828, + 0.005989805002172943, + 0.010621886001899838, + 0.018283220997545868, + 0.007090318002155982, + 0.005040611002186779, + 0.010044545000710059, + 0.004366619003121741, + 0.007521368999732658, + 0.008138336001138669, + 0.005855260002135765, + 0.006993369999690913, + 0.0014571010033250786, + 0.0005440029999590479, + 0.0007443780050380155, + 0.0050039880006806925, + 0.0006875909966765903, + 0.0008669029994052835, + 0.0005267070009722374, + 0.0005272459966363385, + 0.0005005029961466789, + 0.006421720994694624, + 0.0006751140026608482, + 0.0006437669944716617, + 0.0005111139980726875, + 0.0005005890052416362, + 0.0005044490026193671, + 0.0004973409959347919, + 0.007596393006679136, + 0.0006808690013713203, + 0.0014914749990566634, + 0.0015149629980442114, + 0.0050402800043229945, + 0.008014885002921801, + 0.008029242002521642, + 0.007952508000016678, + 0.005990621000819374, + 0.005988559001707472, + 0.007995218998985365, + 0.005999528999382164, + 0.007991838996531442, + 0.007998150002094917, + 0.00799506000475958, + 0.005997803993523121, + 0.005999279004754499, + 0.008001417001651134, + 0.007991824997588992, + 0.0059952500014333054, + 0.007997194996278267, + 0.010159366000152659, + 0.00785266899765702, + 0.002970162000565324, + 0.008014334001927637, + 0.0070014209995861165, + 0.016996696998830885, + 0.007963983000081498, + 0.0250037260047975, + 0.0179822550053359, + 0.014230025000870228, + 0.012733201001537964, + 0.013998372000060044, + 0.008658115999423899, + 0.01431455399870174, + 0.006975834003242198, + 0.01076965299580479, + 0.01320597300218651, + 0.014987090005888604, + 0.004994940994947683, + 0.01099947199691087, + 0.0070905309985391796, + 0.007609396998304874, + 0.0020978449974791147, + 0.0021085639964439906, + 0.0047299939978984185, + 0.003269556000304874, + 0.0057940119950217195, + 0.0018774370037135668, + 0.002108307999151293, + 0.004217319001327269, + 0.0021085219996166416, + 0.0021071439987281337, + 0.0029124450011295266, + 0.003418195999984164, + 0.004219526002998464, + 0.0031654839986003935, + 0.002716859002248384, + 0.003598735995183233, + 0.0021049779970780946, + 0.002108905006025452, + 0.004224109994538594, + 0.002102717997331638, + 0.002225634998467285, + 0.006212686996150296, + 0.002106374995491933, + 0.0045747160038445145, + 0.001389523000398185, + 0.002471509993483778, + 0.005290199005685281, + 0.0020961110058124177, + 0.008443619000900071, + 0.0019817809952655807, + 0.007179264001024421, + 0.0012037829947075807, + 0.0028717370005324483, + 0.005573643000388984, + 0.00210044900450157, + 0.002103441998769995, + 0.00903404699784005, + 0.0014559760020347312, + 0.008427533000940457, + 0.0010524840035941452, + 0.0077359850038192235, + 0.0018589860046631657, + 0.0016758120036683977, + 0.012041800997394603, + 0.01899078700080281, + 0.027720194004359655, + 0.006334364996291697, + 0.00549884200154338, + 0.02540032000251813, + 0.009992733001126908, + 0.020010331005323678, + 0.023079421000147704, + 0.023889434000011533, + 0.01039563099766383, + 0.013558276994444896, + 0.009949986000719946, + 0.01197211100225104, + 0.0116800410032738, + 0.012587748002260923, + 0.011650362001091707, + 0.0037321749987313524, + 0.0063158340053632855, + 0.011234031000640243, + 0.022748417999537196, + 0.006130968999059405, + 0.005073749001894612, + 0.00676139400457032, + 0.004774894994625356, + 0.004190997999103274, + 0.0069922309994581155, + 0.008450638997601345, + 0.004524202005995903, + 0.013016716002312023, + 0.006959984006243758, + 0.007378674003120977, + 0.002133362999302335, + 0.0010484700032975525, + 0.0005984800009173341, + 0.0005823030005558394, + 0.0006238639980438165, + 0.011739699999452569, + 0.007019801996648312, + 0.000615723998635076, + 0.0006097299992688932, + 0.002353111995034851, + 0.0077823500032536685, + 0.0006231859952094965, + 0.0005966989992884919, + 0.0005898099989281036, + 0.0005809009962831624, + 0.0005857979995198548, + 0.007754672995361034, + 0.01105301600182429, + 0.0007138000000850298, + 0.0005811480004922487, + 0.010033489001216367, + 0.0007318249990930781, + 0.0005879629970877431, + 0.0005682699993485585, + 0.000573734003410209, + 0.000574616999074351, + 0.00828221499978099, + 0.0006667429988738149, + 0.007793806005793158, + 0.0006403600054909475, + 0.0005945459997747093, + 0.0005695209983969107, + 0.0005720370027120225, + 0.0007742450034129433, + 0.0059562939932220615, + 0.0006976700024097227, + 0.0005966149983578362, + 0.000633537994872313, + 0.0005661239993060008, + 0.0005700960027752444, + 0.0005626999991363846, + 0.0005665680000674911, + 0.0031778640041011386, + 0.0006143259961390868, + 0.0005940670016570948, + 0.0006538660018122755, + 0.0006792499989387579, + 0.0020138120016781613, + 0.0022927110039745457, + 0.0006754889982403256, + 0.0005820570004289038, + 0.0005660550013999455, + 0.0005681380062014796, + 0.0005618919967673719, + 0.0061545460048364475, + 0.0006998890021350235, + 0.0007126310010789894, + 0.000561273998755496, + 0.0005676210057572462, + 0.0005472560005728155, + 0.0005428419972304255, + 0.008189828993636183, + 0.0007093989988788962, + 0.0005465979993459769, + 0.0005408459983300418, + 0.0005459189997054636, + 0.0005398880020948127, + 0.006925421999767423, + 0.000613666998106055, + 0.0005708460012101568, + 0.0009373069988214411, + 0.0005472150005516596, + 0.0005479539977386594, + 0.006598647996725049, + 0.0006391880015144125, + 0.00058548100059852, + 0.0005513629948836751, + 0.0005429589946288615, + 0.0005500049956026487, + 0.0005850629968335852, + 0.004457117000129074, + 0.0010175209972658195, + 0.0006670750008197501, + 0.0005732969948439859, + 0.0005665279968525283, + 0.0006016670013195835, + 0.0005905950019950978, + 0.0005795839970232919, + 0.0005622050011879764, + 0.0005670800019288436, + 0.0005563439990510233, + 0.0005734639998991042, + 0.0005666049983119592, + 0.0005569849963649176, + 0.0005592350062215701, + 0.0005963369985693134, + 0.000571995995414909, + 0.0005791600051452406, + 0.0005754220037488267, + 0.0005458380037453026, + 0.0005829950023326091, + 0.0005747769973822869, + 0.0008038329979171976, + 0.0005504259970621206, + 0.000569775998883415, + 0.0005600990043603815, + 0.0005866020001121797, + 0.0005705139992642216, + 0.0005792010051663965, + 0.001112601996283047, + 0.0005519160040421411, + 0.0005645689961966127, + 0.0005772889999207109, + 0.000580451000132598, + 0.0005598409989033826, + 0.0005970389975118451, + 0.0005689669997082092, + 0.00058239400095772, + 0.0005537799952435307, + 0.0005516979945241474, + 0.0005590139990090393, + 0.0005790340001112781, + 0.0005579560020123608, + 0.0005595019974862225, + 0.0005473679993883707, + 0.0005720300032407977, + 0.0005738750041928142, + 0.0005804840038763359, + 0.0005865550047019497, + 0.0005598990028374828, + 0.0005811490045743994, + 0.0005468309973366559, + 0.0005725630035158247, + 0.0005667579971486703, + 0.0005642510004690848, + 0.0005782189982710406, + 0.0005630140003631823, + 0.0005764699963037856, + 0.0006167579995235428, + 0.0005710750047001056, + 0.0005739500047639012, + 0.0005819819998578168, + 0.0005841440070071258, + 0.0005941530034760945, + 0.0005810450020362623, + 0.0006465109981945716, + 0.0005530780035769567, + 0.0005706199954147451, + 0.000555638995137997, + 0.000571333002881147, + 0.0005607689963653684, + 0.000577224993321579, + 0.0005833290051668882, + 0.0005777830010629259, + 0.0005567760017584078, + 0.0005862840043846518, + 0.0005670050013577566, + 0.0006121469996287487, + 0.0005589540014625527, + 0.0005697739979950711, + 0.0005540339989238419, + 0.0005742729990743101, + 0.000587424001423642, + 0.0005680999965989031, + 0.0005539590056287125, + 0.0005508340036612935, + 0.002512952996767126, + 0.0011879960002261214, + 0.000574616999074351, + 0.0005649919985444285, + 0.0005428099975688383, + 0.0005506500019691885, + 0.008185247002984397, + 0.0023080129976733588, + 0.003732064003997948, + 0.0024327330029336736, + 0.0006784640063415281, + 0.0006507860016427003, + 0.0015392209970741533, + 0.0005915770016144961, + 0.007828200999938417, + 0.0007310460059670731, + 0.0006030410004314035, + 0.0005648840015055612, + 0.0005373909953050315, + 0.0005352199950721115, + 0.006533519997901749, + 0.0006881549998070113, + 0.0006167860046843998, + 0.0005911920015932992, + 0.0005435569983092137, + 0.0005340079951565713, + 0.0005288040047162212, + 0.005173612000362482, + 0.0006295589992078021, + 0.0005571460060309619, + 0.0005405549964052625, + 0.0005386359989643097, + 0.0005292050045682117, + 0.004466076999960933, + 0.0009522429973003455, + 0.0005489799950737506, + 0.0005440199965960346, + 0.0005310299966367893, + 0.0005304969963617623, + 0.0005311539935064502, + 0.004294058999221306, + 0.0006288680015131831, + 0.0005898320014239289, + 0.0005580760043812916, + 0.0005312690045684576, + 0.0005295009977999143, + 0.003407015996344853, + 0.001492406998295337, + 0.0005473939963849261, + 0.0005345239987946115, + 0.000538278000021819, + 0.0005288079992169514, + 0.0005324430021573789, + 0.0005279130054987036, + 0.005060841001977678, + 0.0022791139999753796, + 0.0005615479967673309, + 0.0032696449998184107, + 0.000597351994656492, + 0.0005979449997539632, + 0.0005365049946703948, + 0.0005214369957684539, + 0.0005113189981784672, + 0.0005689059980795719, + 0.0007209920004243031, + 0.0022765729954699054, + 0.003544889004842844, + 0.0005884040001546964, + 0.000580479005293455, + 0.0010554420005064458, + 0.0005115059975651093, + 0.0034466939978301525, + 0.000599785998929292, + 0.0006419359997380525, + 0.0005199489969527349, + 0.0005203740001888946, + 0.0005246590008027852, + 0.0005164270041859709, + 0.0007886159946792759, + 0.0012780550023308024, + 0.0032144219949259423, + 0.0006139380056993105, + 0.0005228170048212633, + 0.0005968320037936792, + 0.004073264004546218, + 0.0006728480002493598, + 0.0005487369999173097, + 0.00054050099424785, + 0.0005325090023688972, + 0.0007560360027127899, + 0.00285581099888077, + 0.0011344500017003156, + 0.000529780998476781, + 0.0005301130004227161, + 0.000522490001458209, + 0.000579009996727109, + 0.0005660330061800778, + 0.0005225670029176399, + 0.005106713004352059, + 0.0006233530002646148, + 0.0006123880011728033, + 0.0005885780046810396, + 0.0005560319987125695, + 0.0005223069965722971, + 0.0013014659998589195, + 0.0005878099982510321, + 0.0005331490028765984, + 0.0005280760015011765, + 0.0005347190017346293, + 0.0005376740009523928, + 0.0005419689987320453, + 0.0005330630010575987, + 0.0005364969983929768, + 0.0005500949991983362, + 0.0005612540044239722, + 0.0005780719948234037, + 0.0005270370020298287, + 0.0005593349997070618, + 0.0005318109979270957, + 0.0005475869984366, + 0.0005340900024748407, + 0.0006253120009205304, + 0.0005328859988367185, + 0.0005410969970398583, + 0.0007931200016173534, + 0.000539191001735162, + 0.000554875994566828, + 0.01413342599698808, + 0.004224557000270579, + 0.005993480001052376, + 0.004059097002027556, + 0.004934497999784071, + 0.003993958998762537, + 0.007999226996616926, + 0.00600529999792343, + 0.004268120996130165, + 0.0006563770002685487, + 0.0006273759936448187, + 0.0031590729995514266, + 0.0005753940058639273, + 0.0005240039972704835, + 0.0005334689994924702, + 0.0005131510042701848, + 0.000524314003996551, + 0.00134149199584499, + 0.005637961003230885, + 0.0006635730023845099, + 0.0005832020033267327, + 0.0005257099983282387, + 0.0005277900054352358, + 0.0005267879969323985, + 0.0005181279993848875, + 0.007770302996505052, + 0.0012983019987586886, + 0.009059942000021692, + 0.0006469839936471544, + 0.0006381600032909773, + 0.00114534000022104, + 0.0005323860023054294, + 0.006631276999542024, + 0.0007586640058434568, + 0.0005989049968775362, + 0.000520156005222816, + 0.0005092880019219592, + 0.0007743919995846227, + 0.005243039995548315, + 0.0006580219996976666, + 0.0005906359947402962, + 0.0005489839968504384, + 0.0005430390010587871, + 0.0006195390014909208, + 0.0005389050056692213, + 0.0012339259992586449, + 0.0006620950007345527, + 0.0022131020014057867, + 0.001033159001963213, + 0.0005568870037677698, + 0.0005224140040809289, + 0.0005422760004876181, + 0.0006367109963321127, + 0.0005685759970219806, + 0.0005445510032586753, + 0.0005472199991345406, + 0.0005523649961105548, + 0.0006577230014954694, + 0.0005653769985656254, + 0.0005568039996433072, + 0.0005376879998948425, + 0.0005222899999353103, + 0.0005616639973595738, + 0.0005504540022229776, + 0.0005298140022205189, + 0.0005377979978220537, + 0.0005177830025786534, + 0.0005518839971045963, + 0.0005474729987327009, + 0.0014396709957509302, + 0.000622326995653566, + 0.0010748510030680336, + 0.002318990998901427, + 0.0005807880006614141, + 0.0006406839966075495, + 0.0005502560015884228, + 0.0005239230013103224, + 0.0005453390040202066, + 0.0005319429983501323, + 0.0005269520042929798, + 0.0005280719997244887, + 0.0005554559975280426, + 0.000549183001567144, + 0.0005682109986082651, + 0.0005418770015239716, + 0.0005502429994521663, + 0.0005435720013338141, + 0.0005524000007426366, + 0.0005597569979727268, + 0.0005749510019086301, + 0.0005359470014809631, + 0.0005312889988999814, + 0.000556057995709125, + 0.0005327730032149702 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil", + "fullname": "TestApplyNabla2ToVnInLateralBoundary", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.0014347019969136454, + "max": 0.06811486399965361, + "mean": 0.007994707482992896, + "stddev": 0.0056226463559909965, + "rounds": 238, + "median": 0.00699729499683599, + "iqr": 0.0019951159993070178, + "q1": 0.006001934998494107, + "q3": 0.007997050997801125, + "iqr_outliers": 23, + "stddev_outliers": 12, + "outliers": "12;23", + "ld15iqr": 0.003037618997041136, + "hd15iqr": 0.010994338001182768, + "ops": 125.08275032292242, + "total": 1.9027403809523094, + "data": [ + 0.007985402997292113, + 0.009993523002776783, + 0.006993151000642683, + 0.011007699999026954, + 0.007988115001353435, + 0.010104262000822928, + 0.00588547299412312, + 0.017939395002031233, + 0.004087536995939445, + 0.009966363999410532, + 0.007978166002430953, + 0.008001551999768708, + 0.01699503899726551, + 0.0070393279966083355, + 0.007955715002026409, + 0.010978432001138572, + 0.006075367004086729, + 0.006911040996783413, + 0.006999073004408274, + 0.008991479997348506, + 0.007994654995854944, + 0.006995906995143741, + 0.004193683002085891, + 0.007797225996910129, + 0.005993325998133514, + 0.006994330004090443, + 0.005498847000126261, + 0.006495075002021622, + 0.0037655419946531765, + 0.013229272000899073, + 0.006997770004090853, + 0.008002462003787514, + 0.010994338001182768, + 0.005994423001538962, + 0.017082939004467335, + 0.006908132003445644, + 0.005992974998662248, + 0.006996201002039015, + 0.00699418100703042, + 0.0069998969993321225, + 0.004995508999854792, + 0.005997168002068065, + 0.007998900997336023, + 0.009996009001042694, + 0.011038135999115184, + 0.007950437000545207, + 0.005932811000093352, + 0.0070504219984286465, + 0.007995973996003158, + 0.003260641999077052, + 0.0037316320012905635, + 0.0059993389950250275, + 0.0059965469990856946, + 0.005659041002218146, + 0.008349645999260247, + 0.004736792005132884, + 0.006713797993143089, + 0.006535526998050045, + 0.011990901999524795, + 0.006977835997531656, + 0.009313091999501921, + 0.012312163002206944, + 0.005355577006412204, + 0.005107101998873986, + 0.006863142996735405, + 0.006682758998067584, + 0.004310420001274906, + 0.005995553998218384, + 0.007003457998507656, + 0.05503793800016865, + 0.06811486399965361, + 0.008852195998770185, + 0.010968752998451237, + 0.008400856000662316, + 0.005572589005168993, + 0.003037618997041136, + 0.02095632700365968, + 0.007947438003611751, + 0.009032950998516753, + 0.006941204999748152, + 0.007013998998445459, + 0.008982072002254426, + 0.008060260995989665, + 0.007960192997416016, + 0.005961650997051038, + 0.003999249995104037, + 0.016008259997761343, + 0.01999303000047803, + 0.013994057997479104, + 0.007978702000400517, + 0.006037437997292727, + 0.008980914004496299, + 0.007952906998980325, + 0.007976846995006781, + 0.008294725994346663, + 0.00470622000284493, + 0.005990377001580782, + 0.007997360000445042, + 0.006000409004627727, + 0.007992175000254065, + 0.006002628004353028, + 0.011148650999530219, + 0.0068407799990382046, + 0.007990664002136327, + 0.0079964190008468, + 0.005442189001769293, + 0.006555268999363761, + 0.00699679500394268, + 0.006994550996751059, + 0.0069899149966659024, + 0.007992620994627941, + 0.006999113997153472, + 0.006006253999657929, + 0.007142312002542894, + 0.007843214996682946, + 0.007996499000000767, + 0.007000899000559002, + 0.007993958002771251, + 0.006207585000083782, + 0.007787083995935973, + 0.0069963549976819195, + 0.006996185002208222, + 0.005993177997879684, + 0.008004693998373114, + 0.007014021997747477, + 0.01798247799888486, + 0.007984834999660961, + 0.007990982994670048, + 0.0060132090002298355, + 0.005519532001926564, + 0.0014347019969136454, + 0.0049982310010818765, + 0.01505067000107374, + 0.00695005899615353, + 0.005983446004393045, + 0.008039501997700427, + 0.007955818000482395, + 0.007983948002220131, + 0.006989805006014649, + 0.006997409000177868, + 0.009184009999444243, + 0.0068138200003886595, + 0.005995150997478049, + 0.006991264002863318, + 0.010997356002917513, + 0.009987599994929042, + 0.007001787998888176, + 0.004988920998584945, + 0.009997007997299079, + 0.006017835999955423, + 0.007984791998751462, + 0.007003424005233683, + 0.005977941997116432, + 0.007310572000278626, + 0.006682944993372075, + 0.006009391996485647, + 0.007973220002895687, + 0.00799451499915449, + 0.008037209998292383, + 0.004954550000547897, + 0.007997738997801207, + 0.005996311992930714, + 0.006001934998494107, + 0.00599336699815467, + 0.006997180993494112, + 0.0069964270005584694, + 0.006997755001066253, + 0.007002410005952697, + 0.005993118000333197, + 0.004996421994292177, + 0.006911958000273444, + 0.0060834390023956075, + 0.006996290998358745, + 0.007994961000804324, + 0.005997373998980038, + 0.007997118998900987, + 0.006999441000516526, + 0.006008676995406859, + 0.0065792899986263365, + 0.005404704003012739, + 0.005996188003337011, + 0.007998098997632042, + 0.007992544000444468, + 0.0059919799969065934, + 0.005996715997753199, + 0.006997909003985114, + 0.0069976430022506975, + 0.006997632997808978, + 0.00499684799433453, + 0.007997050997801125, + 0.006882517001940869, + 0.005112959006510209, + 0.00893375500163529, + 0.010060048000013921, + 0.006996613003138918, + 0.007996640000783373, + 0.005996913998387754, + 0.007997116998012643, + 0.005998016997182276, + 0.00499765499989735, + 0.006014407998009119, + 0.006984005005506333, + 0.007983249000972137, + 0.006995992996962741, + 0.01254787200014107, + 0.005462999004521407, + 0.006959405000088736, + 0.0070300520019372925, + 0.007980376998602878, + 0.005969108002318535, + 0.006987649998336565, + 0.006986774998949841, + 0.007999034998647403, + 0.006994611001573503, + 0.006995044001087081, + 0.006994526003836654, + 0.00699555900064297, + 0.007997565000550821, + 0.0069943339985911734, + 0.006995138006459456, + 0.006012899000779726, + 0.0069876639972790144, + 0.007989933001226746, + 0.00799363699479727, + 0.006994294002652168, + 0.006990702997427434, + 0.00801062900427496, + 0.010997422999935225, + 0.00597184900107095, + 0.004974215000402182, + 0.010446911001054104, + 0.01055090400041081, + 0.007102935000148136, + 0.003163291999953799, + 0.00771810000151163, + 0.005989331999444403, + 0.002988249994814396, + 0.008007942000404 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil", + "fullname": "TestMoApplyNabla2ToW", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.0009703689938760363, + "max": 0.021006240000133403, + "mean": 0.006861762038649382, + "stddev": 0.0020891229171211413, + "rounds": 334, + "median": 0.006995611500315135, + "iqr": 0.002003421999688726, + "q1": 0.005990964004013222, + "q3": 0.007994386003701948, + "iqr_outliers": 23, + "stddev_outliers": 68, + "outliers": "68;23", + "ld15iqr": 0.0030023319995962083, + "hd15iqr": 0.01100079799653031, + "ops": 145.73516166364064, + "total": 2.2918285209088936, + "data": [ + 0.0020699829983641393, + 0.005125960997247603, + 0.01103598700137809, + 0.00597243600350339, + 0.00495679400046356, + 0.008002058006240986, + 0.006991759000811726, + 0.0061907429990242235, + 0.0057985839957837015, + 0.007993293002073187, + 0.007000350997259375, + 0.0059933929951512255, + 0.007997305001481436, + 0.00600098699942464, + 0.010864024996408261, + 0.006125586005509831, + 0.0049870590009959415, + 0.001910188999318052, + 0.0051974180023535155, + 0.006882449000841007, + 0.009987735000322573, + 0.00799688199913362, + 0.007008653003140353, + 0.008781401003943756, + 0.006183436002174858, + 0.00394815700565232, + 0.00503611599560827, + 0.004990233006537892, + 0.008008311000594404, + 0.007987794997461606, + 0.008993139999802224, + 0.0069913589977659285, + 0.006989465000515338, + 0.008002759001101367, + 0.007987589000549633, + 0.008991048998723272, + 0.005990917998133227, + 0.007984951000253204, + 0.008012913996935822, + 0.007966632998432033, + 0.007977642999321688, + 0.007990518999577034, + 0.007998100001714192, + 0.0069951670011505485, + 0.007011786998191383, + 0.0072359580008196644, + 0.012399139995977748, + 0.0023287209987756796, + 0.003989844000898302, + 0.008650486000988167, + 0.007358834001934156, + 0.007991131002199836, + 0.00697333000425715, + 0.00698862099670805, + 0.006994228999246843, + 0.006994686998950783, + 0.008002663998922799, + 0.004993379996449221, + 0.007995403997483663, + 0.006999040000664536, + 0.007408530000247993, + 0.007607285006088205, + 0.006998376004048623, + 0.006953311996767297, + 0.007209711999166757, + 0.008777991999522783, + 0.00813713399838889, + 0.00784561299951747, + 0.006994453004153911, + 0.006994637995376252, + 0.01108317499893019, + 0.00790803999552736, + 0.0069804040031158365, + 0.006941988998732995, + 0.008074920995568391, + 0.006966371998714749, + 0.006756334005331155, + 0.00824371600174345, + 0.005978256005619187, + 0.008788608000031672, + 0.0037989010015735403, + 0.0053950529982103035, + 0.004982058999303263, + 0.006995610005105846, + 0.007996211003046483, + 0.0069969579999451526, + 0.0070170199978747405, + 0.009392781001224648, + 0.006496939000498969, + 0.006994739000219852, + 0.006978861994866747, + 0.00699431799876038, + 0.006996819000050891, + 0.006997039003181271, + 0.00740686799690593, + 0.0036245709998183884, + 0.006951355004275683, + 0.00699691400222946, + 0.007994386003701948, + 0.008000608999282122, + 0.005993888997181784, + 0.013136133005900774, + 0.007883478996518534, + 0.006951939001737628, + 0.004985977000615094, + 0.007673785999941174, + 0.0015678319978178479, + 0.009760387001733761, + 0.007994431995030027, + 0.005974911000521388, + 0.008007639000425115, + 0.005014747002860531, + 0.007946099001856055, + 0.008746201994654257, + 0.004560171000775881, + 0.00666371999977855, + 0.00599760399927618, + 0.00699539700144669, + 0.0070236469982774, + 0.005001117999199778, + 0.00796632599667646, + 0.014001057999848854, + 0.0012907709970022552, + 0.006676580997009296, + 0.006997039003181271, + 0.007994020001206081, + 0.005064021999714896, + 0.007877782001742162, + 0.0066687829967122525, + 0.0073690680001163855, + 0.01100079799653031, + 0.010984288004692644, + 0.010088067996548489, + 0.008876530002453364, + 0.0069759510006406344, + 0.011997080997389276, + 0.007981704002304468, + 0.009971826999390032, + 0.001979239998036064, + 0.007014899994828738, + 0.009160591995168943, + 0.005819821002660319, + 0.021006240000133403, + 0.0009703689938760363, + 0.010017872999014799, + 0.010981346997141372, + 0.004967597997165285, + 0.004140518001804594, + 0.0028372080050758086, + 0.008011613004782703, + 0.00498796500323806, + 0.005990964004013222, + 0.009640801996283699, + 0.009660000003350433, + 0.004676138996728696, + 0.007950093000545166, + 0.006024508002155926, + 0.006196510003064759, + 0.0027794580018962733, + 0.007006004998402204, + 0.005988871998852119, + 0.003986639996583108, + 0.0039700580018688925, + 0.007030469001620077, + 0.007989574005478062, + 0.00798488599684788, + 0.007995965002919547, + 0.004991765999875497, + 0.004995356997824274, + 0.005131623001943808, + 0.0068727379984920844, + 0.0060019259981345385, + 0.005989239005430136, + 0.0049831789947347715, + 0.006993396003963426, + 0.005209337003179826, + 0.007787030997860711, + 0.008326301998749841, + 0.007666151002922561, + 0.004993659000319894, + 0.007009936998656485, + 0.005983711998851504, + 0.010998246994859073, + 0.0059859050015802495, + 0.009996537999541033, + 0.006996609998168424, + 0.008024490998650435, + 0.006971932998567354, + 0.008023210997635033, + 0.007961825001984835, + 0.005995570005325135, + 0.005004102997190785, + 0.006992170005105436, + 0.008002368995221332, + 0.007001261998084374, + 0.005986887998005841, + 0.005999547000101302, + 0.006993709001108073, + 0.007998448003490921, + 0.006999278994044289, + 0.008997057993838098, + 0.00999754799704533, + 0.007996139996976126, + 0.008006613999896217, + 0.006691567999951076, + 0.008279287998448126, + 0.007994034996954724, + 0.007018549003987573, + 0.0069783080034540035, + 0.00599385799432639, + 0.005005571998481173, + 0.006987779001065064, + 0.007000140998570714, + 0.005990746998577379, + 0.00821197599725565, + 0.006777450995286927, + 0.006985437998082489, + 0.002811417005432304, + 0.005181141001230571, + 0.006000426998070907, + 0.005990994002786465, + 0.004995759001758415, + 0.006997755001066253, + 0.008001039001101162, + 0.006993538001552224, + 0.0069967329982318915, + 0.007997317996341735, + 0.004904822999378666, + 0.004093683994142339, + 0.005994745006319135, + 0.006998697994276881, + 0.007004311999480706, + 0.006990019996010233, + 0.006996629999775905, + 0.007996273998287506, + 0.005996804997266736, + 0.0089968729953398, + 0.006995612995524425, + 0.00699684199935291, + 0.00699750999774551, + 0.007999221998034045, + 0.005995888997858856, + 0.007996856002137065, + 0.005997382999339607, + 0.006996486001298763, + 0.005998203996568918, + 0.004000594002718572, + 0.0049947619991144165, + 0.006995849005761556, + 0.00799906699830899, + 0.00699681799596874, + 0.007016727002337575, + 0.007976310000231024, + 0.00799772099708207, + 0.0039946670003701, + 0.005995392995828297, + 0.0039975589970708825, + 0.005999269000312779, + 0.0069945180002832785, + 0.007002158999966923, + 0.005999843000608962, + 0.008004167997569311, + 0.006983069994021207, + 0.007014406000962481, + 0.007979402995260898, + 0.005997790998662822, + 0.008003725000889972, + 0.0048404460030724294, + 0.007379562004643958, + 0.006764685997040942, + 0.008000237998203374, + 0.0030023319995962083, + 0.003801975995884277, + 0.001857725997979287, + 0.00632067299738992, + 0.00802090499928454, + 0.007988252000359353, + 0.008981498998764437, + 0.007996735999768134, + 0.00599702700128546, + 0.004998782002076041, + 0.0019948970002587885, + 0.007997983004315756, + 0.0059978040007990785, + 0.008003659997484647, + 0.007991311998921447, + 0.007996530999662355, + 0.005998098000418395, + 0.00800464899657527, + 0.0017265799979213625, + 0.0042590509983710945, + 0.005994473001919687, + 0.007008109001617413, + 0.005237605000729673, + 0.003742715998669155, + 0.00801198699628003, + 0.007985470998391975, + 0.005994323997583706, + 0.006096229997638147, + 0.007896581999375485, + 0.006995060000917874, + 0.00699703699501697, + 0.007993651997821871, + 0.006990312998823356, + 0.008003432994883042, + 0.006994115996349137, + 0.008009438002773095, + 0.006398858000466134, + 0.0025703540013637394, + 0.005995221006742213, + 0.004995772003894672, + 0.003977878004661761, + 0.006010031000187155, + 0.007031674998870585, + 0.010950953997962642, + 0.01041783800610574, + 0.0075691789970733225, + 0.006994262999796774, + 0.00699684199935291, + 0.0034278210005140863, + 0.0035554559945012443, + 0.006008786003803834, + 0.007727452997642104, + 0.0031647169962525368, + 0.0030798450025031343, + 0.006997014999797102, + 0.007008478001807816, + 0.004407599997648504, + 0.006615455000428483, + 0.007940900999528822, + 0.01201732299523428, + 0.004988779000996146, + 0.004989982997358311, + 0.005593310001131613 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil", + "fullname": "TestApplyNabla2ToWInUpperDampingLayer", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.00034094200236722827, + "max": 0.06015052899601869, + "mean": 0.0021926930991410095, + "stddev": 0.0031025214623593444, + "rounds": 2441, + "median": 0.00039833600021665916, + "iqr": 0.0036502199982351158, + "q1": 0.0003543577495292993, + "q3": 0.004004577747764415, + "iqr_outliers": 32, + "stddev_outliers": 490, + "outliers": "490;32", + "ld15iqr": 0.00034094200236722827, + "hd15iqr": 0.009606883002561517, + "ops": 456.0601756770025, + "total": 5.352363855003205, + "data": [ + 0.0005411559977801517, + 0.0004379070014692843, + 0.0004158369993092492, + 0.00041172000055667013, + 0.00040860399894881994, + 0.0003909480001311749, + 0.003540161000273656, + 0.0004282590016373433, + 0.00040564400114817545, + 0.0003995379956904799, + 0.00040284600254381076, + 0.00039712600118946284, + 0.0003914660046575591, + 0.00040854999679140747, + 0.0003998690008302219, + 0.0007878050018916838, + 0.0010405959983472712, + 0.0008402099992963485, + 0.0007076710026012734, + 0.0014143730004434474, + 0.0004935909964842722, + 0.001058183996065054, + 0.0004827469965675846, + 0.0004115449992241338, + 0.0004097259952686727, + 0.00039707300311420113, + 0.0014068599994061515, + 0.0009175329978461377, + 0.0013126159974490292, + 0.001836062001530081, + 0.0004544020048342645, + 0.00040832500235410407, + 0.0004041939973831177, + 0.0003986140000051819, + 0.00040151300345314667, + 0.0003956849977839738, + 0.00168190999829676, + 0.0007965400000102818, + 0.0011193160025868565, + 0.0005526740060304292, + 0.0006730120003339835, + 0.0004512049999902956, + 0.00040472399996360764, + 0.0003939259986509569, + 0.0004101550002815202, + 0.0003995200022473, + 0.0006469809959526174, + 0.000494239000545349, + 0.0004215649969410151, + 0.00039630199898965657, + 0.0003989080069004558, + 0.000402379002480302, + 0.0003975430008722469, + 0.00040086000080918893, + 0.00039493200165452436, + 0.0012439359998097643, + 0.0013363360048970208, + 0.0007080860013957135, + 0.003115052000794094, + 0.006996263997280039, + 0.007997303997399285, + 0.010999758000252768, + 0.006995121999352705, + 0.005995307001285255, + 0.007994952000444755, + 0.006996322001214139, + 0.009997727000154555, + 0.007996635999006685, + 0.009996517997933552, + 0.006998046999797225, + 0.007995448002475314, + 0.01100158200279111, + 0.00899280900193844, + 0.00799754499894334, + 0.014000007999129593, + 0.007995938998647034, + 0.005001682999136392, + 0.003989017001003958, + 0.01009269300266169, + 0.008902678004233167, + 0.009033501999510918, + 0.007963422998727765, + 0.009019022996653803, + 0.006980575999477878, + 0.007991559003130533, + 0.003989208002167288, + 0.005995318999339361, + 0.004068549002113286, + 0.007993854000233114, + 0.007929022998723667, + 0.0029945130008854903, + 0.00535256299917819, + 0.006648150003456976, + 0.003984513001341838, + 0.005998374996124767, + 0.007002559999818914, + 0.007990427999175154, + 0.007171101002313662, + 0.007823662999726366, + 0.006995711002673488, + 0.00599547599995276, + 0.006005564995575696, + 0.007991526996192988, + 0.00678035899909446, + 0.008212870998249855, + 0.007996029002242722, + 0.007998510998731945, + 0.006995755997195374, + 0.005997521002427675, + 0.0070277670020004734, + 0.006961165003303904, + 0.00865759199950844, + 0.0031672359982621856, + 0.004165427999396343, + 0.0059918219994870014, + 0.006996108000748791, + 0.005001224999432452, + 0.00799044499581214, + 0.0060003550024703145, + 0.00513321199832717, + 0.005854861999978311, + 0.005999951004923787, + 0.006994654999289196, + 0.004460541997104883, + 0.007548732006398495, + 0.004988501998013817, + 0.0029897300046286546, + 0.005995811996399425, + 0.005188317001739051, + 0.007810492999851704, + 0.007644536999578122, + 0.007341137999901548, + 0.007998154003871605, + 0.006002164001984056, + 0.009996644999773707, + 0.006125032996351365, + 0.007866895000915974, + 0.004077960002177861, + 0.007920418996945955, + 0.008203846999094822, + 0.007010852998064365, + 0.004775208995852154, + 0.00713431800249964, + 0.0068605670021497644, + 0.007997358996362891, + 0.007998571003554389, + 0.004997371994249988, + 0.0060000860030413605, + 0.00431641899922397, + 0.0066762860005837865, + 0.006064502005756367, + 0.010934410995105281, + 0.0010408800008008257, + 0.0003596460010157898, + 0.00035353000566828996, + 0.00034915399737656116, + 0.0003478279977571219, + 0.0003532939954311587, + 0.00038146600127220154, + 0.00048712499847169966, + 0.0004194110006210394, + 0.002181835996452719, + 0.00036446299782255664, + 0.0003473730030236766, + 0.0003479129954939708, + 0.0003474299955996685, + 0.0031911740006762557, + 0.000382637997972779, + 0.00035884699900634587, + 0.0003529169989633374, + 0.0003487069989205338, + 0.0003454289981164038, + 0.0038692029993399046, + 0.00037083300412632525, + 0.00036748599814018235, + 0.0003495980054140091, + 0.00034943499485962093, + 0.0003483569962554611, + 0.005068569000286516, + 0.00042650900286389515, + 0.0012085149937774986, + 0.0006144520011730492, + 0.000364705003448762, + 0.0003457400016486645, + 0.00034406000486342236, + 0.0010237000024062581, + 0.00035291600215714425, + 0.0003429490025155246, + 0.00034330800554016605, + 0.0005819299985887483, + 0.0025864359995466657, + 0.003001933997438755, + 0.003425938994041644, + 0.0065645270005916245, + 0.009996055006922688, + 0.00700181900174357, + 0.007993805993464775, + 0.006997368000156712, + 0.0059971549999318086, + 0.003997384003014304, + 0.0069973819990991615, + 0.004995205999875907, + 0.005997371998091694, + 0.007998293993296102, + 0.005997522996040061, + 0.007997817003342789, + 0.00999705700087361, + 0.004006598006526474, + 0.0011149040001328103, + 0.00038720699376426637, + 0.00039803999970899895, + 0.00040459899901179597, + 0.00039600199670530856, + 0.00038597000093432143, + 0.00039331500011030585, + 0.00038381400372600183, + 0.0004003430003649555, + 0.0003850450011668727, + 0.00038521099486388266, + 0.00040283400448970497, + 0.0004040200001327321, + 0.00038252599915722385, + 0.0004163960038567893, + 0.0003902630051015876, + 0.000394492999475915, + 0.00038967200089246035, + 0.0003867229970637709, + 0.0004117310018045828, + 0.00037303799763321877, + 0.0003984099967055954, + 0.0003947410004911944, + 0.00038214699452510104, + 0.0003968599994550459, + 0.0003808960027527064, + 0.0004099810030311346, + 0.0004013560028397478, + 0.0003786469969782047, + 0.0003845769970212132, + 0.000407414001529105, + 0.00039802800165489316, + 0.00040605599497212097, + 0.00037617799534928054, + 0.0003878499992424622, + 0.0004045200039399788, + 0.0004031289936392568, + 0.00038764400233048946, + 0.0003906049969373271, + 0.00039203499909490347, + 0.0004051529977004975, + 0.00039316499896813184, + 0.00038950800080783665, + 0.0003888569990522228, + 0.0003933409971068613, + 0.00040060700121102855, + 0.0003855460017803125, + 0.00038752200634917244, + 0.0003885360056301579, + 0.0003854349997709505, + 0.0003897260030498728, + 0.0004013249999843538, + 0.0003970789985032752, + 0.00037880199670325965, + 0.00038197499816305935, + 0.00038887400296516716, + 0.00038614200457232073, + 0.00037863000034121796, + 0.0003721030007000081, + 0.0003821119971689768, + 0.0003813399962382391, + 0.0004298730054870248, + 0.0003942969997297041, + 0.00040110300324158743, + 0.0003861639997921884, + 0.0003982400012318976, + 0.0004034449957543984, + 0.00039730000571580604, + 0.0003892110034939833, + 0.0003765769943129271, + 0.000376800999219995, + 0.0003884410034515895, + 0.0003850919965771027, + 0.0003800839986070059, + 0.0003845120008918457, + 0.00037778299883939326, + 0.0003924569973605685, + 0.00040352500218432397, + 0.00039630800165468827, + 0.0003993900027126074, + 0.00040568800613982603, + 0.00039173500408651307, + 0.00039788000140106305, + 0.00038676500116707757, + 0.00039416200161213055, + 0.0007804769993526861, + 0.0007944090029923245, + 0.00038848300027893856, + 0.00037695599894504994, + 0.00037714800419053063, + 0.00039572800596943125, + 0.0004095030017197132, + 0.0004036129976157099, + 0.0003903110045939684, + 0.0005734999940614216, + 0.00039346500125247985, + 0.0004175079957349226, + 0.0004208270038361661, + 0.0004120009980397299, + 0.00041327899816678837, + 0.00040647100104251876, + 0.0003931899991584942, + 0.00040074500429909676, + 0.0004176090005785227, + 0.00040841200097929686, + 0.00039833600021665916, + 0.0004955990007147193, + 0.0004165819991612807, + 0.00039370199374388903, + 0.00039974199899006635, + 0.000413970003137365, + 0.0004187659942544997, + 0.0004949479989591055, + 0.0004994260016246699, + 0.0004582840047078207, + 0.00039359099901048467, + 0.00038795400178059936, + 0.0004031650023534894, + 0.0003933850020985119, + 0.00039114499668357894, + 0.0003886580016114749, + 0.0003934400010621175, + 0.0004162969999015331, + 0.00039913799992064014, + 0.00039103200106183067, + 0.00040920800529420376, + 0.00041564700222807005, + 0.00039710000419290736, + 0.0025135270043392666, + 0.0055895879995659925, + 0.005997521999233868, + 0.005998327993438579, + 0.004997351999918465, + 0.007002502003160771, + 0.006995870004175231, + 0.007996983993507456, + 0.004994830000214279, + 0.005997405998641625, + 0.0036900529958074912, + 0.006305380993580911, + 0.007020491000730544, + 0.008992547998786904, + 0.005993673999910243, + 0.006985343999986071, + 0.007981522001500707, + 0.005997611006023362, + 0.004993902002752293, + 0.00999731000047177, + 0.007001680998655502, + 0.005992992999381386, + 0.0050062439986504614, + 0.0035631330028991215, + 0.007420472000376321, + 0.004992929003492463, + 0.004994408001948614, + 0.006000332999974489, + 0.006991811998886988, + 0.008017946005566046, + 0.006977693003136665, + 0.008011865997104906, + 0.006407318003766704, + 0.007569207999040373, + 0.0010017390013672411, + 0.007992227998329327, + 0.0059973559982609, + 0.00799736400222173, + 0.004997377000108827, + 0.006999835000897292, + 0.009996278000471648, + 0.008069071998761501, + 0.006918756997038145, + 0.007997790999070276, + 0.006415179996110965, + 0.006576339001185261, + 0.005806455999845639, + 0.004188886996416841, + 0.004501281000557356, + 0.005492238997248933, + 0.007997355998668354, + 0.007999670000572223, + 0.006995502000791021, + 0.01116645799629623, + 0.003637169997091405, + 0.004880289998254739, + 0.005302905003190972, + 0.006999067998549435, + 0.007238728998345323, + 0.007746676994429436, + 0.005995544001052622, + 0.006998229997407179, + 0.004494775006605778, + 0.007141464993765112, + 0.00935454000136815, + 0.006999337994784582, + 0.007993307001015637, + 0.0070009269984439015, + 0.00699446599901421, + 0.007998390996363014, + 0.006994274001044687, + 0.006996156997047365, + 0.006997825003054459, + 0.00599722199694952, + 0.004051689000334591, + 0.003943875999539159, + 0.005997849002596922, + 0.006996585005254019, + 0.00799813200137578, + 0.00799615200230619, + 0.006995946998358704, + 0.002999290998559445, + 0.007995854000910185, + 0.008001563997822814, + 0.007993553001142573, + 0.005823287996463478, + 0.006169626998598687, + 0.005998186003125738, + 0.00799630399706075, + 0.005998742002702784, + 0.003995577993919142, + 0.007021277000603732, + 0.002973866998218, + 0.0027280549984425306, + 0.004268028002115898, + 0.005993921004119329, + 0.005999795001116581, + 0.007994076004251838, + 0.006998180004302412, + 0.005997364998620469, + 0.005998619999445509, + 0.0079962469972088, + 0.005997631000354886, + 0.006996746000368148, + 0.00800720199913485, + 0.006986793006944936, + 0.0036289919953560457, + 0.0073658029941725545, + 0.007633384993823711, + 0.009363031000248156, + 0.0070058919955044985, + 0.007986076998349745, + 0.00699637000070652, + 0.00799719199858373, + 0.006005777002428658, + 0.005995751998852938, + 0.007985322998138145, + 0.006005907998769544, + 0.005986224001389928, + 0.006996495001658332, + 0.006751615997927729, + 0.007241160994453821, + 0.007770397001877427, + 0.006357594000292011, + 0.00786371999856783, + 0.0069961659974069335, + 0.008034922000661027, + 0.007960393995745108, + 0.005155968996405136, + 0.007672373001696542, + 0.009166341995296534, + 0.006993841001531109, + 0.0023040259984554723, + 0.0004017989995190874, + 0.0004234410007484257, + 0.0004078150013810955, + 0.00036882900167256594, + 0.0003522240003803745, + 0.00034786900505423546, + 0.00034724699798971415, + 0.002994342998135835, + 0.00042870300239883363, + 0.000405013001000043, + 0.00039059999835444614, + 0.00039049900078680366, + 0.0003449589930824004, + 0.00035683299938682467, + 0.0003462149979895912, + 0.00034702000266406685, + 0.002323899003386032, + 0.0003802409992204048, + 0.00042605099588399753, + 0.00040007800271268934, + 0.0003514019990689121, + 0.00035565199505072087, + 0.00034301800042157993, + 0.000343592997523956, + 0.000350327005435247, + 0.0003431369987083599, + 0.00034452199906809255, + 0.005287301006319467, + 0.00043911200191359967, + 0.00045501499698730186, + 0.0004360029997769743, + 0.0003466560010565445, + 0.00035493399627739564, + 0.00034573900484247133, + 0.000396482995711267, + 0.0003442539964453317, + 0.00034626200067577884, + 0.0030752980019315146, + 0.0005936890011071227, + 0.00040792799700284377, + 0.0003646650002337992, + 0.00035766999644692987, + 0.00036494700179900974, + 0.00036370899761095643, + 0.00036709900450659916, + 0.0003595569942262955, + 0.00036177199945086613, + 0.00034111800050595775, + 0.00042384500557091087, + 0.000344956002663821, + 0.00034673399932216853, + 0.00037649600562872365, + 0.0003565379956853576, + 0.00034820700238924474, + 0.00336743699881481, + 0.0004436409944901243, + 0.0005548799963435158, + 0.0004041279971715994, + 0.00034740300179691985, + 0.00037268899905029684, + 0.0003507049987092614, + 0.0003744179994100705, + 0.00034496799344196916, + 0.0003461939995759167, + 0.00397121599962702, + 0.0004572359976009466, + 0.00041921899537555873, + 0.00035355400177650154, + 0.0003472900061751716, + 0.00035031200241064653, + 0.00034784900344675407, + 0.0003448110001045279, + 0.00035048699646722525, + 0.0047341750032501295, + 0.0004463139994186349, + 0.00043781899876194075, + 0.00038937200588406995, + 0.00034778400004142895, + 0.00034243899426655844, + 0.00034849499934352934, + 0.0003516189972287975, + 0.00034389999927952886, + 0.0003426039984333329, + 0.0024780789972282946, + 0.000424956000642851, + 0.0005385540061979555, + 0.000388355998438783, + 0.0003568869942682795, + 0.00034956700255861506, + 0.0003452540040598251, + 0.000354718002199661, + 0.00034375800169073045, + 0.0003474440018180758, + 0.005440210996312089, + 0.0004405559957376681, + 0.001454500001273118, + 0.0003557460004230961, + 0.00036306100082583725, + 0.00038152699562488124, + 0.0003638159978436306, + 0.00034944300568895414, + 0.00035004299570573494, + 0.000341184000717476, + 0.003178544000547845, + 0.0004274599996278994, + 0.0004237629982526414, + 0.0003531070033204742, + 0.00034434199915267527, + 0.00035250300425104797, + 0.00035838299663737416, + 0.0003454730031080544, + 0.0003465980043984018, + 0.005109472003823612, + 0.006997370001045056, + 0.004394299001432955, + 0.009606883002561517, + 0.006990574998781085, + 0.005003409001801629, + 0.005992379999952391, + 0.007038807001663372, + 0.007952207000926137, + 0.006993542003328912, + 0.007040165000944398, + 0.0069560030024149455, + 0.008003101000213064, + 0.0011558459955267608, + 0.0038307380018522963, + 0.007008574997598771, + 0.006986152999161277, + 0.006997581003815867, + 0.004425916005857289, + 0.002568606003478635, + 0.00800048399833031, + 0.008001451002201065, + 0.007996745000127703, + 0.005988961005641613, + 0.0059950120048597455, + 0.012001211005554069, + 0.005988532997434959, + 0.006997633005084936, + 0.007002948994340841, + 0.006994180002948269, + 0.007997446999070235, + 0.004995920993678737, + 0.00399580699740909, + 0.007130680998670869, + 0.006863559996418189, + 0.008997309996630065, + 0.0069970710028428584, + 0.005983934999676421, + 0.004006732000561897, + 0.0044237620022613555, + 0.0035724209956242703, + 0.009998718000133522, + 0.007995097003004048, + 0.0069974520010873675, + 0.006999293000262696, + 0.006006417999742553, + 0.008023389003938064, + 0.003958369001338724, + 0.00667396900098538, + 0.003319380004541017, + 0.006009513002936728, + 0.0059854159990209155, + 0.006996182004513685, + 0.007999716995982453, + 0.00737704800121719, + 0.009615152004698757, + 0.006998081000347156, + 0.007995833999302704, + 0.004583747999276966, + 0.003271884001151193, + 0.008153147005941719, + 0.003124859002127778, + 0.0038668780034640804, + 0.009985073003917933, + 0.004820357004064135, + 0.007170414995925967, + 0.0059921610009041615, + 0.00799648099928163, + 0.005002184996556025, + 0.004991385001630988, + 0.006065948000468779, + 0.0059292559963068925, + 0.006881704000988975, + 0.0021090239970362745, + 0.0046743979983148165, + 0.006324607995338738, + 0.0069962720008334145, + 0.006997625998337753, + 0.006994462994043715, + 0.007006824998825323, + 0.005987220996757969, + 0.008999552999739535, + 0.001993098994717002, + 0.006133136994321831, + 0.0004518890054896474, + 0.00043085499783046544, + 0.00042491899512242526, + 0.0004000150001957081, + 0.0003650699945865199, + 0.0003523519990267232, + 0.00035310200473759323, + 0.0003618650007410906, + 0.00036024899600306526, + 0.005580191995250061, + 0.00044255800457904115, + 0.0004565880008158274, + 0.00046446899796137586, + 0.0003674199979286641, + 0.0003620530042098835, + 0.00035582800046540797, + 0.00035820199991576374, + 0.00035802600177703425, + 0.0003532269984134473, + 0.0017929239984368905, + 0.00042419700184836984, + 0.00042547599878162146, + 0.0003955919964937493, + 0.00036374999763211235, + 0.00039010999898891896, + 0.00037231799797154963, + 0.00037354599771788344, + 0.00035838899930240586, + 0.005507160996785387, + 0.0004940329963574186, + 0.0004547709977487102, + 0.0004360800012364052, + 0.0003805270025623031, + 0.00036263599758967757, + 0.0003571560009731911, + 0.0003672110033221543, + 0.00047480600187554955, + 0.00035441700310911983, + 0.005857090000063181, + 0.010243620003166143, + 0.0004296409970265813, + 0.0003627359983511269, + 0.0010647279996192083, + 0.00036237199674360454, + 0.0003528259985614568, + 0.00034937600139528513, + 0.00035595100052887574, + 0.00035364799987291917, + 0.005927468002482783, + 0.009461659996304661, + 0.00040736600203672424, + 0.00035765099892159924, + 0.0003631770014180802, + 0.00035422899964032695, + 0.0003500789971440099, + 0.0003532780028763227, + 0.0003875519978464581, + 0.0003444340036367066, + 0.0052875990004395135, + 0.00047127200377872214, + 0.00042693100112956017, + 0.0004112230017199181, + 0.00034818899439414963, + 0.0003582559947972186, + 0.00034602900268509984, + 0.0003543839993653819, + 0.000347825996868778, + 0.0003662870003608987, + 0.00384993899933761, + 0.0016182649997062981, + 0.00040943500061985105, + 0.00040655000339029357, + 0.00041290799708804116, + 0.0003509500020300038, + 0.0003499329977785237, + 0.00034335500095039606, + 0.0003517320001265034, + 0.0003564080034266226, + 0.0003539320023264736, + 0.00035036299959756434, + 0.004852256002777722, + 0.0004870350021519698, + 0.0004206340017844923, + 0.0011270610048086382, + 0.000434485002188012, + 0.000348544999724254, + 0.0003488989968900569, + 0.00035183400177629665, + 0.00034409599902573973, + 0.00034968699765158817, + 0.0030024350053281523, + 0.000455147004686296, + 0.00040127800457412377, + 0.000404826998419594, + 0.0003813619987340644, + 0.00037784300366183743, + 0.00035166599991498515, + 0.0003481050007394515, + 0.004910960997221991, + 0.0004697010008385405, + 0.00044213499495526776, + 0.0003745380017790012, + 0.000353587995050475, + 0.0003461539963609539, + 0.0003469050061539747, + 0.0003540879988577217, + 0.0003446400005486794, + 0.00437142800365109, + 0.0005184050023672171, + 0.00040789099875837564, + 0.00039000800461508334, + 0.00039538599958177656, + 0.00034812000376405194, + 0.00035016900073969737, + 0.0003532209957484156, + 0.0003558069947757758, + 0.0003451469965511933, + 0.0003433820020291023, + 0.00443311600247398, + 0.0004617020022124052, + 0.0004408050008350983, + 0.0004308940042392351, + 0.00040417700074613094, + 0.00037748499744338915, + 0.00036097800330026075, + 0.0003535189971444197, + 0.00034462500480003655, + 0.00034860499727074057, + 0.0003526110012899153, + 0.0003429339994909242, + 0.00035014199966099113, + 0.0023958349993336014, + 0.0004335150006227195, + 0.0004658289981307462, + 0.0004133399997954257, + 0.0003651730003184639, + 0.0003587970059015788, + 0.0003657749984995462, + 0.0003642800002126023, + 0.0003626500038080849, + 0.0003889360013999976, + 0.0003986200026702136, + 0.0003698250002344139, + 0.0003736399958143011, + 0.00040476099820807576, + 0.0003720480017364025, + 0.00036885000008624047, + 0.0003712700054165907, + 0.0003707529976963997, + 0.0003714159975061193, + 0.00036423799610929564, + 0.00036914300289936364, + 0.00036321300285635516, + 0.0003641069997684099, + 0.0003657180059235543, + 0.00036271200224291533, + 0.00036181700124870986, + 0.0003719669985002838, + 0.0003598219991545193, + 0.0003623099983087741, + 0.0003823910010396503, + 0.00036407200241228566, + 0.0003581970013328828, + 0.00036350300069898367, + 0.0003878779971273616, + 0.0003816820026258938, + 0.0003777129968511872, + 0.000376689997210633, + 0.0003776480007218197, + 0.0003717860017786734, + 0.00035789600224234164, + 0.00037041599716758355, + 0.0003559119941201061, + 0.0003723699992406182, + 0.0003632529987953603, + 0.000364061001164373, + 0.00036538299900712445, + 0.0003784969958360307, + 0.00037696800427511334, + 0.0003755309953703545, + 0.0003941529939766042, + 0.0003849019994959235, + 0.00037604800309054554, + 0.00038371499977074564, + 0.00037608700222335756, + 0.00036936400283593684, + 0.00037347900070017204, + 0.00039223900239448994, + 0.0003685650008264929, + 0.00038559599488507956, + 0.0003736889993888326, + 0.0003738449959200807, + 0.0003828129993053153, + 0.00037491200055228546, + 0.00046991900308057666, + 0.00036897899553878233, + 0.0003558329990482889, + 0.00037672300095437095, + 0.0003696120038512163, + 0.00039104800089262426, + 0.00036884399742120877, + 0.00037162299850024283, + 0.00036382700636750087, + 0.0003695249979500659, + 0.00037713000347139314, + 0.00038962300459388644, + 0.0003746619986486621, + 0.00037715899816248566, + 0.0003723879999597557, + 0.0003734799975063652, + 0.0003764650027733296, + 0.0003729599993675947, + 0.00036929799534846097, + 0.0030292260053101927, + 0.006998454002314247, + 0.009999071000493132, + 0.007996173000719864, + 0.006997062999289483, + 0.006996711003012024, + 0.0045772909943480045, + 0.0064184860020759515, + 0.007998873996257316, + 0.0058943860058207065, + 0.0005018080046284012, + 0.0004613619967130944, + 0.0003953399937017821, + 0.0003866810002364218, + 0.0003981650006608106, + 0.00036631099646911025, + 0.0003725829956238158, + 0.0012002410003333353, + 0.003074294996622484, + 0.00043340399861335754, + 0.00042503599979681894, + 0.00040599500061944127, + 0.0003964709976571612, + 0.0003705769995576702, + 0.0003674850013339892, + 0.0003690170051413588, + 0.00037510399852180853, + 0.0003665750045911409, + 0.0003681700036395341, + 0.00037601600342895836, + 0.0003715880011441186, + 0.00037426199560286477, + 0.00037335800152504817, + 0.0003713999976753257, + 0.00037112300196895376, + 0.00040142999932868406, + 0.0003837969998130575, + 0.0003768579990719445, + 0.00036426400038180873, + 0.0003689400036819279, + 0.002868434996344149, + 0.0031273470012820326, + 0.00043459100561449304, + 0.0004088649948243983, + 0.00041952099854825065, + 0.0004156499999226071, + 0.0003824559971690178, + 0.0003642980009317398, + 0.00037572200380964205, + 0.0003758530001505278, + 0.0003855300019495189, + 0.0003872800007229671, + 0.00038684199535055086, + 0.000373980998119805, + 0.00039920100243762136, + 0.0037322040006984025, + 0.006997190001129638, + 0.011002063001797069, + 0.006993913004407659, + 0.006995525000093039, + 0.006998081000347156, + 0.007998350003617816, + 0.007996893997187726, + 0.006181636999826878, + 0.001115904000471346, + 0.00047382699995068833, + 0.0003929469967260957, + 0.00036359400110086426, + 0.0003575040027499199, + 0.0037430320007842965, + 0.0010616610015858896, + 0.00044414700096240267, + 0.0005075660010334104, + 0.0003782009953283705, + 0.00036871200427412987, + 0.0003886900012730621, + 0.006801101000746712, + 0.00047611299669370055, + 0.00044110700400779024, + 0.0003885909973178059, + 0.0003576140006771311, + 0.0003632489970186725, + 0.0003571979978005402, + 0.0003604849989642389, + 0.00035719099832931533, + 0.006923174005351029, + 0.00046294899948406965, + 0.0004379050005809404, + 0.000388505999580957, + 0.00036389799788594246, + 0.0003534779971232638, + 0.00035677199775818735, + 0.00036032199568580836, + 0.00035484699765220284, + 0.003240810998249799, + 0.0028161660011392087, + 0.0004753559987875633, + 0.00044094599434174597, + 0.0003479709994280711, + 0.0003556299998308532, + 0.0003453819954302162, + 0.00035086799471173435, + 0.00034971500281244516, + 0.00034959500044351444, + 0.004440151002199855, + 0.0004752059976453893, + 0.0004422999991220422, + 0.00042529999336693436, + 0.0003582559947972186, + 0.00034687399602262303, + 0.0003472039970802143, + 0.0003503620027913712, + 0.0003473010001471266, + 0.0003674239997053519, + 0.002546517993323505, + 0.0004449170010047965, + 0.0004669109985115938, + 0.0025972029980039224, + 0.0004175649955868721, + 0.00039304199890466407, + 0.00036611600080505013, + 0.0003475940029602498, + 0.00034653599868761376, + 0.003421304005314596, + 0.00044187300227349624, + 0.00041875099850585684, + 0.00041012799920281395, + 0.00038863300142111257, + 0.001983553003810812, + 0.0004215709996060468, + 0.0004045049936394207, + 0.00035585599835030735, + 0.00035423900408204645, + 0.0003530989997670986, + 0.0034694889982347377, + 0.0004879670013906434, + 0.00045573600073112175, + 0.0004508869969868101, + 0.00035375500010559335, + 0.0003607239996199496, + 0.0003548579989001155, + 0.0003484900007606484, + 0.0003490860035526566, + 0.0003450430012890138, + 0.0015972409964888357, + 0.004261293994204607, + 0.00045084899466019124, + 0.00043867000204045326, + 0.0003564319995348342, + 0.00034825399779947475, + 0.00034812199737643823, + 0.0003481519961496815, + 0.00034811400109902024, + 0.0003420409993850626, + 0.0003454369943938218, + 0.0031853420005063526, + 0.00044514399633044377, + 0.00042730599670903757, + 0.00043337600072845817, + 0.0003885019978042692, + 0.00036247100069886073, + 0.00034535099985077977, + 0.00034291300107724965, + 0.0003486540008452721, + 0.0003423420057515614, + 0.00034289299946976826, + 0.0034286250011064112, + 0.00044195800001034513, + 0.0004385639986139722, + 0.00043247800203971565, + 0.0004369410016806796, + 0.0003446909977355972, + 0.003443157998844981, + 0.00045208300434751436, + 0.0004328920040279627, + 0.0017826249968493357, + 0.0004037280014017597, + 0.00042364600085420534, + 0.00039682299393462017, + 0.0003721370012499392, + 0.00034653200418688357, + 0.0003488079964881763, + 0.0003555329967639409, + 0.0003486459972918965, + 0.005273877999570686, + 0.0005746660026488826, + 0.00041888099804054946, + 0.00043725599971367046, + 0.0003491160023258999, + 0.0003458240025793202, + 0.0003489179944153875, + 0.00035371199919609353, + 0.00035645800380734727, + 0.0003748780000023544, + 0.0003541730038705282, + 0.004487320999032818, + 0.0004573900005198084, + 0.0004383019986562431, + 0.0004234259977238253, + 0.00035190700145903975, + 0.0003482120009721257, + 0.0003566049999790266, + 0.00034716800291789696, + 0.0003519320016494021, + 0.0003454180041444488, + 0.004113961003895383, + 0.0005982020011288114, + 0.0004068609996465966, + 0.0003628609993029386, + 0.0003463589964667335, + 0.0003510839960654266, + 0.00035450499854050577, + 0.00034390200016787276, + 0.0003441620065132156, + 0.00035608300095191225, + 0.0003483990003587678, + 0.0008867490032571368, + 0.0003494320044410415, + 0.00035419099731370807, + 0.00034461899485904723, + 0.00034302299900446087, + 0.0003490830058581196, + 0.000344883999787271, + 0.00034881299507105723, + 0.00034805299947038293, + 0.00034806100302375853, + 0.005201321000640746, + 0.0004937720004818402, + 0.0003864419995807111, + 0.00036304300010669976, + 0.00036026100133312866, + 0.00035200999991502613, + 0.00035367200325708836, + 0.0003459330037003383, + 0.00034370500361546874, + 0.00035071999445790425, + 0.00034705200232565403, + 0.0003484370026853867, + 0.0005076090019429103, + 0.0039458879982703365, + 0.0004442389981704764, + 0.00045635800051968545, + 0.0003838159973383881, + 0.00034816900006262586, + 0.00034672800393309444, + 0.0003597000031732023, + 0.0003568819956853986, + 0.00034319899714319035, + 0.00034250599856022745, + 0.005088250996777788, + 0.0005157560008228756, + 0.0004067740010214038, + 0.0012687820053542964, + 0.0005523860018001869, + 0.0051356619951548055, + 0.0005009379965486005, + 0.0004315209953347221, + 0.00035857100010616705, + 0.000359874997229781, + 0.0003507609944790602, + 0.00035408300027484074, + 0.00035267299972474575, + 0.00034909100213553756, + 0.004458387003978714, + 0.0004990960005670786, + 0.0004882840003119782, + 0.0004553679973469116, + 0.0003587829996831715, + 0.00048274200526066124, + 0.00035870300052920356, + 0.00035094100167043507, + 0.00035307399957673624, + 0.005851180001627654, + 0.0005008719963370822, + 0.0004520049988059327, + 0.00039210700197145343, + 0.00035436100006336346, + 0.00044547199649969116, + 0.0003866909974021837, + 0.0003498540027067065, + 0.00035503700200933963, + 0.005460904001665767, + 0.0004962780003552325, + 0.0004938619968015701, + 0.0004611080003087409, + 0.00035359500179765746, + 0.0003458279970800504, + 0.0003436779952608049, + 0.0005905930011067539, + 0.0003425510003580712, + 0.002307360999111552, + 0.000459770999441389, + 0.0004086030021426268, + 0.00041504699765937403, + 0.00039060800190782174, + 0.00035160900006303564, + 0.00034634899930097163, + 0.000494854997668881, + 0.0003487860012683086, + 0.00034348600456723943, + 0.00034572799631860107, + 0.004510152997681871, + 0.0004873100042459555, + 0.0004418079988681711, + 0.00037175299803493544, + 0.000510247002239339, + 0.00038887000118847936, + 0.00036911299685016274, + 0.00036575400008587167, + 0.000349346999428235, + 0.001215306001540739, + 0.004194789005850907, + 0.0004986730054952204, + 0.0004425939987413585, + 0.00042539299465715885, + 0.0003501210012473166, + 0.0003531949987518601, + 0.0003525379943312146, + 0.00035002100048586726, + 0.0003503909974824637, + 0.0003620909992605448, + 0.005746041999373119, + 0.0004662529972847551, + 0.0005185569971217774, + 0.0003829970009974204, + 0.0003556140000000596, + 0.0003431919976719655, + 0.000345571999787353, + 0.00034777399559970945, + 0.0003484090048004873, + 0.00034275899815838784, + 0.0016799779987195507, + 0.003556411000317894, + 0.0004382939951028675, + 0.0003938000008929521, + 0.00037324800359783694, + 0.00036629099486162886, + 0.00035245200706413016, + 0.00034510099794715643, + 0.0003453680037637241, + 0.0003486660061753355, + 0.0003504089982016012, + 0.00034896899887826294, + 0.0004290200013201684, + 0.00516378599422751, + 0.00042619699524948373, + 0.0004289759963285178, + 0.00039900000410852954, + 0.0003503719999571331, + 0.00034994200541405007, + 0.00034476899600122124, + 0.00035057999775744975, + 0.0003577309980755672, + 0.0003432600060477853, + 0.0035823180005536415, + 0.00044810200051870197, + 0.00044810799590777606, + 0.00037862599856453016, + 0.00034666500141611323, + 0.00035463600215734914, + 0.00034951199631905183, + 0.00034794599923770875, + 0.00034918200253741816, + 0.0003485330016701482, + 0.0003737439983524382, + 0.004216084998915903, + 0.00046174300223356113, + 0.00045301699719857424, + 0.00043179199565202, + 0.00035260000004200265, + 0.000343095998687204, + 0.00034250599856022745, + 0.000356894001015462, + 0.000347944995155558, + 0.0003476490019238554, + 0.003555257004336454, + 0.00043294599890941754, + 0.0004487059995881282, + 0.00045555199903901666, + 0.00036384700069902465, + 0.0003748680028365925, + 0.0003502999970805831, + 0.00035140499676344916, + 0.0003456659978837706, + 0.00034094200236722827, + 0.005819464000524022, + 0.00046226000267779455, + 0.00043093800195492804, + 0.000346601998899132, + 0.0003512919938657433, + 0.0003441680019022897, + 0.0003874239992001094, + 0.0003493839976727031, + 0.00035160100378561765, + 0.004781338997418061, + 0.00045905300066806376, + 0.000440818999777548, + 0.00038293900433927774, + 0.0003577330062398687, + 0.00034899399906862527, + 0.0004399429963086732, + 0.0003924890042981133, + 0.00035639099951367825, + 0.003045880002900958, + 0.0007339059957303107, + 0.00037434000114444643, + 0.00036381300014909357, + 0.0003506900029606186, + 0.0003472180032986216, + 0.00035735099663725123, + 0.00034928500099340454, + 0.00034724899887805805, + 0.00035352000122657046, + 0.00034941299963975325, + 0.0003493170006549917, + 0.00035080099769402295, + 0.00034379800490569323, + 0.003228415000194218, + 0.00043634000030579045, + 0.0003948949961340986, + 0.00036731200088979676, + 0.000363745006325189, + 0.0003571399938664399, + 0.0003546439984347671, + 0.00035226999898441136, + 0.00034711299667833373, + 0.0003453980025369674, + 0.0003499809972709045, + 0.004300770997360814, + 0.0004795019995071925, + 0.00039923900476424024, + 0.0003518859957694076, + 0.00035113900230498984, + 0.00034489900281187147, + 0.000356467004166916, + 0.00034590999712236226, + 0.0025961569990613498, + 0.002474524997523986, + 0.00043173499580007046, + 0.0004153420013608411, + 0.0003605070014600642, + 0.00035415199818089604, + 0.0003453899989835918, + 0.0003460040024947375, + 0.00035228799970354885, + 0.00034953000431414694, + 0.005231980998360086, + 0.0004727180057670921, + 0.00039224500505952165, + 0.00040308899770025164, + 0.0003528750021359883, + 0.0003557359959813766, + 0.00045178100117482245, + 0.0003656979970401153, + 0.0047066150000318885, + 0.0005890839966014028, + 0.0003696749990922399, + 0.0003589269981603138, + 0.0004151780012762174, + 0.0003579699987312779, + 0.00034560999483801425, + 0.0003505259956000373, + 0.0003445619950070977, + 0.0003451879965723492, + 0.005456300998048391, + 0.0005220700040808879, + 0.000974212001892738, + 0.0003763779968721792, + 0.0010469849949004129, + 0.0003916630012099631, + 0.005273361995932646, + 0.000671157002216205, + 0.0003598969997256063, + 0.00036385099519975483, + 0.0003478840008028783, + 0.00035717299761017784, + 0.0003570179978851229, + 0.00034748200414469466, + 0.004572891994030215, + 0.0005083350042696111, + 0.00048402699758298695, + 0.00040358799742534757, + 0.0009431029975530691, + 0.000392280999221839, + 0.0003596009992179461, + 0.000350878995959647, + 0.005537527002161369, + 0.000459530005173292, + 0.00044581000111065805, + 0.00042526500328676775, + 0.00034967400279128924, + 0.000346143999195192, + 0.0003454929974395782, + 0.0003543479979271069, + 0.0003492600008030422, + 0.0003499390004435554, + 0.0037718960011261515, + 0.0005431190002127551, + 0.0005040749965701252, + 0.00036624200583901256, + 0.00034676899667829275, + 0.0003864329992211424, + 0.0003649889986263588, + 0.000348888999724295, + 0.0003437409977777861, + 0.0003504960041027516, + 0.0003429670032346621, + 0.00037791200156789273, + 0.0056998619984369725, + 0.00046591100544901565, + 0.00043255700438749045, + 0.00034765000600600615, + 0.00035537000076146796, + 0.0003467229980742559, + 0.0003456539998296648, + 0.0003544159990269691, + 0.0003459010040387511, + 0.00034756500099319965, + 0.00034469400270609185, + 0.00034420099837007, + 0.005315588001394644, + 0.004996349998691585, + 0.004997251002350822, + 0.007998917004442774, + 0.005996653002512176, + 0.006998085998930037, + 0.006001257999741938, + 0.0069910090023768134, + 0.007623971003340557, + 0.008370530005777255, + 0.008007043004909065, + 0.007989651996467728, + 0.006999657998676412, + 0.003989552002167329, + 0.006998178003414068, + 0.006995936004386749, + 0.007996116000867914, + 0.007997624998097308, + 0.007997212996997405, + 0.006999334997090045, + 0.0089954380018753, + 0.006997379998210818, + 0.00514904799638316, + 0.005845216997840907, + 0.00599653999961447, + 0.0069981690030545, + 0.007001518999459222, + 0.007003512997471262, + 0.005991712998365983, + 0.006989823006733786, + 0.00800303700088989, + 0.06015052899601869, + 0.0011747509997803718, + 0.0005344919991330244, + 0.0004157579969614744, + 0.00040030300442595035, + 0.01573724399349885, + 0.00902320699970005, + 0.005525321001186967, + 0.0049930879977182485, + 0.0037250020031933673, + 0.0006687630011583678, + 0.0004244049996486865, + 0.002125696002622135, + 0.0034122900033253245, + 0.005605016005574726, + 0.023001296001893934, + 0.006943124004465062, + 0.0069836179973208345, + 0.006995459996687714, + 0.006999037999776192, + 0.007007986001553945, + 0.006982910002989229, + 0.006997631004196592, + 0.007022757003142033, + 0.005967395998595748, + 0.007994289997441228, + 0.00699754900415428, + 0.006997536002018023, + 0.00703717400028836, + 0.006964133004657924, + 0.006990957997913938, + 0.00699771100335056, + 0.00699743400036823, + 0.006996705997153185, + 0.0069946009971317835, + 0.007998328997928184, + 0.006997668999247253, + 0.005999458000587765, + 0.006995821000600699, + 0.007997340995643754, + 0.006998497003223747, + 0.006997235002927482, + 0.005997571999614593, + 0.00699759599956451, + 0.00799744499818189, + 0.007998312001291197, + 0.00800401399465045, + 0.007991099999344442, + 0.007997339002031367, + 0.006997550000960473, + 0.003999715998361353, + 0.007996913998795208, + 0.006997911994403694, + 0.010997682002198417, + 0.005019973999878857, + 0.003990425000665709, + 0.005006247003620956, + 0.006379394995747134, + 0.007505581001169048, + 0.006076632002077531, + 0.007995478001248557, + 0.008002057002158836, + 0.007991880993358791, + 0.0029960839965497144, + 0.005246001994237304, + 0.008749403001274914, + 0.004004340997198597, + 0.005990069999825209, + 0.005500637002114672, + 0.0035002950025955215, + 0.006993916002102196, + 0.010995024000294507, + 0.011997187000815757, + 0.006997709002462216, + 0.004997042000468355, + 0.005998159002047032, + 0.007998271998076234, + 0.005997877000481822, + 0.007997021995834075, + 0.007997945001989137, + 0.006999908000580035, + 0.005994793995341752, + 0.008027972005947959, + 0.006969757996557746, + 0.007995141000719741, + 0.0039948420017026365, + 0.00699607400019886, + 0.0059975010008201934, + 0.007997494998562615, + 0.006998114004090894, + 0.005000051998649724, + 0.009129835001658648, + 0.010862681003345642, + 0.005997240004944615, + 0.004998411997803487, + 0.004997134004952386, + 0.007997383996553253, + 0.006999377001193352, + 0.005995402003463823, + 0.0060028469961252995, + 0.005993050996039528, + 0.006998728997132275, + 0.005002920996048488, + 0.005991150996123906, + 0.008438719996775035, + 0.005553677001444157, + 0.009021169003972318, + 0.006975192998652346, + 0.0029976029982208274, + 0.0017196529952343553, + 0.001283607998630032, + 0.007239344005938619, + 0.008747038002184127, + 0.006993623996095266, + 0.007996363005077, + 0.006998813994869124, + 0.0059981559970765375, + 0.006996327996603213, + 0.00699854100093944, + 0.004089843001565896, + 0.003904517005139496, + 0.00599989000329515, + 0.007996806998562533, + 0.00699917600286426, + 0.005996766994940117, + 0.005006426996260416, + 0.004992938003852032, + 0.003649113998108078, + 0.007411229002173059, + 0.007454081001924351, + 0.007467604002158623, + 0.005997244996251538, + 0.007130179001251236, + 0.00886481899942737, + 0.006997576005232986, + 0.008008168995729648, + 0.00698767599533312, + 0.0020029760053148493, + 0.00499205900268862, + 0.006001094996463507, + 0.00039874499634606764, + 0.0003760620020329952, + 0.00035622300492832437, + 0.0003476190031506121, + 0.001362692994007375, + 0.0003576500021154061, + 0.0003485159977572039, + 0.00034273700293852016, + 0.0008817869966151193, + 0.002374112998950295, + 0.00038098399818409234, + 0.00037663200055249035, + 0.0003878479983541183, + 0.00037552999856416136, + 0.0003539199969964102, + 0.0003493839976727031, + 0.0003470800002105534, + 0.003409217002626974, + 0.0004247600008966401, + 0.0003586770035326481, + 0.0003513109986670315, + 0.0003446159971645102, + 0.0004375529970275238, + 0.0007732980011496693, + 0.00039301299693761393, + 0.0003610920030041598, + 0.00034524299553595483, + 0.0009586520027369261, + 0.00041370400140294805, + 0.00035194199881516397, + 0.0003532490009092726, + 0.0003451349984970875, + 0.0003454309990047477, + 0.0003545420040609315, + 0.00037095099833095446, + 0.0003511440008878708, + 0.00035199199919588864, + 0.0003433660021983087, + 0.00034751699422486126, + 0.005370618004235439, + 0.0003779840044444427, + 0.00039317899791058153, + 0.000385116996767465, + 0.0003657480046967976, + 0.00035407899849815294, + 0.00034767499892041087, + 0.00034920599864562973, + 0.0003466929993010126, + 0.0003485120032564737, + 0.00034343999868724495, + 0.0031203320031636395, + 0.0030477260006591678, + 0.004938631995173637, + 0.002283912996063009, + 0.005978671004413627, + 0.004723981997813098, + 0.006997279000643175, + 0.006998349002969917, + 0.0070848070026841015, + 0.006920244995853864, + 0.00799449099577032, + 0.0069888750003883615, + 0.007994261002750136, + 0.006996798001637217, + 0.0069982820059522055, + 0.0069968099996913224, + 0.006996906995482277, + 0.006997901000431739, + 0.005999682005494833, + 0.005996713000058662, + 0.00452927200240083, + 0.004467904000193812, + 0.005301665994920768, + 0.007689646998187527, + 0.00118368099356303, + 0.0003581389973987825, + 0.0003521799953887239, + 0.0003531339971232228, + 0.00034633299947017804, + 0.00034535099985077977, + 0.00035817000025417656, + 0.00034322699502808973, + 0.0005421140012913384, + 0.0004237340035615489, + 0.00037502100167330354, + 0.00038602200220339, + 0.00041108400182565674, + 0.0003550749970600009, + 0.00034637199860299006, + 0.00035359100002096966, + 0.0003518639932735823, + 0.0003461130036157556, + 0.00034347600012551993, + 0.00034934999712277204, + 0.0003422149966354482, + 0.0003421740038902499, + 0.0050298770001973026, + 0.00048744100058684126, + 0.0003531010006554425, + 0.00035545900027500466, + 0.00034624399995664135, + 0.0003438910061959177, + 0.0003495580021990463, + 0.00034273700293852016, + 0.00034817300183931366, + 0.0003517989971442148, + 0.00035043399839196354, + 0.00034510299883550033, + 0.005136842002684716, + 0.002729939005803317, + 0.0003851120054605417, + 0.00036683600046671927, + 0.00035416199534665793, + 0.000353683004505001, + 0.00034610399598022923, + 0.0026217780032311566, + 0.0003789260008488782, + 0.00035887300327885896, + 0.0003524190033203922, + 0.00034850699739763513, + 0.00035367600503377616, + 0.00034982400393346325, + 0.0003427839983487502, + 0.00034617300116224214, + 0.00034564700035844, + 0.0003485140041448176, + 0.005211850002524443, + 0.00047646299935877323, + 0.00037726799928350374, + 0.0020925939970766194, + 0.00042126799962716177, + 0.0003989529941463843, + 0.004828848999750335, + 0.000382555001124274, + 0.00037518500175792724, + 0.0003585779995773919, + 0.00035169699549442157, + 0.00034644099650904536, + 0.000347396002325695, + 0.0017175920002046041, + 0.0044415320007828996, + 0.0008022069960134104, + 0.00036669399560196325, + 0.0003556720039341599, + 0.000353540999640245, + 0.0013199020031606779, + 0.003786143999604974, + 0.0003789360052905977, + 0.0003679450019262731, + 0.00035772399860434234, + 0.00035738600126933306, + 0.0003493569965939969, + 0.0003547039959812537, + 0.0003434829995967448, + 0.00034955899900523946, + 0.0035050150036113337, + 0.0016290290004690178, + 0.0003551280024112202, + 0.0003690919984364882, + 0.00037985800008755177, + 0.0003779849939746782, + 0.0003602560027502477, + 0.0011717750021489337, + 0.005202868997002952, + 0.00043906499922741205, + 0.000383929000236094, + 0.0003571619963622652, + 0.000350420996255707, + 0.0003472829994279891, + 0.0003448209972702898, + 0.00034348400367889553, + 0.00035150199983036146, + 0.0003511889954097569, + 0.000342715997248888, + 0.0003603809964261018, + 0.0007935440007713623, + 0.0003621380019467324, + 0.00034293300268473104, + 0.0003553580027073622, + 0.0020675019986811094, + 0.00035119400126859546, + 0.00035253300302429125, + 0.0003487680005491711, + 0.00035386600211495534, + 0.0003422350055188872, + 0.0022612119937548414, + 0.0005957540051895194, + 0.0003632700027083047, + 0.0003536790027283132, + 0.0003440380023675971, + 0.0003544020000845194, + 0.00034594800672493875, + 0.000346526998328045, + 0.00035882199881598353, + 0.0003437160048633814, + 0.0003449220021138899, + 0.003910218001692556, + 0.0003796879973378964, + 0.0010044849987025373, + 0.0003475220000836998, + 0.00035025599936489016, + 0.00035026599653065205, + 0.0003474729965091683, + 0.000347621004038956, + 0.00035023999953409657, + 0.0003428859999985434, + 0.004277639003703371, + 0.00038739499723305926, + 0.00037532499845838174, + 0.0003529800014803186, + 0.0003495010023470968, + 0.0003575290029402822, + 0.00034863100154325366, + 0.00034380899887764826, + 0.00034469799720682204, + 0.005185319001611788, + 0.000884729997778777, + 0.0003496770004858263, + 0.0003471610034466721, + 0.00034678300289670005, + 0.0003600060008466244, + 0.00034845200570998713, + 0.0003484839980956167, + 0.0003536369986250065, + 0.002076431999739725, + 0.003450901996984612, + 0.0003881410011672415, + 0.0003646749973995611, + 0.00034999399940716103, + 0.00034603299718583, + 0.0017996839960687794, + 0.0003576720046112314, + 0.0003714649938046932, + 0.00037333100044634193, + 0.0003651490042102523, + 0.002373426999838557, + 0.0003779699982260354, + 0.0003630880019045435, + 0.0003796969976974651, + 0.00037088199314894155, + 0.00034579999919515103, + 0.0003597449976950884, + 0.0003433439997024834, + 0.0003433199963183142, + 0.0024236219978774898, + 0.00037959399924147874, + 0.0017520239998702891, + 0.0003809449990512803, + 0.00037313500070013106, + 0.00036258299951441586, + 0.0003515270000207238, + 0.0003486580026219599, + 0.0003466439957264811, + 0.00034826800401788205, + 0.00034528200194472447, + 0.0003518760058796033, + 0.0046955150028225034, + 0.0004445770027814433, + 0.0004226910023135133, + 0.0004064210006617941, + 0.00038629000482615083, + 0.00038375800068024546, + 0.00036536000698106363, + 0.0003522550032357685, + 0.000348692003171891, + 0.00034688199957599863, + 0.00035079699591733515, + 0.00035053500323556364, + 0.0003420229986659251, + 0.004130320005060639, + 0.000372824004443828, + 0.00138408099883236, + 0.00036719600029755384, + 0.00034396600676700473, + 0.00034733799839159474, + 0.0003490590024739504, + 0.00035016699985135347, + 0.00034377800329821184, + 0.0003494400007184595, + 0.002701166995393578, + 0.00037889899977017194, + 0.00035258999560028315, + 0.0003508809968479909, + 0.00034641199454199523, + 0.0003458240025793202, + 0.0004300189975765534, + 0.0011867720022564754, + 0.0008420670055784285, + 0.0003601200005505234, + 0.0003500070015434176, + 0.00034511100238887593, + 0.002045878005446866, + 0.0003666019983938895, + 0.0012420439961715601, + 0.000354531999619212, + 0.0003522750048432499, + 0.00034754499938571826, + 0.00035684099566424266, + 0.00034644099650904536, + 0.00035200400452595204, + 0.00034668800071813166, + 0.00034501399932196364, + 0.0003530710018821992, + 0.00034371799847576767, + 0.0003522280021570623, + 0.004192470005364157, + 0.0004128200016566552, + 0.0003528080051182769, + 0.0003550710025592707, + 0.00034339899866608903, + 0.0003433339952607639, + 0.00035152200143784285, + 0.00035265299811726436, + 0.00034368699562037364, + 0.00035206900065531954, + 0.00034346399479545653, + 0.0003496850040392019, + 0.005426871997769922, + 0.0004028349940199405, + 0.00038537500222446397, + 0.0003766369991353713, + 0.00034911499824374914, + 0.0003481739986455068, + 0.0003444399990257807, + 0.00034255899663548917, + 0.00035046499397139996, + 0.0003444889953243546, + 0.004522561997873709, + 0.0003713510013767518, + 0.0009452189988223836, + 0.0003621809955802746, + 0.000353880001057405, + 0.0003588190011214465, + 0.0003451290031080134, + 0.0003500890015857294, + 0.00034650100133148953, + 0.0020395800020196475, + 0.0018920229995273985, + 0.0003768159949686378, + 0.00037125000380910933, + 0.0003624190067057498, + 0.00035508399741956964, + 0.00034928099921671674, + 0.00034525500086601824, + 0.00034952200076077133, + 0.0003462800013949163, + 0.00034678699739743024, + 0.004832562000956386, + 0.0003768919996218756, + 0.0003675900006783195, + 0.0003522460028761998, + 0.00034782600414473563, + 0.0003452019955147989, + 0.00034356999822193757, + 0.0003559160031727515, + 0.00034667200088733807, + 0.00035160100378561765, + 0.0042818010042537935, + 0.00037126700044609606, + 0.0003562140045687556, + 0.0022729519987478852, + 0.008450995999737643, + 0.00037620000512106344, + 0.0003665749973151833, + 0.0003606490063248202, + 0.00035012800071854144, + 0.00034433900145813823, + 0.0003453289973549545, + 0.0003509969974402338, + 0.00034942500496981665, + 0.000343936997523997, + 0.004389365996757988, + 0.000402919999032747, + 0.0003492739997454919, + 0.0003514220006763935, + 0.0003466160051175393, + 0.0003510410024318844, + 0.0003446869959589094, + 0.0003439520005485974, + 0.0005811679948237725, + 0.0007390480022877455, + 0.0008017629952519201, + 0.00044303399772616103, + 0.0009749480013852008, + 0.0003524290004861541, + 0.00035364500217838213, + 0.00035272999957669526, + 0.00035144899447914213, + 0.00034655600029509515, + 0.000352048002241645, + 0.0003451869997661561, + 0.0012898299974040128, + 0.00036546599585562944, + 0.0003521690014167689, + 0.00035669699718710035, + 0.00034842600143747404, + 0.0003447720009717159, + 0.0004931049988954328, + 0.000389271997846663, + 0.00039841100078774616, + 0.0003969809986301698, + 0.0003965819996665232, + 0.0003961379989050329, + 0.0003720259992405772, + 0.00036926900065736845, + 0.00038690600194968283, + 0.0006103039995650761, + 0.00034963800135301426, + 0.00034660699748201296, + 0.0003524489948176779, + 0.00034240699460497126, + 0.0003452649980317801, + 0.00034831099765142426, + 0.00034506899828556925, + 0.00034428099752403796, + 0.00038916100311325863, + 0.0009363839999423362, + 0.0004981469974154606, + 0.0003522989936755039, + 0.00035520600067684427, + 0.00034839200088754296, + 0.0003497259967844002, + 0.00034953000431414694, + 0.0003451069933362305, + 0.0004198649985482916, + 0.00038661500002490357, + 0.0003962179980590008, + 0.0003950139944208786, + 0.0003886699996655807, + 0.0003867720006383024, + 0.00038400499761337414, + 0.0003915880006388761, + 0.0003740519969142042, + 0.0003453760000411421, + 0.00035319599555805326, + 0.0003437870036577806, + 0.0003419339991523884, + 0.000350456997693982, + 0.0003473519973340444, + 0.0003877420022035949, + 0.0003952350016334094, + 0.00038265599869191647, + 0.00038993900670902804, + 0.0003686479976749979, + 0.00034834699908969924, + 0.00034729199978755787, + 0.0003498510050121695, + 0.00034759799746097997, + 0.0009731289974297397, + 0.000439258998085279, + 0.00038134300120873377, + 0.0003767460002563894, + 0.00039412399928551167, + 0.0009269450019928627, + 0.0005220950042712502, + 0.0006620819985982962, + 0.0003509579983074218, + 0.0004803710035048425, + 0.00035427600232651457, + 0.00034964399674208835, + 0.00034772099752444774, + 0.00034476699511287734, + 0.0003472989992587827, + 0.00034715999936452135, + 0.00034774700179696083, + 0.000358994999260176, + 0.0003826870015473105, + 0.00039339000068139285, + 0.0003905379999196157, + 0.00041912800224963576, + 0.000855906997458078, + 0.00042866700096055865, + 0.00039692599966656417, + 0.000406040002417285, + 0.00035940000088885427, + 0.0003518129960866645, + 0.0003510410024318844, + 0.0004669600020861253, + 0.000558130006538704, + 0.00035339400346856564, + 0.0007596650029881857, + 0.00036287299735704437, + 0.00034964400401804596, + 0.00034903400228358805, + 0.00034724899887805805, + 0.0003497119978419505, + 0.0003483319960650988, + 0.0003494100019452162, + 0.0003489090013317764, + 0.00035735999699682, + 0.00034352699731243774, + 0.00035044999822275713, + 0.00040169100248022005, + 0.00040745700243860483, + 0.00039304500387515873, + 0.00038683900493197143, + 0.00037890899693593383, + 0.0016270269989036024, + 0.0014240100063034333, + 0.00046234500041464344, + 0.0005016840004827827, + 0.000375608004105743, + 0.00034486599906813353, + 0.0003441200024099089, + 0.0006362960048136301, + 0.0004112820024602115, + 0.00041528499423293397, + 0.00036040099803358316, + 0.0003540580000844784, + 0.000357051998435054, + 0.0003488570000627078, + 0.0003580109987524338, + 0.0003986449955846183, + 0.00035584400029620156, + 0.00035468900023261085, + 0.0003485039997030981, + 0.00039825099520385265, + 0.00039409100281773135, + 0.0005634840053971857, + 0.0004975769988959655, + 0.0003481819949229248, + 0.0003647330013336614, + 0.0003435399994486943, + 0.00034351900103501976, + 0.0007206610025605187, + 0.0004216129964333959, + 0.00039817200013203546, + 0.00040239800000563264, + 0.00038337300065904856, + 0.00039680399640928954, + 0.00038745900383219123, + 0.0003920879971701652, + 0.0003547040032572113, + 0.00044896899635205045, + 0.0007419859975925647, + 0.00035441500222077593, + 0.0003539219978847541, + 0.00035357299930183217, + 0.0003492620016913861, + 0.00034354699891991913, + 0.000595729004999157, + 0.0011242149994359352, + 0.0007576600037282333, + 0.0005193410033825785, + 0.0003637650006567128, + 0.00034793999657267705, + 0.00035692799428943545, + 0.00034754700027406216, + 0.00035131799813825637, + 0.00034385699837002903, + 0.0013719459966523573, + 0.0004014809965156019, + 0.00039372400351567194, + 0.0011433510007918812, + 0.00035190700145903975, + 0.0003492129981168546, + 0.0003510230017127469, + 0.0003436440019868314, + 0.00034911699913209304, + 0.00035324200143804774, + 0.0003519020028761588, + 0.000364705003448762, + 0.0005056629961472936, + 0.0004931929943268187, + 0.00039966099575394765, + 0.0004115569972782396, + 0.0013985080004204065, + 0.0007867870008340105, + 0.0003683070026454516, + 0.00036798400105908513, + 0.00035547500010579824, + 0.00035493400355335325, + 0.00035686299816006795, + 0.0027418610043241642, + 0.00037664399860659614, + 0.00037269099993864074, + 0.00036389799788594246, + 0.0003614859961089678, + 0.00035524000122677535, + 0.0003570319968275726, + 0.00036375399940880015, + 0.0003541069963830523, + 0.00035597399983089417, + 0.004625460001989268, + 0.0004120540033909492, + 0.0003609240011428483, + 0.00035436300095170736, + 0.00035126700095133856, + 0.0003595360030885786, + 0.00036133299727225676, + 0.0003564729995559901, + 0.00035482500243233517, + 0.00035566900623962283, + 0.004808443001820706, + 0.0003951799953938462, + 0.0003656720000435598, + 0.00036626200017053634, + 0.00035811900306725875, + 0.0003585090016713366, + 0.002688232001673896, + 0.0003786939996643923, + 0.00035653300437843427, + 0.00035827900137519464, + 0.0003548110034898855, + 0.00035307399957673624, + 0.0016703059955034405, + 0.00037207300192676485, + 0.001108358999772463, + 0.000371916001313366, + 0.00035670500074047595, + 0.0003629580023698509, + 0.0003521249964251183, + 0.00035266100167063996, + 0.0003589370026020333, + 0.0003603899967856705, + 0.0003527189983287826, + 0.0029468209977494553, + 0.00037783999869134277, + 0.00036942500446457416, + 0.0003611150023061782, + 0.0003614060042309575, + 0.00035567399754654616, + 0.00035885499528376386, + 0.00035589699837146327, + 0.0003515719945426099, + 0.005974225998215843, + 0.00037939399771858007, + 0.00036838799860561267, + 0.00034271099866600707, + 0.0003569960026652552, + 0.00034526100353104994, + 0.00034599700302351266, + 0.0003517200020723976, + 0.0003477769932942465, + 0.0049267439972027205, + 0.0003803389990935102, + 0.0008794440000201575, + 0.0003604770026868209, + 0.00034770499769365415, + 0.00034431899985065684, + 0.00036136899871053174, + 0.0003511690010782331, + 0.00035609299811767414, + 0.0003522469996823929, + 0.0003433559977565892, + 0.00629315800324548, + 0.006998878998274449, + 0.007995260006282479, + 0.006998752003710251, + 0.006997886004683096, + 0.007996660999197047, + 0.005998475004162174, + 0.004783037999004591, + 0.005219225000473671, + 0.001990837001358159, + 0.003002227997058071, + 0.005219598999246955, + 0.006771094995201565, + 0.006998560995270964, + 0.007072665001032874, + 0.00770352799736429, + 0.006219178998435382, + 0.008994142997835297, + 0.007995375999598764, + 0.00399810700037051, + 0.0030086569968261756, + 0.002008953997574281, + 0.006989667002926581, + 0.007983859999512788, + 0.005125508003402501, + 0.005868061998626217, + 0.007999335997737944, + 0.007996318992809393, + 0.005996665000566281, + 0.004005287999461871, + 0.00399398899753578, + 0.005994056999043096, + 0.0060001689998898655, + 0.007994535000761971, + 0.008118224002828356, + 0.0038777959998697042, + 0.004146761006268207, + 0.005931380997935776, + 0.003916581998055335, + 0.005121836999023799, + 0.007871347996115219, + 0.005241163999016862, + 0.004744315003335942, + 0.001999906002311036, + 0.006990253001276869, + 0.0049970599939115345, + 0.006999189005000517, + 0.010998690995620564, + 0.002093479997711256, + 0.003899923001881689, + 0.004998723998141941, + 0.006996990996412933, + 0.005060047005827073, + 0.002169924002373591, + 0.009762969995790627, + 0.006993637995037716, + 0.006992490998527501, + 0.007999437999387737, + 0.0060095509979873896, + 0.009984916003304534, + 0.006996554999204818, + 0.005993383005261421, + 0.008125872001983225, + 0.006943092004803475, + 0.006918535997101571, + 0.010002603004977573, + 0.006992476002778858, + 0.0032332019982277416, + 0.001035249006235972, + 0.0009619839984225109, + 0.0009822989959502593, + 0.0009145680014626123, + 0.0009780079999472946, + 0.0009408230034750886, + 0.0009674520042608492, + 0.0009048000065376982, + 0.0009326329964096658, + 0.000928841000131797, + 0.0009524240012979135, + 0.0009618330004741438, + 0.0008908520030672662, + 0.000939710000238847, + 0.0009331459950772114, + 0.0009477950006839819, + 0.000928918001591228, + 0.0009271779999835417, + 0.0009444359966437332, + 0.0009536430006846786, + 0.0009483940011705272, + 0.0009329690001322888, + 0.0030504870010190643, + 0.001385980001941789, + 0.0009434930034331046, + 0.0009319899982074276, + 0.0018788850065902807, + 0.0011495660000946373, + 0.0009688499994808808, + 0.0010448059983900748, + 0.0009556439981679432, + 0.0009573439965606667, + 0.0009274459953303449, + 0.001166312002169434, + 0.000901004001207184, + 0.002307315000507515, + 0.0011955809968640096, + 0.0010645930015016347, + 0.0009988960009650327, + 0.0009744909984874539, + 0.0008885779971024022, + 0.0006409790003090166, + 0.00039191500400193036, + 0.0005283520004013553, + 0.00035715099511435255, + 0.00035721799940802157, + 0.0003506180000840686, + 0.00034807599877240136, + 0.0058122990012634546, + 0.0003746980000869371, + 0.0003580009943107143, + 0.000354953997884877, + 0.0003702180038089864, + 0.0003510250026010908, + 0.00034905899519799277, + 0.0003549270040821284, + 0.00034507900272728875, + 0.00035331700200913474, + 0.005999223001708742, + 0.00037824099854333326, + 0.0003573360008886084, + 0.0003619779963628389, + 0.0003563270001905039, + 0.0003515170028549619, + 0.0003519239980960265, + 0.00035073800245299935, + 0.0003476109995972365, + 0.004438700001628604, + 0.0004221889976179227, + 0.0003700969973579049, + 0.0003572490022634156, + 0.0003466140042291954, + 0.000351178998243995, + 0.0003461219966993667, + 0.00034421199961798266, + 0.001099280001653824, + 0.0003853169982903637, + 0.0004004809961770661, + 0.00035433699667919427, + 0.0003514830023050308, + 0.0003426059993216768, + 0.002938441000878811, + 0.0005741380009567365, + 0.0003616499961935915, + 0.00034824600152205676, + 0.0003580150005291216, + 0.00034735599911073223, + 0.0003519180027069524, + 0.000350330003129784, + 0.00034451999817974865, + 0.0020151620046817698, + 0.0019474229993647896, + 0.00038740099989809096, + 0.0003531139955157414, + 0.0003476410056464374, + 0.0019390650049899705, + 0.0018300759984413162, + 0.0003741510008694604, + 0.001386174997605849, + 0.00036797000211663544, + 0.00035827299871016294, + 0.00034868300281232223, + 0.00035912000021198764, + 0.00035803399805445224, + 0.0003462149979895912, + 0.0003534930001478642, + 0.0038442750010290183, + 0.0007552159950137138, + 0.00035167600435670465, + 0.0003532449991325848, + 0.0003523079940350726, + 0.0003495760029181838, + 0.0003524610001477413, + 0.0003439579959376715, + 0.00035027400008402765, + 0.005429645003459882, + 0.007205285000964068, + 0.0006001650035614148, + 0.000674711998726707, + 0.00035391699930187315, + 0.0003464600013103336, + 0.0003478879953036085, + 0.00037680199602618814, + 0.0003633960004663095, + 0.005084693999378942, + 0.0004189280007267371, + 0.00035347299854038283, + 0.0003496999997878447, + 0.0003557839954737574, + 0.0003520500031299889, + 0.0004199329996481538, + 0.00035215199750382453, + 0.0003508650042931549, + 0.005503874002897646, + 0.0004082369996467605, + 0.0003624099990702234, + 0.00035431000287644565, + 0.0003480200030026026, + 0.0003663969982881099, + 0.00035602600109996274, + 0.0003530300018610433, + 0.00036074499803362414, + 0.000349654997990001, + 0.0003434870013734326, + 0.0061460520009859465, + 0.0003772990021388978, + 0.00035667000338435173, + 0.00036287499824538827, + 0.0003656370026874356, + 0.00035073199978796765, + 0.0003518899975460954, + 0.0003526619984768331, + 0.00034533200232544914, + 0.005917376998695545, + 0.00036764000105904415, + 0.00035172500065527856, + 0.00034838999999919906, + 0.0003521780017763376, + 0.000347747998603154 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil", + "fullname": "TestCalculateDiagnosticsForTurbulence", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.0008620799999334849, + "max": 0.01892632200178923, + "mean": 0.006499208834429737, + "stddev": 0.0020825534092384075, + "rounds": 465, + "median": 0.006990721994952764, + "iqr": 0.002739232999374508, + "q1": 0.005250037251244066, + "q3": 0.007989270250618574, + "iqr_outliers": 7, + "stddev_outliers": 116, + "outliers": "116;7", + "ld15iqr": 0.0014779140037717298, + "hd15iqr": 0.012571820996527094, + "ops": 153.86488193800952, + "total": 3.022132108009828, + "data": [ + 0.008456655996269546, + 0.00575521400605794, + 0.0037555399976554327, + 0.0035256189948995598, + 0.007968473000801168, + 0.007834352996724192, + 0.006606032999115996, + 0.009988350997446105, + 0.005999191002047155, + 0.006990850997681264, + 0.010103467000590172, + 0.00500432999979239, + 0.0028689920000033453, + 0.004192112995951902, + 0.0027951689990004525, + 0.00726114799908828, + 0.004808696998225059, + 0.002918116995715536, + 0.005103912997583393, + 0.002242561000457499, + 0.005041762997279875, + 0.0015944830010994337, + 0.006001943998853676, + 0.002512462997401599, + 0.0034750569975585677, + 0.0024270159992738627, + 0.0041683429953991435, + 0.007396637003694195, + 0.008000283000001218, + 0.005983488001220394, + 0.007079926996084396, + 0.006909409006766509, + 0.006002722002449445, + 0.008053311001276597, + 0.004905912996036932, + 0.010000529000535607, + 0.007999601999472361, + 0.006992700000409968, + 0.004995529001462273, + 0.003703932001371868, + 0.004288075004296843, + 0.002831216996128205, + 0.005348997998225968, + 0.012571820996527094, + 0.01892632200178923, + 0.006084300999646075, + 0.004648907997761853, + 0.005540612997720018, + 0.006000995999784209, + 0.009997960005421191, + 0.005989419005345553, + 0.00698322600510437, + 0.007995870000740979, + 0.008992641000077128, + 0.006132797003374435, + 0.007851493995985948, + 0.0042734289963846095, + 0.007706620999670122, + 0.00299807199917268, + 0.005776345999038313, + 0.007215574005385861, + 0.005520858998352196, + 0.006472859000496101, + 0.005004192003980279, + 0.004990725006791763, + 0.00599564299773192, + 0.006996023999818135, + 0.006997240001510363, + 0.0059911700009251945, + 0.006002120004268363, + 0.0018822220008587465, + 0.005968072000541724, + 0.0031356649997178465, + 0.003236573000322096, + 0.007794547003868502, + 0.003959066998504568, + 0.005055234003521036, + 0.0083563750013127, + 0.008643282002594788, + 0.005930769999395125, + 0.008997031996841542, + 0.006991223999648355, + 0.00699718100077007, + 0.007004084000072908, + 0.0059898690014961176, + 0.006993548995524179, + 0.007003953003732022, + 0.007988403005583677, + 0.006996575997618493, + 0.008997721000923775, + 0.006063041000743397, + 0.008809468999970704, + 0.005498006998095661, + 0.00456358099472709, + 0.0022579040014534257, + 0.0027918870036955923, + 0.0019865430003846996, + 0.004011655000795145, + 0.007989149002241902, + 0.011002008002833463, + 0.0059954189928248525, + 0.007995191997906659, + 0.010991284994815942, + 0.008005602001503576, + 0.0070865430025151, + 0.006893239995406475, + 0.006996751995757222, + 0.004995552000764292, + 0.0049941070028580725, + 0.003994480997789651, + 0.005012159002944827, + 0.006983241000853013, + 0.009994988999096677, + 0.004566083996905945, + 0.0034225010022055358, + 0.0030020870035514235, + 0.0039906040037749335, + 0.006998903998464812, + 0.007998095999937505, + 0.0076106170017737895, + 0.010385476998635568, + 0.007990137004526332, + 0.005998016000376083, + 0.007993224004167132, + 0.005996941996272653, + 0.0053068979977979325, + 0.007686237004236318, + 0.002027750000706874, + 0.004960065001796465, + 0.007003771999734454, + 0.006991175003349781, + 0.0074976000032620504, + 0.005818284997076262, + 0.006668226000329014, + 0.0020043240001541562, + 0.0030159349989844486, + 0.0049734529966372065, + 0.008996413998829667, + 0.005996210005832836, + 0.0089975129958475, + 0.0070023009975557216, + 0.005993787002807949, + 0.006994372000917792, + 0.007144226998207159, + 0.006851893005659804, + 0.00499157200101763, + 0.005254340001556557, + 0.003733215999091044, + 0.007000024997978471, + 0.0070257109982776456, + 0.00797537800099235, + 0.006990721994952764, + 0.0049891770031536, + 0.005237129000306595, + 0.003753422999579925, + 0.00701208200189285, + 0.005993008002405986, + 0.006997620999754872, + 0.00698680299683474, + 0.005992040998535231, + 0.006998517994361464, + 0.007993191000423394, + 0.006997890995990019, + 0.007993355000508018, + 0.007996843996807002, + 0.002993649999552872, + 0.004999337004846893, + 0.006994547002250329, + 0.0079948249986046, + 0.006995256000664085, + 0.006998197997745592, + 0.0054239609962678514, + 0.007573875998787116, + 0.005989814002532512, + 0.006002521004120354, + 0.002994107002450619, + 0.006994502000452485, + 0.013723189003940206, + 0.0032692359964130446, + 0.007003182006883435, + 0.00599466499988921, + 0.003998631000285968, + 0.007015703995421063, + 0.007973895000759512, + 0.009802556000067852, + 0.005186170994420536, + 0.007990977996087167, + 0.0029948160008643754, + 0.00599432599847205, + 0.008001709000382107, + 0.008001091999176424, + 0.006991140005993657, + 0.0069948729942552745, + 0.0071151279989862815, + 0.005876147995877545, + 0.006999650999205187, + 0.007997003995114937, + 0.007005008999840356, + 0.0059943119995296, + 0.006996979005634785, + 0.007987700999365188, + 0.007992732003913261, + 0.008994775998871773, + 0.005690530000720173, + 0.009306259998993482, + 0.00699671299662441, + 0.004003442001703661, + 0.0020582960059982724, + 0.003924323995306622, + 0.004752230997837614, + 0.004238811001414433, + 0.0033186649961862713, + 0.00367616000585258, + 0.005994920000375714, + 0.004727677995106205, + 0.00726813299843343, + 0.006940040002518799, + 0.01108098999975482, + 0.00901667399739381, + 0.006934067001566291, + 0.00580048199481098, + 0.006182806995639112, + 0.0029987819943926297, + 0.006167077001009602, + 0.00769057399884332, + 0.013147637000656687, + 0.004355270000814926, + 0.0015945600025588647, + 0.006022359004418831, + 0.011000746999343392, + 0.006992285001615528, + 0.007009196997387335, + 0.005516746001376305, + 0.006464359998062719, + 0.007993475002876949, + 0.003985802999523003, + 0.007001725003647152, + 0.008000554000318516, + 0.006989606001297943, + 0.006999195000389591, + 0.0069936149957356974, + 0.004001803004939575, + 0.001993203994061332, + 0.007998171000508592, + 0.007993944003828801, + 0.009998184003052302, + 0.006990697998844553, + 0.006592619996808935, + 0.010407622998172883, + 0.007994338004209567, + 0.001989917000173591, + 0.0039984699978958815, + 0.013001639999856707, + 0.0060951630002819, + 0.003893924003932625, + 0.0021353949996409938, + 0.005292094996548258, + 0.008568548997573089, + 0.0059959640057059005, + 0.006931965996045619, + 0.006058838000171818, + 0.004994098999304697, + 0.004999960998247843, + 0.0069998269973439164, + 0.004999313001462724, + 0.00999652300379239, + 0.004015632002847269, + 0.0014779140037717298, + 0.005662201998347882, + 0.007821659004548565, + 0.007995626001502387, + 0.006997751996095758, + 0.00799775100313127, + 0.006992194001213647, + 0.006567900003574323, + 0.005804642001749016, + 0.007627726998180151, + 0.007986320997588336, + 0.007995531996130012, + 0.007997988002898637, + 0.005999517998134252, + 0.0049969300016527995, + 0.005991309000819456, + 0.008001804002560675, + 0.00799426800222136, + 0.005010163004044443, + 0.007980820999364369, + 0.008004223003808875, + 0.00798963399574859, + 0.005171823999262415, + 0.006816487999458332, + 0.00901342999713961, + 0.010977947000355925, + 0.00899583900172729, + 0.006994627001404297, + 0.01099719599733362, + 0.0070561089960392565, + 0.008354186000360642, + 0.004587465999065898, + 0.0049189950004802085, + 0.003075200002058409, + 0.0069846810001763515, + 0.001991404002183117, + 0.006002235000778455, + 0.007999174995347857, + 0.007991288002813235, + 0.006001359994115774, + 0.0059946809997200035, + 0.008001303998753428, + 0.006999707002250943, + 0.007982686001923867, + 0.0059958479978377, + 0.0059988729990436696, + 0.00599826800316805, + 0.005993510996631812, + 0.006998863005719613, + 0.00799509699572809, + 0.006996440999500919, + 0.007995854000910185, + 0.006996854004682973, + 0.007998553999641445, + 0.00799399899551645, + 0.007995444000698626, + 0.007997409004019573, + 0.009002516999316867, + 0.007995512998604681, + 0.007990192003489938, + 0.006999488003202714, + 0.006993793998844922, + 0.005005414001061581, + 0.00798590300109936, + 0.006995334995735902, + 0.0059968599962303415, + 0.003018668998265639, + 0.0019680770055856556, + 0.004997422998712864, + 0.006997521995799616, + 0.007011310997768305, + 0.006646972993621603, + 0.00634390000050189, + 0.006986496002355125, + 0.008004315997823142, + 0.005990035002469085, + 0.005995542996970471, + 0.007996466003532987, + 0.006998583994572982, + 0.005998212000122294, + 0.00699610300216591, + 0.0069995189987821504, + 0.00799548900249647, + 0.004000387001724448, + 0.005996564999804832, + 0.005994804007059429, + 0.006998013996053487, + 0.008004657996934839, + 0.008554842999728862, + 0.006440402998123318, + 0.002986177998536732, + 0.005045960999268573, + 0.007946771998831537, + 0.007997448003152385, + 0.00799515100516146, + 0.006997303004027344, + 0.004999599004804622, + 0.006997274998866487, + 0.006995105999521911, + 0.00600043900340097, + 0.008995992997370195, + 0.007996936998097226, + 0.007996845000889152, + 0.006181951997859869, + 0.00781260499934433, + 0.007346473001234699, + 0.009648993000155315, + 0.007998387998668477, + 0.007997107000846881, + 0.005996181003865786, + 0.006249837999348529, + 0.005978959001367912, + 0.0037095289953867905, + 0.007056169997667894, + 0.008991504000732675, + 0.008000251000339631, + 0.006996592994255479, + 0.00625100800243672, + 0.0008620799999334849, + 0.00631148400134407, + 0.0036872959972242825, + 0.004386145999887958, + 0.002768272999674082, + 0.0009268690046155825, + 0.003771850999328308, + 0.006083134001528379, + 0.009916185001202393, + 0.007007115003943909, + 0.00798581699928036, + 0.00699505699594738, + 0.005996331994538195, + 0.0070000470004742965, + 0.004994178998458665, + 0.004162385994277429, + 0.008829997001157608, + 0.00507799399929354, + 0.006917627004440874, + 0.005994450002617668, + 0.010998650002875365, + 0.008002556998690125, + 0.007993277002242394, + 0.0069958500025677495, + 0.007007243999396451, + 0.006987291002587881, + 0.007995620006113313, + 0.007000259000051301, + 0.006999755001743324, + 0.006544903000758495, + 0.00845501200092258, + 0.0059894599980907515, + 0.00699310599884484, + 0.007535876000474673, + 0.0074594479956431314, + 0.006996627998887561, + 0.008998826000606641, + 0.0037371809958131053, + 0.004257284999766853, + 0.008002225993550383, + 0.007993241000804119, + 0.005997106003633235, + 0.005997214997478295, + 0.005998426000587642, + 0.007997979999345262, + 0.0059964170068269596, + 0.008003647999430541, + 0.006989806999627035, + 0.00625215800391743, + 0.00774121200083755, + 0.0059972710005240515, + 0.005010865002986975, + 0.004985420004231855, + 0.007400396003504284, + 0.007604655998875387, + 0.006985125997744035, + 0.007998436005436815, + 0.007996884996828157, + 0.006997133001277689, + 0.005004163998819422, + 0.007993102000909857, + 0.00754887599759968, + 0.0025271889971918426, + 0.0059132690003025346, + 0.007341365999309346, + 0.007651440995687153, + 0.005999214998155367, + 0.007996439999260474, + 0.007998786997632124, + 0.007999884001037572, + 0.006994915995164774, + 0.005994260005536489, + 0.003996416002337355, + 0.005996132000291254, + 0.005999282999255229, + 0.007999143999768421, + 0.00699604600231396, + 0.006994585994107183, + 0.007999351000762545, + 0.007996865999302827 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil", + "fullname": "TestCalculateHorizontalGradientsForTurbulence", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.00037598999915644526, + "max": 0.056945527001516894, + "mean": 0.0024951280792298884, + "stddev": 0.0032387813717274596, + "rounds": 2159, + "median": 0.0005633919936371967, + "iqr": 0.004570067998429295, + "q1": 0.000420880498495535, + "q3": 0.00499094849692483, + "iqr_outliers": 8, + "stddev_outliers": 494, + "outliers": "494;8", + "ld15iqr": 0.00037598999915644526, + "hd15iqr": 0.011923240999749396, + "ops": 400.7810293684988, + "total": 5.3869815230573295, + "data": [ + 0.0005756730024586432, + 0.0005950170016149059, + 0.0005938870017416775, + 0.0010072239965666085, + 0.0005097789980936795, + 0.0004728400017484091, + 0.0004727520063170232, + 0.0004641939958673902, + 0.0005057979942648672, + 0.00047229800111381337, + 0.0005490599942277186, + 0.0007633290006197058, + 0.0009610060005798005, + 0.001501126003859099, + 0.0006464219986810349, + 0.0011828780043288134, + 0.00048573499952908605, + 0.0004719119970104657, + 0.00045843099360354245, + 0.0005684079951606691, + 0.0005894790010643192, + 0.0005989979981677607, + 0.0005510199989657849, + 0.00047035900206537917, + 0.0004717470001196489, + 0.0004630980038200505, + 0.00056561800010968, + 0.0004613530036294833, + 0.0005238659941824153, + 0.0016906689997995272, + 0.0004875390004599467, + 0.00046130800183163956, + 0.0005517259996850044, + 0.000586733003729023, + 0.0006025170005159453, + 0.0009102449985221028, + 0.0004888959956588224, + 0.0008171919980668463, + 0.000691790999553632, + 0.0004884949958068319, + 0.0007162100009736605, + 0.0005035390058765188, + 0.0004728279964183457, + 0.0005049499959568493, + 0.000585602996579837, + 0.0005825339976581745, + 0.0005823620012961328, + 0.0005639359951601364, + 0.0005687739976565354, + 0.0007960350048961118, + 0.00047613100468879566, + 0.0004675939999287948, + 0.0005452089972095564, + 0.00045906299783382565, + 0.0009447929987800308, + 0.000617742000031285, + 0.0005947089957771823, + 0.0005831229937030002, + 0.0005512799980351701, + 0.0004735059992526658, + 0.0004638190002879128, + 0.0004638089958461933, + 0.0004646779998438433, + 0.0004633429998648353, + 0.0004548490032902919, + 0.0010793319961521775, + 0.002214510001067538, + 0.0007744020040263422, + 0.0005413050021161325, + 0.00048417499783681706, + 0.0004941320003126748, + 0.0005037800001446158, + 0.0011787610055762343, + 0.00048109100316651165, + 0.0006782999989809468, + 0.000661213998682797, + 0.00048272000276483595, + 0.0004714900060207583, + 0.0004695899988291785, + 0.0005719929977203719, + 0.0006001720030326396, + 0.0013380689997575246, + 0.0010002599956351332, + 0.00048013299965532497, + 0.0004590880053001456, + 0.00046679499791935086, + 0.0004453799992916174, + 0.0021024069937993772, + 0.0005035360009060241, + 0.0004487529949983582, + 0.0005413059989223257, + 0.00044666499888990074, + 0.00045144899922888726, + 0.0004464519952307455, + 0.00045266400411492214, + 0.00044659799459623173, + 0.0005625560006592423, + 0.0010849259997485206, + 0.0038005039968993515, + 0.0005039270035922527, + 0.00045960200077388436, + 0.0004503460004343651, + 0.0004537149943644181, + 0.00045053900248603895, + 0.0004460679992916994, + 0.00045309399865800515, + 0.0004450089982128702, + 0.0009807559981709346, + 0.0006724630002281629, + 0.0014480399986496195, + 0.0009822650026762858, + 0.0014364199960255064, + 0.0006057189966668375, + 0.00045415999920805916, + 0.0004603099951054901, + 0.00045567699999082834, + 0.0004458269977476448, + 0.0005633919936371967, + 0.000460836999991443, + 0.0007753099998808466, + 0.0007192589982878417, + 0.0010773980029625818, + 0.002680634999705944, + 0.0005013040063204244, + 0.00046943299821577966, + 0.0004541729940683581, + 0.00045101100113242865, + 0.0005683419949491508, + 0.0004509470018092543, + 0.0004516539993346669, + 0.0004426300001796335, + 0.0006121150072431192, + 0.0034514150029281154, + 0.000499007997859735, + 0.00045828700240235776, + 0.00044558499939739704, + 0.0004436719973455183, + 0.0005507950045284815, + 0.0004661900020437315, + 0.0004566060015349649, + 0.00044446500396588817, + 0.0015073499962454662, + 0.001214894997247029, + 0.0020864139951299876, + 0.0006260060035856441, + 0.0005062810014351271, + 0.00046049199590925127, + 0.0005603440004051663, + 0.00046489600208587945, + 0.00045131899969419464, + 0.0004853430000366643, + 0.0004549609948298894, + 0.0006885839975439012, + 0.0006869279968668707, + 0.0029719029989792034, + 0.0010514929963392206, + 0.0005796629993710667, + 0.00047710399667266756, + 0.0004496969995670952, + 0.0004542880051303655, + 0.0004549389996100217, + 0.00045249400136526674, + 0.00044596200314117596, + 0.0004450369960977696, + 0.0018379289977019653, + 0.003497565005091019, + 0.0007114030013326555, + 0.0004978100041626021, + 0.0004345029956311919, + 0.00044081800297135487, + 0.00042941699939547107, + 0.0004371070026536472, + 0.000428086997999344, + 0.0005283100035740063, + 0.0007735480030532926, + 0.000961409998126328, + 0.0007630769978277385, + 0.001327062003838364, + 0.0004868280011578463, + 0.00043330100015737116, + 0.00043709199962904677, + 0.0004344139961176552, + 0.000440361000073608, + 0.0004304190006223507, + 0.0007266039974638261, + 0.003655302003608085, + 0.00048070600314531475, + 0.0007453680009348318, + 0.0004570099990814924, + 0.0004455129965208471, + 0.00043683800322469324, + 0.00043329899926902726, + 0.0004432679997989908, + 0.0004347950016381219, + 0.0007867199965403415, + 0.004334860997914802, + 0.0009557949961163104, + 0.0005060540061094798, + 0.0004381789985927753, + 0.000452868000138551, + 0.0004377799996291287, + 0.00043865800398634747, + 0.0004294460013625212, + 0.00044102199899498373, + 0.0004356199933681637, + 0.0011492470002849586, + 0.0035833099973388016, + 0.0004952059971401468, + 0.0004541320013231598, + 0.0004378500016173348, + 0.00043493600242072716, + 0.00043388400081312284, + 0.0004377820005174726, + 0.00043681199895218015, + 0.00043579900375334546, + 0.0007152729958761483, + 0.000910431997908745, + 0.0008407339992118068, + 0.002822721995471511, + 0.0005302880017552525, + 0.00044935799814993516, + 0.0004403649945743382, + 0.00044441399950301275, + 0.0004369889938971028, + 0.0004363309999462217, + 0.0004755469999508932, + 0.00043522399937501177, + 0.0009965599965653382, + 0.001178376995085273, + 0.00204862800455885, + 0.00048152700037462637, + 0.00044958299986319616, + 0.0004401789992698468, + 0.00043519299651961774, + 0.000437276998127345, + 0.0004283540038159117, + 0.00043764700239989907, + 0.00043416400148998946, + 0.0005403780014603399, + 0.0008028149968595244, + 0.005926193996856455, + 0.0028862820036010817, + 0.0004869580006925389, + 0.00043842499871971086, + 0.0005008989974157885, + 0.00046001099690329283, + 0.00044283999886829406, + 0.0004332530006649904, + 0.004315772996051237, + 0.0004686879983637482, + 0.00044431100104702637, + 0.0004479709969018586, + 0.00043420499423518777, + 0.0004432319983607158, + 0.00048047299787867814, + 0.0030627190062659793, + 0.0012850419952883385, + 0.0004478289993130602, + 0.0004368480003904551, + 0.0004424780054250732, + 0.0004325659974711016, + 0.0004377840014058165, + 0.00043251700117252767, + 0.00047909100248944014, + 0.00048711500130593777, + 0.0058515740020084195, + 0.00048721799976192415, + 0.00044147399603389204, + 0.00042334299359936267, + 0.0004307670023990795, + 0.00045747600233880803, + 0.00043532900599529967, + 0.0004211209961795248, + 0.0036507990007521585, + 0.0011625490005826578, + 0.00042498399852775037, + 0.0012411529969540425, + 0.00043145099334651604, + 0.0005037500013713725, + 0.00044372100092004985, + 0.00043958199967164546, + 0.0004314300022087991, + 0.0004205010045552626, + 0.004545490999589674, + 0.00047085500409593806, + 0.000464078999357298, + 0.0004313690005801618, + 0.00046749199827900156, + 0.000491261002025567, + 0.00046866300544934347, + 0.0015938900032779202, + 0.0025566870026523247, + 0.0032178690016735345, + 0.0007935489993542433, + 0.0027084160028607585, + 0.003466874004516285, + 0.0007571460009785369, + 0.00048016300570452586, + 0.00043403600284364074, + 0.0004258780027157627, + 0.0004275289975339547, + 0.0004660669947043061, + 0.0038265360053628683, + 0.00048479500401299447, + 0.0004253799997968599, + 0.00041766500362427905, + 0.001110883997171186, + 0.000430148997111246, + 0.0004226010059937835, + 0.006010097000398673, + 0.0005255009964457713, + 0.00043780100531876087, + 0.0004425469960551709, + 0.00043176900362595916, + 0.005453423997096252, + 0.0009214210003847256, + 0.00044217299728188664, + 0.0004243020011927001, + 0.00042867699812632054, + 0.000422666002123151, + 0.00046320800174726173, + 0.0004303309979150072, + 0.005404210998676717, + 0.008997502998681739, + 0.010026304997154512, + 0.007963564996316563, + 0.0040001849993132055, + 0.0005100300040794536, + 0.0007521660008933395, + 0.00046559900511056185, + 0.00042895099613815546, + 0.0004237550019752234, + 0.003400457004318014, + 0.006024887996318284, + 0.0007576479983981699, + 0.0005513010037248023, + 0.0004583240006468259, + 0.00043519299651961774, + 0.00042079599370481446, + 0.0004258620028849691, + 0.0004270409990567714, + 0.00041636899550212547, + 0.00640832199860597, + 0.0005824539985042065, + 0.003130951998173259, + 0.0008678160011186264, + 0.000456109999504406, + 0.002596115999040194, + 0.006003661997965537, + 0.0079901740027708, + 0.006989854999119416, + 0.010997864999808371, + 0.007991721002326813, + 0.0021462299991981126, + 0.005844336999871302, + 0.0029964329951326363, + 0.0060183709938428365, + 0.006981901002291124, + 0.007997654996870551, + 0.006984231004025787, + 0.01100198199856095, + 0.005992620004690252, + 0.006992980997893028, + 0.006996984004217666, + 0.008995653006422799, + 0.011037987998861354, + 0.027100560000690166, + 0.0007916690010461025, + 0.056945527001516894, + 0.010118543999851681, + 0.01399031600158196, + 0.006884558002639096, + 0.02415811399987433, + 0.00812597000185633, + 0.003766355999687221, + 0.006962367006053682, + 0.007995025996933691, + 0.009980376999010332, + 0.007997271000931505, + 0.0059994360053678975, + 0.003984771996329073, + 0.006998657001531683, + 0.006997090997174382, + 0.007004205006523989, + 0.0006077010039007291, + 0.0005850629968335852, + 0.000522383998031728, + 0.00045880999823566526, + 0.00044504800462163985, + 0.0005122130023664795, + 0.0022267250024015084, + 0.0005096269960631616, + 0.00046078699961071834, + 0.0004438839969225228, + 0.0005771659998572432, + 0.0027939509964198805, + 0.0005109650010126643, + 0.001718090003123507, + 0.0006100070022512227, + 0.000523478003742639, + 0.0004600890024448745, + 0.0004559110020636581, + 0.0004597149963956326, + 0.0004597750012180768, + 0.00044694699317915365, + 0.0014180279977153987, + 0.0036684860024251975, + 0.0005837169956066646, + 0.0004531640006462112, + 0.00044540899398270994, + 0.00043664000259013847, + 0.00043584999366430566, + 0.0004323940011090599, + 0.003774019001866691, + 0.0007788530056132004, + 0.0004353600015747361, + 0.0004434080037754029, + 0.00043892499525099993, + 0.00046502600162057206, + 0.0004752910026581958, + 0.000444637997134123, + 0.0004383060004329309, + 0.0004411509944475256, + 0.004485476005356759, + 0.0005937640016782098, + 0.00048055799561552703, + 0.0004366080029285513, + 0.0005072499989182688, + 0.00045144899922888726, + 0.004493786000239197, + 0.0005089320038678125, + 0.0007230569972307421, + 0.0004652839998016134, + 0.00046394400123972446, + 0.0004584139969665557, + 0.0005042139964643866, + 0.0004463820005184971, + 0.0004541180023807101, + 0.00043412100058048964, + 0.003478943996015005, + 0.0013198369997553527, + 0.0004461009957594797, + 0.0004341309977462515, + 0.00044370699470164254, + 0.00169189200096298, + 0.0004800099995918572, + 0.00044462500227382407, + 0.0004381119942991063, + 0.00043796699901577085, + 0.000838476000353694, + 0.0017399919961462729, + 0.0009961600007954985, + 0.0005334039960871451, + 0.00045909899927210063, + 0.00048486800369573757, + 0.00045516699901781976, + 0.00043881600140593946, + 0.00044439200428314507, + 0.00044052200246369466, + 0.00043337600072845817, + 0.005968885001493618, + 0.0005593409950961359, + 0.0004406360021675937, + 0.002089241999783553, + 0.0004935300021315925, + 0.0004534270046860911, + 0.0013978290007798932, + 0.00044523299584398046, + 0.001334818996838294, + 0.00045156000123824924, + 0.00048410399904241785, + 0.0004647289970307611, + 0.000429411004006397, + 0.0004362210020190105, + 0.0004346330024418421, + 0.00044624199654208496, + 0.00043541600462049246, + 0.00626232499780599, + 0.0005418919972726144, + 0.0005081420022179373, + 0.0021242759976303205, + 0.00043460799497552216, + 0.0004206140001770109, + 0.0004360710008768365, + 0.00042363199463579804, + 0.0007962139934534207, + 0.002287324001372326, + 0.0006290599994827062, + 0.00048786800471134484, + 0.0004957499986630864, + 0.00044308900396572426, + 0.00043161900248378515, + 0.00042660000326577574, + 0.0017291139956796542, + 0.0030009939946467057, + 0.0005425749986898154, + 0.000938804994802922, + 0.00045687500096391886, + 0.00042826600110856816, + 0.0004344729968579486, + 0.00042194200068479404, + 0.00042991100053768605, + 0.00042362400563433766, + 0.0033467929970356636, + 0.0009570729962433688, + 0.00048107300244737417, + 0.00046473000111291185, + 0.0005189980001887307, + 0.0004457420000107959, + 0.00043208400165895, + 0.00042060700070578605, + 0.002643505999003537, + 0.0004937490011798218, + 0.00048038199747679755, + 0.00042242000199621543, + 0.0004321000014897436, + 0.00042336899787187576, + 0.000427088001742959, + 0.0004190619947621599, + 0.0004210599945508875, + 0.00524283800041303, + 0.000520128000061959, + 0.0004873529978794977, + 0.0004758240029332228, + 0.0004288709969841875, + 0.00042636899888748303, + 0.00042411400499986485, + 0.00042750599823193625, + 0.0004679359990404919, + 0.0057430520027992316, + 0.0005414610059233382, + 0.00046274399937829003, + 0.00042635099816834554, + 0.0004218989997752942, + 0.00046536000445485115, + 0.00043567099783103913, + 0.00042483599827392027, + 0.005306622006173711, + 0.000542205001693219, + 0.0005118730041431263, + 0.000888654998561833, + 0.0005693700004485436, + 0.0007091910010785796, + 0.0004346610003267415, + 0.005436273000668734, + 0.0005406159980338998, + 0.001201318002131302, + 0.0005955939996056259, + 0.00046576099703088403, + 0.00046007199853193015, + 0.00042628200026229024, + 0.0004293370002415031, + 0.0004229380065225996, + 0.0031140400024014525, + 0.0004920919964206405, + 0.0004619430037564598, + 0.0004914840028504841, + 0.00045808000140823424, + 0.00043264800478937104, + 0.0004217259993311018, + 0.004359174003184307, + 0.0005712810016120784, + 0.0006300789973465726, + 0.0004269299970474094, + 0.00041071300074690953, + 0.0004358060032245703, + 0.00041548500303179026, + 0.0008874870036379434, + 0.0015596580051351339, + 0.0021395739968284033, + 0.0005832440001540817, + 0.000490272999741137, + 0.00041946199780795723, + 0.00041496800258755684, + 0.00041010399581864476, + 0.0004131419991608709, + 0.0005867780055268668, + 0.003913713000656571, + 0.0010132489987881854, + 0.0004266359974280931, + 0.00043076599831692874, + 0.00041501499799778685, + 0.0004160820026299916, + 0.000427965002018027, + 0.0004145829952904023, + 0.00044410199916455895, + 0.00262374500016449, + 0.0022298900003079325, + 0.0005073060019640252, + 0.0004843980059376918, + 0.0004458699986571446, + 0.00043078000453533605, + 0.00041795500146690756, + 0.0004216899978928268, + 0.0004475620007724501, + 0.004387435001262929, + 0.0005301729979692027, + 0.0005063619973952882, + 0.00045655699796043336, + 0.000415923997934442, + 0.00041065100231207907, + 0.00041027599945664406, + 0.00041547100408934057, + 0.0004145580023759976, + 0.005404311006714124, + 0.0006763229976058938, + 0.0004925020039081573, + 0.0004521669980022125, + 0.0004135920025873929, + 0.00040613400051370263, + 0.00044318599975667894, + 0.0004304590038373135, + 0.00041235899698222056, + 0.005606316000921652, + 0.0005079300026409328, + 0.0004676630051108077, + 0.00041082499956246465, + 0.003464200002781581, + 0.000494656000228133, + 0.0004731190056190826, + 0.0004094679970876314, + 0.003171929005475249, + 0.0005408540018834174, + 0.00047457699838560075, + 0.00048788400454213843, + 0.00041372200212208554, + 0.000410573004046455, + 0.00040862699825083837, + 0.006175032001920044, + 0.0005734090009354986, + 0.00048807099665282294, + 0.0004806990036740899, + 0.00041606200102251023, + 0.00042168700019828975, + 0.00041144699935102835, + 0.005235445998550858, + 0.0005408619981608354, + 0.00048330199933843687, + 0.0004703309969045222, + 0.0004223119976813905, + 0.00045728400436928496, + 0.0004248140030540526, + 0.005216727004153654, + 0.0005475299985846505, + 0.0005068379978183657, + 0.0004790640014107339, + 0.0004239779955241829, + 0.0004170820029685274, + 0.002695392002351582, + 0.0008292409984278493, + 0.0004773470063810237, + 0.000452943000709638, + 0.0011082259952672757, + 0.0004331460004323162, + 0.00042663299973355606, + 0.00040986499516293406, + 0.0003959480018238537, + 0.00040045299829216674, + 0.0003990539989899844, + 0.000396557996282354, + 0.000402229001338128, + 0.00039701700006844476, + 0.006404580999515019, + 0.0005517440004041418, + 0.00048241399781545624, + 0.00043370600178604946, + 0.0004008840041933581, + 0.0004019089974462986, + 0.00039991800440475345, + 0.000405740000132937, + 0.0004017489991383627, + 0.00040006200288189575, + 0.002699898002902046, + 0.002087072003632784, + 0.0009810660048970021, + 0.0004236250024405308, + 0.00040537400491302833, + 0.000398156997107435, + 0.0013588869987870567, + 0.00041081100062001497, + 0.00040201900264946744, + 0.0004028739931527525, + 0.00039987999480217695, + 0.0019054579970543273, + 0.0004580230015562847, + 0.00040629299473948777, + 0.00040498199814464897, + 0.00040088099922286347, + 0.00040569900011178106, + 0.00040977300523081794, + 0.0004005619994131848, + 0.003620543997385539, + 0.00046585899690398946, + 0.000513416001922451, + 0.0004512589948717505, + 0.00040464699850417674, + 0.0003988539974670857, + 0.0003962069968110882, + 0.004284390000975691, + 0.0004399779936647974, + 0.0005932280037086457, + 0.00042006799776572734, + 0.0013205770010245033, + 0.00046071199903963134, + 0.00040353700023842975, + 0.001184467000712175, + 0.000832910998724401, + 0.0004138629956287332, + 0.00040177800110541284, + 0.0004061479994561523, + 0.00040392799564870074, + 0.00040385499596595764, + 0.00040718199306866154, + 0.0008553640000172891, + 0.0017520830006105825, + 0.002336140001716558, + 0.0004078210040461272, + 0.0004062430016347207, + 0.0004011839992017485, + 0.0004067710033268668, + 0.001025730001856573, + 0.0013024680010857992, + 0.000577087004785426, + 0.00040701199759496376, + 0.0004915600002277642, + 0.00045371799933491275, + 0.0004057810001540929, + 0.0020362669965834357, + 0.0015786320000188425, + 0.0004191330008325167, + 0.0004061500003444962, + 0.0004082039959030226, + 0.0004075760007253848, + 0.0004220319970045239, + 0.0004143469996051863, + 0.0012633250007638708, + 0.0024676710017956793, + 0.0012569829996209592, + 0.0004311379962018691, + 0.00042399499943712726, + 0.0008126050015562214, + 0.0013847589943907224, + 0.0004181459953542799, + 0.00047202599671436474, + 0.0004552529935608618, + 0.00040479499875800684, + 0.00040775499655865133, + 0.00040382699808105826, + 0.0003987580057582818, + 0.004201559000648558, + 0.0018218519981019199, + 0.00044732799869962037, + 0.0004042810032842681, + 0.0009958920054486953, + 0.001468877999286633, + 0.0004258189946995117, + 0.00040515799628337845, + 0.00040177199844038114, + 0.00040160000207833946, + 0.0004029459960293025, + 0.0004053049997310154, + 0.0007949169958010316, + 0.003425710994633846, + 0.0010665849986253306, + 0.0004159090021857992, + 0.00040248900040751323, + 0.000422358003561385, + 0.0004043549997732043, + 0.006421999001759104, + 0.006914705001690891, + 0.006649780996667687, + 0.0023468439976568334, + 0.000448560000222642, + 0.00041852999856928363, + 0.0005151029981789179, + 0.0004311579978093505, + 0.0007740380024188198, + 0.0004100340011063963, + 0.0005440230015665293, + 0.0004107080021640286, + 0.00040907200309447944, + 0.0003989020042354241, + 0.006964713000343181, + 0.00048273600259562954, + 0.0004132409958401695, + 0.0021623779975925572, + 0.0013699120027013123, + 0.00043791999632958323, + 0.0004179660027148202, + 0.0004128560030949302, + 0.0004082360028405674, + 0.0041747129944269545, + 0.0004515789987635799, + 0.0004074680036865175, + 0.0016735300014261156, + 0.0004216719971736893, + 0.0019393850016058423, + 0.0014136700046947226, + 0.00044575500214705244, + 0.0004142299949307926, + 0.0004100459991605021, + 0.0033867310048663057, + 0.00044149200402898714, + 0.0004065750035806559, + 0.0004128099972149357, + 0.0004103800019947812, + 0.00041436699393671006, + 0.0036954120005248114, + 0.0004208350001135841, + 0.0004141340032219887, + 0.0032067790016299114, + 0.00044189300388097763, + 0.0014247119979700074, + 0.00043324400030542165, + 0.0019478690010146238, + 0.0012837369940825738, + 0.0004188719976809807, + 0.00040791100036585703, + 0.00040077499579638243, + 0.0032333279959857464, + 0.00045303499791771173, + 0.00040624800021760166, + 0.0004183830023976043, + 0.0004029769988846965, + 0.0004016640014015138, + 0.0003988879980170168, + 0.00040573500155005604, + 0.004882916000497062, + 0.000418278003053274, + 0.00040160999924410135, + 0.00040245099808089435, + 0.0004325159970903769, + 0.00039797899808036163, + 0.00041354399581905454, + 0.00039898999966681004, + 0.0004027210015919991, + 0.0005032779954490252, + 0.00041842499922495335, + 0.0004417360032675788, + 0.00041015599708771333, + 0.0004152920009801164, + 0.00041011600114870816, + 0.00040556100429967046, + 0.0004003549984190613, + 0.00039933100197231397, + 0.008607367999502458, + 0.00042405500425957143, + 0.0004201600022497587, + 0.0004036940008518286, + 0.0004089700014446862, + 0.004814815001736861, + 0.0004237810062477365, + 0.0004004160000476986, + 0.0034392619927530177, + 0.0004166810031165369, + 0.0004059100028825924, + 0.00040480699681211263, + 0.00040032399556366727, + 0.0004081710067111999, + 0.0003988859971286729, + 0.005959334994258825, + 0.00044900100328959525, + 0.0004063399974256754, + 0.0003958059969590977, + 0.00040747800085227937, + 0.00039903699507704005, + 0.00040150499989977106, + 0.000401080003939569, + 0.00039928800106281415, + 0.005741594002756756, + 0.00041592700290493667, + 0.00040588900446891785, + 0.00040843799797585234, + 0.0004048590053571388, + 0.00039972599915927276, + 0.000400736003939528, + 0.00040322700078831986, + 0.000396363997424487, + 0.007569845001853537, + 0.00039887100138003007, + 0.00039996499981498346, + 0.0003877139970427379, + 0.0003993989957962185, + 0.000392889000067953, + 0.00039296699833357707, + 0.00039407899748766795, + 0.00038850700366310775, + 0.006873178004752845, + 0.0015027449990157038, + 0.00041503499960526824, + 0.0003943030023947358, + 0.0003977469968958758, + 0.0004352309988462366, + 0.0003951879989472218, + 0.0003935389977414161, + 0.0003884110046783462, + 0.0072743900018394925, + 0.00040059700404526666, + 0.0003927149955416098, + 0.00039322800148511305, + 0.0003923460026271641, + 0.0003891239975928329, + 0.0003933789994334802, + 0.00038519600639119744, + 0.0070788619996164925, + 0.000399910997657571, + 0.00039018299867166206, + 0.0003973810016759671, + 0.0003899259972968139, + 0.0003936260036425665, + 0.0003937840010621585, + 0.0003928280057152733, + 0.004688815002737101, + 0.000432438995630946, + 0.00039533499511890113, + 0.0003968009987147525, + 0.0003925339988199994, + 0.000387124004191719, + 0.0003932270046789199, + 0.0009395250017405488, + 0.0008064619978540577, + 0.0007260179991135374, + 0.001220004000060726, + 0.0014512649941025302, + 0.0004082710001966916, + 0.0006296589999692515, + 0.0011328450054861605, + 0.0003981509944424033, + 0.0004006999952252954, + 0.0003982899943366647, + 0.0003951780017814599, + 0.0003976480002165772, + 0.00039423300040652975, + 0.0004414120048750192, + 0.002478063994203694, + 0.001592683001945261, + 0.00117573299939977, + 0.00039105199539335445, + 0.00040134300070349127, + 0.000448899001639802, + 0.00039633599953958765, + 0.002138174997526221, + 0.0008866139978636056, + 0.00040978999459184706, + 0.00039418999949702993, + 0.00039671699778409675, + 0.0003908829967258498, + 0.0004035780002595857, + 0.0003910390005330555, + 0.0003942850016755983, + 0.0007818430021870881, + 0.0008274380015791394, + 0.00040075700235320255, + 0.0004002760033472441, + 0.0003921370007446967, + 0.00039470100455218926, + 0.0017000689986161888, + 0.0023826879987609573, + 0.00043314499635016546, + 0.00039890499465400353, + 0.00040270200406666845, + 0.00039677200402366, + 0.000411225002608262, + 0.0003955040010623634, + 0.0004689659981522709, + 0.004077457000676077, + 0.00042598599975463003, + 0.00039751300209900364, + 0.0004479139970499091, + 0.0003921489987988025, + 0.001141404005466029, + 0.00046265100536402315, + 0.00038808799581602216, + 0.00040183700184570625, + 0.00039213600393850356, + 0.0003980449982918799, + 0.00039039100374793634, + 0.0007997030043043196, + 0.002204286996857263, + 0.0019187470024917275, + 0.0004387910012155771, + 0.00040185699617723003, + 0.0003912449974450283, + 0.0015053859970066696, + 0.0009663840028224513, + 0.00046542500058421865, + 0.0003941950053558685, + 0.00039451899647247046, + 0.00038852899888297543, + 0.000437856993812602, + 0.0003962099945056252, + 0.0003901980016962625, + 0.0016954940001596697, + 0.0024404020005022176, + 0.0003933650004910305, + 0.0005946660021436401, + 0.000401427001634147, + 0.0003875190013786778, + 0.0006772170017939061, + 0.0005089209953439422, + 0.0004990640009054914, + 0.0005071050036349334, + 0.0004948259957018308, + 0.0005018290030420758, + 0.0005033619963796809, + 0.0004955859985784627, + 0.00040689999877940863, + 0.00039326000114670023, + 0.00039853400085121393, + 0.0003879270007018931, + 0.003514994001307059, + 0.0004960950027452782, + 0.0004075030010426417, + 0.0003927220022887923, + 0.0003958630040870048, + 0.00041254299867432564, + 0.00038918400241527706, + 0.0012264339966350235, + 0.002537617001507897, + 0.00046048700460232794, + 0.00039700700290268287, + 0.00039020500116748735, + 0.00044634100049734116, + 0.0003958099987357855, + 0.0005968120021861978, + 0.0037100609988556243, + 0.0018390329933026806, + 0.001293483997869771, + 0.0004239750051056035, + 0.001440997002646327, + 0.0005102259965497069, + 0.00041281099402112886, + 0.003323324999655597, + 0.00245696899946779, + 0.0004486309990170412, + 0.0004095999975106679, + 0.0015531210010522045, + 0.0010156749995076098, + 0.00042962300358340144, + 0.0004065989996888675, + 0.00041650400089565665, + 0.00039934900269145146, + 0.00041899200004991144, + 0.00039970799844013527, + 0.0005395320040406659, + 0.0007534370015491731, + 0.000964988001214806, + 0.0019006680013262667, + 0.001501561993791256, + 0.0009537389996694401, + 0.0004176560032647103, + 0.0008847479984979145, + 0.001103272006730549, + 0.0006105739958002232, + 0.0004102540042367764, + 0.00041328400402562693, + 0.00040128800173988566, + 0.0004001990018878132, + 0.00044689200149150565, + 0.0004010679986095056, + 0.001096685002266895, + 0.001084631003323011, + 0.00043642700620694086, + 0.00040494099812349305, + 0.0025166940031340346, + 0.001182861000415869, + 0.0004177029986749403, + 0.0015504830007557757, + 0.0004113039976800792, + 0.00041487300040898845, + 0.00040111199632519856, + 0.0004098110002814792, + 0.0003996360028395429, + 0.0003981909976573661, + 0.0007758100036880933, + 0.0006246789998840541, + 0.0032420880015706643, + 0.00045408100413624197, + 0.000407768995501101, + 0.00040123499638866633, + 0.0014534160000039265, + 0.0004046809990541078, + 0.0012740959937218577, + 0.00047276599798351526, + 0.000408037994930055, + 0.0005612460008705966, + 0.0004397750017233193, + 0.001302445998589974, + 0.0014783360020373948, + 0.00044882500515086576, + 0.0008114529991871677, + 0.001058212001225911, + 0.00041233999945688993, + 0.0003980020046583377, + 0.0007838689998607151, + 0.0004156970026087947, + 0.0020318070019129664, + 0.000463356998807285, + 0.0004134539994993247, + 0.0004356970021035522, + 0.00041345100180478767, + 0.0013061630015727133, + 0.0014812279987381771, + 0.00044929300202056766, + 0.00040635099867358804, + 0.0003995709994342178, + 0.00044709700159728527, + 0.0004073370000696741, + 0.0034253230041940697, + 0.0005323040022631176, + 0.00042576999840093777, + 0.0004050630013807677, + 0.0003982440030085854, + 0.0004131280002184212, + 0.00041465600224910304, + 0.0013240839980426244, + 0.002301294996868819, + 0.0004311699958634563, + 0.00041468699782853946, + 0.0004030679992865771, + 0.00040826600161381066, + 0.0003985559960710816, + 0.00040312399505637586, + 0.0027332750032655895, + 0.0013652269990416244, + 0.00042101699364138767, + 0.00039845900028012693, + 0.00041092999890679494, + 0.00041081900417339057, + 0.0005988249977235682, + 0.00044029700075043365, + 0.0004339809966040775, + 0.0004151009998167865, + 0.00040437599818687886, + 0.00040227700083050877, + 0.000401545999920927, + 0.0004042350046802312, + 0.0004033479999634437, + 0.000490427999466192, + 0.0018201380007667467, + 0.0024773569966782816, + 0.00043392299994593486, + 0.001866084996436257, + 0.0009626219980418682, + 0.0005480140025611036, + 0.0004093140014447272, + 0.00040239399822894484, + 0.0004082430023117922, + 0.0003988110038335435, + 0.00040849300421541557, + 0.0004024019945063628, + 0.0007454549995600246, + 0.0025954700031434186, + 0.0005506200031959452, + 0.0004182840057183057, + 0.00040600300417281687, + 0.000398231997678522, + 0.00044711700320476666, + 0.00040598299528937787, + 0.0008788139966782182, + 0.0035742570034926757, + 0.00044434700248530135, + 0.00040754899964667857, + 0.0005255590003798716, + 0.0008869250013958663, + 0.0007643600038136356, + 0.0008087860041996464, + 0.0005641270035994239, + 0.0004118939978070557, + 0.00045734299783362076, + 0.0004090410002390854, + 0.0004060399951413274, + 0.0004127389984205365, + 0.0017337780009256676, + 0.0029056749990559183, + 0.0004123309990973212, + 0.0004146049977862276, + 0.0004052540025440976, + 0.0015890659997239709, + 0.001527218999399338, + 0.00044831600098405033, + 0.0004078759957337752, + 0.00040766600432107225, + 0.0004009199983556755, + 0.0004047859983984381, + 0.0036825179995503277, + 0.0013523279994842596, + 0.0004636409939848818, + 0.0004091829978278838, + 0.0009723640032461844, + 0.0007563670005765744, + 0.0015534510021097958, + 0.0004410190013004467, + 0.0004065380053361878, + 0.00041386600059922785, + 0.00039759799983585253, + 0.0013180489986552857, + 0.002608397997391876, + 0.003967023003497161, + 0.005539988000236917, + 0.0004870260017924011, + 0.0004424360013217665, + 0.00040873199759516865, + 0.0007263900042744353, + 0.0004197960006422363, + 0.00040236399945570156, + 0.004161176999332383, + 0.00046819300041534007, + 0.0004680279962485656, + 0.00042151899833697826, + 0.00041333900298923254, + 0.0004083539970451966, + 0.000402115001634229, + 0.00040972699935082346, + 0.004222749004838988, + 0.000482153998746071, + 0.00042382000538054854, + 0.0004053150041727349, + 0.0004106359992874786, + 0.00040811399958329275, + 0.0004320719963288866, + 0.00042915900121442974, + 0.000400017001084052, + 0.004126453000935726, + 0.005998824002745096, + 0.00699350699869683, + 0.005991404999804217, + 0.007996314998308662, + 0.008009664001292549, + 0.005980863999866415, + 0.006995581999944989, + 0.006992411996179726, + 0.007000527999480255, + 0.00599915100610815, + 0.007991259000846185, + 0.002992608002386987, + 0.005004447993997019, + 0.0059900679989368655, + 0.005993288999889046, + 0.00699885499489028, + 0.007996191001439001, + 0.007996870001079515, + 0.006993662995228078, + 0.008999361001770012, + 0.005991865000396501, + 0.005995362997055054, + 0.006995121999352705, + 0.005995733001327608, + 0.006995903997449204, + 0.007002058999205474, + 0.006989301000430714, + 0.006995171999733429, + 0.007995537998795044, + 0.007996003005246166, + 0.0059945309985778295, + 0.005993537000904325, + 0.008001894006156363, + 0.009993270999984816, + 0.010996154000167735, + 0.00799727900448488, + 0.005994579005346168, + 0.0069952060002833605, + 0.0069973300051060505, + 0.0069891829989501275, + 0.008996299002319574, + 0.006994496005063411, + 0.00800233100017067, + 0.007995201005542185, + 0.005995239000185393, + 0.00498672899993835, + 0.006998230004683137, + 0.007009052002103999, + 0.006963070001802407, + 0.006334932004392613, + 0.005677234999893699, + 0.005972897000901867, + 0.00891102400055388, + 0.007009437998931389, + 0.007005706000200007, + 0.007998622000741307, + 0.007969917001901194, + 0.007009286004176829, + 0.005983376999211032, + 0.005992712998704519, + 0.010020137997344136, + 0.00703495999914594, + 0.006932001000677701, + 0.006027567993442062, + 0.006964191001316067, + 0.005975414002023172, + 0.0070129419982549734, + 0.007971107996127103, + 0.005128007003804669, + 0.003546228996128775, + 0.004781026000273414, + 0.007293271999515127, + 0.006235248001758009, + 0.007042101999104489, + 0.006944815002498217, + 0.007405812000797596, + 0.003598512994358316, + 0.00698281500081066, + 0.007001648998993915, + 0.003995184997620527, + 0.0030336920026456937, + 0.00592959899950074, + 0.006997936005063821, + 0.008007267999346368, + 0.005954817999736406, + 0.0052043550022062846, + 0.007796976002282463, + 0.007964544995047618, + 0.0059896489983657375, + 0.006032174998836126, + 0.006942368003365118, + 0.007014861002971884, + 0.00594765799905872, + 0.012994236996746622, + 0.00803839300351683, + 0.006952898998861201, + 0.005990576995827723, + 0.006154871996841393, + 0.006835795997176319, + 0.007995640000444837, + 0.006993168994085863, + 0.007995371001015883, + 0.005994402999931481, + 0.00799512099911226, + 0.006993518996750936, + 0.007994118001079187, + 0.005998395005008206, + 0.004977495998900849, + 0.007004897001024801, + 0.006997577998845372, + 0.005995439998514485, + 0.005988318000163417, + 0.006005722003465053, + 0.007005447994743008, + 0.006980231999477837, + 0.005991625002934597, + 0.006992617003561463, + 0.0100035739960731, + 0.006995080002525356, + 0.006992896000156179, + 0.00799565600027563, + 0.004995612005586736, + 0.009986148004827555, + 0.0059921200008830056, + 0.0069930250028846785, + 0.007993957005965058, + 0.006998850003583357, + 0.004992354995920323, + 0.006996912001341116, + 0.010996525001246482, + 0.007997223998245317, + 0.010016015003202483, + 0.007972660998348147, + 0.0060008830041624606, + 0.00898876699648099, + 0.007993409999471623, + 0.007994961000804324, + 0.005280110002786387, + 0.00770890100102406, + 0.0069985930022085086, + 0.007996057000127621, + 0.007987985001818743, + 0.005000574004952796, + 0.009989225000026636, + 0.0068030280017410405, + 0.008185936996596865, + 0.0069951650002622046, + 0.002238475004560314, + 0.007753814999887254, + 0.005992181002511643, + 0.00799335099873133, + 0.008395384000323247, + 0.007047566999972332, + 0.007544057996710762, + 0.007995093998033553, + 0.007992610000655986, + 0.008005853000213392, + 0.010998516001563985, + 0.007985676005773712, + 0.00798899399524089, + 0.007990847996552475, + 0.006995751995418686, + 0.005995968997012824, + 0.0069969910036888905, + 0.0080166210027528, + 0.007980298003531061, + 0.006980794001719914, + 0.006553148996317759, + 0.0074314919984317385, + 0.007999991001270246, + 0.006997416996455286, + 0.004521432005276438, + 0.004466347003472038, + 0.008099369006231427, + 0.007889190994319506, + 0.005992755002807826, + 0.007002455000474583, + 0.007995156003744341, + 0.007995456995558925, + 0.005995924999297131, + 0.00799791699682828, + 0.006993663999310229, + 0.0031716689991299063, + 0.0048216080031124875, + 0.006995512994762976, + 0.005992672995489556, + 0.007994419996975921, + 0.007997838001756463, + 0.004006094997748733, + 0.005987527001707349, + 0.005991908998112194, + 0.006993675997364335, + 0.01000739099981729, + 0.007016681993263774, + 0.008986421998997685, + 0.005974126004730351, + 0.007994239997060504, + 0.006995179996010847, + 0.008994106996397022, + 0.006995137002377305, + 0.009235154997440986, + 0.0067557450020103715, + 0.007995084000867791, + 0.002567445997556206, + 0.0040000409935601056, + 0.0054189209986361675, + 0.005995207000523806, + 0.004300392000004649, + 0.005691539998224471, + 0.003953706000174861, + 0.007004932995187119, + 0.003983882997999899, + 0.007000724996032659, + 0.00799837499653222, + 0.00500071400165325, + 0.0079893189977156, + 0.008160663994203787, + 0.00782920299388934, + 0.00599240799783729, + 0.008006609001313336, + 0.007997796004929114, + 0.005985607000184245, + 0.011018978999345563, + 0.007019681004749145, + 0.007947623002110049, + 0.006000972003675997, + 0.007005642997683026, + 0.00800482100021327, + 0.005975457999738865, + 0.005992787999275606, + 0.005133518003276549, + 0.003855989998555742, + 0.007993580999027472, + 0.007992365994141437, + 0.005990675002976786, + 0.007997777000127826, + 0.007998200002475642, + 0.005998123000608757, + 0.006990367000980768, + 0.00800003999756882, + 0.006998910001129843, + 0.006835410000348929, + 0.010153837996767834, + 0.005007621002732776, + 0.006975127005716786, + 0.011001517996191978, + 0.006995005001954269, + 0.007002876998740248, + 0.008004242001334205, + 0.008013147999008652, + 0.005956027001957409, + 0.005018577998271212, + 0.003980574998422526, + 0.005988388002151623, + 0.0069914809937472455, + 0.006062506996386219, + 0.006926581001607701, + 0.00711687799775973, + 0.009873374001472257, + 0.008698573998117354, + 0.007294766997802071, + 0.005986310003208928, + 0.0049960579999606125, + 0.0049968989987974055, + 0.00899145400035195, + 0.010008983001171146, + 0.01106339800026035, + 0.011923240999749396, + 0.004782791998877656, + 0.0065520200005266815, + 0.007646218000445515, + 0.0069967300005373545, + 0.00508691200229805, + 0.007031823006400373, + 0.008308721997309476, + 0.007744173999526538, + 0.006801688999985345, + 0.0099898039989057, + 0.006066383000870701, + 0.005926056997850537, + 0.0029972940028528683, + 0.00699052100389963, + 0.008007504999113735, + 0.007993020997673739, + 0.006983186001889408, + 0.006001530004141387, + 0.007989398996869568, + 0.004922191001242027, + 0.0020545349980238825, + 0.005993566999677569, + 0.009367801998450886, + 0.006625866000831593, + 0.005993295002554078, + 0.003303602003143169, + 0.0026921859971480444, + 0.010996680000971537, + 0.007001462996413466, + 0.005995581996103283, + 0.0069950979959685355, + 0.007994874002179131, + 0.006467020000854973, + 0.007515839002735447, + 0.006997374999627937, + 0.007994728002813645, + 0.007995405998372007, + 0.007992552003997844, + 0.006990904999838676, + 0.006999926998105366, + 0.006998070995905437, + 0.005148944997927174, + 0.005833444003656041, + 0.0059984999970765784, + 0.007994546002009884, + 0.006567005999386311, + 0.0034154299937654287, + 0.006998349002969917, + 0.004993381000531372, + 0.006997698001214303, + 0.006993065006099641, + 0.007998651002708357, + 0.006993791997956578, + 0.005001965997507796, + 0.0018735720004769973, + 0.006113468996773008, + 0.007996855994861107, + 0.007994316001713742, + 0.005992458995024208, + 0.005996975996822584, + 0.007995300002221484, + 0.005997396001475863, + 0.007996104002813809, + 0.005998831999022514, + 0.004993942995497491, + 0.006987515000218991, + 0.006995251998887397, + 0.00500505499803694, + 0.003982493995863479, + 0.006001467001624405, + 0.007995781001227442, + 0.007988126999407541, + 0.0060039099989808165, + 0.0060835580006823875, + 0.00686624700028915, + 0.006986155996855814, + 0.008077147998847067, + 0.0059137270000064746, + 0.003988043994468171, + 0.00699578199419193, + 0.014002515003085136, + 0.00799335099873133, + 0.006995522002398502, + 0.005993660997773986, + 0.007003407001320738, + 0.0059445670049171895, + 0.006996551004704088, + 0.006295320999925025, + 0.00571922300150618, + 0.007975859996804502, + 0.005988746001094114, + 0.00490743899717927, + 0.006085494998842478, + 0.007990484999027103, + 0.007999208995897789, + 0.0069934490020386875, + 0.006991941001615487, + 0.0070105359991430305, + 0.007990887999767438, + 0.007987568002135959, + 0.00699343100131955, + 0.006991714995820075, + 0.007000192999839783, + 0.005996086998493411, + 0.005989785997371655, + 0.0070058080018498, + 0.007990553000126965, + 0.006998280005063862, + 0.007995167004992254, + 0.00699520499620121, + 0.006999217999691609, + 0.006994991999818012, + 0.00699110199639108, + 0.010999500002071727, + 0.006018410000251606, + 0.006972025003051385, + 0.006994264003878925, + 0.005996419000439346, + 0.006997165997745469, + 0.006996440002694726, + 0.007996280000952538, + 0.007992762002686504, + 0.00799611399997957, + 0.00799301199731417, + 0.006997194002906326, + 0.0069933380000293255, + 0.006108576999395154, + 0.002886042006139178, + 0.007996446998731699, + 0.007996792999620084, + 0.005994074999762233, + 0.007989808997081127, + 0.005998738000926096, + 0.005459214000438806, + 0.005522196996025741, + 0.0039987359996302985, + 0.007995113999641035, + 0.006995292998908553, + 0.006992992995947134, + 0.007999444998858962, + 0.002987745996506419, + 0.006000998997478746, + 0.007997007000085432, + 0.006995670999458525, + 0.007997009000973776, + 0.005993489001411945, + 0.005995690000418108, + 0.00801628200133564, + 0.007979252004588488, + 0.007997733999218326, + 0.005996334002702497, + 0.007996864995220676, + 0.007992685998033267, + 0.006003613001666963, + 0.007986884003912564, + 0.004404502993565984, + 0.005584159000136424, + 0.013995777997479308, + 0.0069984119982109405, + 0.009830859999055974, + 0.004148111998802051, + 0.007989079000253696, + 0.009192869001708459, + 0.007560451005701907, + 0.004253338003763929, + 0.0056050789935397916, + 0.005369929996959399, + 0.008000717003596947, + 0.007987410004716367, + 0.010993777999829035, + 0.006991472000663634, + 0.006005828996421769, + 0.005982693997793831, + 0.007036924005660694, + 0.005944297998212278, + 0.008001902999239974, + 0.009992225997848436, + 0.007870579000154976, + 0.011303425999358296, + 0.007810023002093658, + 0.00799194400315173, + 0.005146547002368607, + 0.0043506260044523515, + 0.003487244001007639, + 0.00499673900048947, + 0.006021335000696126, + 0.007967106001160573, + 0.007998405999387614, + 0.006993290000536945, + 0.006986298998526763, + 0.00699121700017713, + 0.007002552003541496, + 0.005992434998915996, + 0.006995369003561791, + 0.007988238998223096, + 0.005998132000968326, + 0.006620700005441904, + 0.005367798003135249, + 0.009993328996642958, + 0.004685043997596949, + 0.003335943001729902, + 0.006960108003113419, + 0.0048086509996210225, + 0.009184419999655802, + 0.007004658000369091, + 0.0069664199982071295, + 0.010997762001352385, + 0.008998083001642954, + 0.007999800996913109, + 0.006991725000261795, + 0.006995800998993218, + 0.007996057996933814, + 0.0057716520022950135, + 0.005212623000261374, + 0.007995273001142778, + 0.005996509004035033, + 0.008042089000809938, + 0.006948620000912342, + 0.007993504004843999, + 0.004996228002710268, + 0.008003001996257808, + 0.0079923779994715, + 0.006993968003371265, + 0.0069992010030546226, + 0.0069994310033507645, + 0.001998690000618808, + 0.003991363002569415, + 0.009890187997370958, + 0.005098171001009177, + 0.007999599998584017, + 0.005993000006128568, + 0.005996646999847144, + 0.005999382003210485, + 0.007987142998899799, + 0.006000045999826398, + 0.0069959749962436035, + 0.0069958110034349374, + 0.0069929219971527345, + 0.006994957999268081, + 0.0069972600031178445, + 0.007993478997377679, + 0.007002275997365359, + 0.006991056005063001, + 0.006996333999268245, + 0.004998586999136023, + 0.002817979002429638, + 0.0005455499995150603, + 0.00045913799840491265, + 0.00044558100489666685, + 0.0003830299974652007, + 0.001604386001417879, + 0.003369992999068927, + 0.0009661130025051534, + 0.0004125279956497252, + 0.00038614899676758796, + 0.00037865799822611734, + 0.00041377700108569115, + 0.00038582699926337227, + 0.00038275900442386046, + 0.0005593149980995804, + 0.0003845550018013455, + 0.0008731260022614151, + 0.0007199190004030243, + 0.00041350300307385623, + 0.0003902480020769872, + 0.00038674600364174694, + 0.0003906150013790466, + 0.0004331460004323162, + 0.00039565299812238663, + 0.00039269699482247233, + 0.0003844599996227771, + 0.00073899699782487, + 0.0009413520019734278, + 0.0022623469994869083, + 0.0005260930047370493, + 0.0004698149932664819, + 0.0019888789975084364, + 0.0004251199934515171, + 0.0003890490042977035, + 0.00038752800173824653, + 0.000388925000152085, + 0.0004407359956530854, + 0.00039066500175977126, + 0.00040431899833492935, + 0.0003845779938274063, + 0.0014760330013814382, + 0.0009552190022077411, + 0.0004240670023136772, + 0.00039712800207780674, + 0.0003829990018857643, + 0.0003867009945679456, + 0.00039287300023715943, + 0.00038422199577325955, + 0.0007529959984822199, + 0.0009270279988413677, + 0.001469936003559269, + 0.0005772709992015734, + 0.00048444200365338475, + 0.00039990799996303394, + 0.00038482200034195557, + 0.0006199999988893978, + 0.0010122140010935254, + 0.0018921779992524534, + 0.0004178399976808578, + 0.0003960470057791099, + 0.00037948499812046066, + 0.00039491499774158, + 0.00042322700028307736, + 0.0003850860011880286, + 0.0012211950015625916, + 0.0006862469963380136, + 0.0004086940025445074, + 0.0003828140033874661, + 0.0004342760003055446, + 0.00039330100116785616, + 0.0003780570041271858, + 0.000751154002500698, + 0.0008366100009880029, + 0.0018893340020440519, + 0.0004206580051686615, + 0.00038285000482574105, + 0.00038701999437762424, + 0.00047062600060598925, + 0.00040197499765781686, + 0.00038239199784584343, + 0.00038978399970801547, + 0.00037922900082776323, + 0.0003802999999606982, + 0.0010083230008604005, + 0.0004336120036896318, + 0.0003830640052910894, + 0.000876758000231348, + 0.0008774809975875542, + 0.0007384259952232242, + 0.00040567899850429967, + 0.0003837999975075945, + 0.00038621100247837603, + 0.00038349000533344224, + 0.0003869279971695505, + 0.0003810119960689917, + 0.00037970599805703387, + 0.0009342619960079901, + 0.0014427529968088493, + 0.0014594460008083843, + 0.0004154540001763962, + 0.0003845270039164461, + 0.0004200890034553595, + 0.0003953179984819144, + 0.0003778500031330623, + 0.0006782249984098598, + 0.0007082960000843741, + 0.0004084210013388656, + 0.00038443099765572697, + 0.0005021350007154979, + 0.0003964019997511059, + 0.00039655499858781695, + 0.0003761810003197752, + 0.0003776950034080073, + 0.0005946400051470846, + 0.0004842490016017109, + 0.0006799019975005649, + 0.0007790710005792789, + 0.0009596870004315861, + 0.0004051669966429472, + 0.00038724100159015507, + 0.00038191599742276594, + 0.0004263840019120835, + 0.0003949550009565428, + 0.00040386899490840733, + 0.00048244000208796933, + 0.0004144990016357042, + 0.0005825429980177432, + 0.0008750999986659735, + 0.0006769199972040951, + 0.0004095840049558319, + 0.00038019999919924885, + 0.0015767699951538816, + 0.0006154099974082783, + 0.0004084409956703894, + 0.000384995000786148, + 0.00038390199915738776, + 0.0003771580013562925, + 0.000380425997718703, + 0.0003859330026898533, + 0.0006577000021934509, + 0.0006165989980218001, + 0.000406358994951006, + 0.0003912469983333722, + 0.00038286899507511407, + 0.0003880970034515485, + 0.00038345799839589745, + 0.00038189200131455436, + 0.0010577680004644208, + 0.0012506490020314232, + 0.0028419490045052953, + 0.0005633409964502789, + 0.00038819899782538414, + 0.0009013179951580241, + 0.001025195000693202, + 0.0003979129978688434, + 0.0003819539997493848, + 0.0005975870008114725, + 0.0008177249983418733, + 0.0003996089944848791, + 0.0003830190034932457, + 0.00038447100087068975, + 0.00037823699676664546, + 0.0005240820028120652, + 0.0013162510003894567, + 0.0007158070002333261, + 0.00040965600055642426, + 0.0003902160024154, + 0.00041898000199580565, + 0.0003771440024138428, + 0.0003858959971694276, + 0.00038168900209711865, + 0.0005043409983045422, + 0.0006269150035222992, + 0.00041435199818806723, + 0.0003864699974656105, + 0.00037988399708410725, + 0.0009562289997120388, + 0.0004354600023361854, + 0.0003837870026472956, + 0.0004891040007350966, + 0.0003958529996452853, + 0.0003874219983117655, + 0.0004064879976795055, + 0.0003866120023303665, + 0.0007288029955816455, + 0.002179894996515941, + 0.00041129500459646806, + 0.0003845289975288324, + 0.0003772690033656545, + 0.0014278219969128259, + 0.0007850390029489063, + 0.00041776199941523373, + 0.0004575969942379743, + 0.0003844480015686713, + 0.000383967999368906, + 0.0003848990018013865, + 0.00037991200224496424, + 0.00037598999915644526, + 0.0006038600040483288, + 0.0006712300018989481, + 0.0015986559956218116, + 0.00042319800559198484, + 0.00038770700484747067, + 0.00038436199974967167, + 0.00037862199678784236, + 0.0003833310038316995, + 0.0007941980002215132, + 0.00076980100129731, + 0.0006277530046645552, + 0.0003989610049757175, + 0.0003880420044879429, + 0.000384340004529804, + 0.0003780350016313605, + 0.0010447120002936572, + 0.0007114629988791421, + 0.0004114300027140416, + 0.0003945869975723326, + 0.0003834959934465587, + 0.00038302000029943883, + 0.00075131199992029, + 0.0008204279947676696, + 0.00046258200018201023, + 0.00039680599729763344, + 0.0003798600009758957, + 0.00252178300434025, + 0.0004470169951673597, + 0.0003958340021199547, + 0.0003856389957945794, + 0.00038023099477868527, + 0.00038426600076491013, + 0.0003788699978031218, + 0.00041906499973265454, + 0.0006865079994895495, + 0.00045643900375580415, + 0.0004655779994209297, + 0.000390635002986528, + 0.0003896060006809421, + 0.00038487400161102414, + 0.0008581210058764555, + 0.002230179998150561, + 0.0007464999944204465, + 0.0004263459995854646, + 0.00039233899587998167, + 0.00039117899723351, + 0.00039205900247907266, + 0.00039358499634545296, + 0.0004661509956349619, + 0.0006501009993371554, + 0.0014095799997448921, + 0.00043298499804222956, + 0.0007416970038320869, + 0.0012381980050122365, + 0.00042197299626423046, + 0.00039075200038496405, + 0.0003902809985447675, + 0.0003949250021832995, + 0.0003947349978261627, + 0.0011612230009632185, + 0.004145375998632517, + 0.00046493300033034757, + 0.0004042090004077181, + 0.0004034240046166815, + 0.0003879950018017553, + 0.0003982639973401092, + 0.000392273002944421, + 0.0003937319997930899, + 0.004972427006578073, + 0.000452943000709638, + 0.00040443600300932303, + 0.00039858699892647564, + 0.0003964369971072301, + 0.0003904469995177351, + 0.00039191100222524256, + 0.0003905279954778962, + 0.006588568998267874, + 0.00045723700168309733, + 0.00040241600072477013, + 0.0003930160019081086, + 0.0003923229960491881, + 0.0003938669979106635, + 0.00038910700095584616, + 0.00039063899748725817, + 0.000389840999559965, + 0.0003869050051434897, + 0.006048770002962556, + 0.00046122899948386475, + 0.00040064399945549667, + 0.0004007940005976707, + 0.0003933340049115941, + 0.0003900070005329326, + 0.00039217000448843464, + 0.00039281099452637136, + 0.006339722000120673, + 0.0004383749983389862, + 0.00038608799513895065, + 0.000388198001019191, + 0.00038159000541782007, + 0.0003891520027536899, + 0.00038458000199170783, + 0.0004699140044976957, + 0.00039550500514451414, + 0.0003830619971267879, + 0.008514553999702912, + 0.00045094399683875963, + 0.0003931379978894256, + 0.00040889400406740606, + 0.0003821709979092702, + 0.000382901998818852, + 0.00038539899833267555, + 0.0003853470043395646, + 0.000388159001886379, + 0.00037891599640715867, + 0.006577979002031498, + 0.0015313900003093295, + 0.0014572769941878505, + 0.0014833759996690787, + 0.001661787995544728, + 0.0015343580016633496, + 0.0015185439988272265, + 0.0014757870012545027, + 0.001483025997004006, + 0.0016154949989868328, + 0.0014793019945500419, + 0.0014527449966408312, + 0.0014986640017013997, + 0.001488362999225501, + 0.0015800020046299323, + 0.0015035770047688857, + 0.0014951670018490404, + 0.0014875599954393692, + 0.0014538050018018112, + 0.0014852540043648332, + 0.0016012429987313226, + 0.0014863609976600856, + 0.001502880004409235, + 0.001453051998396404, + 0.0014367519979714416, + 0.001645798998652026, + 0.0014953860008972697, + 0.001473948999773711, + 0.0014846449994365685, + 0.0014744079962838441, + 0.0016374850019929, + 0.0015246849943650886, + 0.0014700870015076362, + 0.0014717930025653914, + 0.0015344490020652302, + 0.0014587920013582334, + 0.0016304050004691817, + 0.0014925719951861538, + 0.0015299700025934726, + 0.0014278890012064949, + 0.001487434004957322, + 0.0015646290048607625, + 0.001515012001618743, + 0.001448397000785917, + 0.0014745480002602562, + 0.001508352994278539, + 0.0015512530007981695, + 0.0018497549972380511, + 0.0014560740019078366, + 0.001445729001716245, + 0.001426281000021845, + 0.001416869999957271, + 0.0016469350011902861, + 0.0014797070034546778, + 0.0014765899977646768, + 0.0015042049999465235, + 0.0014932759950170293, + 0.001662686001509428, + 0.001544710001326166, + 0.0015145700017455965, + 0.001589022002008278, + 0.001564352998684626, + 0.0016054020015872084, + 0.001531178000732325, + 0.001562295998155605, + 0.001550730004964862, + 0.0014788490007049404, + 0.0015430209969053976, + 0.0015883500018389896, + 0.0015110859967535362, + 0.0015203729999484494, + 0.0015118009978323244, + 0.0006338070015772246, + 0.0003910329978680238, + 0.0022013289999449626, + 0.0004408859967952594, + 0.00038687799678882584, + 0.0003950000027543865, + 0.0003833090013358742, + 0.0003789809998124838, + 0.0003888909996021539, + 0.00038298800063785166, + 0.005101198999909684, + 0.00043782099965028465, + 0.00038635700184386224, + 0.0003870979999192059, + 0.0003830920031759888, + 0.0003774049982894212, + 0.00038738900184398517, + 0.0003778550017159432, + 0.00038105499697849154, + 0.00624294799490599, + 0.0021134050039108843, + 0.0005960010021226481, + 0.0009335919967270456, + 0.0005966180033283308, + 0.004289306001737714, + 0.0004533749961410649, + 0.000386412997613661, + 0.0003810440030065365, + 0.000410573004046455, + 0.0003792830029851757, + 0.00046597199980169535, + 0.00042245499935233966, + 0.0010808029983309098, + 0.0007647140009794384, + 0.000779764995968435, + 0.0010729259956860915, + 0.0013970830041216686, + 0.0004318689971114509 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil", + "fullname": "TestCalculateNabla2AndSmagCoefficientsForVn", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.0006627990005654283, + "max": 0.01599517700378783, + "mean": 0.004580611106477362, + "stddev": 0.003070203786632812, + "rounds": 1127, + "median": 0.005403256000136025, + "iqr": 0.006111218497608206, + "q1": 0.0008873767510522157, + "q3": 0.006998595248660422, + "iqr_outliers": 0, + "stddev_outliers": 564, + "outliers": "564;0", + "ld15iqr": 0.0006627990005654283, + "hd15iqr": 0.01599517700378783, + "ops": 218.31148219195418, + "total": 5.162348716999986, + "data": [ + 0.005989123004837893, + 0.006119521000073291, + 0.006877184001496062, + 0.00799317000200972, + 0.007985330004885327, + 0.004990490997442976, + 0.00787348199810367, + 0.011124377000669483, + 0.0032321839971700683, + 0.004760497002280317, + 0.003995115002908278, + 0.005245815002126619, + 0.008750633998715784, + 0.01034987500315765, + 0.007647395999811124, + 0.006367273999785539, + 0.005687756005499978, + 0.00396536599873798, + 0.007327580999117345, + 0.005614620997221209, + 0.006554630999744404, + 0.0022770329960621893, + 0.001999681997403968, + 0.001998607003770303, + 0.0019248159951530397, + 0.0019477250025374815, + 0.0019209290039725602, + 0.001953696002601646, + 0.0018841609999071807, + 0.0018528110012994148, + 0.0018586669975775294, + 0.0019637820005300455, + 0.0019084230007138103, + 0.0019235300060245208, + 0.0018943020040751435, + 0.0019504800002323464, + 0.0019282009961898439, + 0.0025947260000975803, + 0.0018799789977492765, + 0.0019325929970364086, + 0.0019132910019834526, + 0.0018529219960328192, + 0.0018935490006697364, + 0.0018632650026120245, + 0.0018111569952452555, + 0.0018339599992032163, + 0.0019470499973976985, + 0.002004021000175271, + 0.001923192998219747, + 0.0018439900013618171, + 0.006870559001981746, + 0.005996510000841226, + 0.007996078995347489, + 0.007998351997230202, + 0.007996608997927979, + 0.008007604999875184, + 0.007987211996805854, + 0.00799562300380785, + 0.005996220999804791, + 0.006997753000177909, + 0.00601841200113995, + 0.007975404005264863, + 0.006998864002525806, + 0.00508273300511064, + 0.00791084200318437, + 0.008995979995233938, + 0.005995018997055013, + 0.006997356002102606, + 0.0069973330028005876, + 0.007998099004908, + 0.006996837000770029, + 0.005999615998007357, + 0.006992994996835478, + 0.00800204500410473, + 0.006985899999563117, + 0.007996984997589607, + 0.006020132001140155, + 0.007967411998833995, + 0.007994591003807727, + 0.010998828998708632, + 0.007995579995622393, + 0.006999811994319316, + 0.006994160998146981, + 0.006987225999182556, + 0.00799564200133318, + 0.005999572997097857, + 0.0060687580044032075, + 0.007925167999928817, + 0.007999105000635609, + 0.0061225640019983985, + 0.006866688003356103, + 0.006571669000550173, + 0.007409140998788644, + 0.002001392000238411, + 0.006016226994688623, + 0.007987520002643578, + 0.005982100999972317, + 0.004993124995962717, + 0.006997839001996908, + 0.006376997000188567, + 0.00460881299659377, + 0.009846916000242345, + 0.004157669005508069, + 0.007985557000210974, + 0.0099958919963683, + 0.007996624000952579, + 0.008119492005789652, + 0.006872439000289887, + 0.004877322993706912, + 0.006115021998994052, + 0.005998150001687463, + 0.01000697600102285, + 0.007986151002114639, + 0.008017770000151359, + 0.007975751002959441, + 0.0070015270030125976, + 0.004992243004380725, + 0.0031881719987723045, + 0.0038023189990781248, + 0.0049972790002357215, + 0.0052597960020648316, + 0.005738180996559095, + 0.007994669998879544, + 0.007999558998562861, + 0.005998452994390391, + 0.008007661999727134, + 0.014984157001890708, + 0.007984926996869035, + 0.007003992002864834, + 0.002990339999087155, + 0.004978362005203962, + 0.0010891210040426813, + 0.000906988003407605, + 0.0006927200010977685, + 0.0006911500022397377, + 0.0006862720038043335, + 0.0006835410022176802, + 0.004930620001687203, + 0.0008340760032297112, + 0.0007372109976131469, + 0.0006846770047559403, + 0.0006933730037417263, + 0.0015191540005616844, + 0.0006858389970147982, + 0.0006938860024092719, + 0.0006850169957033359, + 0.0006745030041201971, + 0.0006828289988334291, + 0.0020868560022790916, + 0.0015941430028760806, + 0.00106733400025405, + 0.0007480409985873848, + 0.0006800490009482019, + 0.0006917439968674444, + 0.0029656519982381724, + 0.0013292360017658211, + 0.0006927239955984987, + 0.0010652630007825792, + 0.0006867210031487048, + 0.0006681829981971532, + 0.0020061390023329295, + 0.0007864880026318133, + 0.0013054359951638617, + 0.0007095059991115704, + 0.0006907630013301969, + 0.0006886840055813082, + 0.0006746819999534637, + 0.0006858199994894676, + 0.001028398000926245, + 0.0006932700052857399, + 0.0006822560026193969, + 0.0006811090061091818, + 0.0006760609976481646, + 0.0006784959987271577, + 0.0012402779975673184, + 0.0017064089988707565, + 0.0031851450039539486, + 0.0008536800014553592, + 0.0007499150015064515, + 0.0006893559984746389, + 0.006924499997694511, + 0.0008380939980270341, + 0.000701283999660518, + 0.0007308479980565608, + 0.000703294004779309, + 0.0006889590003993362, + 0.007305384999199305, + 0.0009059079966391437, + 0.0007004639992373995, + 0.0006900619991938584, + 0.0006863589951535687, + 0.0006721939935232513, + 0.006488027000159491, + 0.0020606210018740967, + 0.0008312569989357144, + 0.005559928998991381, + 0.0019873070050380193, + 0.0008741979982005432, + 0.0035777919983956963, + 0.0011749070035875775, + 0.0007153499973355792, + 0.0007222870044643059, + 0.0007019190015853383, + 0.005821788996399846, + 0.0008740470002521761, + 0.0007605989958392456, + 0.0006998469980317168, + 0.0006843669980298728, + 0.0049499599990667775, + 0.0008517989990650676, + 0.000680414006637875, + 0.0007233229989651591, + 0.0007075160028762184, + 0.005908403996727429, + 0.0008372500014957041, + 0.0006793679931433871, + 0.0006850569989182986, + 0.0006863360031275079, + 0.0006770779946236871, + 0.005266471998766065, + 0.0008978610057965852, + 0.0007138850050978363, + 0.0006869739954709075, + 0.000685793005686719, + 0.0006789690014556982, + 0.0007166249997681007, + 0.003799879996222444, + 0.0008579339992138557, + 0.0007313850001082756, + 0.0007110470032785088, + 0.0008469399981549941, + 0.005130307996296324, + 0.0008616770064691082, + 0.0008109120026347227, + 0.0007013140057097189, + 0.000705490994732827, + 0.004868854994128924, + 0.0008525369994458742, + 0.0007376659996225499, + 0.0006774809953640215, + 0.0006885729962959886, + 0.005687210999894887, + 0.0009417050023330376, + 0.0006866149997222237, + 0.0006953440024517477, + 0.0006957150035304949, + 0.0044015680032316595, + 0.000928561006730888, + 0.0008888400043360889, + 0.0010050540004158393, + 0.0007408639939967543, + 0.0007102000017766841, + 0.0007248460024129599, + 0.0032853030061232857, + 0.0008294240033137612, + 0.0007213339995360002, + 0.0006905100017320365, + 0.0006856980035081506, + 0.0006828510013292544, + 0.006183035999129061, + 0.0009275709962821566, + 0.0007907629988039844, + 0.0006961760009289719, + 0.0007288979977602139, + 0.0027808220038423315, + 0.0008201499949791469, + 0.0007404260031762533, + 0.0006889249998494051, + 0.0007085460019879974, + 0.0007014720031293109, + 0.005137446001754142, + 0.007371861000137869, + 0.00799684799858369, + 0.006566984004166443, + 0.007433452999975998, + 0.006987598993873689, + 0.006996434000029694, + 0.006674950003798585, + 0.007329972002480645, + 0.008983446001366246, + 0.007989891004399396, + 0.011012698996637482, + 0.0049794260048656724, + 0.006999714998528361, + 0.007991757993295323, + 0.004995546994905453, + 0.0069970049953553826, + 0.005997969004965853, + 0.0069961609988240525, + 0.00799824199930299, + 0.006001187000947539, + 0.006991383001150098, + 0.007996257998456713, + 0.006996210999204777, + 0.0059994030016241595, + 0.009594451003067661, + 0.005403256000136025, + 0.003997687999799382, + 0.005897534996620379, + 0.008604294998804107, + 0.007485718000680208, + 0.0069925049974699505, + 0.008003823000763077, + 0.007990537000296172, + 0.007997979999345262, + 0.00799788299627835, + 0.006990144000155851, + 0.007999314002518076, + 0.005993717000819743, + 0.004996159004804213, + 0.007002819002082106, + 0.00799252599972533, + 0.008998923993203789, + 0.005993355000100564, + 0.005996874002448749, + 0.005995542000164278, + 0.006997334996412974, + 0.007996443004230969, + 0.007998763001523912, + 0.008001611000509001, + 0.010001562994148117, + 0.006986227999732364, + 0.005988974997308105, + 0.006000387002131902, + 0.005997981003019959, + 0.006994801995460875, + 0.008139039004163351, + 0.007853729999624193, + 0.008001381997019053, + 0.006991080997977406, + 0.006995171999733429, + 0.0070037499972386286, + 0.0069929579985910095, + 0.002210898994235322, + 0.004781445000844542, + 0.006002982001518831, + 0.007991311002115253, + 0.006999084995186422, + 0.005995259001792874, + 0.007004357998084743, + 0.005987259995890781, + 0.0070042139996076, + 0.00798952500190353, + 0.0061885200047981925, + 0.007804644003044814, + 0.005996225001581479, + 0.008057514001848176, + 0.007941453994135372, + 0.007992738006578293, + 0.0029943190020276234, + 0.005997658001433592, + 0.005000750999897718, + 0.00799547399947187, + 0.00650745999882929, + 0.0034827609997591935, + 0.0039041389973135665, + 0.0040898170045693405, + 0.005999720997351687, + 0.0069958500025677495, + 0.002996627998072654, + 0.005997947999276221, + 0.00799554099648958, + 0.005998589003866073, + 0.005996408995997626, + 0.007995776002644561, + 0.007003140999586321, + 0.007991730999492574, + 0.006996061994868796, + 0.00799729300342733, + 0.004993820999516174, + 0.010014149003836792, + 0.006979447003686801, + 0.009172791993478313, + 0.006815344997448847, + 0.006003332993714139, + 0.006991116999415681, + 0.007996652995643672, + 0.010998253994330298, + 0.005038864997914061, + 0.004954169999109581, + 0.004997493997507263, + 0.006119188001321163, + 0.006881957997393329, + 0.0069891129969619215, + 0.007037568997475319, + 0.0029619359993375838, + 0.0049991420019068755, + 0.00732265500118956, + 0.007660900999326259, + 0.006994606002990622, + 0.00599699799931841, + 0.010012078004365321, + 0.00699076900491491, + 0.0069808280022698455, + 0.007002131998888217, + 0.0069881919989711605, + 0.0030040849960641935, + 0.0029903899994678795, + 0.005997653002850711, + 0.006999379002081696, + 0.007999162000487559, + 0.006996559997787699, + 0.006993920003878884, + 0.00799656700110063, + 0.004771261003043037, + 0.006221375995664857, + 0.007005058003414888, + 0.006987318993196823, + 0.006575352999789175, + 0.0074221079994458705, + 0.010994842996296939, + 0.007264738000230864, + 0.006712864000292029, + 0.007996489999641199, + 0.007999212997674476, + 0.003993343998445198, + 0.003968921999330632, + 0.005280657998810057, + 0.004740284995932598, + 0.005809777001559269, + 0.009186235998640768, + 0.003069840000534896, + 0.0059216230001766235, + 0.005995807994622737, + 0.005997911001031753, + 0.005999656998028513, + 0.0033075129977078177, + 0.004684046994952951, + 0.00800343699665973, + 0.006993550996412523, + 0.009007095002743881, + 0.006013368998537771, + 0.00796387500304263, + 0.005967242999759037, + 0.009026546002132818, + 0.00799614300194662, + 0.007996029999048915, + 0.0020039619994349778, + 0.007997756001714151, + 0.003992204001406208, + 0.004997668998839799, + 0.007999216002644971, + 0.0069968159950803965, + 0.0051526830065995455, + 0.00684083100350108, + 0.005997075000777841, + 0.007998275999852922, + 0.005995029998302925, + 0.005998408996674698, + 0.007999098997970577, + 0.006000599001708906, + 0.00599066599534126, + 0.008001359994523227, + 0.005991833997541107, + 0.006470795000495855, + 0.006656364996160846, + 0.007867692998843268, + 0.007993580999027472, + 0.007998524000868201, + 0.007995113999641035, + 0.007003147002251353, + 0.006990713998675346, + 0.00799491700308863, + 0.005997621999995317, + 0.008994383002573159, + 0.006995161005761474, + 0.00699777699628612, + 0.006996787997195497, + 0.005997248001222033, + 0.007997141001396812, + 0.005591925997578073, + 0.006402992003131658, + 0.007996496999112424, + 0.006997604999924079, + 0.0030086159968050197, + 0.006990308997046668, + 0.009999663001508452, + 0.005003081998438574, + 0.006989799003349617, + 0.007996441003342625, + 0.007996979999006726, + 0.007997230997716542, + 0.009995989996241406, + 0.0069994310033507645, + 0.006994825002038851, + 0.007998291999683715, + 0.008000456997251604, + 0.005992983999021817, + 0.006994053997914307, + 0.0069978839965187944, + 0.00999315400258638, + 0.009998342997278087, + 0.006996819996857084, + 0.005997613996441942, + 0.00699649299349403, + 0.006997331998718437, + 0.00800400099979015, + 0.0069956870065652765, + 0.007008753003901802, + 0.005993446997308638, + 0.007978351997735444, + 0.0070097369971335866, + 0.005012936999264639, + 0.007963902004121337, + 0.004994948998501059, + 0.008003126000403427, + 0.005988411001453642, + 0.005057695001596585, + 0.004933146999974269, + 0.007997770000656601, + 0.007997265995072667, + 0.006000674002279993, + 0.007993610997800715, + 0.003994599996076431, + 0.004996796997147612, + 0.008007794996956363, + 0.008998487995995674, + 0.007990991005499382, + 0.010991306000505574, + 0.007247626002936158, + 0.0057506459997966886, + 0.007169099000748247, + 0.010814032000780571, + 0.006995862000621855, + 0.005998322005325463, + 0.007036012997559737, + 0.006957753001188394, + 0.006994887000473682, + 0.006999919001827948, + 0.006994093993853312, + 0.007996979002200533, + 0.006997086005867459, + 0.00799894300143933, + 0.005368220998207107, + 0.00462209199758945, + 0.008003535003808793, + 0.008010110999748576, + 0.009986796998418868, + 0.00498851799784461, + 0.004861718996835407, + 0.00520598900038749, + 0.001093063001462724, + 0.00782143300602911, + 0.008193408997613005, + 0.00779820499883499, + 0.0070005849993322045, + 0.0069931050020386465, + 0.005244690997642465, + 0.001742804997775238, + 0.005238309997366741, + 0.0037584399979095906, + 0.003995972998382058, + 0.007002288999501616, + 0.008994877003715374, + 0.007994450999831315, + 0.00641019699833123, + 0.004013607001979835, + 0.0011376269976608455, + 0.00842317299975548, + 0.00798893400497036, + 0.0064728439974715, + 0.007521834995714016, + 0.0049962159973802045, + 0.008997707998787519, + 0.005850552995980252, + 0.005883746998733841, + 0.009256940000341274, + 0.006996757998422254, + 0.005997397005558014, + 0.0069958189997123554, + 0.00699653599440353, + 0.006998441000177991, + 0.006998995995672885, + 0.0036758109999937005, + 0.00431486799789127, + 0.005999941997288261, + 0.006998898999881931, + 0.0069930640020174906, + 0.006998203003604431, + 0.004997110001568217, + 0.006998641001700889, + 0.005001869998523034, + 0.009989304999180604, + 0.0044903779999003746, + 0.006506375997560099, + 0.00799294099851977, + 0.007997428001544904, + 0.004850844998145476, + 0.006075199002225418, + 0.007071109001117293, + 0.005762765002145898, + 0.0052214469978935085, + 0.007995171996299177, + 0.009248007998394314, + 0.008279264999146108, + 0.00646749600127805, + 0.007985905998793896, + 0.0039959549976629205, + 0.005995880004775245, + 0.004995230003260076, + 0.008002217000466771, + 0.006990431000303943, + 0.0070015039964346215, + 0.007997214001079556, + 0.007991790000232868, + 0.0110019809944788, + 0.0010059580017696135, + 0.007985310003277846, + 0.004995334005798213, + 0.004909102004603483, + 0.003109222001512535, + 0.00821706800343236, + 0.007753521997074131, + 0.006739515003573615, + 0.0012537369984784164, + 0.007000464000157081, + 0.006992272996285465, + 0.007997553002496716, + 0.007003309001447633, + 0.003766014997381717, + 0.006218141999852378, + 0.00455583800066961, + 0.005238124998868443, + 0.004195837005681824, + 0.007998203000170179, + 0.005997514999762643, + 0.00499721599771874, + 0.007001676000072621, + 0.0079894350055838, + 0.0059976959964842536, + 0.006996068004809786, + 0.006997433003562037, + 0.006996616997639649, + 0.0059970430011162534, + 0.007005001003562938, + 0.006991943002503831, + 0.005246386004728265, + 0.007748728996375576, + 0.0069944539936841466, + 0.007993664999958128, + 0.0059978819990647025, + 0.005131504003657028, + 0.0024345160054508597, + 0.006747658000676893, + 0.0036730979991261847, + 0.003997766994871199, + 0.007997522996447515, + 0.002001927998207975, + 0.005992853999487124, + 0.005996201995003503, + 0.005996827996568754, + 0.00800016900029732, + 0.00799474900122732, + 0.007996875996468589, + 0.010997215998941101, + 0.006996438998612575, + 0.006998577999183908, + 0.003999173997726757, + 0.005054139997810125, + 0.0019449829997029155, + 0.005999954999424517, + 0.003990295001131017, + 0.006998153003223706, + 0.007996034997631796, + 0.006997894997766707, + 0.005995206993247848, + 0.005997730004310142, + 0.006998356999247335, + 0.0060011910027242266, + 0.006990781002969015, + 0.007997998000064399, + 0.005998070999339689, + 0.008000989000720438, + 0.007990824997250456, + 0.005998291999276262, + 0.007993054001417477, + 0.007990391997736879, + 0.0060009550070390105, + 0.003993421996710822, + 0.0079969149956014, + 0.00599683600012213, + 0.00800184299441753, + 0.009128210003837012, + 0.00786213600076735, + 0.006993227005295921, + 0.006184696998388972, + 0.007806801004335284, + 0.006998167002166156, + 0.006999297002039384, + 0.006995201998506673, + 0.005995158993755467, + 0.006996732001425698, + 0.008190981003281195, + 0.007818810998287518, + 0.007978269997693133, + 0.0070002020001993515, + 0.006234624997887295, + 0.002745050995144993, + 0.007000881996646058, + 0.006995046998781618, + 0.007028025996987708, + 0.007964094998897053, + 0.006997031996434089, + 0.006999342003837228, + 0.005997554995701648, + 0.006997481999860611, + 0.0069964270005584694, + 0.005991878002532758, + 0.0069988200048101135, + 0.005994566003209911, + 0.006005846000334714, + 0.005988051998429, + 0.005996187996061053, + 0.005997681997541804, + 0.0069966520022717305, + 0.006997616997978184, + 0.0069986009984859265, + 0.006993964001594577, + 0.006999391996941995, + 0.006996661999437492, + 0.0069957349987817, + 0.006477725000877399, + 0.007516912002756726, + 0.006996750002144836, + 0.005996974003210198, + 0.006259547000809107, + 0.008731253001315054, + 0.00799748300050851, + 0.007996054999239277, + 0.00699688600434456, + 0.006995403004111722, + 0.006996559000981506, + 0.007000279001658782, + 0.005994571001792792, + 0.007995892003236804, + 0.0026531839976087213, + 0.0007937429982121103, + 0.0006810609993408434, + 0.0006823540024925023, + 0.0006730979948770255, + 0.003378611996595282, + 0.0015208729964797385, + 0.0010897059983108193, + 0.0006967939989408478, + 0.0006708329965476878, + 0.0006791640043957159, + 0.0006916080019436777, + 0.0006634820019826293, + 0.0018997029983438551, + 0.002047467998636421, + 0.0006956159995752387, + 0.000685382001393009, + 0.0006627990005654283, + 0.0006701059974147938, + 0.0020140089982305653, + 0.0019894919969374314, + 0.000864364999870304, + 0.0007255880045704544, + 0.000685542996507138, + 0.000681878998875618, + 0.0006954700002097525, + 0.0006854789971839637, + 0.0023022949972073548, + 0.0017205289987032302, + 0.0007158399967011064, + 0.0006992540002102032, + 0.0007504280001739971, + 0.0009250119983335026, + 0.0008707239976502024, + 0.0008071159973042086, + 0.0010564109979895875, + 0.0007047449980746023, + 0.0006883999958517961, + 0.0006940640014363453, + 0.0009024199971463531, + 0.001194379001390189, + 0.0012556719957501628, + 0.000689488006173633, + 0.0007003149949014187, + 0.000704362006217707, + 0.0007797979997121729, + 0.0009107420046348125, + 0.0017559059997438453, + 0.000729421000869479, + 0.004131326000788249, + 0.0008240669994847849, + 0.0009412350045749918, + 0.0025551680009812117, + 0.0011639929944067262, + 0.002341471001273021, + 0.0007805749992257915, + 0.0016852660046424717, + 0.0027746549967559986, + 0.0007088510028552264, + 0.0006812310020904988, + 0.0009327040024800226, + 0.0015418800030602142, + 0.0007332090026466176, + 0.00070570800016867, + 0.000692442998115439, + 0.0008628590003354475, + 0.0023525549986516126, + 0.0009450209981878288, + 0.0007734890023129992, + 0.0006922510001459159, + 0.0007590429959236644, + 0.0009140120018855669, + 0.0008940589978010394, + 0.001588210005138535, + 0.0007124309995560907, + 0.0006812930005253293, + 0.0006792929998482578, + 0.0023364150038105436, + 0.0010486780010978691, + 0.0008777399998507462, + 0.0008124150044750422, + 0.0007770299998810515, + 0.0013841079999110661, + 0.001439093000954017, + 0.0007601300021633506, + 0.0006899629952386022, + 0.0009069750012713484, + 0.0008993840019684285, + 0.000849798001581803, + 0.0023211070001707412, + 0.000718825998774264, + 0.0006841000067652203, + 0.0006756220027455129, + 0.001104467002733145, + 0.000923464001971297, + 0.0008142809965647757, + 0.0007253139992826618, + 0.000709451996954158, + 0.0006873660022392869, + 0.0008620430016890168, + 0.0012466519983718172, + 0.0017834679965744726, + 0.0007387590012513101, + 0.0007098889982444234, + 0.0006965560023672879, + 0.0008131590002449229, + 0.0008865249983500689, + 0.0014990429990575649, + 0.0010298409979441203, + 0.000716763999662362, + 0.0007067219994496554, + 0.0014031200043973513, + 0.0016864470017026179, + 0.000788598001236096, + 0.0007176130020525306, + 0.0006958269950700924, + 0.0009237769991159439, + 0.0011459480010671541, + 0.0019969109998783097, + 0.002965740997751709, + 0.0008053130004554987, + 0.0007883609941927716, + 0.0007271349968505092, + 0.0009632770015741698, + 0.0009996630033128895, + 0.0009410639977431856, + 0.0007395009961328469, + 0.0007128379947971553, + 0.0017383630038239062, + 0.0033309469945379533, + 0.000828716998512391, + 0.0007670550039620139, + 0.0007181149994721636, + 0.0007317089985008352, + 0.0016493439979967661, + 0.0028708120007649995, + 0.0007422769995173439, + 0.0007058449991745874, + 0.0007119529982446693, + 0.0008922810011426918, + 0.0009674220054876059, + 0.0015645590028725564, + 0.0007401969996863045, + 0.0021431040004245006, + 0.0007442630012519658, + 0.002052839001407847, + 0.0007436700034304522, + 0.0020427639974514022, + 0.0013894100047764368, + 0.0007489159979741089, + 0.0007083579985192046, + 0.0006936660065548494, + 0.0008497079979861155, + 0.002275284001370892, + 0.0007317950003198348, + 0.0010989709990099072, + 0.0007376790017588064, + 0.0020292000044719316, + 0.0007394750064122491, + 0.0007045519960229285, + 0.0006962789993849583, + 0.0007074239983921871, + 0.004503689000557642, + 0.0007709909987170249, + 0.0007119130023056641, + 0.0007101070004864596, + 0.0006995219955570064, + 0.0009669739956734702, + 0.0007993519975570962, + 0.0007374839988187887, + 0.004512156003329437, + 0.0024536630007787608, + 0.0007549450019723736, + 0.0007033559959381819, + 0.0007045920065138489, + 0.005060479998064693, + 0.0007486629983759485, + 0.0007118229987099767, + 0.0006954079944989644, + 0.004104948995518498, + 0.0008718570024939254, + 0.0007225359950098209, + 0.002587686998595018, + 0.0007392019979306497, + 0.0007011420020717196, + 0.0007079690039972775, + 0.0006974630014155991, + 0.000695214002917055, + 0.01128958600020269, + 0.007000094003160484, + 0.008010076002392452, + 0.007015157003479544, + 0.006946350004000124, + 0.007991521000803914, + 0.006000348999805283, + 0.005992491001961753, + 0.007997663000423927, + 0.006044319998181891, + 0.007955182998557575, + 0.007989197998540476, + 0.008000903006177396, + 0.0069941290057613514, + 0.007993860999704339, + 0.0059972300005028956, + 0.008461620003799908, + 0.006532315994263627, + 0.001000481002847664, + 0.005994112005282659, + 0.00799785499839345, + 0.006996853000600822, + 0.006005554998409934, + 0.006990704998315778, + 0.008996580996608827, + 0.00799915699462872, + 0.006995681003900245, + 0.0059968519999529235, + 0.006995504998485558, + 0.0059968830028083175, + 0.006999408004048746, + 0.006995284995355178, + 0.006997272997978143, + 0.00699708100000862, + 0.006997766999120358, + 0.006996251999225933, + 0.00699746300233528, + 0.006996994998189621, + 0.005997906999255065, + 0.007996514003025368, + 0.006998566001129802, + 0.007997512999281753, + 0.00799828000162961, + 0.005995129002258182, + 0.005997387001116294, + 0.007997417000296991, + 0.007998671004315838, + 0.007994788997166324, + 0.005999458000587765, + 0.007997861001058482, + 0.007993760002136696, + 0.005998590997478459, + 0.0059888909963774495, + 0.006998705997830257, + 0.00799559500592295, + 0.005000690995075274, + 0.007995178995770402, + 0.005997051004669629, + 0.006997745003900491, + 0.004993588001525495, + 0.008005704003153369, + 0.006990105001023039, + 0.006995146999543067, + 0.005073426000308245, + 0.004920890998619143, + 0.0059980399964842945, + 0.007995448999281507, + 0.007998480999958701, + 0.005996535997837782, + 0.007998177003173623, + 0.008020055000088178, + 0.0029996860030223615, + 0.0051511749989003874, + 0.007813076001184527, + 0.006992137998167891, + 0.004994652001187205, + 0.009000073994684499, + 0.005999971006531268, + 0.0069903709954814985, + 0.011029333996702917, + 0.006963497995457146, + 0.00800347999756923, + 0.007992267004738096, + 0.007995338994078338, + 0.006995563999225851, + 0.01299818199913716, + 0.006993303999479394, + 0.008004436000192072, + 0.008990827998786699, + 0.006003394999424927, + 0.01599517700378783, + 0.007982932002050802, + 0.005852788999618497, + 0.0021479599963640794, + 0.0029833920052624308, + 0.005007559004297946, + 0.006995424002525397, + 0.0037986339957569726, + 0.005230298003880307, + 0.00796315900515765, + 0.006997393000347074, + 0.0032062879981822334, + 0.003691432997584343, + 0.005789907001599204, + 0.006349739000143018, + 0.0009090469975490123, + 0.000746277000871487, + 0.0006884399990667589, + 0.0006869100034236908, + 0.0006851080033811741, + 0.0059694340015994385, + 0.0008657170037622564, + 0.0007557309945696034, + 0.0006722140024066903, + 0.0006962150000617839, + 0.0032273239994538017, + 0.0008274489955510944, + 0.0014326700038509443, + 0.0006895219994476065, + 0.0006960410028113984, + 0.0006961570034036413, + 0.0006820950002293102, + 0.0006781010015401989, + 0.003402275004191324, + 0.0008694520001881756, + 0.00221835700358497, + 0.0008050780015764758, + 0.0007178400046541356, + 0.0006849049968877807, + 0.003605756995966658, + 0.0008457680014544167, + 0.0007746379997115582, + 0.0007048979969113134, + 0.0007008329994278029, + 0.0006844059971626848, + 0.004356330995506141, + 0.0008868889999575913, + 0.0007678030015085824, + 0.0006801830022595823, + 0.002252910999231972, + 0.0007556990021839738, + 0.0007373370026471093, + 0.000681685000017751, + 0.002459171002556104, + 0.0007965889963088557, + 0.0006795590015826747, + 0.0006857449989183806, + 0.0006838350018369965, + 0.0006798940012231469, + 0.005128924000018742, + 0.0008792980006546713, + 0.0013881859995308332, + 0.0006769930041627958, + 0.0007179349995567463, + 0.0006908819996169768, + 0.004187295999145135, + 0.000846171002194751, + 0.0007155860002967529, + 0.0006807919999118894, + 0.0007119149959180504, + 0.002507345998310484, + 0.0026793809956870973, + 0.0008651879979879595, + 0.0008058780003921129, + 0.0006841579961474054, + 0.0007056450049276464, + 0.006231170002138242, + 0.001000873999146279, + 0.0007408430028590374, + 0.0006897839994053356, + 0.0006977509983698837, + 0.0006829920021118596, + 0.006534415006171912, + 0.0008568129997001961, + 0.0006933609984116629, + 0.0006993349961703643, + 0.0006861590009066276, + 0.004283005000615958, + 0.0008340540007338859, + 0.0008601770023233257, + 0.0007257500037667342, + 0.0006986429943935946, + 0.005902028002310544, + 0.0008340530039276928, + 0.0007802109976182692, + 0.0006797250025556423, + 0.0006838879999122582, + 0.003155979997245595, + 0.0008725090010557324, + 0.0007890009947004728, + 0.002178786999138538, + 0.0040939079990494065, + 0.0012693489989032969, + 0.0007007160020293668, + 0.0006861789952381514, + 0.002230211997812148, + 0.0007200789987109601, + 0.0007376720022875816, + 0.0006964499989408068, + 0.0006927349968464114, + 0.002174620000005234, + 0.002794586995150894, + 0.0008092079951893538, + 0.0006766240039723925 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil", + "fullname": "TestCalculateNabla2ForW", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.0003208999987691641, + "max": 0.06402748999971664, + "mean": 0.003404689042697311, + "stddev": 0.0037307441009177666, + "rounds": 1687, + "median": 0.0015808429961907677, + "iqr": 0.006198535498697311, + "q1": 0.0003456052509136498, + "q3": 0.0065441407496109605, + "iqr_outliers": 5, + "stddev_outliers": 242, + "outliers": "242;5", + "ld15iqr": 0.0003208999987691641, + "hd15iqr": 0.01901425200048834, + "ops": 293.71257916927584, + "total": 5.743710415030364, + "data": [ + 0.0006079320010030642, + 0.000561126995307859, + 0.0005768699993495829, + 0.0005684620045940392, + 0.0006446689949370921, + 0.0005989019991829991, + 0.0005796589975943789, + 0.0005980950008961372, + 0.000573858997086063, + 0.0010199649987043813, + 0.005994811006530654, + 0.0060005850027664565, + 0.006994929994107224, + 0.008032661993638612, + 0.006999516001087613, + 0.00829950999468565, + 0.006643805005296599, + 0.007998031003808137, + 0.007995243999175727, + 0.00799275500321528, + 0.007395872999040876, + 0.006340387000818737, + 0.00424814700090792, + 0.008171932000550441, + 0.007830046997696627, + 0.005990630997985136, + 0.007988632998603862, + 0.008076662001258228, + 0.007928008999442682, + 0.007988360004674178, + 0.007988852004928049, + 0.005996225001581479, + 0.005994807004753966, + 0.0059947920017293654, + 0.0060023030018783174, + 0.007112163002602756, + 0.006883369998831768, + 0.004981685997336172, + 0.0069955090002622455, + 0.007995728999958374, + 0.005999653003527783, + 0.00467156100057764, + 0.006312996003543958, + 0.007028164000075776, + 0.006962283005123027, + 0.006997178999881726, + 0.004207951002172194, + 0.00978377600404201, + 0.007995945001312066, + 0.005513951000466477, + 0.006562123999174219, + 0.00392160900082672, + 0.006979531994147692, + 0.007997863998753019, + 0.005234782001934946, + 0.005820648999360856, + 0.003929541999241337, + 0.005985764000797644, + 0.00611862799996743, + 0.003975550003815442, + 0.004128284999751486, + 0.007763616005831864, + 0.006991772002947982, + 0.007995790998393204, + 0.006998056000156794, + 0.006993318995228037, + 0.007995558997208718, + 0.009998810994147789, + 0.006202839998877607, + 0.006807275000028312, + 0.00233237099746475, + 0.00362934400618542, + 0.0039776530029485, + 0.010020032001193613, + 0.008014694998564664, + 0.009983962998376228, + 0.005985484000120778, + 0.006244430995138828, + 0.008747551000851672, + 0.005994027000269853, + 0.006996905998676084, + 0.00699369400535943, + 0.007004885999776889, + 0.007315121001738589, + 0.007670396000321489, + 0.005992170998069923, + 0.005998152999382, + 0.005992249003611505, + 0.0059954109965474345, + 0.0060000599987688474, + 0.004216436995193362, + 0.007369670995103661, + 0.00639607799530495, + 0.0035196289973100647, + 0.004727105995698366, + 0.0017655740011832677, + 0.006029654003214091, + 0.007915475995105226, + 0.00800113299919758, + 0.0043595189999905415, + 0.004614613004378043, + 0.006987483000557404, + 0.008003244998690207, + 0.005995964995236136, + 0.008012776001123711, + 0.007966185999976005, + 0.005996143001539167, + 0.007014295995759312, + 0.006979084006161429, + 0.006995146002736874, + 0.005712140002287924, + 0.007961376002640463, + 0.007316738003282808, + 0.0039724980015307665, + 0.006132966998848133, + 0.0018912579980678856, + 0.00599178699485492, + 0.004993180999008473, + 0.007003011000051629, + 0.008031125995330513, + 0.007956720000947826, + 0.007992468003067188, + 0.006978138997510541, + 0.006941366002138238, + 0.007996963999175932, + 0.006984994004596956, + 0.007019240001682192, + 0.0069729370006825775, + 0.005983846996969078, + 0.007021023004199378, + 0.007980141999723855, + 0.007898898998973891, + 0.010140145001059864, + 0.0049724850032362156, + 0.005949776001216378, + 0.006038737999915611, + 0.007960400005686097, + 0.007721456000581384, + 0.007193782999820542, + 0.008026884999708273, + 0.005993124002998229, + 0.0020022020034957677, + 0.003806733999226708, + 0.0014629189972765744, + 0.005225802000495605, + 0.0015022349980426952, + 0.006173503999889363, + 0.0037803009981871583, + 0.001982439003768377, + 0.005999212997267023, + 0.004991471003449988, + 0.006000371002301108, + 0.006996470998274162, + 0.006992465001530945, + 0.008003249997273088, + 0.007988504999957513, + 0.005996510000841226, + 0.003004739999596495, + 0.000386570995033253, + 0.0003432890007388778, + 0.0003495959972497076, + 0.00033679199987091124, + 0.00033332500606775284, + 0.00034080399927916005, + 0.00033736800105543807, + 0.00033249899570364505, + 0.0003379640038474463, + 0.00033406099828425795, + 0.0052557210001396015, + 0.0003959200039389543, + 0.0003405269962968305, + 0.00033512099616928026, + 0.00034421400050632656, + 0.00033699499908834696, + 0.0003326599980937317, + 0.0003422540030442178, + 0.0003390260026208125, + 0.00033091400109697133, + 0.006201976997544989, + 0.00037862199678784236, + 0.0003351660052430816, + 0.0003421340006752871, + 0.0003321999975014478, + 0.00033518100099172443, + 0.00033756199991330504, + 0.0003305829959572293, + 0.0003351819977979176, + 0.0003347309975652024, + 0.0074696550000226125, + 0.0008036910003283992, + 0.00034833700192393735, + 0.0003420830034883693, + 0.00033528100175317377, + 0.0003314019995741546, + 0.0003380900016054511, + 0.00033214800350833684, + 0.00035786999796982855, + 0.006271018995903432, + 0.0003992989950347692, + 0.0013763549941359088, + 0.0003787579989875667, + 0.00033927400363609195, + 0.0003329779938212596, + 0.0019533829981810413, + 0.00035313300031702965, + 0.00033387200528522953, + 0.00033364800037816167, + 0.00034895499993581325, + 0.00033541299490025267, + 0.0003283039986854419, + 0.00033832000190159306, + 0.0003328599996166304, + 0.0003361309936735779, + 0.004133356997044757, + 0.0003776030061999336, + 0.00033205700310645625, + 0.000347517998307012, + 0.00033796900243032724, + 0.00033426199661334977, + 0.00038335700082825497, + 0.0003376559980097227, + 0.0003391779973753728, + 0.004925535999063868, + 0.0020490099996095523, + 0.000430410000262782, + 0.00040711299516260624, + 0.0032331009933841415, + 0.0007193459969130345, + 0.00035014199966099113, + 0.0003340600014780648, + 0.0003314649948151782, + 0.0004264650051482022, + 0.00033992499811574817, + 0.00033218700264114887, + 0.0003370609992998652, + 0.0003363379946677014, + 0.0003314780042273924, + 0.003985691997513641, + 0.00043293400085531175, + 0.0003462129971012473, + 0.00033112899836851284, + 0.0003414879975025542, + 0.0003325540019432083, + 0.00033056000393116847, + 0.0003347389938426204, + 0.0003365259981364943, + 0.00033096699917223305, + 0.004507024001213722, + 0.000561530003324151, + 0.00035528400621842593, + 0.00033354799961671233, + 0.0005896849979762919, + 0.0003700930028571747, + 0.0003450559961493127, + 0.00033315100154140964, + 0.00033106400223914534, + 0.005695518004358746, + 0.000399835997086484, + 0.0004098049976164475, + 0.0003398469998501241, + 0.0018629320038598962, + 0.0003946939978050068, + 0.00034306099405512214, + 0.00045792799937771633, + 0.00032674800604581833, + 0.00033087599877035245, + 0.0003288360021542758, + 0.00033019800321199, + 0.0028870559981442057, + 0.00037475599674507976, + 0.0003268270011176355, + 0.00033031900238711387, + 0.0003304880010546185, + 0.00032686600025044754, + 0.0003208999987691641, + 0.0003330830004415475, + 0.0003218339988961816, + 0.00032325799838872626, + 0.003912018997652922, + 0.001295890993787907, + 0.0003293449990451336, + 0.00032220600405707955, + 0.0004515530017670244, + 0.000331805000314489, + 0.00033029200130840763, + 0.0003345850054756738, + 0.00032213299709837884, + 0.00032236400147667155, + 0.004158295996603556, + 0.00039495599776273593, + 0.0003390769998077303, + 0.00035663999733515084, + 0.000339015998179093, + 0.00032774599822005257, + 0.00032987300073727965, + 0.0003298819938208908, + 0.0003274260016041808, + 0.005160971995792352, + 0.0003817879987764172, + 0.00032928700238699093, + 0.000328219000948593, + 0.0011285670043434948, + 0.00039908500184537843, + 0.00033106800401583314, + 0.00033144099870696664, + 0.00032647500484017655, + 0.0003216460027033463, + 0.003235809002944734, + 0.00034389000211376697, + 0.00032526399445487186, + 0.0021649679983966053, + 0.00035353600105736405, + 0.00033157099824165925, + 0.00032282999745802954, + 0.00033598799927858636, + 0.00032328000088455155, + 0.001944390001881402, + 0.00033614000130910426, + 0.0003443609966780059, + 0.0003293529953225516, + 0.0003225780019420199, + 0.003139894994092174, + 0.0003376609965926036, + 0.0003244540057494305, + 0.0018621039998834021, + 0.0003476530037005432, + 0.0003357419991516508, + 0.0003241559970774688, + 0.0003255849951528944, + 0.00032918999932007864, + 0.0003276369970990345, + 0.0003273780021118, + 0.004453412999282591, + 0.00033398800587747246, + 0.00033032499777618796, + 0.00032732999534346163, + 0.00032792399724712595, + 0.00032404999365098774, + 0.00032830899726832286, + 0.00032158400426851586, + 0.00032490299781784415, + 0.00032823999936226755, + 0.0003273539987276308, + 0.00032508900039829314, + 0.003996256004029419, + 0.00034262700501130894, + 0.00033003499993355945, + 0.00033727000118233263, + 0.0003378199980943464, + 0.0003303830017102882, + 0.0003252660026191734, + 0.004682671002228744, + 0.001151872995251324, + 0.00038002599467290565, + 0.0003548620006768033, + 0.0003346399971633218, + 0.0003359609981998801, + 0.0003262309983256273, + 0.00033032899955287576, + 0.0003468389986664988, + 0.0003366210003150627, + 0.00032826800452312455, + 0.005471345997648314, + 0.0003549699977156706, + 0.00033853299828479066, + 0.00032708499929867685, + 0.00032327399821951985, + 0.0003334780049044639, + 0.0003309879975859076, + 0.00032474700128659606, + 0.00035785799991572276, + 0.0004708499982370995, + 0.0003474199984339066, + 0.01308392800274305, + 0.00699561600049492, + 0.008000709000043571, + 0.00799032599752536, + 0.008329307005624287, + 0.008682552994287107, + 0.014000860996020492, + 0.00695665900275344, + 0.006995037998422049, + 0.00798692399985157, + 0.006989351997617632, + 0.007995137995749246, + 0.0069956639999873005, + 0.00999543399666436, + 0.00799971700325841, + 0.007984791998751462, + 0.0089963060017908, + 0.005091300001367927, + 0.005900258998735808, + 0.00999971399869537, + 0.006991481997829396, + 0.0059960470025544055, + 0.006998526994721033, + 0.008017240004846826, + 0.007973492996825371, + 0.006014150996634271, + 0.007977847999427468, + 0.007996795000508428, + 0.008002082999155391, + 0.006988193003053311, + 0.005996125997626223, + 0.007995109001058154, + 0.007322777004446834, + 0.007670508995943237, + 0.006987171997025143, + 0.007001630001468584, + 0.006983853003475815, + 0.005995104998874012, + 0.007995350999408402, + 0.00610569299897179, + 0.0088882409982034, + 0.005407092998211738, + 0.0055816219974076375, + 0.006026750001183245, + 0.007967092999024317, + 0.00699530600104481, + 0.007018377000349574, + 0.005417435000708792, + 0.005547528002352919, + 0.007992769998963922, + 0.0059957060002489015, + 0.012683961002039723, + 0.010828283004229888, + 0.0006085019995225593, + 0.00039033500070217997, + 0.00035183200088795274, + 0.00035363399365451187, + 0.0003333770000608638, + 0.0003335249930387363, + 0.00034430900268489495, + 0.0003368400066392496, + 0.0003339550021337345, + 0.0030058629999984987, + 0.00035058000503340736, + 0.0015120230018510483, + 0.00037589699786622077, + 0.00034309100010432303, + 0.00033290799910901114, + 0.00033738800266291946, + 0.00034115600283257663, + 0.00033042199356714264, + 0.003850280001643114, + 0.00035318799928063527, + 0.00033572699612705037, + 0.00034618900099303573, + 0.00034026500361505896, + 0.0003391479986021295, + 0.00033370200253557414, + 0.00035729700175579637, + 0.00033810500463005155, + 0.0003333209970151074, + 0.005378326000936795, + 0.00035703100002137944, + 0.00033893799991346896, + 0.0003458789942669682, + 0.0003316810034448281, + 0.0003328259990666993, + 0.00033679999614832923, + 0.0003354179934831336, + 0.00033267099934164435, + 0.004661209000914823, + 0.0003377269968041219, + 0.00036383100086823106, + 0.00033514900133013725, + 0.00034325999877182767, + 0.00036431899934541434, + 0.00033401400287402794, + 0.00033925400202861056, + 0.00033149999944726005, + 0.00032996200025081635, + 0.0064745839990791865, + 0.0003821279969997704, + 0.0003353670035721734, + 0.000343929001246579, + 0.0003371390048414469, + 0.000331880000885576, + 0.00034399700234644115, + 0.0003332960041007027, + 0.0003352920030010864, + 0.00034011900424957275, + 0.0003306159997009672, + 0.00033243800135096535, + 0.0047095929985516705, + 0.0003546010048012249, + 0.00033702899963827804, + 0.0003344550059409812, + 0.00033486699976492673, + 0.00033739199716364965, + 0.0003304880010546185, + 0.00033646600059000775, + 0.000341566999850329, + 0.011602451995713636, + 0.0069878840004093945, + 0.006996018004429061, + 0.00799723000091035, + 0.006997385004069656, + 0.006993556002271362, + 0.005033866000303533, + 0.0029606370007968508, + 0.0059956719996989705, + 0.006988449000346009, + 0.0069964610011084005, + 0.00700010600121459, + 0.0059988779976265505, + 0.01118648900592234, + 0.008858707005856559, + 0.005927088001044467, + 0.009992792998673394, + 0.005989933000819292, + 0.00699290099873906, + 0.007992403996468056, + 0.0070063499952084385, + 0.0053065070023876615, + 0.008688170004461426, + 0.007980556998518296, + 0.006996128999162465, + 0.01099701500061201, + 0.0069948230011505075, + 0.008000237998203374, + 0.0069894109983579256, + 0.00799405300494982, + 0.007001058998866938, + 0.0059919670020462945, + 0.006995306997851003, + 0.008017104999453295, + 0.005969876998278778, + 0.004995688999770209, + 0.005995569001242984, + 0.006996830001298804, + 0.00799770699813962, + 0.006996028998401016, + 0.006998387005296536, + 0.007143690003431402, + 0.007843858998967335, + 0.006996014999458566, + 0.00799622900376562, + 0.00699672300106613, + 0.0070002360007492825, + 0.007002963000559248, + 0.008993824005301576, + 0.007001860001764726, + 0.007970226004545111, + 0.00799713499873178, + 0.007996309002919588, + 0.004995144001441076, + 0.007994266001333017, + 0.007665469995117746, + 0.009333302004961297, + 0.007988555997144431, + 0.0045471120029105805, + 0.004453358000318985, + 0.007983224000781775, + 0.005994876999466214, + 0.00800107100076275, + 0.00799778399959905, + 0.006989836998400278, + 0.007999696004844736, + 0.006995518997428007, + 0.006992433001869358, + 0.0037029840023024008, + 0.003325127996504307, + 0.0004945619948557578, + 0.0003414170059841126, + 0.00034870600211434066, + 0.0010123679967364296, + 0.0003356300003360957, + 0.00033359900407958776, + 0.0004003310023108497, + 0.0003478360013104975, + 0.00033916099346242845, + 0.0003395730018382892, + 0.00033609499951126054, + 0.00047478799388045445, + 0.0003431839941185899, + 0.00033354799961671233, + 0.00034544600202934816, + 0.0031429390000994317, + 0.0003861090008285828, + 0.00035232800291851163, + 0.00033426199661334977, + 0.00033926399919437245, + 0.0009990110047510825, + 0.000341867002134677, + 0.0003382099966984242, + 0.0007072119988151826, + 0.004206192999845371, + 0.0004540719965007156, + 0.00034978999610757455, + 0.0003368719990248792, + 0.0004208679965813644, + 0.00036217599699739367, + 0.0003364410003996454, + 0.00035173899959772825, + 0.0003390419951756485, + 0.0003403549999347888, + 0.005493372002092656, + 0.00039081700379028916, + 0.0003583270008675754, + 0.00034500999754527584, + 0.00033808500302257016, + 0.0003737360020750202, + 0.00034824400063371286, + 0.0007989740042830817, + 0.0009072350003407337, + 0.005382843999541365, + 0.0004993910042685457, + 0.0003649440041044727, + 0.0006359889957820997, + 0.0003573640060494654, + 0.0003420069988351315, + 0.0004061090003233403, + 0.0003478010039543733, + 0.0003480039958958514, + 0.0045221109976409934, + 0.0004067799964104779, + 0.0003667759956442751, + 0.0003463179964455776, + 0.0006425770043279044, + 0.00038042500091250986, + 0.00033904299925779924, + 0.00036021000414621085, + 0.00033956800325540826, + 0.0032071250025182962, + 0.00042620900057954714, + 0.0004852560014114715, + 0.0031186659980448894, + 0.00041664400487206876, + 0.00037335600063670427, + 0.00035141099942848086, + 0.00034774900268530473, + 0.000340304002747871, + 0.0027548250000108965, + 0.00039913799992064014, + 0.0016017489979276434, + 0.00041476600017631426, + 0.0003616339963627979, + 0.00038865899841766804, + 0.003844601997116115, + 0.0003819429985014722, + 0.0003475599951343611, + 0.00033919700217666104, + 0.00034231100289616734, + 0.0005155010003363714, + 0.0003397479958948679, + 0.00038018199848011136, + 0.0003429930002312176, + 0.0004860440021730028, + 0.004985670995665714, + 0.0004928000053041615, + 0.00034367700573056936, + 0.0006540960021084175, + 0.0003430029973969795, + 0.00035077699430985376, + 0.0003470650044619106, + 0.0006060300001990981, + 0.0003565769948181696, + 0.00034225000126753, + 0.0037389970020740293, + 0.0012319480010773987, + 0.0014338700057123788, + 0.00035226599720772356, + 0.00046837099944241345, + 0.00040978300239657983, + 0.00035040800139540806, + 0.0006729430024279281, + 0.004383897998195607, + 0.0005079420006950386, + 0.00034340900310780853, + 0.00039595099951839074, + 0.00034986100217793137, + 0.00053587400179822, + 0.0003634449967648834, + 0.0003475500052445568, + 0.00034903200139524415, + 0.0003404809976927936, + 0.0003472600001259707, + 0.0059577260035439394, + 0.00041181199776474386, + 0.0003492989999358542, + 0.00035864800156559795, + 0.0004825979995075613, + 0.0012233529996592551, + 0.0019150070002069697, + 0.0024374049971811473, + 0.0012584659998537973, + 0.0003473150063655339, + 0.00034996599424630404, + 0.00034338900150032714, + 0.0003389249977772124, + 0.00034835700353141874, + 0.00035072000173386186, + 0.0003382280046935193, + 0.003117801999906078, + 0.003328998005599715, + 0.0003630559949669987, + 0.0003904000041075051, + 0.00038360399776138365, + 0.0006178010007715784, + 0.00176658700365806, + 0.003524382002069615, + 0.0003748010058188811, + 0.002202830000896938, + 0.0009884259998216294, + 0.0004189350001979619, + 0.0003402980000828393, + 0.00034873000549850985, + 0.000342480001563672, + 0.0003390840065549128, + 0.0023688270011916757, + 0.0007818160011083819, + 0.0003547099986462854, + 0.0003410639983485453, + 0.0003425719987717457, + 0.00040643400279805064, + 0.0003699079970829189, + 0.0007935440007713623, + 0.0017874260011012666, + 0.005652380998071749, + 0.002977707001264207, + 0.006001362002280075, + 0.006991996000579093, + 0.0070022560030338354, + 0.007005139996181242, + 0.006979753001360223, + 0.008003351002116688, + 0.006584994000149891, + 0.007391286002530251, + 0.0059858050008188, + 0.005990473000565544, + 0.006988585999351926, + 0.006998516997555271, + 0.0059922360014752485, + 0.006996220996370539, + 0.007995473002665676, + 0.006598899999517016, + 0.005397824999818113, + 0.0079924360034056, + 0.0059956980039714836, + 0.005995620995236095, + 0.00601442699553445, + 0.009981451003113762, + 0.006993214999965858, + 0.004995506998966448, + 0.008000255998922512, + 0.007996118998562451, + 0.007277871998667251, + 0.006714181006827857, + 0.005991288002405781, + 0.005995727995468769, + 0.004734663001727313, + 0.006256655004108325, + 0.009572833005222492, + 0.005435596001916565, + 0.007983796000189614, + 0.005995312996674329, + 0.008004328003153205, + 0.007990998004970606, + 0.007987798999238294, + 0.0029944729976705275, + 0.006997393997153267, + 0.007996430002094712, + 0.006999801000347361, + 0.006990829999267589, + 0.0059933820011792704, + 0.006000594003126025, + 0.0069915810017846525, + 0.005996414001856465, + 0.006000459994538687, + 0.007999387999007013, + 0.007988564000697806, + 0.005300400000123773, + 0.0066444929980207235, + 0.006049975003406871, + 0.005990054996800609, + 0.007996451000508387, + 0.003996157000074163, + 0.006996558004175313, + 0.007997762993909419, + 0.006997017997491639, + 0.004013658995972946, + 0.005977084001642652, + 0.004990185996575747, + 0.005004254999221303, + 0.007989577999978792, + 0.0069957349987817, + 0.00600198900065152, + 0.007992862003447954, + 0.005083067997475155, + 0.0069014700056868605, + 0.0059958609999739565, + 0.007998322005732916, + 0.005995178995362949, + 0.0059961329970974475, + 0.0069962670022505336, + 0.007997624001291115, + 0.006000286994094495, + 0.00415893299941672, + 0.00382895499933511, + 0.00039198999729705974, + 0.00034943100035889074, + 0.00036292699951445684, + 0.00034656599746085703, + 0.000341302999004256, + 0.004256861000612844, + 0.0047009749978315085, + 0.0003855030008708127, + 0.00034775100357364863, + 0.0009558849997119978, + 0.005396043001383077, + 0.0004073090021847747, + 0.00035006100370083004, + 0.0003434970058151521, + 0.006401257996913046, + 0.0003873149980790913, + 0.000350488000549376, + 0.00034598100319271907, + 0.00034414300171192735, + 0.0027784310004790314, + 0.007055848000163678, + 0.0023285770002985373, + 0.0003886159975081682, + 0.0036082099977647886, + 0.0011318559991195798, + 0.0005931769992457703, + 0.0003475359990261495, + 0.0003405419993214309, + 0.00033856899972306564, + 0.0008270649996120483, + 0.002070075999654364, + 0.005172204000700731, + 0.0012619090048247017, + 0.0003626160032581538, + 0.0003448140050750226, + 0.00034994599991478026, + 0.00034594199678394943, + 0.0003404669987503439, + 0.0013501919966074638, + 0.0006155540031613782, + 0.0003560960030881688, + 0.0003404379967832938, + 0.0008138530029100366, + 0.0003687409989652224, + 0.00034528799733379856, + 0.0003424820024520159, + 0.0003416620020288974, + 0.0024211150011979043, + 0.0003807199973380193, + 0.00034867799695348367, + 0.00039912600186653435, + 0.0003440309956204146, + 0.003354663000209257, + 0.00037990199780324474, + 0.00034791199868777767, + 0.00036505499883787706, + 0.0003481599997030571, + 0.00034558799961814657, + 0.0035540239987312816, + 0.00037581799551844597, + 0.0003578570031095296, + 0.0003446400005486794, + 0.0003408720003790222, + 0.00038537100044777617, + 0.0003473719989415258, + 0.0003555750008672476, + 0.0003488669972284697, + 0.0003396180036361329, + 0.00034445899655111134, + 0.0029436379991238937, + 0.00038655000389553607, + 0.00034122900251531973, + 0.0003469479997875169, + 0.0003451979937381111, + 0.0003431130026001483, + 0.0003491079987725243, + 0.0003459310028119944, + 0.0003415970058995299, + 0.000665783001750242, + 0.0038963120023254305, + 0.000396932999137789, + 0.00035540899989428, + 0.0003448369971010834, + 0.00034691499604377896, + 0.0007295550021808594, + 0.0003468970026005991, + 0.004232778999721631, + 0.00037365599564509466, + 0.00034948199754580855, + 0.0003499649974401109, + 0.0003453009994700551, + 0.0003402360016480088, + 0.0004066769979544915, + 0.00034662699908949435, + 0.0003403230002732016, + 0.004085421998752281, + 0.005970776001049671, + 0.003994259997853078, + 0.005998203996568918, + 0.006994022995058913, + 0.005997513995680492, + 0.005997478998324368, + 0.007997818000148982, + 0.005996901003527455, + 0.006997103002504446, + 0.00899800500337733, + 0.007996560001629405, + 0.0060003190010320395, + 0.006993844996031839, + 0.0057115950039587915, + 0.00528409999969881, + 0.008006527998077217, + 0.00799275899771601, + 0.007000473000516649, + 0.00799037900287658, + 0.006989966997934971, + 0.007999831999768503, + 0.00799555499543203, + 0.006007224001223221, + 0.008011503006855492, + 0.01901425200048834, + 0.007745226997940335, + 0.006814860003942158, + 0.014391746997716837, + 0.040002701003686525, + 0.06402748999971664, + 0.008943223998358008, + 0.024204378998547327, + 0.007741574998362921, + 0.007184078000136651, + 0.004786650002643, + 0.004096472002856899, + 0.004901552005321719, + 0.009057937997567933, + 0.004836994005017914, + 0.006035561003955081, + 0.0018834860020433553, + 0.004099577003216837, + 0.00395853299414739, + 0.00695017799444031, + 0.0069628280034521595, + 0.003968842996982858, + 0.009907469000609126, + 0.0040763709985185415, + 0.003990640994743444, + 0.006998917000601068, + 0.006994586001383141, + 0.007997974993486423, + 0.005989559998852201, + 0.006068900001992006, + 0.004960814003425185, + 0.0029032279999228194, + 0.0020055770000908524, + 0.0019949529960285872, + 0.0031015029962873086, + 0.007898049996583723, + 0.006030497999745421, + 0.007963652002217714, + 0.0059932550066150725, + 0.003003220001119189, + 0.0029834360029781237, + 0.0029845200042473152, + 0.005005873004847672, + 0.0037488620000658557, + 0.004231840001011733, + 0.0050031189966830425, + 0.004989109002053738, + 0.0029865360047551803, + 0.007998398003110196, + 0.005997761996695772, + 0.002982243000587914, + 0.006997592994594015, + 0.0050008649996016175, + 0.003987349002272822, + 0.007008797001617495, + 0.006994805997237563, + 0.003990329001680948, + 0.004993396993086208, + 0.0060049710009479895, + 0.003976135994889773, + 0.006000102002872154, + 0.003994906001025811, + 0.004131052002776414, + 0.003861065997625701, + 0.0019984079990535975, + 0.007248100999277085, + 0.0037144779998925515, + 0.007005748004303314, + 0.005970552003418561, + 0.006982782993873116, + 0.005000488999939989, + 0.00598558100318769, + 0.004651408002246171, + 0.0015199219997157343, + 0.007890052998845931, + 0.006929865005076863, + 0.0029566489974968135, + 0.005018317999201827, + 0.0059767870043287985, + 0.006995269002800342, + 0.00598833299591206, + 0.004273892998753581, + 0.0037085610019857995, + 0.00699457099835854, + 0.008003130002180114, + 0.003986056995927356, + 0.004996277995815035, + 0.0049982620039372705, + 0.005003090998798143, + 0.00398404600127833, + 0.004999769997084513, + 0.006990196998231113, + 0.006992144997639116, + 0.0069991260024835356, + 0.02220677800505655, + 0.007768221003061626, + 0.003967384996940382, + 0.008005844996660016, + 0.007974235006258823, + 0.007990893995156512, + 0.00601404099870706, + 0.007968099002027884, + 0.0030201320041669533, + 0.0004088220011908561, + 0.00036636999720940366, + 0.00037117199826752767, + 0.0003634189997683279, + 0.0003634669992607087, + 0.000370935995306354, + 0.0003661839946289547, + 0.0003667880009743385, + 0.0003621120049501769, + 0.0003620260031311773, + 0.007315823997487314, + 0.0069251239983714186, + 0.006369351001922041, + 0.007620943004440051, + 0.006997975993726868, + 0.006994278999627568, + 0.003805975000432227, + 0.004184800993243698, + 0.003993451995484065, + 0.008002817994565703, + 0.00799490200006403, + 0.008994117000838742, + 0.007999512999958824, + 0.010993601004884113, + 0.0059996749987476505, + 0.010996719996910542, + 0.006990532005147543, + 0.006059437997464556, + 0.0019408379957894795, + 0.003995334998762701, + 0.005995772997266613, + 0.007992177001142409, + 0.006000369001412764, + 0.006991116002609488, + 0.006992873997660354, + 0.004996354997274466, + 0.006583873000636231, + 0.005408908000390511, + 0.004991009998775553, + 0.004991360001440626, + 0.007996903004823253, + 0.007992503997229505, + 0.007997396001883317, + 0.0069923080009175465, + 0.007000530000368599, + 0.006994200994085986, + 0.006992889997491147, + 0.007000242003414314, + 0.0049960629985434934, + 0.006995497999014333, + 0.006004242000926752, + 0.006992423004703596, + 0.005001697994885035, + 0.007994504005182534, + 0.006987407999986317, + 0.007006623003690038, + 0.008023807997233234, + 0.00700074100313941, + 0.005914276996918488, + 0.006982658000197262, + 0.0069879669972578995, + 0.005769253002654295, + 0.0022195179990376346, + 0.006100703998527024, + 0.0068956740069552325, + 0.0022707190000801347, + 0.004713260001153685, + 0.006008926997310482, + 0.007994865998625755, + 0.006989701003476512, + 0.006999610006459989, + 0.00038019999919924885, + 0.0003607379985623993, + 0.000359678000677377, + 0.000342024999554269, + 0.0003340600014780648, + 0.00034275500365765765, + 0.00033444000291638076, + 0.0003347550009493716, + 0.0003397879991098307, + 0.00032829499832587317, + 0.00033385199640179053, + 0.0032400960044469684, + 0.00267910700495122, + 0.0003622719959821552, + 0.0004232760038576089, + 0.00035042800300288945, + 0.0003514380005071871, + 0.0003385780000826344, + 0.0003336180016049184, + 0.0003391999998711981, + 0.00033151800016639754, + 0.00033411799813620746, + 0.004674144998716656, + 0.00037822000012965873, + 0.00036399300006451085, + 0.00033979499858105555, + 0.0003456570048001595, + 0.00034225299896206707, + 0.0003358670001034625, + 0.00034326599416090176, + 0.0003315639987704344, + 0.00033470900234533474, + 0.004648852998798247, + 0.0003947529985453002, + 0.00034836499980883673, + 0.00033140899904537946, + 0.00033136000274680555, + 0.0003468650029390119, + 0.00033352999889757484, + 0.00033622099726926535, + 0.00033960000291699544, + 0.0003304639976704493, + 0.00033253299625357613, + 0.0033019360053003766, + 0.0028909070024383254, + 0.00036094000097364187, + 0.00034945300285471603, + 0.0003576300005079247, + 0.000340604005032219, + 0.00033118599822046235, + 0.0003422679947107099, + 0.00033715899917297065, + 0.0003371359998709522, + 0.0003392059952602722, + 0.009871415000816341, + 0.0004598699961206876, + 0.00036238999746274203, + 0.0044453559967223555, + 0.000391028996091336, + 0.007933651002531406, + 0.00043638899660436437, + 0.0003691480014822446, + 0.00036464699951466173, + 0.0003646529949037358, + 0.00033874799555633217, + 0.00033858299866551533, + 0.00035280900192447007, + 0.0003860529977828264, + 0.0003385149975656532, + 0.005235490003542509, + 0.0005137510015629232, + 0.0003747999944607727, + 0.0003435970065766014, + 0.00033700900530675426, + 0.0003351990017108619, + 0.0003770569965126924, + 0.00037473500560736284, + 0.000366891996236518, + 0.000335090997396037, + 0.00033330100268358365, + 0.0038808640019851737, + 0.0003600959971663542, + 0.00034705899452092126, + 0.0003400389978196472, + 0.00037820199941052124, + 0.00034555100137367845, + 0.0003353710053488612, + 0.00033880999399116263, + 0.00033762300154194236, + 0.0003342970012454316, + 0.004795862005266827, + 0.00041822600178420544, + 0.0003875639959005639, + 0.0003597350005293265, + 0.0003843809972750023, + 0.0003396219981368631, + 0.0003342199997860007, + 0.0003431549994274974, + 0.0003396170068299398, + 0.0003413459999137558, + 0.0003403649971005507, + 0.0003317910013720393, + 0.000331916002323851, + 0.0047830339972279035, + 0.0004215219960315153, + 0.0006291540048550814, + 0.0006908770010340959, + 0.0003311749969725497, + 0.0003379789995960891, + 0.00033209400135092437, + 0.0003322529955767095, + 0.0003369330006535165, + 0.00033322799572488293, + 0.00033398000232409686, + 0.005139573993801605, + 0.007993505001650192, + 0.006997499003773555, + 0.005997887994453777, + 0.007997788998181932, + 0.005543473998841364, + 0.006455506001657341, + 0.0059923549997620285, + 0.009000566002214327, + 0.006992834001721349, + 0.006998245997237973, + 0.006999686003837269, + 0.006994905001192819, + 0.005996961997880135, + 0.007004384002357256, + 0.005055669003922958, + 0.003930754006432835, + 0.005997523003316019, + 0.005999368004268035, + 0.00799819399981061, + 0.005989244004013017, + 0.005998091997753363, + 0.004995991999749094, + 0.007003121994785033, + 0.001014364002912771, + 0.00035623699659481645, + 0.00033610100217629224, + 0.00034059100289596245, + 0.0003591780041460879, + 0.0003583719953894615, + 0.00034291699557797983, + 0.0003401989961275831, + 0.0003293350018793717, + 0.0021546900024986826, + 0.0012101069951313548, + 0.0003408539996598847, + 0.00033855299989227206, + 0.00033564700424904004, + 0.0003339480026625097, + 0.000329869995766785, + 0.00034163700183853507, + 0.000335248994815629, + 0.0004392349947011098, + 0.00035336000291863456, + 0.000402956000471022, + 0.0020952170016244054, + 0.0031527499959338456, + 0.00040201299998443574, + 0.0003464960027486086, + 0.00035235499672126025, + 0.0003339810064062476, + 0.00033491800422780216, + 0.0018938560024253093, + 0.0003596980022848584, + 0.0003440849977778271, + 0.00033852000342449173, + 0.0003516039942041971, + 0.0003495989949442446, + 0.000336951001372654, + 0.000342751998687163, + 0.00034346199390711263, + 0.00033041000278899446, + 0.005269553999823984, + 0.0003772729978663847, + 0.0003476799975032918, + 0.00034643400431377813, + 0.0003411349971429445, + 0.0003346419980516657, + 0.00034132500150008127, + 0.00033746700501069427, + 0.00039903500146465376, + 0.00038175599911483005, + 0.0004014200021629222, + 0.006077894999179989, + 0.00039378300425596535, + 0.0003647460034699179, + 0.0007571269961772487, + 0.0003702429967233911, + 0.0003381489950697869, + 0.00034007099748123437, + 0.0003304999991087243, + 0.005017094998038374, + 0.00044784799683839083, + 0.0003641460061771795, + 0.00044414900185074657, + 0.0003833470036624931, + 0.00034342599974479526, + 0.003236902004573494, + 0.00039766699774190784, + 0.0003715429993462749, + 0.0015808429961907677, + 0.0003577600000426173, + 0.00033825499849626794, + 0.00033480000274721533, + 0.0003303320045233704, + 0.00034094799775630236, + 0.0003482749962131493, + 0.00033188700035680085, + 0.0003332490014145151, + 0.0003362460047355853, + 0.00033337600325467065, + 0.005469229996378999, + 0.00044032500591129065, + 0.0003924830016330816, + 0.00033893399813678116, + 0.0009361509946756996, + 0.00036659000033978373, + 0.0006687290006084368, + 0.00036625799839384854, + 0.0034799009954440407, + 0.0003762699998333119, + 0.0003551049958332442, + 0.00033677800092846155, + 0.00033271100255660713, + 0.000333559000864625, + 0.0003391619975445792, + 0.0003354069995111786, + 0.0003414370003156364, + 0.000340379003318958, + 0.00033191499824170023, + 0.0003343360003782436, + 0.005299935000948608, + 0.0005588900021393783, + 0.0005593329988187179, + 0.00034158400376327336, + 0.0003470129959168844, + 0.00033811300090746954, + 0.0003354170039528981, + 0.00033582199830561876, + 0.00035420100175542757, + 0.00034639400109881535, + 0.004854209000768606, + 0.000408816005801782, + 0.0003626260004239157, + 0.00035066699638264254, + 0.0003407279946259223, + 0.000333186995703727, + 0.000329132002661936, + 0.00033963700116146356, + 0.00033384200651198626, + 0.0003325529978610575, + 0.0003363049981999211, + 0.000331564006046392, + 0.000333447998855263, + 0.003812912000284996, + 0.001285672995436471, + 0.0004290040014893748, + 0.00037456300196936354, + 0.0006105889988248236, + 0.00035072000173386186, + 0.0008358830018551089, + 0.00034709899773588404, + 0.00033726400579325855, + 0.0003391520003788173, + 0.00033822099794633687, + 0.0003386390017112717, + 0.00033286799589404836, + 0.0003294669950264506, + 0.0029699339938815683, + 0.00037547899410128593, + 0.0003631110012065619, + 0.0003601459975470789, + 0.0003648019992397167, + 0.0003356909946887754, + 0.0022922220014152117, + 0.0003728119991137646, + 0.00035180600389139727, + 0.0003969649987993762, + 0.0003314279965707101, + 0.00032994200591929257, + 0.00037567999970633537, + 0.0003479879960650578, + 0.0003328360035084188, + 0.004702327001723461, + 0.0004018280014861375, + 0.0003665550029836595, + 0.0003409609998925589, + 0.0003377940010977909, + 0.0003789360052905977, + 0.00037018700095359236, + 0.0007959619979374111, + 0.0003891440064762719, + 0.002614908000396099, + 0.00036073500086786225, + 0.0003588279942050576, + 0.00033485300082247704, + 0.00033470900234533474, + 0.00033754600008251145, + 0.00033177300065290183, + 0.00036730599822476506, + 0.0006849120036349632, + 0.00259893399925204, + 0.0003611189968069084, + 0.00035471100272843614, + 0.00034490899997763336, + 0.0003339670001878403, + 0.00037026599602540955, + 0.0003331420011818409, + 0.00033969400101341307, + 0.00041550499736331403, + 0.00034713599598035216, + 0.0003303679986856878, + 0.005075136999948882, + 0.0009563219937263057, + 0.00036515599640551955, + 0.00033766999695217237, + 0.00033126900234492496, + 0.00034037000295938924, + 0.0003348910031490959, + 0.0003285869970568456, + 0.00033874100336106494, + 0.0033243580037378706, + 0.0013972380038467236, + 0.00034245300048496574, + 0.0003298470037407242, + 0.00035028000274905935, + 0.0003343270000186749, + 0.00032949799788184464, + 0.001198190002469346, + 0.0007047780018183403, + 0.0010975190016324632, + 0.0006642439984716475, + 0.0005068199970992282, + 0.0003804599982686341, + 0.001363026000035461, + 0.001266707004106138, + 0.00040148400148609653, + 0.0009699019938125275, + 0.0014226780040189624, + 0.00035355400177650154, + 0.00034656200296012685, + 0.0003322729971841909, + 0.00040867699863156304, + 0.0011120729977847077, + 0.0010177989970543422, + 0.0003477339996607043, + 0.0003312900007585995, + 0.0003360230039106682, + 0.0003313410052214749, + 0.0003311179971206002, + 0.0003546880034264177, + 0.00033046200405806303, + 0.000550528995518107, + 0.00034687399602262303, + 0.0003316630027256906, + 0.0030811129981884733, + 0.0003972119957325049, + 0.00036533999809762463, + 0.0018188899994129315, + 0.0007284880048246123, + 0.0005999049972160719, + 0.0003538269957061857, + 0.00033275199530180544, + 0.0010128290014108643, + 0.0004099259967915714, + 0.00048682299529900774, + 0.0003428719937801361, + 0.000345947002642788, + 0.00033115900441771373, + 0.00033829399762907997, + 0.002136458999302704, + 0.0003580739939934574, + 0.00035494000621838495, + 0.00035566599399317056, + 0.0003374419975443743, + 0.00033380100649083033, + 0.0003340300027048215, + 0.00046059900341788307, + 0.0003366089949849993, + 0.0003396460015210323, + 0.0004900620042462833, + 0.0004729959982796572, + 0.0004402699996717274, + 0.00038046999543439597, + 0.0007025980012258515, + 0.00041856700408970937, + 0.000543087997357361, + 0.0009719109948491678, + 0.0021323830005712807, + 0.0003671590020530857, + 0.00034345199674135074, + 0.00043884000479010865, + 0.0003524049971019849, + 0.0003333000058773905, + 0.0006396320022759028, + 0.00035688700154423714, + 0.0003381989954505116, + 0.005003923004551325, + 0.005997938002110459, + 0.005999710003379732, + 0.00534947599953739, + 0.006650882001849823, + 0.007989257996086963, + 0.006994721996306907, + 0.007997549000720028, + 0.006000475004839245, + 0.007992471997567918, + 0.007000270001299214, + 0.0069940649991622195, + 0.006123029001173563, + 0.007876436997321434, + 0.005992509999487083, + 0.005996497995511163, + 0.005996592000883538, + 0.008998211000289302, + 0.006997957003477495, + 0.007997235996299423, + 0.005996894993586466, + 0.007997346001502592, + 0.006999543998972513, + 0.00799531300435774, + 0.006996903000981547, + 0.007997668006282765, + 0.00699722900026245, + 0.006997065000177827, + 0.0021210720005910844, + 0.005873653994058259, + 0.005996575993776787, + 0.007998480003152508, + 0.00799729199934518, + 0.005998668995744083, + 0.0069971159973647445, + 0.006995693998760544, + 0.005997504005790688, + 0.0059976850025122985, + 0.006997693999437615, + 0.0059973619936499745, + 0.006997626995143946, + 0.0060047179940738715, + 0.006989596004132181, + 0.007996979999006726, + 0.005997313004627358, + 0.006007017000229098, + 0.007990345999132842, + 0.006000303998007439, + 0.00799194900173461, + 0.005996451996907126, + 0.00800452999828849, + 0.005991685997287277, + 0.005995174004056025, + 0.0069953999991412275, + 0.0036488920013653114, + 0.0063485140053671785, + 0.00799561199528398, + 0.0029933170008007437, + 0.004827908000152092, + 0.0061674959943047725, + 0.008998534001875669, + 0.005993191996822134, + 0.006999769000685774, + 0.006989927998802159, + 0.006998434997512959, + 0.007997590997547377, + 0.008221493000746705, + 0.006822995994298253, + 0.007967829995322973, + 0.00818151599378325, + 0.007781528001942206, + 0.007996264001121745, + 0.004000038999947719, + 0.007992355996975675, + 0.00803933700080961, + 0.003953842999180779, + 0.007995790001587011, + 0.006997349002631381, + 0.007995181003934704, + 0.006994249997660518, + 0.004003722999186721, + 0.0019910820046789013, + 0.006014858001435641, + 0.00797651699394919, + 0.006995037001615856, + 0.0069948669988662004, + 0.005936735004070215, + 0.004056620004121214, + 0.007998044995474629, + 0.007998270004463848, + 0.005988349003018811, + 0.009012355003505945, + 0.00698074400133919, + 0.010996320001140703, + 0.005303003003064077, + 0.009051724002347328, + 0.0068764719981118105, + 0.00774690000253031, + 0.00400462000106927, + 0.0021951539965812117, + 0.0004182010015938431, + 0.00039402599941240624, + 0.0003839780038106255, + 0.00037358299596235156, + 0.00036949400237062946, + 0.0005243510022410192, + 0.0004104570034542121, + 0.0003876589980791323, + 0.0003887369966832921, + 0.00039736100006848574, + 0.0003906290003214963, + 0.0003792569987126626, + 0.0003695279956446029, + 0.0003657159977592528, + 0.00037451000389410183, + 0.0003658909990917891, + 0.000364948995411396, + 0.011098492999735754, + 0.006995621995883994, + 0.007995075000508223, + 0.007996273998287506, + 0.006995712996285874, + 0.006998084005317651, + 0.007996428998012561, + 0.0069956919978722, + 0.005997360000037588, + 0.005994390994601417, + 0.00800274599896511, + 0.00799267399997916, + 0.006995401003223378, + 0.0080002750037238, + 0.007995681997272186, + 0.007992900005774572, + 0.007995371997822076, + 0.007051690001389943, + 0.005965917996945791, + 0.005963916002656333, + 0.008504183002514765, + 0.008149733002937865, + 0.004430597000464331, + 0.0029058610016363673, + 0.0011942669952986762, + 0.007604354002978653, + 0.005183182001928799, + 0.00500052300048992, + 0.006009607001033146, + 0.011053734997403808, + 0.008925728994654492, + 0.004025303001981229, + 0.009668286998930853, + 0.00691161699796794, + 0.004373660995042883, + 0.004575676997774281, + 0.004413971000758465, + 0.0036455899971770123, + 0.006490191000921186, + 0.004859070999373216, + 0.0029956060025142506, + 0.007995987005415373, + 0.00799724899843568, + 0.008040572996833362, + 0.00594420400011586, + 0.009001247999549378, + 0.008029924996662885, + 0.007952166000904981, + 0.007001766003668308, + 0.006979294994380325, + 0.006114135998359416, + 0.004955919001076836, + 0.009919262003677431, + 0.004778567003086209, + 0.005205548004596494, + 0.00599804800003767, + 0.00599569999758387, + 0.007992536993697286, + 0.007995772997674067, + 0.00599810600397177, + 0.0071082560025388375, + 0.003726499999174848, + 0.004033803998026997, + 0.006968291003431659, + 0.007355195004492998, + 0.004782460993737914, + 0.006984003004617989, + 0.007093458996678237, + 0.007893705995229539, + 0.007994055005838163, + 0.004994954004359897, + 0.001986459996260237, + 0.007000956000410952, + 0.0059957080011372454 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil", + "fullname": "TestCalculateNabla2ForZ", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.00035895800101570785, + "max": 0.016018908994738013, + "mean": 0.0027472810595105318, + "stddev": 0.0031088569312507455, + "rounds": 2338, + "median": 0.0005580219985859003, + "iqr": 0.005586072002188303, + "q1": 0.00038300600135698915, + "q3": 0.005969078003545292, + "iqr_outliers": 3, + "stddev_outliers": 591, + "outliers": "591;3", + "ld15iqr": 0.00035895800101570785, + "hd15iqr": 0.014607814999180846, + "ops": 363.9962487777514, + "total": 6.4231431171356235, + "data": [ + 0.0004140509990975261, + 0.00044366600195644423, + 0.0004188559978501871, + 0.00041580700053600594, + 0.005287389998557046, + 0.0005029899984947406, + 0.0005002940015401691, + 0.0004611670010490343, + 0.00041869200504152104, + 0.00042370099981781095, + 0.0004143650003243238, + 0.0004359239974291995, + 0.00042533400119282305, + 0.0038606399975833483, + 0.001633487998333294, + 0.0004825510040973313, + 0.0005229309972492047, + 0.0004471479987842031, + 0.00047233199438778684, + 0.0004306800037738867, + 0.00041627299651736394, + 0.0004221319977659732, + 0.0004166260041529313, + 0.0034631289963726886, + 0.0005176039994694293, + 0.0005098589972476475, + 0.0004349829978309572, + 0.00042922399734379724, + 0.0004168619998381473, + 0.0004268080010660924, + 0.00042930500057991594, + 0.00042749400017783046, + 0.006008299998939037, + 0.0005888410014449619, + 0.0005254650022834539, + 0.0011446429998613894, + 0.00043248000292805955, + 0.00042646699876058847, + 0.0004225630036671646, + 0.0010899629996856675, + 0.0023013120007817633, + 0.00045683000644203275, + 0.00043338200339348987, + 0.00043205700058024377, + 0.0004286640032660216, + 0.0004370280003058724, + 0.0004323449975345284, + 0.00042588100041029975, + 0.002596524005639367, + 0.0018057289998978376, + 0.0004780590024893172, + 0.000468391997856088, + 0.0004647800014936365, + 0.00042600299639161676, + 0.0004159840027568862, + 0.001228666995302774, + 0.0006767410013708286, + 0.0004356989957159385, + 0.0004229130063322373, + 0.000421370001276955, + 0.003198434002115391, + 0.00046781599667156115, + 0.0004300419968785718, + 0.0004553840044536628, + 0.002133403002517298, + 0.0007502570006181486, + 0.0004495730026974343, + 0.0028173550017527305, + 0.00045386199781205505, + 0.0004539360015769489, + 0.0004373980045784265, + 0.0006019520005793311, + 0.0004231409984640777, + 0.0004594280035234988, + 0.00042466699960641563, + 0.002762434996839147, + 0.0024598960008006543, + 0.00043407100019976497, + 0.00041407099342904985, + 0.00041877300100168213, + 0.0004063140004291199, + 0.00045782300003338605, + 0.0012933019970660098, + 0.002371221999055706, + 0.000522616996022407, + 0.0004471749998629093, + 0.003189793002093211, + 0.0004897629987681285, + 0.00047349300439236686, + 0.0004688070039264858, + 0.00041774199780775234, + 0.00041000400233315304, + 0.00043254900083411485, + 0.001120640998124145, + 0.007742198999039829, + 0.0005203279943089001, + 0.0004594529964379035, + 0.00044730299850925803, + 0.00041459599742665887, + 0.00040717799856793135, + 0.0005344809978851117, + 0.00042000700341304764, + 0.00743685600173194, + 0.0005363379968912341, + 0.00045368399878498167, + 0.00041070699808187783, + 0.00042312999721616507, + 0.0004169890016783029, + 0.00041188299655914307, + 0.00041509799484629184, + 0.00041877500189002603, + 0.007403035000606906, + 0.0005669579986715689, + 0.0005736629973398522, + 0.00045636600407306105, + 0.0014895029962644912, + 0.0004575099956127815, + 0.00920232100179419, + 0.0008042279951041564, + 0.0018714010002440773, + 0.008289607998449355, + 0.0006571490011992864, + 0.0004916659963782877, + 0.00042833499901462346, + 0.004470268002478406, + 0.003996019993792288, + 0.007993464001629036, + 0.007006854997598566, + 0.00798574999498669, + 0.007993809995241463, + 0.007992188002390321, + 0.008012352998775896, + 0.006981681006436702, + 0.010151741997106, + 0.0038314729972626083, + 0.01099706300010439, + 0.007055682995996904, + 0.007931519001431298, + 0.006980485995882191, + 0.008237932001065928, + 0.00575435500650201, + 0.005990351994114462, + 0.008006046002265066, + 0.005989697005134076, + 0.006985066000197548, + 0.007000675999734085, + 0.006986558000789955, + 0.007997806002094876, + 0.00799000099505065, + 0.005004209000617266, + 0.0060467719958978705, + 0.007936084002722055, + 0.009087471997190733, + 0.0059015250008087605, + 0.011090057996625546, + 0.0068985129983047955, + 0.006997726995905396, + 0.006995210998866241, + 0.007996459004061762, + 0.005102041002828628, + 0.006896781997056678, + 0.004989832996216137, + 0.003995190003479365, + 0.0070019519989728, + 0.008055153994064312, + 0.006939474995306227, + 0.006998568998824339, + 0.005405789001088124, + 0.008592287005740218, + 0.007063969002047088, + 0.006919307001226116, + 0.006991959999140818, + 0.007004003004112747, + 0.006991059999563731, + 0.005992942999000661, + 0.006993287999648601, + 0.004992825000954326, + 0.005008669999369886, + 0.006987597997067496, + 0.005039597999711987, + 0.003943995005101897, + 0.010000472000683658, + 0.005943233998550568, + 0.00405624600534793, + 0.003983121001510881, + 0.009005366002384108, + 0.007165765004174318, + 0.00637686499976553, + 0.004438549003680237, + 0.005253150004136842, + 0.0037401330046122894, + 0.006982700004300568, + 0.007999766996363178, + 0.005997486994601786, + 0.007993342995177954, + 0.006560968002304435, + 0.0054232699985732324, + 0.005995608997181989, + 0.007001988000411075, + 0.004034286001115106, + 0.0029534819987020455, + 0.006000722001772374, + 0.00909722299547866, + 0.003919066999515053, + 0.00047703299787826836, + 0.00045021699770586565, + 0.00042362399835838005, + 0.0003818460027105175, + 0.0020734770005219616, + 0.0004013919970020652, + 0.00038493899774039164, + 0.003833574999589473, + 0.0005231989998719655, + 0.00045532100193668157, + 0.0003863829988404177, + 0.005379233996791299, + 0.00975620200188132, + 0.004988663000403903, + 0.00508182100020349, + 0.007916356000350788, + 0.00600334900082089, + 0.00798862199735595, + 0.007999109002412297, + 0.0029962150001665577, + 0.006995124000241049, + 0.005019011005060747, + 0.0029705820052186027, + 0.005988978999084793, + 0.007000502999289893, + 0.006998679004027508, + 0.005995452003844548, + 0.004922141000861302, + 0.011071179993450642, + 0.010001913004089147, + 0.0059880420012632385, + 0.00799023600120563, + 0.006985281994275283, + 0.005342881995602511, + 0.004654676005884539, + 0.006708577006065752, + 0.0033227650055778213, + 0.00405949899868574, + 0.00522220300626941, + 0.003677854998386465, + 0.006977454999287147, + 0.004993960996216629, + 0.0070042020015534945, + 0.007991542996023782, + 0.00799761700181989, + 0.011998729001788888, + 0.006986483997025061, + 0.007998117005627137, + 0.006991756999923382, + 0.008996274002129212, + 0.006994299001235049, + 0.006997849995968863, + 0.007996533997356892, + 0.006995314004598185, + 0.007000033001531847, + 0.005992521000734996, + 0.007997998000064399, + 0.005995808998704888, + 0.010005408999859355, + 0.007989370998984668, + 0.008372561998839956, + 0.006631907002883963, + 0.004820007998205256, + 0.0031683370034443215, + 0.003985973999078851, + 0.0050001690033241175, + 0.007291185000212863, + 0.007695932996284682, + 0.009992970997700468, + 0.007000969002547208, + 0.006680530997982714, + 0.005310260006808676, + 0.005998469001497142, + 0.006992440998146776, + 0.007997619999514427, + 0.005997407002723776, + 0.0069947509982739575, + 0.008001032001629937, + 0.009015193005325273, + 0.006964349995541852, + 0.002990836997923907, + 0.004999479999241885, + 0.006002023998007644, + 0.0070967869978630915, + 0.008894918995792978, + 0.006716226998833008, + 0.006260681999265216, + 0.004000174994871486, + 0.0005162379966350272, + 0.000446805999672506, + 0.00042101899452973157, + 0.00038525900163222104, + 0.0009576969969202764, + 0.0004785639976034872, + 0.00039591499808011577, + 0.007426702999509871, + 0.0004925419998471625, + 0.0004554859988274984, + 0.0004198989990982227, + 0.0003831879948847927, + 0.0003871300068567507, + 0.0006031629964127205, + 0.00047457100299652666, + 0.0003988349999417551, + 0.008095345998299308, + 0.0004911629948765039, + 0.000431597996794153, + 0.0005431540048448369, + 0.00037205099943093956, + 0.00037054499989608303, + 0.000371279995306395, + 0.0003675010011647828, + 0.006601565000892151, + 0.0005023050034651533, + 0.00045903900172561407, + 0.00037458999577211216, + 0.0003755930010811426, + 0.004293347999919206, + 0.00048046900337794796, + 0.0004466960017452948, + 0.0004096850025234744, + 0.002286572998855263, + 0.00042334199679316953, + 0.0006345540023176, + 0.00044504999823402613, + 0.00037472300027729943, + 0.00038213100197026506, + 0.0003714639969985001, + 0.000372456997865811, + 0.00037282499397406355, + 0.00037400000292109326, + 0.0003761970001505688, + 0.00040711300243856385, + 0.000409926004067529, + 0.0004354639968369156, + 0.0004063399974256754, + 0.0004020970009150915, + 0.0003857749979943037, + 0.00039673099672654644, + 0.00041930199950002134, + 0.00040859400178305805, + 0.00039602400647709146, + 0.00037990800046827644, + 0.00037522999627981335, + 0.0003735190039151348, + 0.0003698680011439137, + 0.0013711850042454898, + 0.00043276999349473044, + 0.00041099100053543225, + 0.0005583059974014759, + 0.0003968129967688583, + 0.0003834180024568923, + 0.0014161520011839457, + 0.00038436999602708966, + 0.00037100099871167913, + 0.0003752140037249774, + 0.0003847600019071251, + 0.0003707860014401376, + 0.0031067880045156926, + 0.0004649599941330962, + 0.0004370840033516288, + 0.00044596799853025004, + 0.0004209020044072531, + 0.00037809700006619096, + 0.00037441300082718953, + 0.0003733589983312413, + 0.005021436998504214, + 0.00047339600132545456, + 0.000403789003030397, + 0.0004259729976183735, + 0.0003830420027952641, + 0.00038196099922060966, + 0.00038751200190745294, + 0.0003748720046132803, + 0.0003712269972311333, + 0.004436781004187651, + 0.00045262800267664716, + 0.00044063299719709903, + 0.00041149599564960226, + 0.0004173500055912882, + 0.00038459899951703846, + 0.0003734459969564341, + 0.00037860700103919953, + 0.0004138079966651276, + 0.003187724003510084, + 0.00044432499998947605, + 0.0004359540034784004, + 0.003351890998601448, + 0.0004867919997195713, + 0.00044638399413088337, + 0.00044518199865706265, + 0.00039282899524550885, + 0.003055229004530702, + 0.00045554599637398496, + 0.00042501799907768145, + 0.0009560520047671162, + 0.00039523200393887237, + 0.0003944220006815158, + 0.00041568199958419427, + 0.00041326000064145774, + 0.0003843099984806031, + 0.00333593699906487, + 0.0005085590019007213, + 0.0004513300009421073, + 0.00043278600060148165, + 0.00037455900019267574, + 0.0003749859970412217, + 0.0003717029976542108, + 0.0003765419969568029, + 0.00041334699926665053, + 0.00038061100349295884, + 0.0030726309996680357, + 0.0004449190018931404, + 0.00043227399874012917, + 0.00043478500447236, + 0.00038915300683584064, + 0.0003676210035337135, + 0.000371841000742279, + 0.0003703280017361976, + 0.005234926000412088, + 0.0004880820051766932, + 0.0004709369968622923, + 0.0004779480004799552, + 0.0004016839957330376, + 0.0003816539974650368, + 0.0003701519963215105, + 0.0003687549979076721, + 0.0006257210043258965, + 0.003848644999379758, + 0.0005417880020104349, + 0.0004994330010958947, + 0.00037106799572939053, + 0.0003791510025621392, + 0.00037546100065810606, + 0.00037531700218096375, + 0.0003843399972538464, + 0.0003697080028359778, + 0.00041938100184779614, + 0.0005174370016902685, + 0.0004850650002481416, + 0.00044155100476928055, + 0.0005484049979713745, + 0.0006291190002229996, + 0.0029671660013264045, + 0.00044863000221084803, + 0.00040304900176124647, + 0.0004024989975732751, + 0.0003937499932362698, + 0.00037947300006635487, + 0.0026863800012506545, + 0.00048358899948652834, + 0.00042644399945857003, + 0.00039236299926415086, + 0.00037748699833173305, + 0.0019357109995326027, + 0.007078795002598781, + 0.0016596900022705086, + 0.00566951999644516, + 0.0018750140006886795, + 0.0004440789998625405, + 0.00039388800360029563, + 0.0003890369989676401, + 0.004531346996373031, + 0.0011955500012845732, + 0.000605918001383543, + 0.0071289260013145395, + 0.0006917010032339022, + 0.00046417499834205955, + 0.0008108300025924109, + 0.003696810999826994, + 0.0016490959969814867, + 0.00047164699935819954, + 0.0005271510017337278, + 0.00039817200013203546, + 0.006119832003605552, + 0.0005535480013350025, + 0.001832829999329988, + 0.0004921150029986165, + 0.005663279000145849, + 0.0023323849964071997, + 0.005370434999349527, + 0.0005251860056887381, + 0.0029006459953961894, + 0.0036518920023809187, + 0.0032011270013754256, + 0.0005239019956206903, + 0.0006195989990374073, + 0.004058310994878411, + 0.0010147129942197353, + 0.00047914800234138966, + 0.00040908200026024133, + 0.002113497997925151, + 0.005144672002643347, + 0.0011181020017829724, + 0.000633118994301185, + 0.002138050003850367, + 0.002395135998085607, + 0.000770108999859076, + 0.002512959996238351, + 0.0008428569999523461, + 0.0006617850012844428, + 0.0004497659974731505, + 0.004221559996949509, + 0.0014367510011652485, + 0.0008608889984316193, + 0.00046497499715769663, + 0.004852925005252473, + 0.0017218450011569075, + 0.001360777001536917, + 0.0004069110000273213, + 0.005697462001990061, + 0.0019709530024556443, + 0.003067439000005834, + 0.0029213170055299997, + 0.0012776119983755052, + 0.0005419920053100213, + 0.0005085560042061843, + 0.004000357999757398, + 0.007411423997837119, + 0.0007262400031322613, + 0.0004448079998837784, + 0.00701927300542593, + 0.0015292329990188591, + 0.0005912029955652542, + 0.0023391270005959086, + 0.004260970999894198, + 0.0013137080022715963, + 0.0007042680008453317, + 0.0004685160020017065, + 0.002293539000675082, + 0.0007311989975278266, + 0.0012841510033467785, + 0.0005899280004086904, + 0.002732073000515811, + 0.006643168002483435, + 0.0006460590011556633, + 0.0035388070027693175, + 0.0032595989978290163, + 0.0032976099973893724, + 0.004710697001428343, + 0.007538799996837042, + 0.0036577430000761524, + 0.0011083299978054129, + 0.006336974001897033, + 0.0006778679962735623, + 0.0004269880009815097, + 0.00040404299943475053, + 0.00041804399370448664, + 0.00039801600360078737, + 0.005278955002722796, + 0.006996829004492611, + 0.0072004190005827695, + 0.008788207000179682, + 0.010236914000415709, + 0.0008257970039267093, + 0.0013575209959526546, + 0.007510394003475085, + 0.0006660069993813522, + 0.00046391999785555527, + 0.0004260120040271431, + 0.0014344890005304478, + 0.006867966003483161, + 0.002306941998540424, + 0.0004818330053240061, + 0.0077698790046270005, + 0.0011752060017897747, + 0.0033538440038682893, + 0.006264814997848589, + 0.0006612220022361726, + 0.0005438320004031993, + 0.0005004809936508536, + 0.00041933000466087833, + 0.00040901800093706697, + 0.0004993190013919957, + 0.0004193109998595901, + 0.008265176002169028, + 0.0016342390008503571, + 0.0007132280006771907, + 0.005612319000647403, + 0.0026217100021312945, + 0.0005591770022874698, + 0.0004446340026333928, + 0.006918474995472934, + 0.0008524849981768057, + 0.0004452949942788109, + 0.0004235079977661371, + 0.0004092109957127832, + 0.0051381769953877665, + 0.0006321320033748634, + 0.0005467570008477196, + 0.00046061100147198886, + 0.0004221990020596422, + 0.0004259710040059872, + 0.00040692000038689, + 0.004883347006398253, + 0.0037136789978831075, + 0.0024061520016402937, + 0.0010941309956251644, + 0.0004674359952332452, + 0.0004327339993324131, + 0.00042928699986077845, + 0.0004223180003464222, + 0.0005450440003187396, + 0.00047538199578411877, + 0.0004660600025090389, + 0.0004607880036928691, + 0.00047118299698922783, + 0.0004140960008953698, + 0.0004090729999006726, + 0.0030885240048519336, + 0.0005257860029814765, + 0.0004529699945123866, + 0.0004607459995895624, + 0.0005112689977977425, + 0.00041189700277755037, + 0.00040510699909646064, + 0.0006782370037399232, + 0.0004626079971785657, + 0.0037757160025648773, + 0.002997572002641391, + 0.0004980010053259321, + 0.00043811099749291316, + 0.0004135009949095547, + 0.000407152998377569, + 0.00043478600127855316, + 0.00044380599865689874, + 0.006973716997890733, + 0.0006104500062065199, + 0.00045188099466031417, + 0.0006069959999877028, + 0.0005332700020517223, + 0.0004251040008966811, + 0.00040936000004876405, + 0.00040821100265020505, + 0.007251574999827426, + 0.0005337359980330803, + 0.00043636400368995965, + 0.0004115800038562156, + 0.00041839899495244026, + 0.0004104810068383813, + 0.00040689200250199065, + 0.0004119930017623119, + 0.0004058400008943863, + 0.00040150699351215735, + 0.00816755200503394, + 0.0005207549984334037, + 0.00047659099800512195, + 0.0004472540022106841, + 0.00039542200102005154, + 0.0003901529998984188, + 0.00038995000068098307, + 0.0007088830025168136, + 0.00045397099893307313, + 0.005952709005214274, + 0.000522521004313603, + 0.0005918890019529499, + 0.0006295420025708154, + 0.000444516001152806, + 0.0004103290048078634, + 0.0003985980001743883, + 0.005972048995317891, + 0.0007233290016301908, + 0.00041706399497343227, + 0.0006839470006525517, + 0.00044549599988386035, + 0.00039563899917993695, + 0.0004026020033052191, + 0.0066707980004139245, + 0.0004651589988498017, + 0.0004059530037920922, + 0.00039631599793210626, + 0.00040371900104219094, + 0.0003996820014435798, + 0.0004017570026917383, + 0.0003884250036207959, + 0.00039489300252171233, + 0.008062321001489181, + 0.005989365999994334, + 0.006995048999669962, + 0.00549262399727013, + 0.009500183005002327, + 0.007034439004200976, + 0.007976654000231065, + 0.006966836997889914, + 0.007998660003067926, + 0.006998270000622142, + 0.005994515995553229, + 0.007788286995491944, + 0.00814127099874895, + 0.008383763997699134, + 0.009061459997610655, + 0.0072888430004240945, + 0.0076302029992803, + 0.004655575001379475, + 0.0050043709998135455, + 0.007055861999106128, + 0.007431532001646701, + 0.006494246001238935, + 0.008014195002033375, + 0.007959402006235905, + 0.005904103003558703, + 0.006946082001377363, + 0.006112585993832909, + 0.0073237659962615, + 0.009799597995879594, + 0.007870788002037443, + 0.006952035997528583, + 0.00599970299663255, + 0.00899260299775051, + 0.008031704004679341, + 0.007469238000339828, + 0.00557438100076979, + 0.005762018001405522, + 0.0011323130020173267, + 0.0017735719957272522, + 0.005514348995347973, + 0.0020801889986614697, + 0.002931732997240033, + 0.005659451999235898, + 0.0019531579964677803, + 0.0005477060039993376, + 0.002013669000007212, + 0.007304304002900608, + 0.00696612899628235, + 0.004509796002821531, + 0.002382320999458898, + 0.002334796001377981, + 0.005328131002897862, + 0.005749876996560488, + 0.003648542995506432, + 0.004370138005469926, + 0.0009830800045165233, + 0.0005173140016268007, + 0.0012401150015648454, + 0.007537277000665199, + 0.0039038280010572635, + 0.0005995229948894121, + 0.0004459850024431944, + 0.0025335269965580665, + 0.0063028099975781515, + 0.0014205199986463413, + 0.005969078003545292, + 0.0029240090007078834, + 0.00048398699436802417, + 0.00043285900028422475, + 0.006426681997254491, + 0.0018499639991205186, + 0.007073485998262186, + 0.0007830870017642155, + 0.006390301998180803, + 0.001863947996753268, + 0.0004560679954010993, + 0.0005385429976740852, + 0.0069591079954989254, + 0.0007190760006778874, + 0.0005039409952587448, + 0.00044664100278168917, + 0.006279884997638874, + 0.0006280640009208582, + 0.0004427640014910139, + 0.00043123800423927605, + 0.0004198809983790852, + 0.005387548997532576, + 0.0008363640008610673, + 0.0004454970039660111, + 0.004930656003125478, + 0.008288304001325741, + 0.011370198997610714, + 0.003674066007079091, + 0.0009259719954570755, + 0.0005051629996160045, + 0.0004419430042617023, + 0.006557111002621241, + 0.010046265000710264, + 0.0023348600007011555, + 0.00898833400424337, + 0.005144144997757394, + 0.0006461419980041683, + 0.00618706300156191, + 0.0027538770009414293, + 0.006450879001931753, + 0.00959150600101566, + 0.0067002759969909675, + 0.010779655996884685, + 0.00690123100503115, + 0.008162703998095822, + 0.003496016994176898, + 0.005688471996109001, + 0.008859174005920067, + 0.014607814999180846, + 0.005277606003801338, + 0.0024553809998906218, + 0.007817917001375463, + 0.007527359004598111, + 0.01041611500113504, + 0.007827055997040588, + 0.00702271300542634, + 0.006624101995839737, + 0.010486819002835546, + 0.009321360004832968, + 0.012841766998462845, + 0.01473491499928059, + 0.01000237599510001, + 0.010249975995975547, + 0.009740733003127389, + 0.00698519100114936, + 0.011409195001760963, + 0.005558388002100401, + 0.010003848001360893, + 0.00696829999651527, + 0.008986134998849593, + 0.00841199400019832, + 0.011303933002636768, + 0.006248430996492971, + 0.008989720001409296, + 0.007985351003299002, + 0.010375309000664856, + 0.005603840996627696, + 0.007997354005055968, + 0.009008136003103573, + 0.004969843997969292, + 0.008013299004232977, + 0.00887489099841332, + 0.006097329001931939, + 0.005988717995933257, + 0.00883997899654787, + 0.004148940999584738, + 0.007996148997335695, + 0.007994526000402402, + 0.005993952996504959, + 0.0050044070012518205, + 0.006989510002313182, + 0.00900041899876669, + 0.006989186003920622, + 0.0034071879999828525, + 0.011005698004737496, + 0.002569539996329695, + 0.006366059998981655, + 0.007679649002966471, + 0.007944228003907483, + 0.006993106995651033, + 0.006996018004429061, + 0.006176098999276292, + 0.007964346004882827, + 0.005842778002261184, + 0.0029894340041209944, + 0.006470146996434778, + 0.0075279939992469735, + 0.007993496998096816, + 0.00799939099670155, + 0.006995066003582906, + 0.007994835999852512, + 0.011913241003639996, + 0.004077363002579659, + 0.006995370997174177, + 0.007997713000804652, + 0.005990571997244842, + 0.005000704004487488, + 0.0079947539998102, + 0.007995161002327222, + 0.006994935996772256, + 0.00599286599754123, + 0.0035776980002992786, + 0.0054189910006243736, + 0.0069947509982739575, + 0.0049954450005316176, + 0.006996461997914594, + 0.008992773000500165, + 0.008725902000151109, + 0.007270561000041198, + 0.007023370002571028, + 0.006963202002225444, + 0.009001382000860758, + 0.0056517819975852035, + 0.009340860997326672, + 0.007992571998329367, + 0.008004889001313131, + 0.0070893119991524145, + 0.006881989000248723, + 0.006997436998062767, + 0.006996012998570222, + 0.00799529400683241, + 0.006994034993113019, + 0.006992627997533418, + 0.007997740998689551, + 0.00999571599822957, + 0.006767796003259718, + 0.0062208279996411875, + 0.005994321996695362, + 0.002992472000187263, + 0.007001494996075053, + 0.007002696998824831, + 0.00665695500356378, + 0.004327804999775253, + 0.0029971209951327182, + 0.004994697002985049, + 0.006998012002441101, + 0.00745626100251684, + 0.007532178999099415, + 0.013996752000821289, + 0.007997300002898555, + 0.0069953999991412275, + 0.005992387996229809, + 0.01047622799524106, + 0.005517865996807814, + 0.0046886179989087395, + 0.008016909996513277, + 0.005344783996406477, + 0.006935541998245753, + 0.010995270997227635, + 0.005134498002007604, + 0.006875686995044816, + 0.005974987005174626, + 0.005994104998535477, + 0.005996063002385199, + 0.005997107997245621, + 0.006996482006798033, + 0.004997452997486107, + 0.006997823002166115, + 0.0069967300005373545, + 0.0068677290037157945, + 0.004130178000195883, + 0.004993798000214156, + 0.007998056993528735, + 0.006997111995588057, + 0.007000444995355792, + 0.006993735005380586, + 0.007997902001079638, + 0.006997313997999299, + 0.005998165004712064, + 0.006629903000430204, + 0.005364264994568657, + 0.007997745000466239, + 0.007996460000867955, + 0.006999862998782191, + 0.007864503000746481, + 0.0061467380000976846, + 0.007978166999237146, + 0.007001380996371154, + 0.005002827994758263, + 0.004985327999747824, + 0.00541704199713422, + 0.00557714299793588, + 0.007995622996531893, + 0.009003184000903275, + 0.008008247998077422, + 0.00797773899830645, + 0.007996735002961941, + 0.00799719199858373, + 0.008998534001875669, + 0.006030694996297825, + 0.003967287004343234, + 0.0069941429974278435, + 0.009993180996389128, + 0.005998447006277274, + 0.007995533000212163, + 0.005998717999318615, + 0.005004856000596192, + 0.007987110999238212, + 0.006997810000029858, + 0.00599985099688638, + 0.006993399998464156, + 0.007997865999641363, + 0.007999525005288888, + 0.006996068004809786, + 0.006995349998760503, + 0.006998127995757386, + 0.0020052110048709437, + 0.008000182999239769, + 0.006989338995481376, + 0.007990912003151607, + 0.007000974997936282, + 0.003964764000556897, + 0.006022254005074501, + 0.007995833999302704, + 0.006998090000706725, + 0.0070028720001573674, + 0.006001690999255516, + 0.009992915001930669, + 0.00399707599717658, + 0.006562319002114236, + 0.0034257969964528456, + 0.006999000004725531, + 0.007997633998456877, + 0.0059921950014540926, + 0.005992139995214529, + 0.008998379998956807, + 0.005992656006128527, + 0.003994323000370059, + 0.005997034000756685, + 0.00799977699352894, + 0.006991725997067988, + 0.0069170350034255534, + 0.005725265000364743, + 0.009348410996608436, + 0.005998212000122294, + 0.008995969998068176, + 0.007188987001427449, + 0.007806802001141477, + 0.00486449299933156, + 0.005130035999172833, + 0.003996728999482002, + 0.006996216005063616, + 0.006000123001285829, + 0.00899270400259411, + 0.00729529200179968, + 0.004699049997725524, + 0.006997424003202468, + 0.006997382006375119, + 0.007978147004905622, + 0.0070180080001591705, + 0.006992296999669634, + 0.014003683005284984, + 0.006990604997554328, + 0.00699647500005085, + 0.006997975004196633, + 0.006996720003371593, + 0.006997084004979115, + 0.007998585002496839, + 0.006996557000093162, + 0.0079996070053312, + 0.005993729995680042, + 0.008002107999345753, + 0.016018908994738013, + 0.007956580004247371, + 0.007991065001988318, + 0.003995364000729751, + 0.005998191001708619, + 0.008003605995327234, + 0.0069917729997541755, + 0.007998335997399408, + 0.004637555997760501, + 0.009358353003335651, + 0.007078882001223974, + 0.0039032880013110116, + 0.007999789995665196, + 0.009994915999413934, + 0.005998887994792312, + 0.005997133994242176, + 0.008022601003176533, + 0.008964499000285286, + 0.006688927998766303, + 0.007303152997337747, + 0.006993086994043551, + 0.00800195799820358, + 0.007989841993548907, + 0.007997066000825725, + 0.005358565998903941, + 0.002855907005141489, + 0.008778176001214888, + 0.006987063999986276, + 0.0059985620027873665, + 0.004624547000275925, + 0.00571625999873504, + 0.0056465169982402585, + 0.010287511002388783, + 0.007631568005308509, + 0.006503120996057987, + 0.001539037999464199, + 0.0011194329999852926, + 0.0010202689954894595, + 0.0010065560054499656, + 0.0009719079971546307, + 0.0009888390050036833, + 0.0032464730029460043, + 0.0011274319986114278, + 0.0010194529968430288, + 0.0009601269994163886, + 0.0010391370014986023, + 0.0010504489982849918, + 0.0010544649994699284, + 0.0010741779988165945, + 0.0009973650012398139, + 0.0009776389997568913, + 0.0010499589989194646, + 0.0011051399997086264, + 0.0010293230006936938, + 0.0009976910005207174, + 0.000963730999501422, + 0.0009805700028664432, + 0.0010448430039105006, + 0.0011197819985682145, + 0.0029723469997406937, + 0.006998197000939399, + 0.0069974520010873675, + 0.006010705001244787, + 0.007983839997905307, + 0.007007628999417648, + 0.007986927004822064, + 0.006996832002187148, + 0.00799792700127, + 0.006996332005655859, + 0.006999247001658659, + 0.00699710899789352, + 0.006064532994059846, + 0.0071882560005178675, + 0.008067789996857755, + 0.0076751290034735575, + 0.0046990629998617806, + 0.006108765002863947, + 0.004161983000813052, + 0.00900230900151655, + 0.006987210996157955, + 0.0070042489969637245, + 0.007980497997778002, + 0.006990553003561217, + 0.0069951729965396225, + 0.005987491997075267, + 0.0059873259961022995, + 0.006000003995723091, + 0.004437174000486266, + 0.006629466995946132, + 0.004944615000567865, + 0.004987460000847932, + 0.0071674849968985654, + 0.004801838003913872, + 0.005992183003399987, + 0.004341022999142297, + 0.006646655005170032, + 0.007987685996340588, + 0.006995536001340952, + 0.005990466997900512, + 0.006991727001150139, + 0.008002793001651298, + 0.00798414199380204, + 0.008010363002540544, + 0.0059735969989560544, + 0.005038477000198327, + 0.007966582001245115, + 0.007978023997566197, + 0.007001748999755364, + 0.006999564997386187, + 0.007988513003510889, + 0.00998583500040695, + 0.006997862001298927, + 0.006984675994317513, + 0.007003232996794395, + 0.00698351099708816, + 0.004483938995690551, + 0.006503951997729018, + 0.005993316997773945, + 0.00499188200046774, + 0.0070048259949544445, + 0.010992589996021707, + 0.006994458999542985, + 0.006998730998020619, + 0.006996983000135515, + 0.007118180001270957, + 0.00685453400365077, + 0.006000301000312902, + 0.007990139005414676, + 0.007997962995432317, + 0.006991435002419166, + 0.006998580000072252, + 0.0069975270016584545, + 0.0038444230012828484, + 0.006150745000923052, + 0.007014331000391394, + 0.007980241003679112, + 0.0070277729973895475, + 0.006961405000765808, + 0.007000356999924406, + 0.007994601997779682, + 0.007998184002644848, + 0.007994333995156921, + 0.008000680005352478, + 0.007991922997462098, + 0.007997346001502592, + 0.002998274998390116, + 0.004007684998214245, + 0.004986879997886717, + 0.00699568800337147, + 0.008010163001017645, + 0.005989019002299756, + 0.005994058003125247, + 0.007990846002940089, + 0.004998556003556587, + 0.0029608370023197494, + 0.007030155997199472, + 0.007003514998359606, + 0.005989031997160055, + 0.0059980009973514825, + 0.006006595998769626, + 0.007988241995917633, + 0.005997776002914179, + 0.007059878000291064, + 0.007930842999485321, + 0.00799679200281389, + 0.003564033002476208, + 0.006432709000364412, + 0.0059941010040347464, + 0.008999154000775889, + 0.0051271410047775134, + 0.004868085001362488, + 0.007994580999366008, + 0.007999264998943545, + 0.007995744002982974, + 0.007996330001333263, + 0.005996018000587355, + 0.005998203996568918, + 0.004997166004613973, + 0.0038777750014560297, + 0.006116178003139794, + 0.007999042005394585, + 0.007703183997364249, + 0.001600689000042621, + 0.0006335260040941648, + 0.0004048870032420382, + 0.0003695210034493357, + 0.00037437499850057065, + 0.000368420995073393, + 0.000606647998210974, + 0.0032171540005947463, + 0.0004377650038804859, + 0.0004393780036480166, + 0.00042078100523212925, + 0.0003915310007869266, + 0.0004600999964168295, + 0.00037130699638510123, + 0.00037943000643281266, + 0.0003712899997481145, + 0.00037198799691395834, + 0.003222731000278145, + 0.0004586709983414039, + 0.0004656849996536039, + 0.00042689199472079054, + 0.0003949000019929372, + 0.000384079001378268, + 0.0003802389983320609, + 0.00037139200139790773, + 0.0003702469985000789, + 0.0003720919994520955, + 0.000373823000700213, + 0.00037000000156695023, + 0.0003677479980979115, + 0.00037311099731596187, + 0.0003777329984586686, + 0.004401366000820417, + 0.0004561730020213872, + 0.000448337996203918, + 0.00043945600191364065, + 0.0003964770003221929, + 0.0003762619962799363, + 0.00037120799970580265, + 0.00037028999940957874, + 0.0003768110036617145, + 0.00037370400241343305, + 0.005293395995977335, + 0.0005094230000395328, + 0.00044086600246373564, + 0.0004367619985714555, + 0.00038017999759176746, + 0.00037802000588271767, + 0.00036696300230687484, + 0.00036983600148232654, + 0.0003792139978031628, + 0.0003677790009533055, + 0.00040965199877973646, + 0.003933099003916141, + 0.0005755879974458367, + 0.0006007750052958727, + 0.0005046410005888902, + 0.00040627399721415713, + 0.0003826219981419854, + 0.000375659998098854, + 0.00045753100130241364, + 0.0004857559979427606, + 0.0005104369993205182, + 0.0036487909965217113, + 0.0004647409950848669, + 0.00043814500531880185, + 0.00042429099994478747, + 0.0003694519982673228, + 0.0007313299938687123, + 0.00036828599695581943, + 0.00043959999311482534, + 0.0004888839976047166, + 0.0005479179963003844, + 0.00043467200157465413, + 0.0004678490004152991, + 0.0008695120050106198, + 0.0005476970036397688, + 0.0005063589997007512, + 0.000500185000419151, + 0.0007951640000101179, + 0.0007217500024125911, + 0.0003711300014401786, + 0.00038306299393298104, + 0.002090355999825988, + 0.0014058500019018538, + 0.0003926289937226102, + 0.00037849799991818145, + 0.0003698110012919642, + 0.0003671000013127923, + 0.00037461199826793745, + 0.00036892899515805766, + 0.000406348000979051, + 0.008908864998375066, + 0.0006992049966356717, + 0.0004943929961882532, + 0.0004166020007687621, + 0.0010372550023021176, + 0.0005182179957046174, + 0.0004124180049984716, + 0.0028490510012488812, + 0.0004489499988267198, + 0.0005522249994101003, + 0.0005527199973585084, + 0.0005113099978188984, + 0.00040452399844070897, + 0.00039076599932741374, + 0.00038160000258358195, + 0.00039837000076659024, + 0.0003897040005540475, + 0.0003802409992204048, + 0.0006240500006242655, + 0.0004929970018565655, + 0.006986663996940479, + 0.006995612995524425, + 0.007994899999175686, + 0.006999596000241581, + 0.008006661999388598, + 0.007995512998604681, + 0.007985559001099318, + 0.005993644001137, + 0.007007587002590299, + 0.006989276997046545, + 0.007452541001839563, + 0.0075358489993959665, + 0.0069956669976818375, + 0.0069964719950803556, + 0.0058726689967443235, + 0.005132131998834666, + 0.003984253999078646, + 0.004996541996661108, + 0.006996055999479722, + 0.004996586001652759, + 0.006995878000452649, + 0.007996756998181809, + 0.006995780000579543, + 0.007996897998964414, + 0.006032063000020571, + 0.007960445997014176, + 0.004001803004939575, + 0.003991113000665791, + 0.006995368996285833, + 0.007997232998604886, + 0.006995840005401988, + 0.003997024999989662, + 0.0029971699987072498, + 0.004996110001229681, + 0.006995115996687673, + 0.005997502004902344, + 0.005613226006971672, + 0.00537868600076763, + 0.005997516003844794, + 0.008009897996089421, + 0.006011566998495255, + 0.006993884002440609, + 0.006970517002628185, + 0.005995929001073819, + 0.008995210002467502, + 0.006996331998379901, + 0.006999656994594261, + 0.006002850997901987, + 0.006994789000600576, + 0.008041141998546664, + 0.007067257996823173, + 0.003534023002430331, + 0.004290420001780149, + 0.00695964100304991, + 0.005996669999149162, + 0.007003573999099899, + 0.003345555000123568, + 0.00043788199400296435, + 0.0003941269969800487, + 0.0003810799971688539, + 0.0003777759993681684, + 0.00038190999475773424, + 0.0003728950032382272, + 0.0033458000034443103, + 0.00040585299575468525, + 0.00037987399991834536, + 0.0036005779984407127, + 0.0004214209984638728, + 0.0003798260004259646, + 0.0003772810014197603, + 0.0027418960016802885, + 0.0004193150016362779, + 0.000408358006097842, + 0.00037299899850040674, + 0.0003646340046543628, + 0.00036960899888072163, + 0.0003692219979711808, + 0.0003647679986897856, + 0.0003636230030679144, + 0.00036400100361788645, + 0.005443250003736466, + 0.0004109559959033504, + 0.00038788900565123186, + 0.00036343800456961617, + 0.0003634020031313412, + 0.00036722000368172303, + 0.0003594349982449785, + 0.0003617100010160357, + 0.00036628200177801773, + 0.0003650090002338402, + 0.0003721349930856377, + 0.003115703002549708, + 0.00044712500675814226, + 0.0003752310003619641, + 0.0034322479987167753, + 0.00037896600406384096, + 0.0003692720056278631, + 0.0003733409976121038, + 0.0003645449978648685, + 0.00036656800511991605, + 0.000363152998033911, + 0.00036432999331736937, + 0.0040755359950708225, + 0.0004056050020153634, + 0.00037533500290010124, + 0.0004115880001336336, + 0.00036586499481927603, + 0.00036329199792817235, + 0.0003760290055652149, + 0.00036017000093124807, + 0.00036568900395650417, + 0.00036102399462834, + 0.00036031300260219723, + 0.006232289000763558, + 0.0004096830016351305, + 0.0003714400008902885, + 0.00036977600393584, + 0.00036927300243405625, + 0.00036235399602446705, + 0.0003672519960673526, + 0.00037069700192660093, + 0.00036635099968407303, + 0.0003650069993454963, + 0.00036081000143894926, + 0.0036529339995468035, + 0.0017211790036526509, + 0.00038301199674606323, + 0.000368066001101397, + 0.0004232279970892705, + 0.00036457600072026253, + 0.00039790399750927463, + 0.00037336399691412225, + 0.00036190600076224655, + 0.0010250699997413903, + 0.003206482004316058, + 0.00040145399543689564, + 0.00036540700239129364, + 0.00037824500032002106, + 0.00039083300362108275, + 0.0004279929999029264, + 0.0003765380024560727, + 0.0003660240035969764, + 0.00037540400080615655, + 0.0003625030003604479, + 0.00036111800000071526, + 0.005445697002869565, + 0.0005187219940125942, + 0.00037142300425330177, + 0.0003666420016088523, + 0.0003970729958382435, + 0.00037042400072095916, + 0.0003688400029204786, + 0.0003772999989450909, + 0.005212340998696163, + 0.00040933999844128266, + 0.0003727509974851273, + 0.00036093100061407313, + 0.0003632240041042678, + 0.00036143499892205, + 0.00036700799682876095, + 0.00039114899846026674, + 0.0003848850028589368, + 0.000365411993698217, + 0.00510140199912712, + 0.00041671900544315577, + 0.0003647720004664734, + 0.0003662189992610365, + 0.00036139100120635703, + 0.0003646969998953864, + 0.00036777200148208067, + 0.00036017000093124807, + 0.00036573900433722883, + 0.0034111839995603077, + 0.0004376010037958622, + 0.00038286999915726483, + 0.0004559780063573271, + 0.001340415001322981, + 0.0003949730016756803, + 0.0003724089983734302, + 0.0003691120000439696, + 0.00036683399957837537, + 0.00036188800004310906, + 0.001451856005587615, + 0.000764926997362636, + 0.0003865289982059039, + 0.00037138599873287603, + 0.00036849900061497465, + 0.0003858099953504279, + 0.0003654079991974868, + 0.00036702999932458624, + 0.00036360800004331395, + 0.00036455399822443724, + 0.0003595180023694411, + 0.00035936700442107394, + 0.0015772109982208349, + 0.002601000996946823, + 0.0004702730002463795, + 0.0022448740055551752, + 0.00040729900501901284, + 0.00037150400021346286, + 0.0003639760034275241, + 0.0003655399996205233, + 0.00036493899824563414, + 0.00036656600423157215, + 0.0003843200029223226, + 0.00037002500175731257, + 0.0004278470005374402, + 0.0012268150021554902, + 0.0011823259992524981, + 0.002719372001593001, + 0.0004051610012538731, + 0.0003766230001929216, + 0.00036795000050915405, + 0.0003656589979073033, + 0.0003684510011225939, + 0.00035979300446342677, + 0.0003618600021582097, + 0.00036824799462920055, + 0.00035985999420518056, + 0.0018368360033491626, + 0.0009445399991818704, + 0.002929930000391323, + 0.0007241880011861213, + 0.00038164200668688864, + 0.0003703379989019595, + 0.0003751240001292899, + 0.00036777400237042457, + 0.000365199004590977, + 0.0003766369991353713, + 0.00036814699706155807, + 0.0013691130006918684, + 0.0013593969997600652, + 0.0010212279958068393, + 0.0006232150044525042, + 0.00042591399687808007, + 0.000372260001313407, + 0.0003687860007630661, + 0.00035948699951404706, + 0.00036105899926042184, + 0.0003655260006780736, + 0.00036445300065679476, + 0.00036301799991633743, + 0.0003609709965530783, + 0.0005978679982945323, + 0.0005777300029876642, + 0.0014775390009162948, + 0.0005321880016708747, + 0.0011671409956761636, + 0.0017253920013899915, + 0.00040429700311506167, + 0.00037061899638501927, + 0.00036737499613082036, + 0.00039917499816510826, + 0.0003620450006565079, + 0.0003656629996839911, + 0.00036793600156670436, + 0.0020148680050624534, + 0.0007285480023710988, + 0.0006314089987426996, + 0.00041896200127666816, + 0.0005608400024357252, + 0.00039259400364244357, + 0.0003805129963438958, + 0.00035970599856227636, + 0.0003716079954756424, + 0.00036710700078401715, + 0.0003616680041886866, + 0.0003637790068751201, + 0.0003619859999162145, + 0.00036856399674434215, + 0.001370888996461872, + 0.001327385994954966, + 0.0028646430000662804, + 0.000494122999953106, + 0.0019554789978428744, + 0.00038325100467773154, + 0.0003851019937428646, + 0.0005250260001048446, + 0.0009731649988680147, + 0.0006577039966941811, + 0.0005948129983153194, + 0.0015450889986823313, + 0.0004072520023328252, + 0.0003720899985637516, + 0.00041172299825120717, + 0.0003654300016933121, + 0.00036144199839327484, + 0.0003682969982037321, + 0.0003694460028782487, + 0.00223918299889192, + 0.0012733559997286648, + 0.000957137999648694, + 0.0004770500017912127, + 0.000374399998690933, + 0.00036731399450218305, + 0.0003598019975470379, + 0.0003652780069387518, + 0.0003641170042101294, + 0.00036325599648989737, + 0.0003645389951998368, + 0.00036422399716684595, + 0.00036309900315245613, + 0.0070947940039332025, + 0.006999965000431985, + 0.007432170998072252, + 0.007555486001365352, + 0.006367672998749185, + 0.007276303003891371, + 0.007344992001890205, + 0.0079963910029619, + 0.006995326002652291, + 0.007996555003046524, + 0.005996104999212548, + 0.006535824999446049, + 0.008463412996206898, + 0.007990891004737932, + 0.007993942002940457, + 0.007996466993063223, + 0.0035585570003604516, + 0.007433637001668103, + 0.005996380998112727, + 0.007996312997420318, + 0.005996357002004515, + 0.004998254000383895, + 0.00999882599717239, + 0.005992797996441368, + 0.004997759002435487, + 0.009996795000915881, + 0.011995179003861267, + 0.006994572002440691, + 0.0053104010003153235, + 0.009682629999588244, + 0.0010310689976904541, + 0.0004061270010424778, + 0.0003670650039566681, + 0.0003753689961740747, + 0.0003607749968068674, + 0.0010691600036807358, + 0.0003898549985024147, + 0.00036209400423103943, + 0.0003622450021794066, + 0.00036709299456560984, + 0.00036464699951466173, + 0.0012542820040835068, + 0.004260190995410085, + 0.0020556640010909177, + 0.0003998539978056215, + 0.00040022200118983164, + 0.00044168900058139116, + 0.00039117300184443593, + 0.00036818700027652085, + 0.00036614500277210027, + 0.00036807199649047107, + 0.002074005999020301, + 0.002233412000350654, + 0.00038300600135698915, + 0.00041949300066335127, + 0.004008162002719473, + 0.0030723849995411, + 0.0027539840011741035, + 0.0025276169981225394, + 0.002484085001924541, + 0.0004066749970661476, + 0.0003658620044006966, + 0.0003631190047599375, + 0.0003683760005515069, + 0.0003613049993873574, + 0.0014687889997730963, + 0.0003960820031352341, + 0.00036609199742088094, + 0.0003678380016935989, + 0.0003607239996199496, + 0.0003630330029409379, + 0.0003626550023909658, + 0.00037075000000186265, + 0.003347447003761772, + 0.0018770809983834624, + 0.0003901269956259057, + 0.0003712580000865273, + 0.002255613995657768, + 0.00040072200499707833, + 0.0003741850014193915, + 0.0003732940022018738, + 0.0003649540012702346, + 0.000359409001248423, + 0.0003717380022862926, + 0.00036335500044515356, + 0.004371881004772149, + 0.00039906299934955314, + 0.0003642509982455522, + 0.0003691989986691624, + 0.0003639010028564371, + 0.0003735579957719892, + 0.00037416999839479104, + 0.00036220999754732475, + 0.0003715249986271374, + 0.0003653129970189184, + 0.0003619959970819764, + 0.0007902900033514015, + 0.0006890189979458228, + 0.0003720640015671961, + 0.00036048400215804577, + 0.0019521279973560013, + 0.0004131359964958392, + 0.00036775100306840613, + 0.00037057799636386335, + 0.00036262300272937864, + 0.0009828779948293231, + 0.0022786129993619397, + 0.00040567200630903244, + 0.000375540999812074, + 0.0003680669979075901, + 0.00037674700433854014, + 0.00036274299782235175, + 0.00038859000051161274, + 0.000375494004401844, + 0.0003612989967223257, + 0.00036368500150274485, + 0.0003700909946928732, + 0.00035982899862574413, + 0.0016153820033650845, + 0.001892235995910596, + 0.0003938040026696399, + 0.0012264389952179044, + 0.00038842600042698905, + 0.0003705409981193952, + 0.00037134800368221477, + 0.00036559000000124797, + 0.0003650979997473769, + 0.00037797500408487394, + 0.0003706259958562441, + 0.0005591559965978377, + 0.002155182999558747, + 0.00111412099795416, + 0.0003733100020326674, + 0.0011496560036903247, + 0.0005635790002997965, + 0.0003730509997694753, + 0.0013078080010018311, + 0.00036657199962064624, + 0.0003744120040209964, + 0.0003622879958129488, + 0.00036831099714618176, + 0.00036781800008611754, + 0.0003643320014816709, + 0.0003643650052254088, + 0.0018416169987176545, + 0.0013942190053057857, + 0.0004908099945168942, + 0.0013150140002835542, + 0.00040371100476477295, + 0.0003663440002128482, + 0.0007001619960647076, + 0.0003671750018838793, + 0.0012795040020137094, + 0.0018052889936370775, + 0.0007430420009768568, + 0.0010243430006084964, + 0.0003961849943152629, + 0.00036929100315319374, + 0.0003620100033003837, + 0.002273605998198036, + 0.0004129349981667474, + 0.00036254600126994774, + 0.0003751149997697212, + 0.0003650860016932711, + 0.00036092599475523457, + 0.0003656920016510412, + 0.000365282001439482, + 0.0007036930037429556, + 0.004074411997862626, + 0.00039759599894750863, + 0.0018991179967997596, + 0.0013765379990218207, + 0.0003757500016945414, + 0.0003721999964909628, + 0.0003691989986691624, + 0.00036455199733609334, + 0.0003632500011008233, + 0.00036251400160836056, + 0.0031864190023043193, + 0.0006442500016419217, + 0.0003751839976757765, + 0.00036781599919777364, + 0.00037386400072136894, + 0.0003612449945649132, + 0.0003627149999374524, + 0.00036584700137609616, + 0.00036477299727266654, + 0.001992136996705085, + 0.0009754819984664209, + 0.00037242400139803067, + 0.0003613660010159947, + 0.0003637350018834695, + 0.0011725780059350654, + 0.0014394520039786585, + 0.00037589400017168373, + 0.0003860180004267022, + 0.0003623059965320863, + 0.0003671279991976917, + 0.00036422000266611576, + 0.0003692699974635616, + 0.00036932600050931796, + 0.00036168900260236114, + 0.0003673899991554208, + 0.001571098000567872, + 0.0013810829987050965, + 0.000381181002012454, + 0.00036360599915497005, + 0.00036279799678595737, + 0.0011700909963110462, + 0.001967837000847794, + 0.0003892029999406077, + 0.00036679099866887555, + 0.0003705660055857152, + 0.00036230300611350685, + 0.00037097700260346755, + 0.00037118600448593497, + 0.0003653149979072623, + 0.0006831399950897321, + 0.0022579259966732934, + 0.0007091880033840425, + 0.000373926006432157, + 0.0013439250033115968, + 0.0011661850003292784, + 0.0003874000030918978, + 0.00043122399802086875, + 0.001336676999926567, + 0.0003718959997058846, + 0.00037580200296361, + 0.00035904099786421284, + 0.0003645389951998368, + 0.00036659200122812763, + 0.00036540900327963755, + 0.0004826579970540479, + 0.0028712099956464954, + 0.0015067009953781962, + 0.0004767989958054386, + 0.00037182400410529226, + 0.0003606000027502887, + 0.0017811239958973601, + 0.00040004600305110216, + 0.0006048079958418384, + 0.00039337199996225536, + 0.005964534997474402, + 0.0004674559968407266, + 0.0004144480044487864, + 0.001310762992943637, + 0.0019100950012216344, + 0.0015566280053462833, + 0.0008717739983694628, + 0.00036753099993802607, + 0.0005156010010978207, + 0.00037018899456597865, + 0.0022796789999119937, + 0.0009493469988228753, + 0.0003770670009544119, + 0.0003648470010375604, + 0.00037020499439677224, + 0.000370671994460281, + 0.00036012700002174824, + 0.0012980150058865547, + 0.0015409170009661466, + 0.0003828150001936592, + 0.000367038999684155, + 0.00035895800101570785, + 0.00036617700243368745, + 0.000366161999409087, + 0.0003604339945013635, + 0.0009848699992289767, + 0.0003855199975077994, + 0.0003688889992190525, + 0.00036521499714581296, + 0.0003638560010585934, + 0.00036592400283552706, + 0.00036741499934578314, + 0.0003628720005508512, + 0.0003654040046967566, + 0.0017635160038480535, + 0.0019487140016281046, + 0.00039230800030054525, + 0.0003684330004034564, + 0.0003659089998109266, + 0.001350140999420546, + 0.0005309340049279854, + 0.0006627120019402355, + 0.0009950919993571006, + 0.00036488499608822167, + 0.00037161499494686723, + 0.0003694560000440106, + 0.0003632459993241355, + 0.0003651220031315461, + 0.0003669059951789677, + 0.00036252200516173616, + 0.001060669994330965, + 0.0005525189990294166, + 0.0006341490006889217, + 0.0006738879965269007, + 0.0013270019990159199, + 0.0005095050000818446, + 0.00046680100058438256, + 0.00039445199945475906, + 0.00036648099921876565, + 0.0020418429994606413, + 0.0004069540009368211, + 0.000395139999454841, + 0.00037649000296369195, + 0.0003641660005087033, + 0.00036459299735724926, + 0.00036987299972679466, + 0.0003667800046969205, + 0.0003974029968958348, + 0.000477920999401249, + 0.0005297890020301566, + 0.0005010359964217059, + 0.0011155199972563423, + 0.0003684920011437498, + 0.0003657220004242845, + 0.0003646059994935058, + 0.0006821959977969527, + 0.0005705430012312718, + 0.0007310070068342611, + 0.00038678900455124676, + 0.0003746959991985932, + 0.0003694650004035793, + 0.0006682530001853593, + 0.0008362239968846552, + 0.0005033769994042814, + 0.00039050299528753385, + 0.0003681520029203966, + 0.00036462300340645015, + 0.0003610240019042976, + 0.000363370003469754, + 0.00037160099600441754, + 0.0003596919996198267, + 0.00036757699854206294, + 0.00045699600013904274, + 0.00047878499754006043, + 0.0011182170055690221, + 0.0010721569997258484, + 0.0014958890023990534, + 0.0004134500049985945, + 0.0003802130013355054, + 0.00036843000270891935, + 0.00037233200418995693, + 0.00037425900518428534, + 0.0003735170030267909, + 0.000453905995527748, + 0.0017509789977339096, + 0.0017361279969918542, + 0.00040217099740402773, + 0.0003784549990086816, + 0.0003770219991565682, + 0.00038217399560380727, + 0.00037438899744302034, + 0.00042582800233503804, + 0.0009014129973365925, + 0.0005011399989598431, + 0.0005337389957276173, + 0.00039439100510207936, + 0.0003758890015888028, + 0.0014500469987979159, + 0.0003950909958803095, + 0.0004047109978273511, + 0.00040055200224742293, + 0.00038095399941084906, + 0.0005975250023766421, + 0.0006656800032942556, + 0.0005345949975890107, + 0.0005066340017947368, + 0.00046584800293203443, + 0.0003943539995816536, + 0.0003737360020750202, + 0.00037098299799254164, + 0.00038143999699968845, + 0.00037227800203254446, + 0.0003754639983526431, + 0.0005116080064908601, + 0.0009914740003296174, + 0.0003803490035352297, + 0.0003797210010816343, + 0.0003742379994946532, + 0.0003704549963003956, + 0.0003766840018215589, + 0.000371848997019697, + 0.0009938899966073222, + 0.0005026150029152632, + 0.0005107359975227155, + 0.0004437320021679625, + 0.00037743400025647134, + 0.00037661399983335286, + 0.00048789300490170717, + 0.0005061549964011647, + 0.0004860090048168786, + 0.00040779599657980725, + 0.00038290899829007685, + 0.00037743800203315914, + 0.001621804003661964, + 0.0004363709958852269, + 0.0015707499987911433, + 0.0004154419948463328, + 0.00040340099803870544, + 0.0003823489969363436, + 0.0003670690057333559, + 0.0003780900005949661, + 0.00038348000089172274, + 0.00037760600389447063, + 0.00037772399809909984, + 0.00037146500108065084, + 0.005977310000162106, + 0.0004243599978508428, + 0.00038277499697869644, + 0.00037186200643191114, + 0.00037807899934705347, + 0.00037541599886026233, + 0.0003771459960262291, + 0.0003765150031540543, + 0.000370609006495215, + 0.003084600997681264, + 0.002301788001204841, + 0.0005803960011689924, + 0.00037981700006639585, + 0.00037733699718955904, + 0.0003701899986481294, + 0.00037272199551807716, + 0.0003751949989236891, + 0.0003744260029634461, + 0.00037508400419028476, + 0.00036943899613106623, + 0.0021946210035821423, + 0.0031664760026615113, + 0.0004098720019101165, + 0.0003833390001091175, + 0.000380418001441285, + 0.0003746049987967126, + 0.0003824340019491501, + 0.0003699160006362945, + 0.0003704619957716204, + 0.003511104005156085, + 0.000666888001433108, + 0.0004135790004511364, + 0.0003785399967455305, + 0.0014757579992874525, + 0.0004081349979969673, + 0.0003951450053136796, + 0.00037421699380502105, + 0.00037461600004462525, + 0.00037585399695672095, + 0.0003726490031112917, + 0.00037192500167293474, + 0.0003777859965339303, + 0.0003711300014401786, + 0.003895068002748303, + 0.0005003659971407615, + 0.001984049995371606, + 0.0004056429970660247, + 0.00037715499638579786, + 0.00038094100455055013, + 0.00037079100002301857, + 0.0004568049989757128, + 0.0022070710037951358, + 0.0004983489998267032, + 0.00040772299689706415, + 0.0003750869946088642, + 0.00037037699803477153, + 0.0020814070012420416, + 0.0004067190020577982, + 0.00037364300078479573, + 0.00038002399378456175, + 0.00037627599522238597, + 0.00036961500154575333, + 0.00037448800139827654, + 0.000373869996110443, + 0.0003793279975070618, + 0.0003850330031127669, + 0.00037842700112378225, + 0.005014601003495045, + 0.0016021679984987713, + 0.0004215439985273406, + 0.000391934001527261, + 0.0003734799975063652, + 0.000413590001699049, + 0.0015045089967316017, + 0.00040691200410947204, + 0.0003819389967247844, + 0.00037405900366138667, + 0.0003801970015047118, + 0.00036919400008628145, + 0.000375737996364478, + 0.000518875996931456, + 0.0005154919999768026, + 0.0010643139976309612, + 0.0010014479994424619, + 0.00043667799764079973, + 0.0003747769951587543, + 0.0003880459989886731, + 0.00037220699596218765, + 0.0003707879950525239, + 0.0003806910026469268, + 0.0003702100002556108, + 0.00043989199912175536, + 0.0005169760042917915, + 0.0004710239954874851, + 0.001475363998906687, + 0.0006531610051752068, + 0.0003755060024559498, + 0.0007521099978475831, + 0.0005577379997703247, + 0.00038330600364133716, + 0.0003701449968502857, + 0.0016143319953698665, + 0.0006999030010774732, + 0.0003881579978042282, + 0.00037509999674512073, + 0.0003785640001296997, + 0.0003715290004038252, + 0.00037515500298468396, + 0.0003758449965971522, + 0.00037149399577174336, + 0.0033180630052811466, + 0.0004538569992291741, + 0.0012509750013123266, + 0.000400630000513047, + 0.0003746309957932681, + 0.0003741970067494549, + 0.0017424290053895675, + 0.0003962240007240325, + 0.000371848997019697, + 0.0003756420046556741, + 0.0003796620003413409, + 0.00037770999915665016, + 0.0003741109976544976, + 0.0003773540011025034, + 0.00037608699494739994, + 0.001151086005847901, + 0.0023179160052677616, + 0.00038217499968595803, + 0.00036879099934594706, + 0.0003864980026264675, + 0.0003886050035362132, + 0.001051132996508386, + 0.00043656699563143775, + 0.0003819209960056469, + 0.000524319002579432, + 0.0016547010018257424, + 0.0003931699975510128, + 0.00037088400131324306, + 0.00037441099993884563, + 0.0003795669981627725, + 0.0003703159964061342, + 0.0003738070008694194, + 0.0003718699954333715, + 0.00037821599835297093, + 0.0013888739995309152, + 0.004214919994410593, + 0.0003995340011897497, + 0.0003822270009550266, + 0.0003690750017995015, + 0.00037303799763321877, + 0.0006709350054734387, + 0.0017347600005450659, + 0.0004150040040258318, + 0.00040538900066167116, + 0.0003884410034515895, + 0.00036968800122849643, + 0.0003805899978033267, + 0.00037483100459212437, + 0.0010623830021359026, + 0.0013794270053040236, + 0.0003850919965771027, + 0.0003735610007424839, + 0.0003743620036402717, + 0.0003747420050785877, + 0.0003766999943763949, + 0.00037023999902885407, + 0.00037307800084818155, + 0.001635459004319273, + 0.00039320399810094386, + 0.00038369700632756576, + 0.0003828589979093522, + 0.001405964998411946, + 0.0010328330026823096, + 0.00039123000169638544, + 0.0003720540044014342, + 0.001393113998346962, + 0.00039111699879867956, + 0.0003822979997494258, + 0.00037410900404211134, + 0.0003740379979717545, + 0.0003802270002779551, + 0.00037191300361882895, + 0.0003708560034283437, + 0.004016879000118934, + 0.00038832499558338895, + 0.000374324998119846, + 0.0003759340033866465, + 0.0003710099990712479, + 0.0003703900001710281, + 0.0013920510027674027, + 0.0018650310012162663, + 0.0003947299992432818, + 0.0003811349961324595, + 0.0003820709971478209, + 0.00037638100184267387, + 0.0003889700019499287, + 0.0003851020010188222, + 0.00037388200144050643, + 0.0013146880010026507, + 0.0005469220050144941, + 0.0023067420042934828, + 0.0004117009957553819, + 0.00037545899976976216, + 0.00037921700277365744, + 0.000377212003513705, + 0.000367683001968544, + 0.003143187001114711, + 0.0003995520019088872, + 0.0003778730024350807, + 0.00038077500357758254, + 0.00037138199695618823, + 0.0003994729995611124, + 0.0003838920019916259, + 0.0003735210048034787, + 0.0037342220020946115, + 0.00040378399717155844, + 0.00038693899841746315, + 0.0003757719969144091, + 0.0003762540000025183, + 0.0003752369957510382, + 0.00037457900180015713, + 0.003417459003685508, + 0.0004059490020154044, + 0.0003845329993055202, + 0.0003804620064329356, + 0.0003682819951791316, + 0.00037455900019267574, + 0.0003680279987747781, + 0.00037002700264565647, + 0.00037283299752743915, + 0.0003697280044434592, + 0.0003718699954333715, + 0.004990822999388911, + 0.0004068260022904724, + 0.0003810199996223673, + 0.0013842560001648962, + 0.00037642499955836684, + 0.000388917003874667, + 0.0004309940050006844, + 0.0003777879974222742, + 0.0003790709961322136, + 0.00036924299638485536, + 0.0003710039964062162, + 0.004317398001148831, + 0.00045600999874295667, + 0.00045712799328612164, + 0.00044775499554816633, + 0.00038920400402275845, + 0.000413297995692119, + 0.00039670099795330316, + 0.0004134920018259436, + 0.0003785900044022128, + 0.0004003040012321435, + 0.00037847799831070006, + 0.0004153100016992539, + 0.0003816379976342432, + 0.00037774199881823733, + 0.0003864640020765364, + 0.0004007980023743585, + 0.00038593800127273425, + 0.0003755290017579682, + 0.00038249399949563667, + 0.0003788230023928918, + 0.00040149699634639546, + 0.0004019090047222562, + 0.0003803409999818541, + 0.0003723029949469492, + 0.0003876610062434338, + 0.0004102729944861494, + 0.00041203499858966097, + 0.000402575999032706, + 0.0003869240026688203, + 0.00038180000410648063, + 0.00038969200249994174, + 0.0003828129993053153, + 0.00037781900027766824, + 0.0004295089966035448, + 0.0004026300011901185, + 0.00041524400148773566, + 0.0003981529953307472, + 0.0003771269985008985, + 0.0003859770004055463, + 0.0004034279991174117, + 0.00040859800355974585, + 0.0003816210009972565, + 0.00039868600288173184, + 0.00038156300433911383, + 0.00039616399590158835, + 0.0003894809997291304, + 0.00037822200101800263, + 0.00037947200326016173, + 0.0003922890027752146, + 0.0003822959988610819, + 0.00041611299820942804, + 0.0003985920047853142, + 0.0004058939957758412, + 0.00038145799771882594, + 0.00037401999725261703, + 0.00042539300193311647, + 0.00037587000406347215, + 0.000375165996956639, + 0.00041127699660137296, + 0.00039568400097778067, + 0.00043003899918403476, + 0.000378586002625525, + 0.0003735840000445023, + 0.0003966069998568855, + 0.0004047999973408878 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil", + "fullname": "TestCalculateNabla2OfTheta", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.00032102699333336204, + "max": 0.01520376700500492, + "mean": 0.004275450127302015, + "stddev": 0.003297357114006507, + "rounds": 999, + "median": 0.004997969001124147, + "iqr": 0.006618793997404282, + "q1": 0.0003801100010605296, + "q3": 0.006998903998464812, + "iqr_outliers": 0, + "stddev_outliers": 541, + "outliers": "541;0", + "ld15iqr": 0.00032102699333336204, + "hd15iqr": 0.01520376700500492, + "ops": 233.89350132147166, + "total": 4.271174677174713, + "data": [ + 0.006984973995713517, + 0.006992690003244206, + 0.006997310003498569, + 0.007996078005817253, + 0.00700949600286549, + 0.005983716997434385, + 0.0069981410051696, + 0.006994752999162301, + 0.007997875000000931, + 0.006997529999352992, + 0.006000068999128416, + 0.010994078998919576, + 0.006998311997449491, + 0.0070006470050429925, + 0.0069955450017005205, + 0.005997569998726249, + 0.006994153998675756, + 0.006994381001277361, + 0.004996586998458952, + 0.006997406999289524, + 0.007997730004717596, + 0.0069984199944883585, + 0.007996056003321428, + 0.006996420001087245, + 0.0020023620018037036, + 0.007994080995558761, + 0.006998621996899601, + 0.006998946002568118, + 0.007993274994078092, + 0.006997928998316638, + 0.00737710500106914, + 0.006619380998017732, + 0.00799547399947187, + 0.006005750001349952, + 0.007997777996934019, + 0.009986240002035629, + 0.006995626004936639, + 0.006997351003519725, + 0.0079969859943958, + 0.007994388004590292, + 0.00436803499906091, + 0.003628400998422876, + 0.007997287997568492, + 0.006994522002059966, + 0.005998131004162133, + 0.006995939998887479, + 0.006997491997026373, + 0.007997339002031367, + 0.006997274998866487, + 0.007007005006016698, + 0.010997275996487588, + 0.006989007000811398, + 0.007991954000317492, + 0.008995993004646152, + 0.007995328000106383, + 0.00705143999948632, + 0.007943524993606843, + 0.00699511499988148, + 0.006998199001827743, + 0.0029975979996379465, + 0.007013187001575716, + 0.007986879005329683, + 0.007989890997123439, + 0.007998652996320743, + 0.007994134997716174, + 0.004997682000976056, + 0.00399577799544204, + 0.006996984004217666, + 0.007999215005838778, + 0.00699679899844341, + 0.009002370999951381, + 0.007991680002305657, + 0.006998955002927687, + 0.007993758001248352, + 0.007000559999141842, + 0.006994221999775618, + 0.00727083799574757, + 0.005721720997826196, + 0.007997401000466198, + 0.005997225001920015, + 0.007998459004738834, + 0.007995143998414278, + 0.007996895001269877, + 0.005996945001243148, + 0.005998986998747569, + 0.006000043998938054, + 0.0009992349951062351, + 0.007993629995326046, + 0.006000631001370493, + 0.005989610006508883, + 0.0060017700016032904, + 0.006991946000198368, + 0.00701313200261211, + 0.007985910000570584, + 0.006996627002081368, + 0.006991652997385245, + 0.004300234002585057, + 0.009699831003672443, + 0.005992489001073409, + 0.006157001000246964, + 0.00889921100315405, + 0.007983283001522068, + 0.006035528997017536, + 0.009906015999149531, + 0.008013913997274358, + 0.00797904199862387, + 0.006994355004280806, + 0.007994355000846554, + 0.00699586900009308, + 0.005996217005304061, + 0.007997107000846881, + 0.005060413001046982, + 0.006941678999282885, + 0.0029939379965071566, + 0.005997730004310142, + 0.006993203001911752, + 0.006995746000029612, + 0.005995924999297131, + 0.007998164001037367, + 0.006998535005550366, + 0.007995258005394135, + 0.007998916000360623, + 0.005996933003189042, + 0.007995954001671635, + 0.008000782996532507, + 0.007993911000085063, + 0.005997617001412436, + 0.003995906001364347, + 0.007000036996032577, + 0.00750819099630462, + 0.009043978003319353, + 0.004448690000572242, + 0.006978816003538668, + 0.007996804000867996, + 0.007994648003659677, + 0.007495939004002139, + 0.006498335002106614, + 0.00799965899932431, + 0.007994935003807768, + 0.0070095420014695264, + 0.008983485997305252, + 0.004992276000848506, + 0.004995490999135654, + 0.0069972739947843365, + 0.007999446002941113, + 0.007998583001608495, + 0.007143599999835715, + 0.009847595996689051, + 0.007996608997927979, + 0.007996115004061721, + 0.00899668299825862, + 0.006996894000621978, + 0.006998589000431821, + 0.007997364999027923, + 0.007996872998774052, + 0.00799685299716657, + 0.007141705995309167, + 0.006860362002043985, + 0.00799654299771646, + 0.007164488000853453, + 0.00781908399949316, + 0.006997321004746482, + 0.006996441996307112, + 0.007999550995009486, + 0.007001357000262942, + 0.0069915400017634965, + 0.00801001800573431, + 0.006988752997131087, + 0.006994379000389017, + 0.007003781000094023, + 0.009017794000101276, + 0.007969828002387658, + 0.006997386000875849, + 0.006996762997005135, + 0.008001395006431267, + 0.005031955995946191, + 0.004960555997968186, + 0.007992225997440983, + 0.00699774399981834, + 0.006996303993219044, + 0.006997028001933359, + 0.011003243002051022, + 0.0059994420007569715, + 0.009072877997823525, + 0.0061119779929867946, + 0.008064431996899657, + 0.005731060002290178, + 0.009996307002438698, + 0.0060002440004609525, + 0.005993490005494095, + 0.00799779299995862, + 0.0059992140013491735, + 0.005062194999482017, + 0.003928201003873255, + 0.007007134998275433, + 0.006988651999563444, + 0.0069951369951013476, + 0.006000138004310429, + 0.00799426800222136, + 0.007996979002200533, + 0.004001512999820989, + 0.007991644997673575, + 0.008001713002158795, + 0.006991638001636602, + 0.007098807000147644, + 0.00589173399930587, + 0.004996985000616405, + 0.008003962997463532, + 0.006990112000494264, + 0.006999953999184072, + 0.006993978000537027, + 0.009996578999562189, + 0.007996202002686914, + 0.004014504003862385, + 0.004986134998034686, + 0.007990580001205672, + 0.006433636001020204, + 0.007561573998827953, + 0.006999944998824503, + 0.006994946998020168, + 0.0110012570003164, + 0.006993249000515789, + 0.005996589999995194, + 0.008017694002774078, + 0.0059750720029114746, + 0.006999523997365031, + 0.005994585997541435, + 0.006997551994572859, + 0.0069976740051060915, + 0.006995832998654805, + 0.0069982929999241605, + 0.006997109005169477, + 0.007996995002031326, + 0.005996954001602717, + 0.006997267002589069, + 0.007002095000643749, + 0.005010242995922454, + 0.007981183000083547, + 0.0059952090014121495, + 0.006000854999001604, + 0.007992412996827625, + 0.005997938998916652, + 0.006998265002039261, + 0.007997253000212368, + 0.005996977000904735, + 0.004560000001220033, + 0.0024287759952130727, + 0.007002170001214836, + 0.004994868002540898, + 0.00799673000437906, + 0.005005580002034549, + 0.006994899005803745, + 0.007991498998308089, + 0.005016288996557705, + 0.0019765850011026487, + 0.00800045499636326, + 0.005997906999255065, + 0.0080940970001393, + 0.004899724001006689, + 0.0015261240041581914, + 0.010470002998772543, + 0.007997354994586203, + 0.005042529999627732, + 0.0059499880007933825, + 0.003995086000941228, + 0.005996343999868259, + 0.006447238003602251, + 0.007548109002527781, + 0.006996978998358827, + 0.006995614996412769, + 0.00799677599570714, + 0.00799864800501382, + 0.006997980999585707, + 0.007994789993972518, + 0.006997131000389345, + 0.006999655997788068, + 0.005993085003865417, + 0.00499778900120873, + 0.005994610000925604, + 0.006997074997343589, + 0.00799996400019154, + 0.007994651998160407, + 0.0095213020031224, + 0.007479905005311593, + 0.007991297003172804, + 0.007003617000009399, + 0.010985806002281606, + 0.0079964060059865, + 0.00599595299718203, + 0.003991832003521267, + 0.003600643998652231, + 0.005395983003836591, + 0.003996906001702882, + 0.00699585500115063, + 0.007007544998486992, + 0.005983262999507133, + 0.0035851760039804503, + 0.006409059998986777, + 0.004995536000933498, + 0.007998100998520385, + 0.005995566003548447, + 0.007997132001037244, + 0.004997969001124147, + 0.007988192002812866, + 0.008629534997453447, + 0.0063666700007161126, + 0.008061868000368122, + 0.006928741997398902, + 0.00700378500187071, + 0.006989779001742136, + 0.009001348000310827, + 0.003991306002717465, + 0.004996572999516502, + 0.007997462002094835, + 0.006997313997999299, + 0.007998305998626165, + 0.00799674400332151, + 0.006998251999903005, + 0.006996869997237809, + 0.007998863999091554, + 0.006998390002991073, + 0.006994835996010806, + 0.0065555590044823475, + 0.006444970997108612, + 0.005986995005514473, + 0.006996379997872282, + 0.006996894997428171, + 0.00699731500208145, + 0.005996874999254942, + 0.006997557997237891, + 0.008001848000276368, + 0.0069948089949321, + 0.010997270001098514, + 0.010997033001331147, + 0.007996424996235874, + 0.006990556001255754, + 0.007014104994595982, + 0.007980907997989561, + 0.00800008400256047, + 0.00863332399603678, + 0.0053523440001299605, + 0.006991460999415722, + 0.006995628995355219, + 0.008001070003956556, + 0.006992939997871872, + 0.006995149997237604, + 0.006994864001171663, + 0.00699690000328701, + 0.00599609700293513, + 0.007006995998381171, + 0.00599614399834536, + 0.006988907000049949, + 0.009011039997858461, + 0.008993674004159402, + 0.007982639996043872, + 0.007992679995368235, + 0.008000273002835456, + 0.006986266998865176, + 0.005994237995764706, + 0.00700410900026327, + 0.006991027003095951, + 0.005010911998397205, + 0.00697817699983716, + 0.005995539002469741, + 0.009997004002798349, + 0.006388171997969039, + 0.007610621003550477, + 0.007991521000803914, + 0.010997260993462987, + 0.007996054002433084, + 0.003997037005319726, + 0.004996643998310901, + 0.007996225001988932, + 0.009014486000523902, + 0.004978967001079582, + 0.0059974269970553, + 0.007998761997441761, + 0.005997489002766088, + 0.00702854800329078, + 0.007966092001879588, + 0.006995124997047242, + 0.007997109998541418, + 0.005007817002478987, + 0.0069872990061412565, + 0.007997166001587175, + 0.008995699005026836, + 0.005997788000968285, + 0.007002974998613354, + 0.0060012839967384934, + 0.004986289000953548, + 0.007998549001058564, + 0.007999689994903747, + 0.007993926003109664, + 0.007994346000486985, + 0.007012278001639061, + 0.010988633999659214, + 0.004855423998378683, + 0.00717923000047449, + 0.007950148996314965, + 0.005997567001031712, + 0.00699833600083366, + 0.009999597001296934, + 0.005993667000439018, + 0.005917317001149058, + 0.007078119000652805, + 0.0049949999956879765, + 0.007997544002137147, + 0.007997332002560142, + 0.007997218999662437, + 0.00633523100259481, + 0.009255267003027257, + 0.007402013005048502, + 0.006997216994932387, + 0.008001776004675776, + 0.008007846998225432, + 0.006986673994106241, + 0.007992044993443415, + 0.006997775999479927, + 0.007997232998604886, + 0.006997634001891129, + 0.00699747700127773, + 0.0069975940059521236, + 0.0050088620046153665, + 0.0079862319980748, + 0.005996829000650905, + 0.007997667999006808, + 0.00800661499670241, + 0.005154282000148669, + 0.003885659003572073, + 0.008020697998290416, + 0.003917261004971806, + 0.003994628001237288, + 0.011001033999491483, + 0.006994262002990581, + 0.0034946810046676546, + 0.0004527990022324957, + 0.00043163500231457874, + 0.0004298269996070303, + 0.0003974389983341098, + 0.0003523900013533421, + 0.0003519420060911216, + 0.0003916970017598942, + 0.0003672239981824532, + 0.0003677380009321496, + 0.00037817199336132035, + 0.00037580500065814704, + 0.0003639409987954423, + 0.00036331800220068544, + 0.0003621830037445761, + 0.00036697100586025044, + 0.00034526700619608164, + 0.0003698470027302392, + 0.00034986899845534936, + 0.00035766700602835044, + 0.0003563779973774217, + 0.00035997099621454254, + 0.00036033400101587176, + 0.00038186399615369737, + 0.0003582139979698695, + 0.0003507090004859492, + 0.0003692760001285933, + 0.00035535799543140456, + 0.00036249400000087917, + 0.00036737200571224093, + 0.0026363150027464144, + 0.005998158005240839, + 0.006903517998580355, + 0.00910072099941317, + 0.007996668995474465, + 0.006990565998421516, + 0.0036763090029126033, + 0.0033135400008177385, + 0.005999979002808686, + 0.008004757000890095, + 0.005998123000608757, + 0.00799141400057124, + 0.006991515998379327, + 0.005997537002258468, + 0.006085228000301868, + 0.00790819300164003, + 0.006998870994721074, + 0.007002361002378166, + 0.008004479997907765, + 0.007983091003552545, + 0.0069978320025256835, + 0.005114170002343599, + 0.005880386001081206, + 0.0059985230036545545, + 0.006997960997978225, + 0.006996653995884117, + 0.00799751700105844, + 0.0056184549976023845, + 0.005383323994465172, + 0.006014660000801086, + 0.007965069999045227, + 0.006997013995714951, + 0.006997486998443492, + 0.008003274000657257, + 0.007991194004716817, + 0.006998914999712724, + 0.004222789000777993, + 0.005770982999820262, + 0.00799618499877397, + 0.006062578999262769, + 0.008186512997781392, + 0.006972081006097142, + 0.007765162998111919, + 0.009997977998864371, + 0.00799778399959905, + 0.006996959004027303, + 0.008005524003237952, + 0.007998436005436815, + 0.004215786997519899, + 0.003767068003071472, + 0.005998326996632386, + 0.006997775999479927, + 0.007003165999776684, + 0.003990388002421241, + 0.008997739998449106, + 0.004201996001938824, + 0.0073382999980822206, + 0.006450009997934103, + 0.007995913001650479, + 0.007996518994332291, + 0.006000936002237722, + 0.007996950000233483, + 0.00799791300232755, + 0.007062894001137465, + 0.006930089999514166, + 0.006994170995312743, + 0.006999815996096004, + 0.0069929269957356155, + 0.008003308001207188, + 0.0035697380008059554, + 0.007416977001412306, + 0.007997213004273362, + 0.007998068998858798, + 0.004996790004952345, + 0.006997861994022969, + 0.005997413994919043, + 0.005997175998345483, + 0.006999025004915893, + 0.005995204002829269, + 0.007996757994988002, + 0.006997291995503474, + 0.005996813997626305, + 0.005999391003570054, + 0.005997704000037629, + 0.005994649996864609, + 0.006999368000833783, + 0.008134334995702375, + 0.006857931999547873, + 0.006996639000135474, + 0.007997575004992541, + 0.006997185999352951, + 0.0060008649961673655, + 0.007994351995876059, + 0.0059353579999879, + 0.004058509999595117, + 0.005998549997457303, + 0.006995648000156507, + 0.00900121599988779, + 0.006992977003392298, + 0.00599863899697084, + 0.006995611998718232, + 0.008003529001143761, + 0.007990815000084694, + 0.006995383002504241, + 0.007997829001396894, + 0.007997339002031367, + 0.006998051001573913, + 0.00799703899974702, + 0.005997230997309089, + 0.007997969994903542, + 0.002997398994921241, + 0.005997535998176318, + 0.006010589000652544, + 0.0008661829997436143, + 0.0003530999965732917, + 0.0003322789998492226, + 0.00033334000181639567, + 0.0007260739948833361, + 0.00032644900056766346, + 0.00032974100031424314, + 0.0003259049990447238, + 0.00032102699333336204, + 0.0011691049949149601, + 0.00413738999486668, + 0.0017375999959767796, + 0.0034710460022324696, + 0.01520376700500492, + 0.0047395259971381165, + 0.0011269399983575568, + 0.001818118995288387, + 0.00033739000355126336, + 0.004491113999392837, + 0.0012041230002068914, + 0.0003456170015851967, + 0.0022464819994638674, + 0.006000230001518503, + 0.0015569419992971234, + 0.00036522899608826265, + 0.0004555949999485165, + 0.0018946039999718778, + 0.006195572001161054, + 0.0012625320014194585, + 0.0021339170052669942, + 0.0021223619987722486, + 0.002946824999526143, + 0.0047316600030171685, + 0.003625165998528246, + 0.0027611109981080517, + 0.00042310500430176035, + 0.0004185660000075586, + 0.0004274580060155131, + 0.0003483240070636384, + 0.004307762996177189, + 0.0038111059984657913, + 0.0027202019991818815, + 0.0024305550032295287, + 0.0003975789950345643, + 0.0003932050021830946, + 0.002763243996014353, + 0.009257906000129879, + 0.0006581400011782534, + 0.0004849680044571869, + 0.0020241759993950836, + 0.004847732998314314, + 0.0004962129969499074, + 0.00045683400094276294, + 0.0010477190007804893, + 0.0007689110061619431, + 0.005152023004484363, + 0.0005109410049044527, + 0.0005039410025347024, + 0.0031840930023463443, + 0.0034398179996060207, + 0.0025307660034741275, + 0.0026709020021371543, + 0.0034371229994576424, + 0.0014445790002355352, + 0.004465276004339103, + 0.001990336000744719, + 0.00042533600208116695, + 0.00043008299689972773, + 0.00042331599979661405, + 0.00034913299896288663, + 0.0003479820006759837, + 0.003233905001252424, + 0.0006590659977518953, + 0.0010565600023255683, + 0.001259722004760988, + 0.003471005999017507, + 0.003374254003574606, + 0.0004449219995876774, + 0.00040633699973113835, + 0.0011635550035862252, + 0.0003654929969343357, + 0.0003600389973144047, + 0.00454403900221223, + 0.0005896549992030486, + 0.0030390199972316623, + 0.00041127100121229887, + 0.00040589099808130413, + 0.002301972999703139, + 0.00045099700218997896, + 0.006417743999918457, + 0.004343672997492831, + 0.0004620770050678402, + 0.0003801220009336248, + 0.00036718399496749043, + 0.002468377999321092, + 0.003875064998283051, + 0.0010827139994944446, + 0.00043372400250518695, + 0.0035635969979921356, + 0.0023208560014609247, + 0.0004123170001548715, + 0.0022204749984666705, + 0.0009259030048269778, + 0.003927190999093, + 0.0004570099990814924, + 0.0004425510051078163, + 0.00037357200199039653, + 0.0003537479933584109, + 0.00035489599395077676, + 0.00036464900040300563, + 0.0003520619939081371, + 0.00034689299354795367, + 0.001930587000970263, + 0.0007647169986739755, + 0.002492441999493167, + 0.0005839560035383329, + 0.0003972510021412745, + 0.0003618199989432469, + 0.001416247003362514, + 0.0003649700011010282, + 0.0004189949977444485, + 0.0003784500004258007, + 0.0003609449995565228, + 0.00035427200054982677, + 0.003056800000194926, + 0.0004564519986161031, + 0.00041762299952097237, + 0.0003780190018005669, + 0.0003505729982862249, + 0.00035253499663667753, + 0.0003643870004452765, + 0.0003507189976517111, + 0.00035011600266443565, + 0.002727311999478843, + 0.0020096239968552254, + 0.0009702740062493831, + 0.0003600510026444681, + 0.0003668470017146319, + 0.00034989499545190483, + 0.00035194300289731473, + 0.00035339199530426413, + 0.00038584099820582196, + 0.0008637219943921082, + 0.000884501998370979, + 0.002734488996793516, + 0.00039407900476362556, + 0.00036482799623627216, + 0.0003517639997880906, + 0.00035326099896337837, + 0.0003531789989210665, + 0.0003551660047378391, + 0.0003475519988569431, + 0.004730702996312175, + 0.0004120939993299544, + 0.0003747609953279607, + 0.00035058600042248145, + 0.0003565039951354265, + 0.0003529759997036308, + 0.00034792600490618497, + 0.00035280700103612617, + 0.0003533929993864149, + 0.0003464580004219897, + 0.003503068001009524, + 0.001816238000174053, + 0.00037608099955832586, + 0.0003530280009726994, + 0.00035545099672162905, + 0.0003529979949234985, + 0.00034844600304495543, + 0.0003618650007410906, + 0.00034896300348918885, + 0.00034738900285447016, + 0.0022737760009476915, + 0.0004112800015718676, + 0.00037227400025585666, + 0.00034924699866678566, + 0.00034901400067610666, + 0.00038977300573606044, + 0.0003567809981177561, + 0.00034674800554057583, + 0.0003586200036806986, + 0.0003513610063237138, + 0.0003574820002540946, + 0.0011878509976668283, + 0.004361118997621816, + 0.00038928700087126344, + 0.0003906719939550385, + 0.00038560599932679906, + 0.0003822370053967461, + 0.0003624679957283661, + 0.0003501499959384091, + 0.0003546719963196665, + 0.0003532940027071163, + 0.0003531180045683868, + 0.0029839519993402064, + 0.0015895980031928048, + 0.0003626630059443414, + 0.00038997599767753854, + 0.000393501999496948, + 0.0004219250040478073, + 0.0003754110002773814, + 0.00035290999949211255, + 0.0003578910036594607, + 0.0003534499992383644, + 0.00034692100598476827, + 0.001747299000271596, + 0.004023920999316033, + 0.00043043200275860727, + 0.0005720129993278533, + 0.00044277499546296895, + 0.0007121189992176369, + 0.0004164909987594001, + 0.0003535169962560758, + 0.0003567949970602058, + 0.003023934994416777, + 0.0004761259988299571, + 0.0003556119991117157, + 0.0003628900012699887, + 0.0003534130009938963, + 0.0003490009985398501, + 0.00035562099947128445, + 0.00035536199720809236, + 0.00034765000600600615, + 0.00035335800203029066, + 0.00034834499820135534, + 0.0003472860043984838, + 0.005248487999779172, + 0.0012642079964280128, + 0.00035950299934484065, + 0.00215434399433434, + 0.00038352100091287866, + 0.00035168600152246654, + 0.00037793899537064135, + 0.0003528710003593005, + 0.0003535880023264326, + 0.002543146001698915, + 0.0003775079967454076, + 0.0003655520049505867, + 0.0010267900070175529, + 0.00036328800342744216, + 0.0003538719975040294, + 0.00035993299388792366, + 0.0003888989958795719, + 0.0047141679970081896, + 0.0009351390035590157, + 0.0003712070028996095, + 0.0005298830001265742, + 0.0003801340062636882, + 0.0003569210020941682, + 0.00036620799801312387, + 0.0004509959981078282, + 0.00036050199560122564, + 0.0003502389954519458, + 0.0003488750007818453, + 0.003595347996451892, + 0.0012843879958381876, + 0.0003516020005918108, + 0.0003413009981159121, + 0.0003412369987927377, + 0.0003459450017544441, + 0.0003421200017328374, + 0.0003391049976926297, + 0.0003817190008703619, + 0.00034119900374207646, + 0.00033787600114010274, + 0.005184669003938325, + 0.0004814870044356212, + 0.00044363200140651315, + 0.0003449630021350458, + 0.0003443989990046248, + 0.00034702599805314094, + 0.00034597899502841756, + 0.00034583499655127525, + 0.0003532190021360293, + 0.0003465469999355264, + 0.0003381830028956756, + 0.004252855993399862, + 0.0003719039959833026, + 0.00034612000308698043, + 0.0003500720049487427, + 0.00034161800431320444, + 0.00033888900361489505, + 0.0003479879960650578, + 0.0003475859994068742, + 0.00033856299705803394, + 0.003944708005292341, + 0.0003777960009756498, + 0.00034446499921614304, + 0.00034251400211360306, + 0.00035402199864620343, + 0.00033962899760808796, + 0.000342930004990194, + 0.0003473739998298697, + 0.00033912199432961643, + 0.0003437889972701669, + 0.001682207002886571, + 0.0024013669972191565, + 0.0006232640007510781, + 0.00038544800190720707, + 0.0003485149936750531, + 0.00033859799441415817, + 0.0003497030047583394, + 0.00034574799792608246, + 0.0003375930027686991, + 0.00035222100268583745, + 0.00034340099955443293, + 0.00033771400194382295, + 0.0021279079956002533, + 0.002630884999234695, + 0.00035462200321489945, + 0.00039554799877805635, + 0.0003819419944193214, + 0.0015745139971841127, + 0.00037266899744281545, + 0.000346180000633467, + 0.00034654000046430156, + 0.0003375080050318502, + 0.00034973500441992655, + 0.0003388920013094321, + 0.0009435870015295222, + 0.003124517999822274, + 0.00036971100053051487, + 0.0003576200033421628, + 0.00034505700023146346, + 0.00034270600008312613, + 0.00036047599860467017, + 0.0003401990034035407, + 0.0003449020005064085, + 0.0003464839974185452, + 0.00034109599801013246, + 0.0004170239990344271, + 0.0027174480055691674, + 0.001506355001765769, + 0.001335121996817179, + 0.000367023007129319, + 0.0003440729997237213, + 0.0004510930011747405, + 0.0003442840024945326, + 0.0003382970025995746, + 0.0003408860065974295, + 0.0029606810057885014, + 0.000971839006524533, + 0.00038410400156863034, + 0.0003450049989623949, + 0.00034451999817974865, + 0.0003618039991124533, + 0.0003503689949866384, + 0.00033865799923660234, + 0.0003482039974187501, + 0.0003391009959159419, + 0.00034070200490532443, + 0.0036719550043926574, + 0.00042298399785067886, + 0.0003703109978232533, + 0.0003415989995119162, + 0.00035257599665783346, + 0.00033920299756573513, + 0.0003371920029167086, + 0.00034641799720702693, + 0.00034966199746122584, + 0.00033905999589478597, + 0.0029907779971836135, + 0.0011875810014316812, + 0.0003747040027519688, + 0.0003538980017765425, + 0.0003451950033195317, + 0.00034227500145789236, + 0.0003419619970372878, + 0.00034339100238867104, + 0.00034250199678353965, + 0.00034131499705836177, + 0.00035098299849778414, + 0.00034236400097142905, + 0.0003495090059004724, + 0.004321589003666304, + 0.0004049639974255115, + 0.0003470559968263842, + 0.0003654709944385104, + 0.00033889499900396913, + 0.00034641000092960894, + 0.0003513769988785498, + 0.00034269000025233254, + 0.0003443630048423074, + 0.0003443320019869134, + 0.0015906819971860386, + 0.0004619510000338778, + 0.0004587030052789487, + 0.001660018999245949, + 0.0007079730057739653, + 0.0004549670047708787, + 0.00037070100370328873, + 0.000343779000104405, + 0.0003425350005272776, + 0.0003532459959387779, + 0.00033976600388996303, + 0.00034017599682556465, + 0.0010180190001847222, + 0.002102543003275059, + 0.0023904840054456145, + 0.0003801060011028312, + 0.0003495089986245148, + 0.0003507920046104118, + 0.000339202000759542, + 0.00036809599987464026, + 0.0003376649983692914, + 0.0003433940000832081, + 0.0003611720021581277, + 0.00034242699621245265, + 0.0003445940019446425, + 0.005047108999860939, + 0.0004053719967487268, + 0.0003835640018223785, + 0.00035122400004183874, + 0.00034347400651313365, + 0.0003377210014150478, + 0.000347546003467869, + 0.0003386470052646473, + 0.0003424850001465529, + 0.0031757479955558665, + 0.00038355300057446584, + 0.0003575270020519383, + 0.0003905909979948774 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil[none]", + "fullname": "TestCalculateNabla4[none]", + "params": { + "static_variant": [ + "none", + null + ] + }, + "param": "none", + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.00047410900151589885, + "max": 0.020068984995305073, + "mean": 0.004361228765102155, + "stddev": 0.003397980529931836, + "rounds": 1868, + "median": 0.004725374499685131, + "iqr": 0.00642587999755051, + "q1": 0.0005770005009253509, + "q3": 0.007002880498475861, + "iqr_outliers": 2, + "stddev_outliers": 1019, + "outliers": "1019;2", + "ld15iqr": 0.00047410900151589885, + "hd15iqr": 0.01715502899605781, + "ops": 229.29317718938242, + "total": 8.146775333210826, + "data": [ + 0.0007394010026473552, + 0.0015758150038891472, + 0.0008321380009874701, + 0.0005799630016554147, + 0.0005317830000421964, + 0.000610720002441667, + 0.0005400189984356984, + 0.000521087997185532, + 0.000520650006365031, + 0.0005336539979907684, + 0.0028133050000178628, + 0.0009912879977491684, + 0.001233732997206971, + 0.0005386670018197037, + 0.0005188860013731755, + 0.0005368119964259677, + 0.0005163239984540269, + 0.0005330700005288236, + 0.0005153439997229725, + 0.001604570003109984, + 0.0009312179972766899, + 0.0012574610009323806, + 0.0005350260034902021, + 0.0019517420005286112, + 0.0005285850056679919, + 0.0005284509970806539, + 0.0005236290016910061, + 0.0022395409978344105, + 0.0006843059964012355, + 0.0005359589995350689, + 0.000519512002938427, + 0.0005288009997457266, + 0.0005195050034672022, + 0.0005256699951132759, + 0.0005205730049056001, + 0.002160669995646458, + 0.001965328003279865, + 0.0005421160021796823, + 0.0005302299978211522, + 0.0005150509969098493, + 0.0005286950035952032, + 0.0005161020017112605, + 0.0005241930048214272, + 0.0005145040049683303, + 0.0024454379963572137, + 0.0005321610005921684, + 0.0013459019974106923, + 0.0007870170011301525, + 0.0007725800023763441, + 0.0005569310014834628, + 0.0005201360036153346, + 0.0006209660059539601, + 0.0011220890010008588, + 0.0007597830044687726, + 0.000540070999704767, + 0.0005662549956468865, + 0.0018649199992069043, + 0.0005397950008045882, + 0.0005292779969749972, + 0.000514482002472505, + 0.0012727540015475824, + 0.0006016389961587265, + 0.0005313540023053065, + 0.0005153789970790967, + 0.0007122550014173612, + 0.000901696999790147, + 0.0008243659976869822, + 0.0005817140045110136, + 0.0005363160016713664, + 0.0005149520002305508, + 0.0005371320003177971, + 0.0005237039949861355, + 0.0012805660007870756, + 0.00308199899882311, + 0.0011627689964370802, + 0.0011035990028176457, + 0.0011801840009866282, + 0.0025037439991137944, + 0.0026382560026831925, + 0.0005869849992450327, + 0.0005326440004864708, + 0.0005281970006763004, + 0.0005248759989626706, + 0.0009825550005189143, + 0.004952340997988358, + 0.0005555809984798543, + 0.0005230499955359846, + 0.0005211379975662567, + 0.0006192779983393848, + 0.0005356219990062527, + 0.004568520002067089, + 0.0005422289978014305, + 0.0005203820037422702, + 0.0005266520020086318, + 0.0005319529955158941, + 0.0005205310008022934, + 0.0005287570020300336, + 0.0005125410025357269, + 0.0030275259996415116, + 0.005999230001179967, + 0.006998839999141637, + 0.005998171000101138, + 0.006991921996814199, + 0.008000019995961338, + 0.005991736004943959, + 0.007996804000867996, + 0.00799907800683286, + 0.005997635998937767, + 0.007998537999810651, + 0.006992436996370088, + 0.0069938660017214715, + 0.008003507995454129, + 0.006998034994467162, + 0.0079962319941842, + 0.006997337004577275, + 0.007993222003278788, + 0.005000899996957742, + 0.0069946369985700585, + 0.007543596999312285, + 0.009444023002288304, + 0.008022879999771249, + 0.00597681499493774, + 0.006987303000641987, + 0.005995384999550879, + 0.008001119997061323, + 0.007993680002982728, + 0.008001571994100232, + 0.006997517004492693, + 0.009995334999985062, + 0.007995929001481272, + 0.005993648002913687, + 0.007238513993797824, + 0.003751688993361313, + 0.005996831998345442, + 0.005997209998895414, + 0.006998507000389509, + 0.016001390999008436, + 0.005997054002364166, + 0.007992537000973243, + 0.008000048997928388, + 0.006007675001455937, + 0.006103829997300636, + 0.008932659002311993, + 0.008054345002165064, + 0.004845805997319985, + 0.006985775995417498, + 0.008012418998987414, + 0.005967571996734478, + 0.007003199003520422, + 0.007985030002600979, + 0.00699576400074875, + 0.008037208004679997, + 0.007585111998196226, + 0.006323320994852111, + 0.007980141999723855, + 0.005994867999106646, + 0.005989452998619527, + 0.006988893997913692, + 0.0070235009989119135, + 0.006969524998567067, + 0.006996787000389304, + 0.006975360003707465, + 0.00698752400057856, + 0.006997413998760749, + 0.0074142609955742955, + 0.005576016003033146, + 0.006974996002099942, + 0.00600399800168816, + 0.0069897470020805486, + 0.006991515001573134, + 0.008000258996617049, + 0.006971882001380436, + 0.008003827002539765, + 0.007983365998370573, + 0.005638588001602329, + 0.003763446002267301, + 0.0035612620049505495, + 0.008813262000330724, + 0.00470346200017957, + 0.007499544997699559, + 0.00629791399842361, + 0.003628696002124343, + 0.006018798005243298, + 0.005669760001183022, + 0.0042990820002160035, + 0.005991208003251813, + 0.00599773800058756, + 0.007001161000516731, + 0.007992459999513812, + 0.006989556997723412, + 0.007996127002115827, + 0.006993675000558142, + 0.0069984090005164035, + 0.0069994310033507645, + 0.006995541996730026, + 0.007004037004662678, + 0.00798375599697465, + 0.00800085200171452, + 0.005993594997562468, + 0.008002349000889808, + 0.007996139996976126, + 0.005001612000341993, + 0.004006869996374007, + 0.003979383000114467, + 0.0080072310011019, + 0.006014840997522697, + 0.007966730001498945, + 0.006997908996709157, + 0.00799202499911189, + 0.008008156000869349, + 0.007979248999617994, + 0.0070704350000596605, + 0.008498552000673953, + 0.007862418999138754, + 0.004516435001278296, + 0.005007182000554167, + 0.0021029639974585734, + 0.003993264996097423, + 0.005874155998753849, + 0.008006543001101818, + 0.006980313999520149, + 0.007002368001849391, + 0.007974412001203746, + 0.007001960002526175, + 0.007974590000230819, + 0.006997664997470565, + 0.006999953002377879, + 0.006983687999309041, + 0.007992939004907385, + 0.00798309800302377, + 0.0059990360023221, + 0.006990342000790406, + 0.006991656999161933, + 0.006838437002443243, + 0.006142352998722345, + 0.0058164749934803694, + 0.007189365001977421, + 0.004063937005412299, + 0.008676698998897336, + 0.0062235839941422455, + 0.00926838000304997, + 0.006716698997479398, + 0.007991365004272666, + 0.00798825600213604, + 0.008001387002877891, + 0.010065725997264963, + 0.006940082996152341, + 0.006928339003934525, + 0.006982150000112597, + 0.007000021003477741, + 0.0069788100008736365, + 0.008710979003808461, + 0.004834398998355027, + 0.006935326004168019, + 0.007669817001442425, + 0.006792485000914894, + 0.003109230994596146, + 0.0038825960000394844, + 0.007995507003215607, + 0.0068332630035001785, + 0.007146903000830207, + 0.005992420003167354, + 0.00802031400235137, + 0.005963767995126545, + 0.007005652005318552, + 0.006982154001889285, + 0.006998655000643339, + 0.005076544999610633, + 0.00790771200263407, + 0.007007410000369418, + 0.00697839700296754, + 0.007998747001693118, + 0.006996491996687837, + 0.010519561998080462, + 0.006456151997554116, + 0.005988826000248082, + 0.007009521003055852, + 0.0069800679993932135, + 0.007001605001278222, + 0.007009549997746944, + 0.005966764998447616, + 0.005040048999944702, + 0.003947356999560725, + 0.007993849001650233, + 0.006999949000601191, + 0.00698650699632708, + 0.007013092996203341, + 0.007981440998264588, + 0.00506303200381808, + 0.007926219994260464, + 0.004984769999282435, + 0.007005814004514832, + 0.006981194994295947, + 0.008008106000488624, + 0.0050638589964364655, + 0.0045076480018906295, + 0.005776196005172096, + 0.00635845800570678, + 0.0063308069948107, + 0.006930650997674093, + 0.006986891996348277, + 0.006990251000388525, + 0.0059828880039276555, + 0.009994077001465484, + 0.008004103001439944, + 0.006981110003835056, + 0.007023090001894161, + 0.006961810002394486, + 0.006999874000030104, + 0.009381782998389099, + 0.013631627996801399, + 0.006953860000066925, + 0.008010110999748576, + 0.006964041996980086, + 0.0070003500004531816, + 0.005985859999782406, + 0.007000923003943171, + 0.007993115999852307, + 0.006970941001782194, + 0.007012545000179671, + 0.00697467899590265, + 0.007005378996836953, + 0.007977681001648307, + 0.006010995995893609, + 0.007989413999894168, + 0.006983691004279535, + 0.008022889000130817, + 0.011570770002435893, + 0.0033909829944605008, + 0.009974494998459704, + 0.008017422005650587, + 0.005996189000143204, + 0.008002898997801822, + 0.007960344999446534, + 0.008020567998755723, + 0.007971530998474918, + 0.008008885000890587, + 0.007979475005413406, + 0.0079940430005081, + 0.0079800730018178, + 0.008038589003263041, + 0.009948269995220471, + 0.0059943009982816875, + 0.00921421499515418, + 0.004747286999190692, + 0.01002882000466343, + 0.007850508998672012, + 0.00619264499982819, + 0.003890799001965206, + 0.005978920999041293, + 0.008036346000153571, + 0.0069487279979512095, + 0.008025212002394255, + 0.006963223000639118, + 0.007007995998719707, + 0.00797129800048424, + 0.00699872199766105, + 0.00475746799929766, + 0.004317598999477923, + 0.007908425999630708, + 0.006467265004175715, + 0.006528167999931611, + 0.009941983997123316, + 0.005030653002904728, + 0.006013794001773931, + 0.006232940002519172, + 0.007109372003469616, + 0.0016172589967027307, + 0.020068984995305073, + 0.01495579699985683, + 0.010956115998851601, + 0.013024546999076847, + 0.007959997994475998, + 0.003984212002251297, + 0.00847266599885188, + 0.010555030996329151, + 0.010955822996038478, + 0.014991067997470964, + 0.015023434003524017, + 0.014083371999731753, + 0.01087433999782661, + 0.007944409997435287, + 0.007999622997886036, + 0.005967748998955358, + 0.007004086997767445, + 0.015993039996828884, + 0.007979075002367608, + 0.010254036998958327, + 0.005706294999981765, + 0.007982702998560853, + 0.00633415599440923, + 0.003747164002561476, + 0.002901416999520734, + 0.007011663001321722, + 0.006975504002184607, + 0.006000936002237722, + 0.007993595994776115, + 0.0080047969968291, + 0.011133744999824557, + 0.0038561250039492734, + 0.007006316998740658, + 0.006982197999604978, + 0.0069960059990989976, + 0.007000255995080806, + 0.0069908209989080206, + 0.007001013997069094, + 0.004204311000648886, + 0.0077869290034868754, + 0.005997127002046909, + 0.005998171996907331, + 0.006997136995778419, + 0.006993508999585174, + 0.0070014150041970424, + 0.004019807995064184, + 0.005243682004220318, + 0.0038924019972910173, + 0.007827761000953615, + 0.0029889060024288483, + 0.0030801399989286438, + 0.002912312003900297, + 0.004322585002228152, + 0.007670503997360356, + 0.008530535000318196, + 0.007462116001988761, + 0.006962541003304068, + 0.007345788995735347, + 0.007683853997150436, + 0.006012974001350813, + 0.008986825996544212, + 0.007989270998223219, + 0.006313840996881481, + 0.010677440004656091, + 0.00699343100131955, + 0.006009092001477256, + 0.00599286599754123, + 0.006987411994487047, + 0.006651761999819428, + 0.006798276997869834, + 0.006539449997944757, + 0.007995857995410915, + 0.006987173001107294, + 0.008008226002857555, + 0.007992537000973243, + 0.006988238004851155, + 0.009083791002922226, + 0.0032121170006575994, + 0.01031359799526399, + 0.008377373000257649, + 0.004079580998222809, + 0.0029058259970042855, + 0.0036051880015293136, + 0.00738855099916691, + 0.005994762999762315, + 0.006119735997344833, + 0.007359267998253927, + 0.006631023999943864, + 0.0031263950004358776, + 0.003634535001765471, + 0.008634316000097897, + 0.0006319390013231896, + 0.0005526129971258342, + 0.013224150003225077, + 0.006399597004929092, + 0.0006918229992152192, + 0.0005204679982853122, + 0.00808299399795942, + 0.000765503995353356, + 0.0005363560048863292, + 0.013496302002749871, + 0.0071198939986061305, + 0.0028120030037825927, + 0.0005508730027941056, + 0.0020471139941946603, + 0.010363253000832628, + 0.0007369049999397248, + 0.0005361310031730682, + 0.010021766000136267, + 0.0006948510053916834, + 0.0005135270039318129, + 0.0030572089963243343, + 0.007441387999278959, + 0.00655068299965933, + 0.010029773999121971, + 0.005974184001388494, + 0.008652053998957854, + 0.0012315109997871332, + 0.004267575000994839, + 0.008292330996482633, + 0.0007263699953909963, + 0.0005568319975282066, + 0.010053079000499565, + 0.008857414999511093, + 0.005079226000816561, + 0.0050685050009633414, + 0.0007818619997124188, + 0.0032963979974738322, + 0.0055241830050363205, + 0.0014400069994735532, + 0.004497620997426566, + 0.004270277997420635, + 0.0008020390014280565, + 0.0005778159975307062, + 0.0031904200004646555, + 0.00764632999926107, + 0.0007341220043599606, + 0.0005659599992213771, + 0.0005828619978274219, + 0.006295387000136543, + 0.004800255999725778, + 0.006993160997808445, + 0.0006780949988751672, + 0.001259887001651805, + 0.0010430530019220896, + 0.0012911000012536533, + 0.0005622810058412142, + 0.0034172470041085035, + 0.0006206409962032922, + 0.000539273998583667, + 0.0005278500029817224, + 0.0005322610013536178, + 0.0022555239993380383, + 0.00309545000345679, + 0.0031236220020218752, + 0.0019189580052625388, + 0.001540174002002459, + 0.0022525970052811317, + 0.001349915997707285, + 0.0029210020002210513, + 0.0005582200028584339, + 0.001615563000086695, + 0.0007027370011201128, + 0.0005899989992030896, + 0.002755761001026258, + 0.00775825600430835, + 0.0006130749970907345, + 0.0010027210009866394, + 0.0005515779994311742, + 0.0010576490021776408, + 0.003761883002880495, + 0.0005761850043199956, + 0.0015480610018130392, + 0.002860959000827279, + 0.0005892099943594076, + 0.0006676630000583827, + 0.0026068369988934137, + 0.004124061000766233, + 0.0006287930009420961, + 0.0005326489990693517, + 0.0008319210028275847, + 0.0006633480006712489, + 0.0005543899969779886, + 0.0005170169970369898, + 0.0018736800047918223, + 0.003804977000982035, + 0.0036141729942755774, + 0.0010415729993837886, + 0.0005657189976773225, + 0.0005316040042089298, + 0.0005221889950917102, + 0.0005268789973342791, + 0.0007119539950508624, + 0.004032645003462676, + 0.0006183619989315048, + 0.004379064994282089, + 0.0008348129995283671, + 0.0023583010042784736, + 0.0005503649954334833, + 0.0029464980034390464, + 0.0009717659995658323, + 0.0006319989988696761, + 0.0006697320059174672, + 0.0006822330033173785, + 0.0013186470023356378, + 0.0005329679988790303, + 0.0034664760023588315, + 0.0005849579974892549, + 0.0010998799989465624, + 0.0008302990027004853, + 0.0009784210051293485, + 0.0005533880030270666, + 0.000528094002220314, + 0.0005318580006132834, + 0.0009541289982735179, + 0.004067782996571623, + 0.0012356879960861988, + 0.0017335299999103881, + 0.0006818149995524436, + 0.0006986119988141581, + 0.0006666200060863048, + 0.0008083760039880872, + 0.0006231110019143671, + 0.0006058380022295751, + 0.0005315169983077794, + 0.0007271639988175593, + 0.000685689999954775, + 0.0012037679989589378, + 0.0005584069949691184, + 0.0005973219958832487, + 0.0005709820034098811, + 0.0005616509952233173, + 0.0008254679996753111, + 0.0005718350003007799, + 0.0005207380017964169, + 0.0005141170040587895, + 0.0005038599992985837, + 0.004565392002405133, + 0.0007244579974212684, + 0.0010427160013932735, + 0.0005707579985028133, + 0.0049377299947082065, + 0.0035482520033838227, + 0.0006457170020439662, + 0.0005368170022848062, + 0.0008682289990247227, + 0.0005657349975081161, + 0.0006086749999667518, + 0.0005065539953648113, + 0.0070189679972827435, + 0.0007099889990058728, + 0.000547543000720907, + 0.0005153360034455545, + 0.0005132470032549463, + 0.0005103830044390634, + 0.004897726001217961, + 0.0006717129945172928, + 0.0005983500013826415, + 0.0005355780012905598, + 0.0005123509981785901, + 0.0005123700029798783, + 0.0005151080040377565, + 0.004880778993538115, + 0.0007126029959181324, + 0.0005593059977400117, + 0.0005513709984370507, + 0.0005175679980311543, + 0.0005117440014146268, + 0.000513809995027259, + 0.0005053229979239404, + 0.003949432997615077, + 0.0008446270003332756, + 0.000548490002984181, + 0.0005179859945201315, + 0.000516728003276512, + 0.0005072330022812821, + 0.0006839779962319881, + 0.0045930160049465485, + 0.0007261680002557114, + 0.0005145879986230284, + 0.0023417330012307502, + 0.0005807790003018454, + 0.0006109050009399652, + 0.0005399199944804423, + 0.0005128959965077229, + 0.0032796310042613186, + 0.0006383410000125878, + 0.0005450269964057952, + 0.0005093910003779456, + 0.000517225002113264, + 0.0005103720031911507, + 0.0037088409953867085, + 0.0006087229994591326, + 0.0005376560002332553, + 0.000525085000845138, + 0.0005066689991508611, + 0.0005357900008675642, + 0.0005148879936314188, + 0.005255652002233546, + 0.0016208559973165393, + 0.0005383120005717501, + 0.0005191349991946481, + 0.0005128479970153421, + 0.000511136997374706, + 0.000518192995514255, + 0.0005097609973745421, + 0.004434854999999516, + 0.0005852600006619468, + 0.0005214510019868612, + 0.00051921699923696, + 0.0005134590028319508, + 0.0005174649995751679, + 0.0005121599970152602, + 0.0005045389989390969, + 0.007201606000307947, + 0.0006212950029294007, + 0.0005197439968469553, + 0.0005075340013718233, + 0.003562635996786412, + 0.0006148839966044761, + 0.0005986260002828203, + 0.004181133997917641, + 0.0006000290013616905, + 0.0006968600064283237, + 0.000557343999389559, + 0.000516878004418686, + 0.0005033280031057075, + 0.004152428999077529, + 0.0022827999928267673, + 0.0020054480046383105, + 0.0006314979982562363, + 0.00208354299684288, + 0.002383307000854984, + 0.0005381829978432506, + 0.0005550360001507215, + 0.0005204059998504817, + 0.002971568006614689, + 0.0006868860000395216, + 0.0005564629973378032, + 0.0005208689981373027, + 0.0005237790028331801, + 0.0005126690011820756, + 0.000798786997620482, + 0.0036861479966319166, + 0.0006205890022101812, + 0.0005196249985601753, + 0.0005058050010120496, + 0.0005105220043333247, + 0.0004889979973086156, + 0.0016162980027729645, + 0.0005010910026612692, + 0.0004977540011168458, + 0.0005024230049457401, + 0.000495462998514995, + 0.0005231410032138228, + 0.0005708089956897311, + 0.003226884000468999, + 0.0006054639961803332, + 0.0005672859988408163, + 0.0004929759961669333, + 0.0009966860015993007, + 0.0005017900039092638, + 0.0005036939983256161, + 0.000500381996971555, + 0.0004994830014766194, + 0.0006237159977899864, + 0.0005070909974165261, + 0.0005012309993617237, + 0.001675055995292496, + 0.0005745050002587959, + 0.0005577720003202558, + 0.002243944996735081, + 0.0006148279935587198, + 0.0005269859975669533, + 0.000702931996784173, + 0.0005306510065565817, + 0.0004925339962937869, + 0.0004987509964848869, + 0.0004933610034640878, + 0.002072317998681683, + 0.0005508350004674867, + 0.0005106550015625544, + 0.0004939290010952391, + 0.0005065229997853749, + 0.0005008779990021139, + 0.0044686620021821, + 0.0006342510023387149, + 0.0005231230024946854, + 0.0004917620026390068, + 0.0005008779990021139, + 0.0004941939987475052, + 0.0005027130027883686, + 0.0004933500022161752, + 0.0006386419991031289, + 0.0008447450018138625, + 0.0005226209977990948, + 0.0018002729993895628, + 0.0005357420013751835, + 0.0005563840022659861, + 0.0005053080021752976, + 0.000490355996589642, + 0.0044113849944551475, + 0.0006304880007519387, + 0.0005346959951566532, + 0.0005039790048613213, + 0.0005290739936754107, + 0.0004979160003131256, + 0.0005496710000443272, + 0.0004966839987901039, + 0.0005033039997215383, + 0.0004988319997210056, + 0.0004985300038242713, + 0.0004967590066371486, + 0.004242101997078862, + 0.0006386929962900467, + 0.0005166889968677424, + 0.0004955129988957196, + 0.0007376599969575182, + 0.0005496449957718141, + 0.0007589430024381727, + 0.0005330880012479611, + 0.0004954609976266511, + 0.0005751310018240474, + 0.0004960459991707467, + 0.0005054049979662523, + 0.0004984290062566288, + 0.0011796789985965006, + 0.000532445999851916, + 0.001146750000771135, + 0.0005021580000175163, + 0.0005015300048398785, + 0.0004991020032321103, + 0.00048824900295585394, + 0.0005156550032552332, + 0.0005007740037399344, + 0.0019376820055185817, + 0.0005709569959435612, + 0.0005517879981198348, + 0.0005131389989401214, + 0.0004968659995938651, + 0.0032197749969782308, + 0.0009554569987813011, + 0.0005980189962428994, + 0.0005274330032989383, + 0.0005088620018796064, + 0.003423825000936631, + 0.0005907809972995892, + 0.0005106620010337792, + 0.0004939439968438819, + 0.0005039480020059273, + 0.0004990460001863539, + 0.0006733929985784926, + 0.0005545579988393001, + 0.0005058000024291687, + 0.0004909060007776134, + 0.0005022960031055845, + 0.0004932250012643635, + 0.003742332999536302, + 0.0008724369981791824, + 0.0005158549975021742, + 0.0004964110048604198, + 0.0011276680015726015, + 0.0012013439991278574, + 0.0005522510036826134, + 0.0004922279986203648, + 0.0005579299977398477, + 0.000498125002195593, + 0.000529708995600231, + 0.0005009820015402511, + 0.0004919749990222044, + 0.000499302004755009, + 0.000495116000820417, + 0.0005158919957466424, + 0.0014470019959844649, + 0.0005537480028579012, + 0.0004999150041840039, + 0.0005001449972041883, + 0.0004948569985572249, + 0.0010359649968449958, + 0.0019460120020085014, + 0.0008464310012641363, + 0.0005499420003616251, + 0.0006354040015139617, + 0.0005647579964715987, + 0.0031342860020231456, + 0.0006944460037630051, + 0.0006789340040995739, + 0.0010383659973740578, + 0.000777467001171317, + 0.0004971619928255677, + 0.0004919109996990301, + 0.0004982050013495609, + 0.0005534369993256405, + 0.0005538840050576255, + 0.0005043330020271242, + 0.0020176260004518554, + 0.0005516829987755045, + 0.0005849029985256493, + 0.0005200899977353401, + 0.0005088140023872256, + 0.0005071089981356636, + 0.0005986699979985133, + 0.002448512001137715, + 0.0014399930005311035, + 0.000624947999313008, + 0.0005150970027898438, + 0.0005096220047562383, + 0.0016716280006221496, + 0.000521302004926838, + 0.0005076399975223467, + 0.0004972530005034059, + 0.0013601899991044775, + 0.0007734169994364493, + 0.0005036760048824362, + 0.0004922709995298646, + 0.0004992429967387579, + 0.0004906050016870722, + 0.0004964679974364117, + 0.0005169440046302043, + 0.000499652000144124, + 0.0005014459966332652, + 0.0006344800058286637, + 0.0005033859997638501, + 0.000497902998176869, + 0.0004953580064466223, + 0.000614461001532618, + 0.0005921539996052161, + 0.0005069000035291538, + 0.0005018150040996261, + 0.0004956249977112748, + 0.004628852002497297, + 0.0006249159996514209, + 0.0017076070071198046, + 0.0005790710056317039, + 0.0005207729991525412, + 0.0004989300068700686, + 0.0004903129956801422, + 0.0005012570036342368, + 0.0004937109988532029, + 0.0014041780013940297, + 0.0005353379965526983, + 0.0014553299988619983, + 0.0004982259997632354, + 0.0005704770010197535, + 0.0005159610009286553, + 0.0005472230041050352, + 0.0005128230041009374, + 0.0005161029985174537, + 0.0005086800010758452, + 0.0004944910033373162, + 0.0005026789949624799, + 0.000493977997393813, + 0.0004907210022793151, + 0.0005045989964855835, + 0.0005013479967601597, + 0.0005194190016482025, + 0.0005258469973341562, + 0.0011191079975105822, + 0.0005785020039184019, + 0.000513687999045942, + 0.000726341997506097, + 0.0005119889974594116, + 0.0004919459970551543, + 0.0004905210007564165, + 0.0005017179937567562, + 0.0004978310025762767, + 0.0006332470002234913, + 0.0005479610044858418, + 0.0007682560026296414, + 0.0005265709987725131, + 0.0005177809944143519, + 0.0004901000065729022, + 0.000502314003824722, + 0.0004979559962521307, + 0.0005168460047570989, + 0.0005523340005311184, + 0.0005662130060954951, + 0.0005175600017537363, + 0.000496477005071938, + 0.0005139810018590651, + 0.0005061400006525218, + 0.0005051960033597425, + 0.0005031170003348961, + 0.0005179810032132082, + 0.0005211859970586374, + 0.0005161000008229166, + 0.0008437910000793636, + 0.0004965789994457737, + 0.0016457040037494153, + 0.0008901360051822849, + 0.0005027340012020431, + 0.0004953729949193075, + 0.0004952150047756732, + 0.0005276509982650168, + 0.0005100670023239218, + 0.0007287209955393337, + 0.0008740760022192262, + 0.0004963500032317825, + 0.0006995750009082258, + 0.0010580590023892, + 0.0004942279992974363, + 0.0005566330000874586, + 0.0004987630018149503, + 0.0008639169973321259, + 0.0005476829974213615, + 0.0006178089970489964, + 0.0005158679996384308, + 0.00050605800061021, + 0.0004992480025975965, + 0.0004984000042895786, + 0.0004969990040990524, + 0.000540821005415637, + 0.000525770999956876, + 0.0005130839999765158, + 0.001187476998893544, + 0.0005320700001902878, + 0.0005028640007367358, + 0.0004913899974781089, + 0.0005033420020481572, + 0.0016586299971095286, + 0.0005008240041206591, + 0.0005042859993409365, + 0.000490430997160729, + 0.0005450809985632077, + 0.0009657300033723004, + 0.0005334079978638329, + 0.0005547429973375984, + 0.000499802001286298, + 0.0013712709987885319, + 0.010081985994474962, + 0.007927642996946815, + 0.0013628539963974617, + 0.005661258001055103, + 0.0032291309980791993, + 0.0037014319968875498, + 0.005668764999427367, + 0.009357442999316845, + 0.006957977995625697, + 0.006352076001348905, + 0.002593732002424076, + 0.00655027199536562, + 0.005377114001021255, + 0.00834187600412406, + 0.006820123002398759, + 0.007870754998293705, + 0.007982650997291785, + 0.007172319004894234, + 0.005679552996298298, + 0.006119910998677369, + 0.005981959999189712, + 0.0069857350026723, + 0.005999373002850916, + 0.005982301998301409, + 0.011014181000064127, + 0.00396653100324329, + 0.0070135030036908574, + 0.007967146993905772, + 0.006982630002312362, + 0.005906190002860967, + 0.004074220996699296, + 0.008072970995272044, + 0.005968752004264388, + 0.005932615000347141, + 0.004986324995115865, + 0.006994994000706356, + 0.009000170000945218, + 0.006940400002349634, + 0.003018930998223368, + 0.005998194006679114, + 0.004000229004304856, + 0.007994589999725576, + 0.007990026999323163, + 0.007983555995451752, + 0.007995915999345016, + 0.005986448006296996, + 0.0029764439968857914, + 0.007005829000263475, + 0.007992497994564474, + 0.005982940005196724, + 0.007994669998879544, + 0.005979402994853444, + 0.007017926996923052, + 0.007977173001563642, + 0.006985626998357475, + 0.007998059001693036, + 0.00801621100254124, + 0.007962420997500885, + 0.006022688998200465, + 0.0006855609972262755, + 0.0005729389959014952, + 0.0006028049974702299, + 0.0019549299977370538, + 0.0010334489998058416, + 0.0020144149966654368, + 0.0006032330056768842, + 0.0005519999976968393, + 0.000542135996511206, + 0.0005444959970191121, + 0.0035940939997090027, + 0.0006457350027631037, + 0.0013301790022524074, + 0.0005625630001304671, + 0.0005439480009954423, + 0.0005452069963212125, + 0.0005357019981602207, + 0.0005426589996204711, + 0.0005823899991810322, + 0.004436088995134924, + 0.0006396559983841144, + 0.0005532270006369799, + 0.000551744997210335, + 0.0005430259989225306, + 0.0005465259964694269, + 0.0006441410005209036, + 0.005220683000516146, + 0.0007224729997687973, + 0.0006001580040901899, + 0.0005946269957348704, + 0.0005473009950947016, + 0.0005559600031119771, + 0.0005466740039992146, + 0.004547736003587488, + 0.0006345060028252192, + 0.0005829639994772151, + 0.0005602359960903414, + 0.0005408829965745099, + 0.0005499949984368868, + 0.00053252399811754, + 0.005510820003109984, + 0.0006714069968438707, + 0.000609780996455811, + 0.0007430640034726821, + 0.0005781209983979352, + 0.0005520199993043207, + 0.005832536000525579, + 0.0006586399977095425, + 0.0005472470002132468, + 0.0005288120009936392, + 0.0005172780001885258, + 0.0026106160003109835, + 0.0006140779951238073, + 0.0013874910000595264, + 0.0005504709988599643, + 0.0005283100035740063, + 0.0005179400031920522, + 0.0005242059996817261, + 0.0005271049958537333, + 0.0005248660017969087, + 0.000517576998390723, + 0.0031594619940733537, + 0.000626329994702246, + 0.0005297019961290061, + 0.0005236979995970614, + 0.0005263150014798157, + 0.0005275990042719059, + 0.0005303349971654825, + 0.0005700709953089245, + 0.005895873997360468, + 0.0006953760021133348, + 0.0005819910002173856, + 0.0005236150027485564, + 0.0005275570001685992, + 0.0005199049992370419, + 0.005033016997913364, + 0.00067352999758441, + 0.0005585050021181814, + 0.0005321520002325997, + 0.0005344180026440881, + 0.0005253099952824414, + 0.0034527489988249727, + 0.007996607993845828, + 0.007996651002031285, + 0.005996921994665172, + 0.005762587999925017, + 0.006230942999536637, + 0.005995340005028993, + 0.005995615996653214, + 0.007017299001745414, + 0.008004402996448334, + 0.005977321998216212, + 0.008006315998500213, + 0.007990689999132883, + 0.007995415006007534, + 0.005988144999719225, + 0.00799887599714566, + 0.006989321998844389, + 0.010546183002588805, + 0.010429057001601905, + 0.010127574001671746, + 0.008160028999554925, + 0.004648727001040243, + 0.007004259998211637, + 0.00797134100139374, + 0.00801604799926281, + 0.007996742999239359, + 0.0069855550027568825, + 0.006970105001528282, + 0.007996366999577731, + 0.006998705997830257, + 0.007977025998116005, + 0.005812533003336284, + 0.006172341003548354, + 0.006003205002343748, + 0.012989143004233483, + 0.006986221000261139, + 0.006988451001234353, + 0.005969132005702704, + 0.006989786001213361, + 0.0077943729993421584, + 0.008185516999219544, + 0.00585776699881535, + 0.004081742998096161, + 0.007994816005520988, + 0.0070736069974373095, + 0.0078979590034578, + 0.004191761996480636, + 0.012244944999110885, + 0.010156027004995849, + 0.00636435500200605, + 0.005006986997614149, + 0.007979212998179719, + 0.0060148129996377975, + 0.010021133995905984, + 0.0059533800013014115, + 0.007989964993612375, + 0.0010016859960160218, + 0.005989823002892081, + 0.006004605005728081, + 0.007987361997948028, + 0.008044160000281408, + 0.0029535780049627647, + 0.004992266003682744, + 0.006994831994234119, + 0.005995769999572076, + 0.005992637001327239, + 0.0069950870019965805, + 0.004120655998121947, + 0.007873500995629001, + 0.005993753002258018, + 0.007999121000466403, + 0.007997082000656519, + 0.0029928329968242906, + 0.004995572002371773, + 0.00799886099412106, + 0.006002817994158249, + 0.007993017003173009, + 0.0069953150014043786, + 0.00799505000031786, + 0.00799699700291967, + 0.007997246997547336, + 0.007994448998942971, + 0.008040721993893385, + 0.0069546650047414005, + 0.005995308005367406, + 0.0069992919961805455, + 0.007994390005478635, + 0.006994467003096361, + 0.007997758002602495, + 0.005994340004690457, + 0.007999044995813165, + 0.006995827003265731, + 0.005995462997816503, + 0.006996894000621978, + 0.007998429995495826, + 0.005999464003252797, + 0.005992489001073409, + 0.00800378600251861, + 0.005990061006741598, + 0.007995391002623364, + 0.0018314729968551546, + 0.0005138319975230843, + 0.0005005269995308481, + 0.0004835989966522902, + 0.0004773960026795976, + 0.00047840000479482114, + 0.00047410900151589885, + 0.000480472001072485, + 0.0004793479965883307, + 0.0012727950015687384, + 0.0026000090001616627, + 0.00057496899535181, + 0.0005666219949489459, + 0.0004924730019411072, + 0.0005307659957907163, + 0.0009394280059495941, + 0.0019289609990664758, + 0.0005951610000920482, + 0.0015346799991675653, + 0.0004883939982391894, + 0.0005697250016964972, + 0.0005142119989614002, + 0.0005053600034443662, + 0.0038869650015840307, + 0.0006703499966533855, + 0.0015207010001176968, + 0.0008763239966356196, + 0.0005032709959778003, + 0.0004911959986202419, + 0.0004944910033373162, + 0.00047605799773009494, + 0.0004866179951932281, + 0.00047928199637681246, + 0.0030969290019129403, + 0.0005918680035392754, + 0.0005297860043356195, + 0.0004923029991914518, + 0.00048651900578988716, + 0.00048044299910543486, + 0.0004890389973297715, + 0.00048726400564191863, + 0.00048057000094559044, + 0.003068243997404352, + 0.0006350270050461404, + 0.0006524410055135377, + 0.0026321999976062216, + 0.0006189060004544444, + 0.0005502839994733222, + 0.00047715999971842393, + 0.0007548600042355247, + 0.00244519799889531, + 0.0005725150040234439, + 0.0005187230053707026, + 0.00047987099969759583, + 0.0004791040046256967, + 0.0004867060051765293, + 0.0004848159951507114, + 0.004689861001679674, + 0.0006202030053827912, + 0.0005725360024371184, + 0.0004955120020895265, + 0.00048148600035347044, + 0.0004840789988520555, + 0.0004753519970108755, + 0.0034294010038138367, + 0.001585136000358034, + 0.0005427790019894019, + 0.0005130490026203915, + 0.0004971950038452633, + 0.0035531030007405207, + 0.0006017439955030568, + 0.0010947820046567358, + 0.0009624859958421439, + 0.0004977659991709515, + 0.0004847409945796244, + 0.00048435700591653585, + 0.0004969610017724335, + 0.0005020539974793792, + 0.0004887149989372119, + 0.0004842349953833036, + 0.0009547150038997643, + 0.004106669999600854, + 0.00704060000134632, + 0.00695593799900962, + 0.005995116996928118, + 0.005998895001539495, + 0.006579913999303244, + 0.004407380001794081, + 0.006068251001124736, + 0.007917313996586017, + 0.008396101999096572, + 0.007991109996510204, + 0.0060898090014234185, + 0.01453382900217548, + 0.007988813005795237, + 0.007989395002368838, + 0.007998057000804693, + 0.00636661200405797, + 0.0074818100038100965, + 0.0070855320009286515, + 0.006997848002356477, + 0.0052690069933305494, + 0.00871587800065754, + 0.007999089997611009, + 0.006982736995269079, + 0.004987913001968991, + 0.007001644997217227, + 0.006284319999394938, + 0.006682127001113258, + 0.007004807994235307, + 0.009983050003938843, + 0.006224540004041046, + 0.0037272170011419803, + 0.00630442699912237, + 0.005809383001178503, + 0.002834872000676114, + 0.007028154999716207, + 0.01715502899605781, + 0.005816879005578812, + 0.009589287998096552, + 0.009155782005109359, + 0.0031799280041013844, + 0.0052002159936819226, + 0.005526769004063681, + 0.008885057999577839, + 0.002915213000960648, + 0.003955786000005901, + 0.003890660998877138, + 0.011014404997695237, + 0.006974937001359649, + 0.005991011996229645, + 0.0054878099981579, + 0.003541696998581756, + 0.006862946996989194, + 0.002574641002865974, + 0.005465225003717933, + 0.007945586999994703, + 0.007000270001299214, + 0.0053780829985043965, + 0.003604845995141659, + 0.006993672999669798, + 0.007996232001460157, + 0.007996952001121826, + 0.00800117600010708, + 0.007994184998096898, + 0.008994194002298173, + 0.004996987001504749, + 0.009004853003716562, + 0.006989694004005287, + 0.008002104004845023, + 0.0039913700020406395, + 0.005996337000397034, + 0.005995455998345278, + 0.008012801001314074, + 0.006277558000874706, + 0.003710229000716936, + 0.00500086099782493, + 0.007998943998245522, + 0.007157651998568326, + 0.006817950998083688, + 0.005966009004623629, + 0.004031914002553094, + 0.005492534997756593, + 0.009075683999981266, + 0.005401962996984366, + 0.010013305996835697, + 0.006994050003413577, + 0.0019434909991105087, + 0.004998345997591969, + 0.009012515001813881, + 0.0058396700042067096, + 0.005183774999750312, + 0.0029638309933943674, + 0.0033433190037612803, + 0.0010972529998980463, + 0.002505853997718077, + 0.010284798001521267, + 0.010737712000263855, + 0.009992626997700427, + 0.00971870400098851, + 0.011130496000987478, + 0.007102784002199769, + 0.014085866998357233, + 0.005298078998748679, + 0.0022641289979219437, + 0.0032537579972995445, + 0.003702248002809938, + 0.006272013000852894, + 0.007034112000837922, + 0.006999315999564715, + 0.004916607998893596, + 0.004054587996506598, + 0.004992084999685176, + 0.00404078999417834, + 0.003947420002077706, + 0.0028779540007235482, + 0.0013368060026550665, + 0.003771422001591418, + 0.0027704699969035573, + 0.004219585003738757, + 0.0033101310036727227, + 0.0009340629985672422, + 0.002086962995235808, + 0.006861129004391842, + 0.0016219099998124875, + 0.00580018699838547, + 0.008365756999410223, + 0.0022286349994828925, + 0.00976655999693321, + 0.012005224998574704, + 0.012758668002788909, + 0.011216245999094099, + 0.01276543400308583, + 0.005176562997803558, + 0.010999460995662957, + 0.006987286004005, + 0.008230022998759523, + 0.006738842996128369, + 0.006984241997997742, + 0.00644435100548435, + 0.005537445002119057, + 0.008491768996464089, + 0.004487902006076183, + 0.01109063400508603, + 0.010919578999164514, + 0.006112568997195922, + 0.006766243001038674, + 0.005829902002005838, + 0.007128367004042957, + 0.005009533997508697, + 0.00630234200070845, + 0.0046741289988858625, + 0.008129968999128323, + 0.004858182001044042, + 0.0060007539941580035, + 0.008379675004107412, + 0.008626593997178134, + 0.007208920003904495, + 0.007744941998680588, + 0.0020401780056999996, + 0.00494050399720436, + 0.004336007004894782, + 0.0036537650012178347, + 0.009984847005398478, + 0.003003031997650396, + 0.004215897002723068, + 0.0029123650019755587, + 0.005866660998435691, + 0.007004409999353811, + 0.008027420000871643, + 0.009577871998772025, + 0.004355413999292068, + 0.00798952500190353, + 0.01046536799549358, + 0.003522957005770877, + 0.003917444999387953, + 0.012068736999935936, + 0.00896650899812812, + 0.008024224000109825, + 0.007838944999093655, + 0.005094163003377616, + 0.00698383100097999, + 0.005988302000332624, + 0.006992648006416857, + 0.006005212999298237, + 0.011924813996301964, + 0.005049182000220753, + 0.00799643900245428, + 0.00699255800282117, + 0.011002156003087293, + 0.006986892003624234, + 0.007012747002590913, + 0.00597944000037387, + 0.010000884001783561, + 0.005987644006381743, + 0.008002616996236611, + 0.008024841001315508, + 0.005960000999039039, + 0.012000869995972607, + 0.00799370899767382, + 0.006994062001467682, + 0.010000965994549915, + 0.006991989001107868, + 0.0019959970013587736, + 0.00599872899329057, + 0.0020021730015287176, + 0.005988584998704027, + 0.005001939003705047, + 0.006992167000134941, + 0.0039619499948457815, + 0.0040372849980485626, + 0.009991041995817795, + 0.009996120003052056, + 0.008000939997145906, + 0.00799420499970438, + 0.005993254002532922, + 0.008996151002065744, + 0.006996112002525479, + 0.0069983729990781285, + 0.007364974997472018, + 0.007627771003171802, + 0.004997497002477758, + 0.010999828999047168, + 0.006992861002800055, + 0.0070008020047680475, + 0.006069944000046235, + 0.00892199900408741, + 0.007994833002157975, + 0.00610433299880242, + 0.007888781998190098, + 0.00502070999937132, + 0.0016877599991858006, + 0.0022807470013503917, + 0.004657313002098817, + 0.009339504002127796, + 0.007999590001418255, + 0.011001544000464492, + 0.008072231998085044, + 0.005956490997050423, + 0.007933034998131916, + 0.010005280993937049, + 0.007996312997420318, + 0.006973970004764851, + 0.007992749000550248, + 0.0074526509997667745, + 0.006499630006146617, + 0.005151516001205891, + 0.00787606299854815, + 0.004007725998235401, + 0.010428636000142433, + 0.006538374997035135, + 0.008006732998182997, + 0.006979813995712902, + 0.005993313003273215, + 0.007988268000190146, + 0.00799568399816053, + 0.0059964300016872585, + 0.00801206399773946, + 0.006976657998166047, + 0.0049896050040842965, + 0.008641170999908354, + 0.009712799997942057, + 0.01143495400174288, + 0.00919833700027084, + 0.010832031002792064, + 0.006152105997898616, + 0.0070025619934313, + 0.006987190994550474, + 0.00799828399613034, + 0.005993735998345073, + 0.00599569700716529, + 0.004991952999262139, + 0.0070023009975557216, + 0.00799066499894252, + 0.007999413996003568, + 0.006937451005796902, + 0.0035341890034032986, + 0.004521375005424488, + 0.005993476996081881, + 0.00699436499417061, + 0.005996235995553434, + 0.007998631001100875, + 0.003997205996711273, + 0.004905781999696046, + 0.004087528999662027, + 0.007779794999805745, + 0.007215837998955976, + 0.005996396001137327, + 0.007997966000402812, + 0.007995275002031121, + 0.008000625995919108, + 0.0069934519997332245, + 0.007997389002412092, + 0.00899731800018344, + 0.00799592299881624, + 0.006997832999331877, + 0.005997661006404087, + 0.006996466996497475, + 0.007997358996362891, + 0.005004190999898128, + 0.008992916002171114, + 0.010998178993759211, + 0.00799195299623534, + 0.0059974520045216195, + 0.004997044998162892, + 0.005030856002122164, + 0.003741566004464403, + 0.006223771997611038, + 0.00798541700351052, + 0.007997018001333345, + 0.006001341003866401, + 0.009993081002903637, + 0.00799661500059301, + 0.0040215660046669655, + 0.007971260994963814, + 0.006000865003443323, + 0.008002889997442253, + 0.006981922997510992, + 0.007018454001809005, + 0.007209501003671903, + 0.0017464850025135092, + 0.003018930998223368, + 0.0064329469969379716, + 0.007537308993050829, + 0.005996402003802359, + 0.006998418997682165, + 0.0069918420049361885, + 0.007994527004484553, + 0.009009538000100292, + 0.005370662998757325, + 0.007929485000204295, + 0.005900617004954256, + 0.007764419999148231, + 0.007992186001501977, + 0.005414474006101955, + 0.001569842999742832, + 0.0060031859975424595, + 0.008002187001693528, + 0.009412167994014453, + 0.004562098998576403, + 0.005998543005262036, + 0.006990915004280396, + 0.006998331002250779, + 0.007993138999154326, + 0.005705491996195633, + 0.00429517800512258, + 0.007018474003416486, + 0.007181521003076341, + 0.0077720629997202195, + 0.007002313999691978, + 0.0059775580011773854, + 0.006928491995495278, + 0.005060566996689886, + 0.00599687900103163, + 0.0070141709948075, + 0.009591582995199133, + 0.004379929006972816, + 0.007011830995907076, + 0.0069796099996892735, + 0.007848856002965476, + 0.005177806997380685, + 0.006962099003430922, + 0.007990769001480658, + 0.006994139002927113, + 0.005040536001615692, + 0.010946690999844577, + 0.007996674998139497, + 0.006996498996159062, + 0.006986647997109685, + 0.007995403997483663, + 0.008001146001333836, + 0.007998035005584825, + 0.00699046299996553, + 0.007999822999408934, + 0.006996055999479722, + 0.007992904000275303, + 0.0059940290011581965, + 0.010996430006343871, + 0.008002456001122482, + 0.010004759002185892, + 0.007983534000231884, + 0.007991015001607593, + 0.007991553000465501, + 0.00999761600542115, + 0.006991836002271157, + 0.007165472001361195, + 0.0073406590017839335, + 0.006478931005403865, + 0.0040280190005432814, + 0.005946752993622795, + 0.0049357789976056665, + 0.007081062998622656, + 0.0069717880032840185, + 0.007996071995876264, + 0.003988649004895706, + 0.010006186996179167, + 0.007985192998603452, + 0.0079995490013971, + 0.005992911996145267, + 0.00799916800315259, + 0.005932520994974766, + 0.004061883999384008, + 0.007990846002940089, + 0.004995374001737218, + 0.00499885300087044, + 0.00799400100368075, + 0.004995684001187328, + 0.0059973330062348396, + 0.006995910000114236, + 0.0068002219995833, + 0.0061930739975650795, + 0.007999928006029222, + 0.005987730000924785, + 0.008002112997928634, + 0.005993847000354435, + 0.00594705199910095, + 0.006042098997568246, + 0.007996514003025368, + 0.00799954000103753, + 0.007994419996975921, + 0.006997898999543395, + 0.007996331005415414, + 0.009000285004731268, + 0.007987536999280564, + 0.006000305998895783, + 0.005993796999973711, + 0.006988258995988872, + 0.008000140995136462, + 0.0069926889991620556, + 0.007996585001819767, + 0.00596805100212805, + 0.006024416004947852, + 0.006994434996158816, + 0.00800204899860546, + 0.007991136000782717, + 0.007006877000094391, + 0.007983338997291867, + 0.004995469003915787, + 0.007997314001841005, + 0.007997351000085473, + 0.007996185006049927, + 0.007998891996976454, + 0.004994146998797078, + 0.009000716003356501, + 0.007997296001121867, + 0.007991798003786243, + 0.004994357994291931, + 0.004996064999431837, + 0.006998496006417554, + 0.006996348005486652, + 0.007005848005064763, + 0.007992742001079023, + 0.007993457002157811, + 0.007993018996785395, + 0.00799348500004271, + 0.007994683997821994, + 0.0070467860059579834, + 0.007955574998049997, + 0.005995102001179475, + 0.00898672600305872, + 0.004976998003257904, + 0.007990491001692135, + 0.006990309004322626, + 0.0060161940054968, + 0.009986344004573766, + 0.007984252995811403, + 0.007992603997990955, + 0.007991598999069538, + 0.007192427001427859, + 0.0041777160004130565, + 0.002551345001847949, + 0.005021859004045837, + 0.008411338996666018, + 0.006603191002795938, + 0.008097873003862333, + 0.0027088429997093044, + 0.0042375980046927, + 0.006938415004697163, + 0.004314541001804173, + 0.0020167830007267185, + 0.0046545329969376326, + 0.0018365440046181902, + 0.00914630700572161, + 0.006997784999839496, + 0.00598726899625035, + 0.008005449999473058, + 0.007991657003003638, + 0.005320367999956943, + 0.00466614100150764, + 0.0060342180004226975, + 0.007967779005412012, + 0.0019962809965363704, + 0.006396839999069925, + 0.002413619004073553, + 0.004173381996224634, + 0.005997095002385322, + 0.004338079001172446, + 0.005670648999512196, + 0.00798859799397178, + 0.0069919669986120425, + 0.0050759419973474, + 0.009914877999108285, + 0.006005524999636691, + 0.006332716999168042, + 0.007651239000551868, + 0.006996506999712437, + 0.005998999004077632, + 0.007005813000432681, + 0.004323608998674899, + 0.004660295002395287, + 0.005992965001496486, + 0.004402030004712287, + 0.002449307001370471, + 0.0041371059996890835, + 0.0032451079969177954, + 0.006757465002010576, + 0.007990853999217506, + 0.008001274996786378, + 0.007001860001764726, + 0.005984833995171357, + 0.005141576999449171, + 0.0068532999939634465, + 0.003991375997429714, + 0.005998152002575807, + 0.002169856998079922, + 0.0005292900023050606, + 0.000507662000018172, + 0.0004958450008416548, + 0.0004967380009475164, + 0.0005288890024530701, + 0.0033688079929561354, + 0.0006284429982770234, + 0.0004957030032528564, + 0.00047619500401197, + 0.0006343670029309578, + 0.0004924609966110438, + 0.0004959060024702922, + 0.00048599800356896594, + 0.0017200299989781342, + 0.001611024999874644, + 0.0004933469972456805, + 0.000490771999466233, + 0.0004839160028495826, + 0.0004870510019827634, + 0.00047933799942256883, + 0.0030964869947638363, + 0.0006509719969471917, + 0.0005806929984828457, + 0.0004883159999735653, + 0.0005116990068927407, + 0.0004865160008193925 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil[compile_time_domain]", + "fullname": "TestCalculateNabla4[compile_time_domain]", + "params": { + "static_variant": [ + "compile_time_domain", + [ + "horizontal_start", + "horizontal_end", + "vertical_start", + "vertical_end" + ] + ] + }, + "param": "compile_time_domain", + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.00046226000267779455, + "max": 0.010999489997630008, + "mean": 0.00587169544827639, + "stddev": 0.002629615661344385, + "rounds": 201, + "median": 0.006990451001911424, + "iqr": 0.002989786247781012, + "q1": 0.004998075502953725, + "q3": 0.007987861750734737, + "iqr_outliers": 19, + "stddev_outliers": 45, + "outliers": "45;19", + "ld15iqr": 0.0005246520013315603, + "hd15iqr": 0.010999489997630008, + "ops": 170.30856058679706, + "total": 1.1802107851035544, + "data": [ + 0.005879707001440693, + 0.0029924499976914376, + 0.00599567099561682, + 0.0069937080043018796, + 0.007001646998105571, + 0.007995829000719823, + 0.007997226995939855, + 0.00799333699978888, + 0.007996729997103103, + 0.005997448002744932, + 0.007996221000212245, + 0.00799657199968351, + 0.00600060499709798, + 0.007999983004992828, + 0.0060135450039524585, + 0.005975626001600176, + 0.0060025749990018085, + 0.007992749000550248, + 0.00799916699907044, + 0.00799494500097353, + 0.005993551996652968, + 0.0059974629984935746, + 0.005996026004140731, + 0.006998340999416541, + 0.0069970710028428584, + 0.008003531002032105, + 0.007996400003321469, + 0.007997838998562656, + 0.003992122001363896, + 0.004998385004000738, + 0.008001025002158713, + 0.009992678002163302, + 0.0069963470014045015, + 0.006301901004917454, + 0.007694865002122242, + 0.006995060997724067, + 0.006999372002610471, + 0.005995259998599067, + 0.006996748001256492, + 0.007002188998740166, + 0.006993192997470032, + 0.008000257999810856, + 0.006992627000727225, + 0.007003309001447633, + 0.0069922779948683456, + 0.00599809100094717, + 0.006992724993324373, + 0.0069971129996702075, + 0.006998631994065363, + 0.006997204000072088, + 0.007997406995855272, + 0.005997021995426621, + 0.007037445000605658, + 0.010959307001030538, + 0.008992159004264977, + 0.004992802001652308, + 0.007443670998327434, + 0.007552561000920832, + 0.00623557699873345, + 0.004823118993954267, + 0.007934261993796099, + 0.007994428000529297, + 0.008995389005576726, + 0.0029961340042063966, + 0.007999401997949462, + 0.00799698300397722, + 0.005995986000925768, + 0.007996690001164097, + 0.008051806995354127, + 0.0077058849929017015, + 0.006233462001546286, + 0.008000407004146837, + 0.00699450899992371, + 0.006997554002737161, + 0.005997936001222115, + 0.006001771005685441, + 0.0062146780037437566, + 0.005772820004494861, + 0.007004440005403012, + 0.00798716300050728, + 0.008003205999557395, + 0.010993026997311972, + 0.0066997650064877234, + 0.005287652995320968, + 0.005996983003569767, + 0.005996944004436955, + 0.00904589000128908, + 0.005971897000563331, + 0.0049795790037023835, + 0.004363877000287175, + 0.0066294610005570576, + 0.003988098003901541, + 0.004988390006474219, + 0.006995556999754626, + 0.007994685001904145, + 0.006009087999700569, + 0.0029883989991503768, + 0.00399132700113114, + 0.0059987249987898394, + 0.008002952999959234, + 0.008000746995094232, + 0.006987572000070941, + 0.006995160001679324, + 0.008996663003927097, + 0.007996238004125189, + 0.007996811000339221, + 0.006997085998591501, + 0.007003491002251394, + 0.00799124199693324, + 0.007001277001108974, + 0.00799218699830817, + 0.00699805399926845, + 0.010999489997630008, + 0.006990909998421557, + 0.007996145999641158, + 0.006999235003604554, + 0.006010536999383476, + 0.002981050005473662, + 0.003996644998551346, + 0.0039980790024856105, + 0.0059969789945171215, + 0.006996971002081409, + 0.007323489000555128, + 0.00767282499873545, + 0.0029951070027891546, + 0.0070003780056140386, + 0.007001926998782437, + 0.006990451001911424, + 0.0070041940052760765, + 0.007993673003511503, + 0.007989958001417108, + 0.007998467997822445, + 0.006008105003274977, + 0.005993944003421348, + 0.0059900600026594475, + 0.008004317001905292, + 0.006988209999690298, + 0.00799684900266584, + 0.007995733001735061, + 0.009302718004619237, + 0.006692304996249732, + 0.006003520997182932, + 0.0059926940011791885, + 0.007000110002991278, + 0.006991146998188924, + 0.005998713000735734, + 0.006996267999056727, + 0.006996083997364622, + 0.0038977930016699247, + 0.006094568001572043, + 0.006996232004894409, + 0.006016744002408814, + 0.006979163001233246, + 0.005997991000185721, + 0.00599528600287158, + 0.005427557000075467, + 0.007569247994979378, + 0.00799190499674296, + 0.00799738299974706, + 0.004996807001589332, + 0.004997146999812685, + 0.006997058000706602, + 0.00699678499950096, + 0.010998465004377067, + 0.006998607997957151, + 0.007995640000444837, + 0.00371095599984983, + 0.0006368269969243556, + 0.0006570960031240247, + 0.000490534002892673, + 0.0004639079998014495, + 0.00047927199921105057, + 0.00046651899901917204, + 0.004659404999983963, + 0.0006018599960952997, + 0.0005882080004084855, + 0.0005246520013315603, + 0.00047045199607964605, + 0.0004705519968410954, + 0.00047603800339857116, + 0.005322265002178028, + 0.0006261819944484159, + 0.000532285004737787, + 0.00046465500054182485, + 0.0004750200023408979, + 0.00046341899724211544, + 0.0004703439990407787, + 0.0004676610042224638, + 0.005487378999532666, + 0.0006017209962010384, + 0.0005281549965729937, + 0.0004712630034191534, + 0.0004707930056611076, + 0.0004683530059992336, + 0.00046947700320743024, + 0.0004725960025098175, + 0.0053124500045669265, + 0.0005711749981855974, + 0.00056279799900949, + 0.0004704289967776276, + 0.00046226000267779455 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil[compile_time_vertical]", + "fullname": "TestCalculateNabla4[compile_time_vertical]", + "params": { + "static_variant": [ + "compile_time_vertical", + [ + "vertical_start", + "vertical_end" + ] + ] + }, + "param": "compile_time_vertical", + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.0004770460000145249, + "max": 0.05026232600357616, + "mean": 0.004794945990073818, + "stddev": 0.004631223442377899, + "rounds": 295, + "median": 0.004902200002106838, + "iqr": 0.0061894879981991835, + "q1": 0.0008117057514027692, + "q3": 0.007001193749601953, + "iqr_outliers": 7, + "stddev_outliers": 18, + "outliers": "18;7", + "ld15iqr": 0.0004770460000145249, + "hd15iqr": 0.0163589250005316, + "ops": 208.55292261271225, + "total": 1.4145090670717764, + "data": [ + 0.007137049004086293, + 0.00684582599933492, + 0.0069967580056982115, + 0.0060692240003845654, + 0.0069221090016071685, + 0.00530778500251472, + 0.008063152999966405, + 0.007256682998558972, + 0.002352803996473085, + 0.004993515998648945, + 0.007994608000444714, + 0.004072004994668532, + 0.007919269999547396, + 0.00800405699555995, + 0.007988495999597944, + 0.00799554099648958, + 0.016001743999368045, + 0.020018267998239025, + 0.0080606940027792, + 0.005755172001954634, + 0.006465951002610382, + 0.0004966080014128238, + 0.0022576810006285086, + 0.0005393889950937591, + 0.0005659749949700199, + 0.0005341940050129779, + 0.00048635499842930585, + 0.005001200996048283, + 0.0005059659961261787, + 0.0020630740036722273, + 0.0005009599990444258, + 0.00048472400521859527, + 0.0004830469988519326, + 0.0004861839988734573, + 0.0004893000004813075, + 0.0004803460033144802, + 0.0054404069960583, + 0.0005641469979309477, + 0.0004880250053247437, + 0.00048005599819589406, + 0.00048595700354781, + 0.0004770460000145249, + 0.0019983739985036664, + 0.005394217005232349, + 0.0024059479983407073, + 0.0005504310029209591, + 0.0020809890047530644, + 0.0011438220026320778, + 0.0004838090026169084, + 0.0004897879989584908, + 0.0004808119992958382, + 0.0163589250005316, + 0.003898666000168305, + 0.0020181139989290386, + 0.004329633004090283, + 0.0023032130047795363, + 0.003520837999531068, + 0.002509514000848867, + 0.0009956470021279529, + 0.0006967180015635677, + 0.0005274309951346368, + 0.0005430579985841177, + 0.0005190399970160797, + 0.000494989006256219, + 0.004902200002106838, + 0.0010331579978810623, + 0.002316696998605039, + 0.0019601920066634193, + 0.0009617469986551441, + 0.0022438310043071397, + 0.0005540679994737729, + 0.0005129130004206672, + 0.002872598997782916, + 0.0016857480004546233, + 0.0009172999998554587, + 0.0014660669985460117, + 0.0014938009990146384, + 0.0005511709969141521, + 0.006539677000546362, + 0.020539933997497428, + 0.002902250998886302, + 0.009907497005769983, + 0.004075721000845078, + 0.006495897003333084, + 0.010411124996608123, + 0.01038513399544172, + 0.013831720003508963, + 0.05026232600357616, + 0.0116395660006674, + 0.010186127998167649, + 0.010040342996944673, + 0.01725114400323946, + 0.006244434000109322, + 0.0013356659983401187, + 0.001719653999316506, + 0.0014041030008229427, + 0.0008047890005400404, + 0.0006100199971115217, + 0.0005548530025407672, + 0.0008391700030188076, + 0.0005985190000501461, + 0.0008009180019143969, + 0.0008335430029546842, + 0.0012335570063441992, + 0.0009021379955811426, + 0.0008709499961696565, + 0.0008324560039909557, + 0.0017583509979886003, + 0.003801519997068681, + 0.004018353996798396, + 0.003974027997173835, + 0.004012217003037222, + 0.0026324229984311387, + 0.005376260996854398, + 0.003998118998424616, + 0.004075926000950858, + 0.001821718004066497, + 0.0024635410009068437, + 0.004694048002420459, + 0.0058699239962152205, + 0.002158035000320524, + 0.00383406100445427, + 0.005167246999917552, + 0.002831375000823755, + 0.005994599006953649, + 0.0020010539956274442, + 0.005033477005781606, + 0.002947214998130221, + 0.007691500999499112, + 0.0009218659979524091, + 0.007294686998648103, + 0.0007242170031531714, + 0.0006327240043901838, + 0.0005993330050841905, + 0.0007046190003165975, + 0.0006460419972427189, + 0.0005945160010014661, + 0.0006139949982753024, + 0.0006072989999665879, + 0.0006330660035018809, + 0.0006153610011097044, + 0.000747146004869137, + 0.0005840699959662743, + 0.0005736189996241592, + 0.0005496730009326711, + 0.0006354430006467737, + 0.0006136570009402931, + 0.0005812739982502535, + 0.0005773489974671975, + 0.0005623360048048198, + 0.0005792760057374835, + 0.0005694939973182045, + 0.0005670140017173253, + 0.0007226849993458018, + 0.0006400700003723614, + 0.0006861529982415959, + 0.000617511999735143, + 0.0005996260006213561, + 0.0005836349955643527, + 0.0005691480037057772, + 0.000607560999924317, + 0.0005842839964316227, + 0.0005848680011695251, + 0.0005749139963882044, + 0.0005716619998565875, + 0.000575615995330736, + 0.0005847740030731075, + 0.0005774330056738108, + 0.0005743069996242411, + 0.0006348989991238341, + 0.0006836549946456216, + 0.006104185995354783, + 0.023009905002254527, + 0.01653653199900873, + 0.008784257996012457, + 0.006903753994265571, + 0.0030825770008959807, + 0.002975693001644686, + 0.007054506997519638, + 0.006940392006072216, + 0.007992837003257591, + 0.007994809995579999, + 0.006005950999679044, + 0.006185893995279912, + 0.0021820769979967736, + 0.006715337003697641, + 0.007890558001236059, + 0.0010787280043587089, + 0.0039058179972926155, + 0.009050659005879425, + 0.007979904003150295, + 0.006949811999220401, + 0.004996126001060475, + 0.008001784000953194, + 0.007998278000741266, + 0.007987598997715395, + 0.007994719002454076, + 0.007006672996794805, + 0.0069990530028007925, + 0.003252406997489743, + 0.005732953002734575, + 0.00498962499841582, + 0.005997446998662781, + 0.007005744002526626, + 0.010989810005412437, + 0.0059878040046896785, + 0.00100208700314397, + 0.0020723319976241328, + 0.007415889005642384, + 0.0054905970027903095, + 0.006050415999197867, + 0.008261948001745623, + 0.0017828150012064725, + 0.006295002000115346, + 0.0037843370009795763, + 0.003279700002167374, + 0.00451088699628599, + 0.008003101997019257, + 0.007993809995241463, + 0.004993319002096541, + 0.007998581997526344, + 0.006222267002158333, + 0.007792795993736945, + 0.010980032995576039, + 0.007703174000198487, + 0.0072787680037436076, + 0.006983986000705045, + 0.005994124003336765, + 0.00700521600083448, + 0.006988240995269734, + 0.005996778003463987, + 0.007996555999852717, + 0.005992961996525992, + 0.005991799996991176, + 0.007997428001544904, + 0.002991362001921516, + 0.00600293400202645, + 0.007997344997420441, + 0.007994543004315346, + 0.0046153800067259, + 0.0043745799994212575, + 0.001728390998323448, + 0.0032625649982946925, + 0.006999420002102852, + 0.007993956001882907, + 0.007996349006134551, + 0.0069938379965606146, + 0.006997468000918161, + 0.007994330000656191, + 0.007994036001036875, + 0.0070028670015744865, + 0.006991825001023244, + 0.007990569996763952, + 0.008003356000699569, + 0.004986152998753823, + 0.004003321002528537, + 0.010993066003720742, + 0.006002747002639808, + 0.008011675003217533, + 0.0079753059981158, + 0.00713815900235204, + 0.005858003998582717, + 0.0069820629942114465, + 0.007000388999585994, + 0.006999678997090086, + 0.007992819002538454, + 0.008008197997696698, + 0.007000017998507246, + 0.006976855998800602, + 0.006996937998337671, + 0.008236128000135068, + 0.004755375994136557, + 0.0069912089966237545, + 0.005996743006107863, + 0.006001637004374061, + 0.005982467999274377, + 0.007002068996371236, + 0.007995617001142818, + 0.006993629998760298, + 0.00699410900415387, + 0.006995506999373902, + 0.006992925002123229, + 0.007995951003977098, + 0.005812378003611229, + 0.004181711999990512, + 0.0070014619996072724, + 0.007989810001163278, + 0.0061094159973436035, + 0.006880783999804407, + 0.007993921004526783, + 0.00599906500428915, + 0.005990644000121392, + 0.005993978993501514, + 0.00600642099743709, + 0.00498650899680797 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil", + "fullname": "TestEnhanceDiffusionCoefficientForGridPointColdPools", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.00032675699912942946, + "max": 0.02308472900040215, + "mean": 0.005575253752455827, + "stddev": 0.004989503482430912, + "rounds": 319, + "median": 0.005988128999888431, + "iqr": 0.007595933497213991, + "q1": 0.00039615625246369746, + "q3": 0.007992089749677689, + "iqr_outliers": 4, + "stddev_outliers": 138, + "outliers": "138;4", + "ld15iqr": 0.00032675699912942946, + "hd15iqr": 0.019948140004999004, + "ops": 179.36403335175783, + "total": 1.778505947033409, + "data": [ + 0.006990530004259199, + 0.008011363999685273, + 0.007984112999110948, + 0.005995724000968039, + 0.005992070997308474, + 0.00999461900210008, + 0.00599915500060888, + 0.005988128999888431, + 0.0039968269993551075, + 0.00398686999687925, + 0.008006832002138253, + 0.005193342003622092, + 0.009827343004872091, + 0.005099105001136195, + 0.009053759000380524, + 0.006787723999877926, + 0.00799335099873133, + 0.006995679999818094, + 0.006008814998494927, + 0.007988617006049026, + 0.006985851003264543, + 0.005993574995954987, + 0.006960134000109974, + 0.010004557996580843, + 0.007983142000739463, + 0.00798075000056997, + 0.008729814995604102, + 0.007241919003718067, + 0.006979917001444846, + 0.00702725000155624, + 0.00697073400078807, + 0.006950850001885556, + 0.008002223999937996, + 0.005912470005569048, + 0.008072659999015741, + 0.006996232004894409, + 0.005129640005179681, + 0.007854882001993246, + 0.006990925001446158, + 0.008004032002645545, + 0.006996452997555025, + 0.004982154001481831, + 0.006999956000072416, + 0.007994580002559815, + 0.006997727003181353, + 0.006993479997618124, + 0.004995670999051072, + 0.015187088996754028, + 0.00882696799817495, + 0.008987290995719377, + 0.017006351001327857, + 0.004955503005476203, + 0.006972403003601357, + 0.008998045996122528, + 0.010990208000293933, + 0.00598137699853396, + 0.007988531004230026, + 0.012992788004339673, + 0.010112870993907563, + 0.008869654004229233, + 0.007981614995514974, + 0.01901512900076341, + 0.014980136998929083, + 0.020981080997444224, + 0.01396004200069001, + 0.013992708001751453, + 0.009445616997254547, + 0.017545544003951363, + 0.02308472900040215, + 0.01585110700398218, + 0.013978128001326695, + 0.014220035998732783, + 0.010755169001640752, + 0.011131314000522252, + 0.015856104000704363, + 0.014992936005000956, + 0.014985130001150537, + 0.01700440899730893, + 0.0060284249993856065, + 0.009946557998773642, + 0.015986610000254586, + 0.016170169998076744, + 0.012791940003808122, + 0.01296895900304662, + 0.01598095900408225, + 0.013980620002257638, + 0.013543672001105733, + 0.016452370000479277, + 0.016968137002550066, + 0.01196940999943763, + 0.005963436000456568, + 0.01898782900389051, + 0.017989705003856216, + 0.021972690003167372, + 0.007371469997451641, + 0.006598294996365439, + 0.00598206900031073, + 0.007999414003279526, + 0.006986209002207033, + 0.007992936996743083, + 0.007992011000169441, + 0.007992115999513771, + 0.0069858709975960664, + 0.007007523003267124, + 0.006986118998611346, + 0.005988993994833436, + 0.00699617899954319, + 0.007997000000614207, + 0.0056007830062299035, + 0.007378855996648781, + 0.010010566002165433, + 0.007999878995178733, + 0.006988319000811316, + 0.007986532997165341, + 0.007983491006598342, + 0.01119328800268704, + 0.007278004995896481, + 0.006897195999044925, + 0.007602302001032513, + 0.006991982998442836, + 0.00799241600179812, + 0.007992253995325882, + 0.006998880999162793, + 0.0060043550038244575, + 0.00399054899753537, + 0.0029820030031260103, + 0.013000737002585083, + 0.006995513002038933, + 0.010000784001022112, + 0.0059952249939669855, + 0.006055912002921104, + 0.006922979002411012, + 0.00699458499730099, + 0.007998747998499312, + 0.00698975599516416, + 0.00700706199859269, + 0.006972426999709569, + 0.0069962489942554384, + 0.008003569004358724, + 0.006988336994254496, + 0.006983455998124555, + 0.007998788998520467, + 0.008004565999726765, + 0.007985476004250813, + 0.01099274899752345, + 0.003992608995758928, + 0.005021883996960241, + 0.007973218002007343, + 0.005008276006265078, + 0.006998028002271894, + 0.005981820002489258, + 0.0059912139986408874, + 0.007994486004463397, + 0.005986449999909382, + 0.002999976997671183, + 0.00699211799656041, + 0.006665054999757558, + 0.00732279299700167, + 0.008017346997803543, + 0.006973376002861187, + 0.006967276000068523, + 0.0069851829975959845, + 0.006992969996645115, + 0.007018674004939385, + 0.006961560997297056, + 0.00799658100004308, + 0.007984359996044077, + 0.007035455004370306, + 0.019948140004999004, + 0.0150564569994458, + 0.008906843002478126, + 0.009734052000567317, + 0.010623919995850883, + 0.001754100994730834, + 0.000358658995537553, + 0.0003879689975292422, + 0.004888009003479965, + 0.00812591199792223, + 0.006490635001682676, + 0.002095473995723296, + 0.0009534450000501238, + 0.00040359400009037927, + 0.00047569999878760427, + 0.016232754998782184, + 0.005279530996631365, + 0.006174891001137439, + 0.003896269998222124, + 0.007949286002258305, + 0.0057760010022320785, + 0.0007660199989913963, + 0.00041854400478769094, + 0.000482161995023489, + 0.00039510200440417975, + 0.0003798409961746074, + 0.0003417189946048893, + 0.0008952060015872121, + 0.000366694999684114, + 0.0049784770017140545, + 0.0005400269947131164, + 0.0003584440055419691, + 0.0003572429995983839, + 0.00035052900057053193, + 0.0018337299989070743, + 0.00038951800524955615, + 0.00035957599902758375, + 0.00034600499930093065, + 0.0003435120015637949, + 0.002735018999374006, + 0.00037224799598334357, + 0.010286965996783692, + 0.0006361740015563555, + 0.004280385001038667, + 0.0005077250025351532, + 0.0015107170038390905, + 0.0006287740034167655, + 0.0012837699978263117, + 0.0003794169970205985, + 0.00034147000405937433, + 0.0003485209963400848, + 0.00034138500632252544, + 0.00033619000168982893, + 0.0003453169993008487, + 0.0003393360020709224, + 0.000338596997607965, + 0.0027553870022529736, + 0.00039931899664225057, + 0.00035090000164927915, + 0.0003426269977353513, + 0.000393196998629719, + 0.00040060500032268465, + 0.00035201099672121927, + 0.00035752799885813147, + 0.00033633800194365904, + 0.0003446910050115548, + 0.005115212996315677, + 0.0003832749935099855, + 0.0003441340013523586, + 0.00040847000491339713, + 0.001033707994793076, + 0.002365585998632014, + 0.0005363700038287789, + 0.0003583590005291626, + 0.0003447900016908534, + 0.0013381859971559606, + 0.00035584800207288936, + 0.0003743819979717955, + 0.0003533019989845343, + 0.00033931100188056007, + 0.00034875600249506533, + 0.0004523050010902807, + 0.00035462199593894184, + 0.0003545400031725876, + 0.0003447400013101287, + 0.0036605630011763424, + 0.0004805910066352226, + 0.0009017880001920275, + 0.0011612730013439432, + 0.0029923359979875386, + 0.0051763499941444024, + 0.00036451800406211987, + 0.0003312939952593297, + 0.0003328319944557734, + 0.0003329089959152043, + 0.0004481899959500879, + 0.00034066200169036165, + 0.00032995399669744074, + 0.00032675699912942946, + 0.0046758079988649115, + 0.000880372004758101, + 0.0003379260015208274, + 0.0003396869942662306, + 0.0003350870028953068, + 0.0004139010052313097, + 0.0003768110036617145, + 0.0003363389987498522, + 0.00035370799741940573, + 0.005641416006255895, + 0.0004364119959063828, + 0.0003430939977988601, + 0.00032941300014499575, + 0.00032760600151959807, + 0.000335512995661702, + 0.00032967400329653174, + 0.00033939399872906506, + 0.0003383049988769926, + 0.0003274510017945431, + 0.0003303240009699948, + 0.004657896002754569, + 0.0009201900029438548, + 0.00037760000122943893, + 0.0003339729955769144, + 0.00044322699977783486, + 0.00036473299405770376, + 0.0003359500042279251, + 0.0003300809985375963, + 0.00033434500073781237, + 0.00032932800240814686, + 0.00032947800355032086, + 0.005056998998043127, + 0.0008358019986189902, + 0.0003508160007186234, + 0.0003400799978408031, + 0.0003296919967397116, + 0.00032891299633774906, + 0.0022865089995320886, + 0.00036510900099528953, + 0.00033676799648674205, + 0.00033469699701527134, + 0.00033059400448109955, + 0.0003290029999334365, + 0.002880165004171431, + 0.014814965004916303, + 0.0068359880024218, + 0.004813850006030407, + 0.004957701996318065, + 0.003302805998828262, + 0.0017566320020705462, + 0.0009305509956902824, + 0.0005061899937572889 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil", + "fullname": "TestTemporaryFieldForGridPointColdPoolsEnhancement", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.0009878880009637214, + "max": 0.010999470003298484, + "mean": 0.0066737883276331445, + "stddev": 0.0017856254882766165, + "rounds": 372, + "median": 0.006996029998845188, + "iqr": 0.0019980180004495196, + "q1": 0.005995185998472152, + "q3": 0.007993203998921672, + "iqr_outliers": 24, + "stddev_outliers": 76, + "outliers": "76;24", + "ld15iqr": 0.0030675300004077144, + "hd15iqr": 0.0109974009974394, + "ops": 149.83993361902887, + "total": 2.4826492578795296, + "data": [ + 0.005993339000269771, + 0.006996619995334186, + 0.008281041002192069, + 0.006704343999444973, + 0.007995282001502346, + 0.006995339004788548, + 0.002685065999685321, + 0.004304442001739517, + 0.006001161003950983, + 0.006992818998696748, + 0.0069995169978938065, + 0.006991820999246556, + 0.005997134001518134, + 0.007610255001054611, + 0.009389061997353565, + 0.006991111004026607, + 0.0109974009974394, + 0.005993252001644578, + 0.006997030999627896, + 0.007996421001735143, + 0.006213474000105634, + 0.0017723139972076751, + 0.007001316000241786, + 0.007058518000121694, + 0.007118503999663517, + 0.006805659999372438, + 0.006993797003815416, + 0.005991634003294166, + 0.007872592999774497, + 0.0064812229975359514, + 0.003631610998127144, + 0.00542544500058284, + 0.007569599001726601, + 0.005989469005726278, + 0.0070000830019125715, + 0.007995226995262783, + 0.006002697999065276, + 0.005993822000164073, + 0.0019868800009135157, + 0.005523749001440592, + 0.006503607997728977, + 0.005967853001493495, + 0.009994117994210683, + 0.006993046001298353, + 0.006991472000663634, + 0.006002586000249721, + 0.0019819019944407046, + 0.006006055999023374, + 0.007076586000039242, + 0.006921123000211082, + 0.006995011994149536, + 0.009991367005568463, + 0.0058256569973309524, + 0.007163946000218857, + 0.007996589003596455, + 0.007994529994903132, + 0.004992220994608942, + 0.0044128279987489805, + 0.004575088998535648, + 0.007999803994607646, + 0.006997671000135597, + 0.006996033996983897, + 0.007997909000550862, + 0.00699297699611634, + 0.006996509000600781, + 0.006996903000981547, + 0.006994014998781495, + 0.009861658996669576, + 0.005128580996824894, + 0.010999470003298484, + 0.007997184999112505, + 0.007994658000825439, + 0.007995505999133456, + 0.004328023002017289, + 0.001667498996539507, + 0.005002542995498516, + 0.001986488998227287, + 0.006498646995169111, + 0.007597811003506649, + 0.007890058994235005, + 0.00985801599745173, + 0.006134955998277292, + 0.006297029001871124, + 0.009696658002212644, + 0.00799659600306768, + 0.006992267997702584, + 0.0079946959958761, + 0.007994851999683306, + 0.005861095996806398, + 0.00413301499793306, + 0.007995476000360213, + 0.005045807003625669, + 0.007951276005769614, + 0.001994915997784119, + 0.003652733998023905, + 0.008337295999808703, + 0.007997280001291074, + 0.007311840003239922, + 0.00768156999401981, + 0.003679761000967119, + 0.010362057997554075, + 0.006945073000679258, + 0.007997196000360418, + 0.006995533003646415, + 0.005724536000343505, + 0.005266643995128106, + 0.0059965549953631125, + 0.003997548999905121, + 0.0070036280012573116, + 0.006988164001086261, + 0.0051855349956895225, + 0.0028076129965484142, + 0.004996784002287313, + 0.007733876998827327, + 0.0052619819980463944, + 0.005996605003019795, + 0.006996675998379942, + 0.010998908001056407, + 0.005995471998176072, + 0.006995679999818094, + 0.007998958993994165, + 0.00799624300270807, + 0.007998155997483991, + 0.006998164994001854, + 0.005995784005790483, + 0.00699775799876079, + 0.00799736799672246, + 0.005997161999403033, + 0.006739383999956772, + 0.006254006999370176, + 0.00599906500428915, + 0.009999740999774076, + 0.007989548998011742, + 0.006995012998231687, + 0.004997611002181657, + 0.009000243997434154, + 0.006996026000706479, + 0.007997255997906905, + 0.005993145998218097, + 0.0059970770016661845, + 0.007997830005479045, + 0.008013720995222684, + 0.0039779939979780465, + 0.005996014995616861, + 0.007015581999439746, + 0.007974599997396581, + 0.0069955760045559146, + 0.007998960994882509, + 0.007995760002813768, + 0.008996632997877896, + 0.008006774995010346, + 0.007987728997250088, + 0.004996331997972447, + 0.006997783995757345, + 0.006996511998295318, + 0.007000302000960801, + 0.007010183995589614, + 0.007982066003023647, + 0.006011215999023989, + 0.007983452000189573, + 0.007999746005225461, + 0.007994685998710338, + 0.007999360001122113, + 0.00747067500196863, + 0.009632132001570426, + 0.00688524099678034, + 0.006997370997851249, + 0.006995233001362067, + 0.006997729004069697, + 0.006997569995291997, + 0.007997225002327468, + 0.006996072996116709, + 0.004997357995307539, + 0.007995455001946539, + 0.007997556000191253, + 0.00699607799469959, + 0.006998208998993505, + 0.006996377996983938, + 0.006998039003519807, + 0.003996797000581864, + 0.004997099997126497, + 0.00799729300342733, + 0.008002303002285771, + 0.006998776996624656, + 0.008991732000140473, + 0.007995011997991242, + 0.005998323002131656, + 0.0069975320002413355, + 0.007995678999577649, + 0.005996920997858979, + 0.007782640001096297, + 0.0072130889966501854, + 0.0050014839944196865, + 0.00999788699846249, + 0.006992697002715431, + 0.006997270000283606, + 0.008003470000403468, + 0.009000806996482424, + 0.007041308002953883, + 0.007939508999697864, + 0.005995808998704888, + 0.00799314599862555, + 0.005996978004986886, + 0.007997834996785969, + 0.007996663996891584, + 0.007997183995030355, + 0.005996229003358167, + 0.005998528999043629, + 0.005997047999699134, + 0.008001316993613727, + 0.007993261999217793, + 0.0076784900011261925, + 0.00833239899657201, + 0.0009878880009637214, + 0.006052620999980718, + 0.006932129996130243, + 0.006001629000820685, + 0.007993326005816925, + 0.006558918998052832, + 0.002433943998767063, + 0.005999674001941457, + 0.007232138996187132, + 0.0030944099999032915, + 0.0076568759977817535, + 0.003999565997219179, + 0.003992794001533184, + 0.006248577999940608, + 0.0077373640015139244, + 0.006999873003223911, + 0.00992064899764955, + 0.0030675300004077144, + 0.0050000240007648245, + 0.009999025001889095, + 0.0069930719982949086, + 0.00799718700000085, + 0.0059933049997198395, + 0.00599637400591746, + 0.005011684996134136, + 0.004980872996384278, + 0.008000802998139989, + 0.0069937130028847605, + 0.008055436999711674, + 0.00693396099813981, + 0.007996781998372171, + 0.006997741998929996, + 0.006996041003731079, + 0.006996932002948597, + 0.006996466996497475, + 0.003999047999968752, + 0.008994430994789582, + 0.002949540998088196, + 0.004042694999952801, + 0.006874605001939926, + 0.002116584997565951, + 0.006001836998621002, + 0.007996982996701263, + 0.005995880004775245, + 0.006561019996297546, + 0.008432829003140796, + 0.00406036800268339, + 0.0029317489970708266, + 0.006996792995778378, + 0.006994762996328063, + 0.0069986779999453574, + 0.006996768002863973, + 0.0069943260023137555, + 0.0070007690010243095, + 0.005115406995173544, + 0.007876370997109916, + 0.003991957004473079, + 0.005996838001010474, + 0.010008625002228655, + 0.007988231998751871, + 0.008477714996843133, + 0.007514860000810586, + 0.00799465399904875, + 0.007002512997132726, + 0.006997342003160156, + 0.007991950005816761, + 0.005999310997140128, + 0.005990137004118878, + 0.006240373993932735, + 0.0027498990020831116, + 0.0055172180000226945, + 0.006478862997028045, + 0.005993841004965361, + 0.00599476999923354, + 0.006140505000075791, + 0.007854837000195403, + 0.0035907860001316294, + 0.003402213995286729, + 0.007997392996912822, + 0.007997351996891666, + 0.005998949003696907, + 0.0013490150013240054, + 0.005646443001751322, + 0.007995398998900782, + 0.009996740998758469, + 0.00699727200117195, + 0.00699756400717888, + 0.005555924006330315, + 0.0025710839981911704, + 0.007862487000238616, + 0.009997509994718712, + 0.005997041997034103, + 0.006308424999588169, + 0.0066873209943878464, + 0.009001396996609401, + 0.0052305540011730045, + 0.008756814000662416, + 0.006095189004554413, + 0.007909141000709496, + 0.00698696100153029, + 0.003994527003669646, + 0.005003822996513918, + 0.0069913979968987405, + 0.005995691004500259, + 0.006995757998083718, + 0.0039972459999262355, + 0.006998315999226179, + 0.006998304001172073, + 0.006997440003033262, + 0.0069956610022927634, + 0.0069980650005163625, + 0.006995998999627773, + 0.0069981010019546375, + 0.006996519005042501, + 0.00599653999961447, + 0.009998289002396632, + 0.006999350000114646, + 0.006997563999902923, + 0.008014341001398861, + 0.005976365995593369, + 0.00699650299793575, + 0.006994492003286723, + 0.002671956994163338, + 0.004331809999712277, + 0.007982579001691192, + 0.0050024719967041165, + 0.009996715001761913, + 0.005992678001348395, + 0.007995678999577649, + 0.007996933993126731, + 0.006994470000790898, + 0.0072802610011422075, + 0.002715952003200073, + 0.0067521200035116635, + 0.007235794997541234, + 0.006001703004585579, + 0.005994899998768233, + 0.008055714999500196, + 0.010942159999103751, + 0.006101341998146381, + 0.006161362995044328, + 0.008062776003498584, + 0.0026459470027475618, + 0.005324484998709522, + 0.008674017000885215, + 0.005994054998154752, + 0.008000349000212736, + 0.004992526999558322, + 0.009997563000069931, + 0.007837821001885459, + 0.007149466997361742, + 0.004999924000003375, + 0.005381415998272132, + 0.001128861003962811, + 0.007477123006538022, + 0.005871414003195241, + 0.0061207019971334375, + 0.001990708005905617, + 0.007002815000305418 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil", + "fullname": "TestTemporaryFieldsForTurbulenceDiagnostics", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.00043558600009419024, + "max": 0.016960643995844293, + "mean": 0.0011528848425870221, + "stddev": 0.0016986498828167875, + "rounds": 1913, + "median": 0.0004994359987904318, + "iqr": 0.00029274399821588304, + "q1": 0.00046294100138766225, + "q3": 0.0007556849996035453, + "iqr_outliers": 347, + "stddev_outliers": 181, + "outliers": "181;347", + "ld15iqr": 0.00043558600009419024, + "hd15iqr": 0.0012006299948552623, + "ops": 867.389320303704, + "total": 2.2054687038689735, + "data": [ + 0.002500215996406041, + 0.0005546089960262179, + 0.0005493860007845797, + 0.0005240680038696155, + 0.0009673000022303313, + 0.0023072110052453354, + 0.0005586560000665486, + 0.0005295049995766021, + 0.0005271539994282648, + 0.0005280129989841953, + 0.0005271929985610768, + 0.001106764997530263, + 0.0010908250042120926, + 0.0005344899982446805, + 0.0005233769988990389, + 0.0005224329943303019, + 0.0005233049960224889, + 0.002658787001564633, + 0.0008739629993215203, + 0.0005311649947543629, + 0.0005268149980111048, + 0.00053804500203114, + 0.0005216499994276091, + 0.001169491995824501, + 0.001207651999720838, + 0.0005210170056670904, + 0.0007723200033069588, + 0.0017566279930179007, + 0.0012209849955979735, + 0.000511959005962126, + 0.001345530996331945, + 0.0007736919942544773, + 0.0005096699969726615, + 0.0005091169950901531, + 0.0005199180013732985, + 0.0005445160059025511, + 0.0038229109995882027, + 0.0005453490011859685, + 0.0005121559952385724, + 0.0005098100009490736, + 0.0005122179936734028, + 0.0027845990043715574, + 0.0009355090005556121, + 0.0005201149979257025, + 0.0005058940005255863, + 0.0005134480015840381, + 0.0005192679964238778, + 0.003234133000660222, + 0.001413938996847719, + 0.0005230640017543919, + 0.0005237980003585108, + 0.0005073150023235939, + 0.0024662939977133647, + 0.0005257969969534315, + 0.0005194339973968454, + 0.0005094820007798262, + 0.0005197190039325505, + 0.0006441730001824908, + 0.0026548050009296276, + 0.0011918979944312014, + 0.0005167729977983981, + 0.0005161919980309904, + 0.0005108199984533712, + 0.0013693439977942035, + 0.0022909620020072907, + 0.0005286500017973594, + 0.0005247950030025095, + 0.0005062619966338389, + 0.0006612040015170351, + 0.0015561160034849308, + 0.0007556290001957677, + 0.0005350630017346703, + 0.000508270000864286, + 0.0005169529977138154, + 0.0005138900014571846, + 0.0005151669975020923, + 0.0005051199987065047, + 0.0006356119993142784, + 0.0006422669976018369, + 0.0011603780003497377, + 0.0007270420028362423, + 0.0005323969962773845, + 0.0013935070019215345, + 0.0005907329978072084, + 0.0007605659993714653, + 0.0006240910006454214, + 0.0005201000021770597, + 0.0005161349981790408, + 0.0005124000017531216, + 0.0005201269959798083, + 0.0005031860055169091, + 0.0005463870038511232, + 0.0006682370003545657, + 0.0006572220008820295, + 0.0006450689979828894, + 0.0020939860041835345, + 0.000535341001523193, + 0.0005135029932716861, + 0.000504837000335101, + 0.0005389500001911074, + 0.0005181679953238927, + 0.0005659270027535968, + 0.0005103200019220822, + 0.0005378689966164529, + 0.0005114560044603422, + 0.000506937998579815, + 0.0005127580006956123, + 0.000654414005111903, + 0.002115827999659814, + 0.003614158005802892, + 0.0005688019955414347, + 0.0005809989961562678, + 0.0005423249967861921, + 0.0005037450027884915, + 0.0024104030017042533, + 0.0008976839962997474, + 0.0005091480052215047, + 0.0004894230005447753, + 0.0012908669959870167, + 0.0025790110012167133, + 0.0005913630011491477, + 0.0005156150000402704, + 0.0005023449994041584, + 0.0005010520035284571, + 0.003880220996506978, + 0.0005530319976969622, + 0.0005316289971233346, + 0.0004975389965693466, + 0.0006224280004971661, + 0.0016220149991568178, + 0.0005162430024938658, + 0.0004922610023641028, + 0.0023644770044484176, + 0.000563677996979095, + 0.0005257329976302572, + 0.0004937859994242899, + 0.0005128690027049743, + 0.0004974890034645796, + 0.0004942399973515421, + 0.0004944690008414909, + 0.0004997079959139228, + 0.000488634999783244, + 0.0004958780045853928, + 0.0004893170043942519, + 0.0004985659979865886, + 0.0005039770039729774, + 0.000494697000249289, + 0.0004963689934811555, + 0.004418094002176076, + 0.0006521729956148192, + 0.0005367939957068302, + 0.0005113989973324351, + 0.000507768003444653, + 0.0004995820054318756, + 0.0005017119983676821, + 0.000499012996442616, + 0.0005042340053478256, + 0.0004964749969076365, + 0.000496572996780742, + 0.000487571996927727, + 0.0005025460050092079, + 0.0004906579997623339, + 0.004279706001398154, + 0.0005519480037037283, + 0.0005184570036362857, + 0.0004880479973508045, + 0.0004949599970132113, + 0.000494241998239886, + 0.0004994330010958947, + 0.0025898789972416125, + 0.0005299030017340556, + 0.0005178569990675896, + 0.0004933759992127307, + 0.0004935310062137432, + 0.0035572640044847503, + 0.0005690889956895262, + 0.000589032999414485, + 0.0005256889999145642, + 0.000759775000915397, + 0.0005016479990445077, + 0.0004906750036752783, + 0.000499809997563716, + 0.0004899119958281517, + 0.0004930029972456396, + 0.0005917689995840192, + 0.004215330998704303, + 0.0006386460008798167, + 0.0005117300024721771, + 0.0005016110008000396, + 0.00049790799675975, + 0.000494815998536069, + 0.0004992530011804774, + 0.0004918260019621812, + 0.0012950450036441907, + 0.0005201229942031205, + 0.0004962929961038753, + 0.0008216540009016171, + 0.0004944009997416288, + 0.0004954250034643337, + 0.0004986540006939322, + 0.00048593599785817787, + 0.0004979420045856386, + 0.0004953309980919585, + 0.0006558029999723658, + 0.001479046004533302, + 0.0005946650053374469, + 0.0005972430008114316, + 0.0020166389949736185, + 0.0005148769996594638, + 0.0006245569966267794, + 0.0005902789998799562, + 0.0005271069967420772, + 0.0005007559957448393, + 0.0022566480038221925, + 0.0005376769986469299, + 0.0005034470013924874, + 0.0004970520021743141, + 0.002326339999854099, + 0.00052143499488011, + 0.0009600870034773834, + 0.0005003859987482429, + 0.0004934050011797808, + 0.0005063000062364154, + 0.00048701299965614453, + 0.0005273239949019626, + 0.005464560003019869, + 0.0005709030010621063, + 0.0005305070008034818, + 0.00048160000005736947, + 0.0004892229990218766, + 0.0004892290016869083, + 0.00047993199405027553, + 0.0036285339956521057, + 0.000930781003262382, + 0.0004984570041415282, + 0.0004910619973088615, + 0.0004899940031464212, + 0.0004889070041826926, + 0.0004833569983020425, + 0.0005478410021169111, + 0.0004752490058308467, + 0.004668460998800583, + 0.0010048650001408532, + 0.0005144919996382669, + 0.00048747500113677233, + 0.0004753719986183569, + 0.00048820700612850487, + 0.0004812359984498471, + 0.004248584002198186, + 0.0013303020023158751, + 0.0004905140012851916, + 0.0004840339970542118, + 0.00048028300079749897, + 0.0004832400009036064, + 0.0004840850015170872, + 0.0037776950048282743, + 0.0004889709962299094, + 0.0004800870010512881, + 0.0004898039987892844, + 0.00048187100037466735, + 0.000482490002468694, + 0.0004785980054293759, + 0.004095159994903952, + 0.0022518269979627803, + 0.0005039560055593029, + 0.0004883399960817769, + 0.000482461997307837, + 0.00047875099699012935, + 0.0005035440044593997, + 0.00047976199857657775, + 0.005350382998585701, + 0.00048727299872552976, + 0.0004818660017917864, + 0.00047771700337762013, + 0.00048579199938103557, + 0.0004851459962083027, + 0.013012787996558473, + 0.00728095299564302, + 0.007306749997951556, + 0.00868252199870767, + 0.005994970000756439, + 0.007001367994234897, + 0.006993047994910739, + 0.008002595001016743, + 0.003774708995479159, + 0.0005326769969542511, + 0.00048111700016306713, + 0.0004709390050265938, + 0.0004762500029755756, + 0.00047140999959083274, + 0.0004657419995055534, + 0.0005264529972919263, + 0.0005296899980749004, + 0.0005119929992360994, + 0.00047305099724326283, + 0.00048041199625004083, + 0.00046542700147256255, + 0.000469496997538954, + 0.00048345999675802886, + 0.003924541000742465, + 0.0005320260024745949, + 0.0004802350013051182, + 0.0005072109997854568, + 0.0004696489995694719, + 0.0004726070037577301, + 0.000465149998490233, + 0.003642707997641992, + 0.000549869000678882, + 0.0004924629974993877, + 0.0004625450019375421, + 0.0004708380001829937, + 0.0004653899959521368, + 0.0004856710002059117, + 0.0004669339978136122, + 0.0010669059993233532, + 0.003821632999461144, + 0.0005963319999864325, + 0.00048349199641961604, + 0.00046768800530117005, + 0.00046260900126071647, + 0.0004681330028688535, + 0.00047862700012046844, + 0.00046163200022419915, + 0.0028026049985783175, + 0.000686266997945495, + 0.0018114129998139106, + 0.0004974929979653098, + 0.0004723449965240434, + 0.0004609140014508739, + 0.0004756170019390993, + 0.0004666899985750206, + 0.002746491998550482, + 0.0005113779989187606, + 0.0004774310000357218, + 0.0004630269977496937, + 0.0005808600035379641, + 0.00047632500354666263, + 0.0004777529975399375, + 0.0051534850063035265, + 0.0005301970013533719, + 0.00048433799383929, + 0.0007312330053537153, + 0.00046202300291042775, + 0.0004763959950651042, + 0.0004673240036936477, + 0.003424980997806415, + 0.0006488279977929778, + 0.0006247549972613342, + 0.000616295998042915, + 0.0005560439967666753, + 0.00048103399603860453, + 0.0004615970028680749, + 0.00046134800504660234, + 0.0004749290019390173, + 0.00046105000365059823, + 0.0013044549996266142, + 0.002343343003303744, + 0.0005197099962970242, + 0.0004646259985747747, + 0.0004677679971791804, + 0.0004911439973511733, + 0.0004841230038437061, + 0.000463245996797923, + 0.0004599509993568063, + 0.0026399139969726093, + 0.0005082909992779605, + 0.0022733599980711006, + 0.0018722110035014339, + 0.0005234999989625067, + 0.0004821940019610338, + 0.00047252499643946066, + 0.0004656099990825169, + 0.000468956000986509, + 0.002322252999874763, + 0.002222355004050769, + 0.0005056430018157698, + 0.0004758100039907731, + 0.0004635880031855777, + 0.00046636699698865414, + 0.0004643079955712892, + 0.00045166399650042877, + 0.00044813800195697695, + 0.00044326899660518393, + 0.0038046519985073246, + 0.000492366001708433, + 0.00045340199721977115, + 0.00045643900375580415, + 0.00044586700096260756, + 0.000449972998467274, + 0.00045539200073108077, + 0.0006221129951882176, + 0.003464292996795848, + 0.0010907140022027306, + 0.0004987569991499186, + 0.00045991600200068206, + 0.00046140699851093814, + 0.0009676519985077903, + 0.0005223929983912967, + 0.001976794002985116, + 0.003084547999606002, + 0.0005287860039970838, + 0.0004672939976444468, + 0.00046424999891314656, + 0.0004598900050041266, + 0.0004578180014505051, + 0.0004506519981077872, + 0.0032458589994348586, + 0.0007860530022298917, + 0.0006419450000976212, + 0.0006052269964129664, + 0.0005546370011870749, + 0.00046509299863828346, + 0.00045445100113283843, + 0.00045898599637439474, + 0.00045501899876398966, + 0.00045192900142865255, + 0.0042948989939759485, + 0.0007209479954326525, + 0.0005396729975473136, + 0.00046027699863770977, + 0.0004599479943863116, + 0.000444366000010632, + 0.00046103000204311684, + 0.0004515230029937811, + 0.00402059900079621, + 0.0005085090015199967, + 0.0004711439978564158, + 0.0004627660018741153, + 0.00045240300096338615, + 0.0004539299989119172, + 0.00045735100138699636, + 0.0004559330045594834, + 0.00420877799479058, + 0.0007347439968725666, + 0.0004916160032735206, + 0.0004653549985960126, + 0.00045588200009660795, + 0.0004523070019786246, + 0.00045878399396315217, + 0.0004572429970721714, + 0.0007676759996684268, + 0.0037465670029632747, + 0.0005089350015623495, + 0.0011168909986736253, + 0.0005307729952619411, + 0.00048070599586935714, + 0.00046188099804567173, + 0.0004504390017245896, + 0.0004590640019159764, + 0.000463313001091592, + 0.0006032210003468208, + 0.0030921629950171337, + 0.0004959370053256862, + 0.0004664860025513917, + 0.0004496949986787513, + 0.00045162700553191826, + 0.00044548000005306676, + 0.00045533700176747516, + 0.00045295399468159303, + 0.009440199995879084, + 0.008995810996566433, + 0.005995127001369838, + 0.007997086002433207, + 0.00799639499746263, + 0.007006609994277824, + 0.010986007997416891, + 0.007002136997471098, + 0.0005082420029793866, + 0.00047118899965425953, + 0.0004552789978333749, + 0.00045960400166222826, + 0.0004597610022756271, + 0.00045326999679673463, + 0.000464115000795573, + 0.0004651509952964261, + 0.005922935997659806, + 0.0009477419953327626, + 0.0005463300039991736, + 0.0004745050027850084, + 0.0004665119995479472, + 0.00044817400339525193, + 0.008081576001131907, + 0.0005021229953854345, + 0.00044674999662674963, + 0.00044633499783230945, + 0.000450888997875154, + 0.0004531239974312484, + 0.00045223299821373075, + 0.005395601001509931, + 0.0005767749971710145, + 0.0005216360004851595, + 0.0004515839973464608, + 0.0004470269996090792, + 0.0004560000015771948, + 0.00044276200060267, + 0.004364204003650229, + 0.00045591600064653903, + 0.000490345002617687, + 0.0011551379939191975, + 0.0005350630017346703, + 0.00046427400229731575, + 0.00044062099914299324, + 0.0005040579999331385, + 0.0004598419982357882, + 0.0004564489936456084, + 0.0004416349984239787, + 0.00043711999751394615, + 0.002841401001205668, + 0.0006397249962901697, + 0.00045744200178887695, + 0.0015790839970577508, + 0.00045609300286741927, + 0.00044837300083599985, + 0.0004500270006246865, + 0.000747653997677844, + 0.0005393859973992221, + 0.0004471360007300973, + 0.0004662339997594245, + 0.00045107999903848395, + 0.0004454750014701858, + 0.0004420490004122257, + 0.00043824200110975653, + 0.001428718001989182, + 0.0016085710012703203, + 0.0018761440005619079, + 0.0004635909936041571, + 0.00045227900409372523, + 0.00044358099694363773, + 0.0004378040030132979, + 0.0006460770018748008, + 0.0014852669992251322, + 0.0010332239980925806, + 0.0007989329969859682, + 0.00047990400344133377, + 0.0004542839960777201, + 0.00044012699800077826, + 0.0004464880039449781, + 0.000450474995886907, + 0.002372319002461154, + 0.001351812003122177, + 0.00045085899910191074, + 0.0004416639931150712, + 0.0004482179938349873, + 0.0004546749987639487, + 0.0007605610007885844, + 0.0008454120034002699, + 0.0008326209936058149, + 0.0004498630005400628, + 0.0004513540043262765, + 0.0004409949979162775, + 0.00044303599861450493, + 0.00044586500007426366, + 0.0004386059954413213, + 0.001426277005521115, + 0.0005404760013334453, + 0.0005442429974209517, + 0.0004593070043483749, + 0.0004476929971133359, + 0.00044849899859400466, + 0.00047492599696852267, + 0.00044503000390250236, + 0.0012864079981227405, + 0.0016559220021008514, + 0.00048145199980353937, + 0.0004403470011311583, + 0.00046484799531754106, + 0.00044535000051837415, + 0.0004377860022941604, + 0.00044255400280235335, + 0.0004366869980003685, + 0.002160411997465417, + 0.00046601700159953907, + 0.0004478450064198114, + 0.001621879004233051, + 0.001499496996984817, + 0.0005339370036381297, + 0.00046715899952687323, + 0.0004596250000759028, + 0.00043831700168084353, + 0.00046386699978029355, + 0.00044216100650373846, + 0.00044508600694825873, + 0.00044216700189281255, + 0.00044264699681662023, + 0.00044240600254852325, + 0.0007434819999616593, + 0.0020464449989958666, + 0.003162611996231135, + 0.0008043900015763938, + 0.0004610959949786775, + 0.0008291740014101379, + 0.0010812479958985932, + 0.0007499350031139329, + 0.00044930900185136124, + 0.00044502800301415846, + 0.0019157099959556945, + 0.0006636010002694093, + 0.0009213900048052892, + 0.00045943199802422896, + 0.0004521529990597628, + 0.00045450600009644404, + 0.00045064000005368143, + 0.00045959399722050875, + 0.0004596329963533208, + 0.0009389340048073791, + 0.0004808879966731183, + 0.00046064500202191994, + 0.0004689449997385964, + 0.0004516890039667487, + 0.0006479630028479733, + 0.0008434529954683967, + 0.0016159160004463047, + 0.001677633001236245, + 0.0013642760022776201, + 0.0004824679999728687, + 0.0004593150006257929, + 0.0008370860014110804, + 0.0009457790001761168, + 0.000687565996486228, + 0.00047238499973900616, + 0.0004808119992958382, + 0.0004609399984474294, + 0.0004715000031865202, + 0.0004556260028039105, + 0.0004519380017882213, + 0.0043368339975131676, + 0.00047119500231929123, + 0.00045835199853172526, + 0.0004553459948510863, + 0.001004586993076373, + 0.0018210960042779334, + 0.0004900209969491698, + 0.0004612280026776716, + 0.00045018000673735514, + 0.00045769099961034954, + 0.0004554900006041862, + 0.003316497000923846, + 0.0007062320000841282, + 0.00048165800399146974, + 0.0004543510003713891, + 0.00045854000200051814, + 0.00045999900612514466, + 0.0012351569966995157, + 0.0018465379980625585, + 0.0007081540024955757, + 0.0004636470039258711, + 0.0004498389971558936, + 0.00044731799425790086, + 0.00045231399417389184, + 0.0004576499995891936, + 0.0017877729987958446, + 0.00047467800322920084, + 0.0004544299954432063, + 0.00046173999726306647, + 0.0004543050017673522, + 0.000452756998129189, + 0.0017734479988575913, + 0.0009885929976007901, + 0.00046968200331320986, + 0.00045463399874279276, + 0.0004569610027829185, + 0.00045332299487199634, + 0.0007021909987088293, + 0.0016900199989322573, + 0.0004935309989377856, + 0.0004624190041795373, + 0.0004578769949148409, + 0.0004545760020846501, + 0.0017447339996579103, + 0.0005749289994128048, + 0.00046516599832102656, + 0.000462314004835207, + 0.00044702400191454217, + 0.00045563499588752165, + 0.00045443300041370094, + 0.00046581999777117744, + 0.0015195250016404316, + 0.0013146929995855317, + 0.0004802900002687238, + 0.0004599269959726371, + 0.0004569200027617626, + 0.0004499220012803562, + 0.0017734189968905412, + 0.0020787460016435944, + 0.0005595059992629103, + 0.0004571409954223782, + 0.0004700499994214624, + 0.000460948002000805, + 0.0004523899988271296, + 0.003800415994192008, + 0.0004909260023850948, + 0.00046385700261453167, + 0.0004567829964798875, + 0.000453934000688605, + 0.0009167949974653311, + 0.0008757579998928122, + 0.0019347240013303235, + 0.0004822349947062321, + 0.00045874500210629776, + 0.0004496530018514022, + 0.0005500370025401935, + 0.0004611029944499023, + 0.0012411480056471191, + 0.0006245859985938296, + 0.0004617590020643547, + 0.0004503540039877407, + 0.000463964999653399, + 0.00045330099965212867, + 0.00044813600106863305, + 0.0024093320025713183, + 0.0004906389949610457, + 0.00045465599396266043, + 0.00044971800525672734, + 0.002022979002504144, + 0.0006575299994437955, + 0.00047777799773029983, + 0.00047256299876607955, + 0.0004542880051303655, + 0.00045617800060426816, + 0.0004460120035219006, + 0.0004751720043714158, + 0.00045840899838367477, + 0.0010351720047765411, + 0.0010775769987958483, + 0.004507588004344143, + 0.0005474810022860765, + 0.001636310997128021, + 0.0006929939991096035, + 0.0025333059966214933, + 0.0006067659996915609, + 0.00048429600428789854, + 0.0015911119990050793, + 0.001382935995934531, + 0.00048209400119958445, + 0.00046326700248755515, + 0.0004773230029968545, + 0.00045047399908071384, + 0.0005293799986247905, + 0.0007715059982729144, + 0.0028124359960202128, + 0.000520435998623725, + 0.0004651869967347011, + 0.00047644699952797964, + 0.00047545599954901263, + 0.0007576899952255189, + 0.0006036129998392425, + 0.0006016309998813085, + 0.0005920969997532666, + 0.0004731919980258681, + 0.0018115940038114786, + 0.0009381789932376705, + 0.0006944679989828728, + 0.00046459399891318753, + 0.00045908300671726465, + 0.00045506500464398414, + 0.0004557779975584708, + 0.0004465849997359328, + 0.0008922500055632554, + 0.0017154809975181706, + 0.0004578849984682165, + 0.0004639550024876371, + 0.0008468070009257644, + 0.001612021995242685, + 0.0007449359982274473, + 0.0004599029998644255, + 0.0018686770054046065, + 0.0004768190046888776, + 0.00045418400259222835, + 0.0006866019975859672, + 0.0004596389990183525, + 0.00045954099914524704, + 0.0004544990006252192, + 0.0006398750047083013, + 0.0006360120023600757, + 0.0016760400030761957, + 0.0007517320045735687, + 0.00047586199798388407, + 0.0004626520021702163, + 0.00045432800106937066, + 0.0019297150065540336, + 0.0004978740034857765, + 0.0005050370018579997, + 0.00045351300650509074, + 0.0004492590014706366, + 0.00045795299956807867, + 0.00045158200373407453, + 0.0007713489976595156, + 0.0017269460004172288, + 0.001654789004533086, + 0.0008234289998654276, + 0.0004630730036296882, + 0.0004493769956752658, + 0.00045440299436450005, + 0.0004578619991661981, + 0.0004590719981933944, + 0.0026022980018751696, + 0.004608885996276513, + 0.0005818050049128942, + 0.000488748999487143, + 0.00046126500092213973, + 0.0004624279972631484, + 0.00044784800411434844, + 0.0004512659943429753, + 0.0004538869980024174, + 0.0004578700027195737, + 0.003875088004861027, + 0.0012565479992190376, + 0.0004785270066349767, + 0.004098010998859536, + 0.0006097020013839938, + 0.0004696039977716282, + 0.004248498997185379, + 0.0007939259958220646, + 0.0004800300011993386, + 0.0004567700016195886, + 0.0004714329988928512, + 0.0004635090008378029, + 0.004298405998270027, + 0.0004953819952788763, + 0.00045909700565971434, + 0.00048505399900022894, + 0.0004525840049609542, + 0.004098473000340164, + 0.001065848999132868, + 0.0006027150011505, + 0.0009301329992013052, + 0.0007746250048512593, + 0.000461338997411076, + 0.0010042169960797764, + 0.00045869400491937995, + 0.00046203500096453354, + 0.0012497409989009611, + 0.0016333960011252202, + 0.00047972099855542183, + 0.0004635130026144907, + 0.002252015001431573, + 0.0004964519976056181, + 0.000461331001133658, + 0.0006840179994469509, + 0.0013136029956513084, + 0.0005172929959371686, + 0.00046535600267816335, + 0.0004568639997160062, + 0.00045948199840495363, + 0.000456868001492694, + 0.0004472150030778721, + 0.0010162639955524355, + 0.0028136990003986284, + 0.0010849649988813326, + 0.0004676870012190193, + 0.00045902000420028344, + 0.0004544809999060817, + 0.00045628500083694234, + 0.0011708820020430721, + 0.001237775999470614, + 0.0009239460050594062, + 0.0009699080037535168, + 0.0007232110001496039, + 0.000461563999124337, + 0.0004594389974954538, + 0.0008516180023434572, + 0.0020971460035070777, + 0.00046340299741132185, + 0.0004527630007942207, + 0.0004584260022966191, + 0.0004946159970131703, + 0.0006407570035662502, + 0.0014023770054336637, + 0.0006997549935476854, + 0.001381478999974206, + 0.0012772909976774827, + 0.00046254799963207915, + 0.001096506995963864, + 0.0007457180035999045, + 0.0011061510012950748, + 0.0005991830039420165, + 0.0004518189962254837, + 0.0004643180000130087, + 0.0004516460030572489, + 0.0004971039961674251, + 0.0013907609973102808, + 0.0007670409977436066, + 0.0015016580000519753, + 0.0004693119990406558, + 0.0004511130027822219, + 0.00046005599870113656, + 0.0004535249972832389, + 0.0010578259971225634, + 0.0017833800011430867, + 0.002007388000492938, + 0.00045968100312165916, + 0.00046417699923040345, + 0.00045379099901765585, + 0.0013491960053215735, + 0.000848046998726204, + 0.0009851659997366369, + 0.0004628410024452023, + 0.00046387899783439934, + 0.0004654909935197793, + 0.00045299500197870657, + 0.0011623230020632036, + 0.00177189199894201, + 0.0007367360012722202, + 0.00047731200174894184, + 0.00047760999586898834, + 0.00046728100278414786, + 0.00046055599523242563, + 0.0004475800014915876, + 0.0037456179998116568, + 0.0004877519968431443, + 0.00045427499571815133, + 0.00045443700219038874, + 0.00045372400199994445, + 0.0004543369941529818, + 0.0006959039965295233, + 0.0011946919985348359, + 0.0012283520045457408, + 0.0005757870021625422, + 0.00047166100557660684, + 0.0004517999987001531, + 0.0004658230027416721, + 0.0005285869992803782, + 0.0016871879997779615, + 0.001833210997574497, + 0.001054783999279607, + 0.000486001001263503, + 0.00048168699868256226, + 0.0004523759998846799, + 0.001055932996678166, + 0.002087809996737633, + 0.0004707379994215444, + 0.0004503329982981086, + 0.0004565720009850338, + 0.0004593190024024807, + 0.000981190001766663, + 0.002861000997654628, + 0.00050381499750074, + 0.00045381999370874837, + 0.00045300300553208217, + 0.00069213799724821, + 0.004876448001596145, + 0.000515522995556239, + 0.0004595220016199164, + 0.00045633199624717236, + 0.00046171100257197395, + 0.00045093000517226756, + 0.0004577349973260425, + 0.00045141200098441914, + 0.0032522980036446825, + 0.007936868998513091, + 0.00699567799892975, + 0.006995073999860324, + 0.0069955110011505894, + 0.006995408999500796, + 0.006996374999289401, + 0.005999491004331503, + 0.006994656003371347, + 0.007994719999260269, + 0.007996340005774982, + 0.00699608900322346, + 0.006996823001827579, + 0.00799674900190439, + 0.006996381998760626, + 0.006996572999923956, + 0.007996426997124217, + 0.007997840002644807, + 0.004996335999749135, + 0.004993663002096582, + 0.006996829004492611, + 0.007995694999408443, + 0.006993324001086876, + 0.006994664996454958, + 0.00800841399905039, + 0.00713530600478407, + 0.006838492001406848, + 0.008000134999747388, + 0.007070294996083248, + 0.006875767998280935, + 0.00895590699656168, + 0.007941555995785166, + 0.006947462999960408, + 0.007992585000465624, + 0.005999380999128334, + 0.008080681996943895, + 0.009004013998492155, + 0.007988008001120761, + 0.003982483998697717, + 0.006891662000271026, + 0.0070838160027051345, + 0.007941274001495913, + 0.003542523001669906, + 0.010457783995661885, + 0.007809755006746855, + 0.007715156003541779, + 0.010311013997124974, + 0.004928082002152223, + 0.008031402998312842, + 0.006048629998986144, + 0.00586709899653215, + 0.005970295002043713, + 0.016960643995844293, + 0.007664700999157503, + 0.008317312000144739, + 0.0057630419978522696, + 0.004221736002364196, + 0.007995873005711474, + 0.007995309999387246, + 0.0059971910013700835, + 0.007996371998160612, + 0.007996382002602331, + 0.007838252000510693, + 0.003189328999724239, + 0.0005027320003136992, + 0.0004761890013469383, + 0.00044869099656352773, + 0.00044442799844546244, + 0.0006317170045804232, + 0.0004693869996117428, + 0.00044154100032756105, + 0.0004397809971123934, + 0.002801043003273662, + 0.00044863000221084803, + 0.00044198500108905137, + 0.00044224999874131754, + 0.00043749299948103726, + 0.00044607799645746127, + 0.0004468869956326671, + 0.0004359199956525117, + 0.0037526169980992563, + 0.0004818599991267547, + 0.0004439410040504299, + 0.00044185099977767095, + 0.00044915499893249944, + 0.0004458390030777082, + 0.003670144003990572, + 0.0004913649972877465, + 0.00044449100096244365, + 0.0004967939967173152, + 0.00043897799332626164, + 0.0004741649972856976, + 0.0004451020067790523, + 0.00043990400445181876, + 0.005299377000483219, + 0.0004845969961024821, + 0.0004614490026142448, + 0.000440787996922154, + 0.00044149299355922267, + 0.0004808970043086447, + 0.00044215899833943695, + 0.0004576540013658814, + 0.000440855001215823, + 0.006010131997754797, + 0.0005191420059418306, + 0.0004550199955701828, + 0.0004454080044524744, + 0.000446236997959204, + 0.0004463360019144602, + 0.0004467659964575432, + 0.00043955799628747627, + 0.004799529000592884, + 0.0004957809942425229, + 0.00046028300130274147, + 0.0012720730010187253, + 0.0005680340036633424, + 0.00062081800570013, + 0.0005044400022597983, + 0.0004509409991442226, + 0.0004505930046434514, + 0.0004446140010259114, + 0.0004408589957165532, + 0.00043662100506480783, + 0.0005858709992025979, + 0.0005949769983999431, + 0.0006170859996927902, + 0.0009835290038608946, + 0.0004532570019364357, + 0.000439835996075999, + 0.00044793300185119733, + 0.0004399919998832047, + 0.0004408930035424419, + 0.0004453840010683052, + 0.00043558600009419024, + 0.0011671690008370206, + 0.000672187001327984, + 0.001299160998314619, + 0.0006985460058785975, + 0.0011649599982774816, + 0.00048260299809044227, + 0.0004477880065678619, + 0.00045138200221117586, + 0.0004370359965832904, + 0.0006541369948536158, + 0.0006370929986587726, + 0.0006141800040495582, + 0.0006032070014043711, + 0.000488290999783203, + 0.0024642760035931133, + 0.0005280510013108142, + 0.0009146980009973049, + 0.0007840399994165637, + 0.0005434879931272008, + 0.0004602590051945299, + 0.00044849699770566076, + 0.00044940000225324184, + 0.0004459209958440624, + 0.000442386997747235, + 0.00044001900096191093, + 0.0004805529970326461, + 0.0005819060024805367, + 0.0006038450010237284, + 0.000593951997871045, + 0.0006407999971997924, + 0.0013633080016006716, + 0.0004500249997363426, + 0.0004506130062509328, + 0.000566182003240101, + 0.00047509999421890825, + 0.0004609399984474294, + 0.00045168799988459796, + 0.0005636109999613836, + 0.0009029110005940311, + 0.0007137529973988421, + 0.0005452569967019372, + 0.00047751099918968976, + 0.0005824850013596006, + 0.0005486599984578788, + 0.0018798529999912716, + 0.00046054999984335154, + 0.0004530399965005927, + 0.006594334998226259, + 0.0012520370000856929, + 0.0008944080036599189, + 0.0004950759976054542, + 0.000458736001746729, + 0.001966736999747809, + 0.0005986910036881454, + 0.0008183640020433813, + 0.0008116289973258972, + 0.00047701299627078697, + 0.0004518450004979968, + 0.00045733700244454667, + 0.0004476989997783676, + 0.0006645039975410327, + 0.001153732999227941, + 0.0007519490027334541, + 0.0012976799989701249, + 0.0005849219960509799, + 0.00045650800166185945, + 0.000468226004159078, + 0.00045578699791803956, + 0.00045302099897526205, + 0.0004572420002659783, + 0.00046075899444986135, + 0.0010280749993398786, + 0.0006065199995646253, + 0.0006360679981298745, + 0.0006229849968804047, + 0.0012006299948552623, + 0.0005002550024073571, + 0.00047032200382091105, + 0.00045369999861577526, + 0.0004551799938781187, + 0.0005052720007370226, + 0.0005071050036349334, + 0.0005995290048304014, + 0.0011685719946399331, + 0.0004916459947708063, + 0.0004604720015777275, + 0.000508498000272084, + 0.00047525799891445786, + 0.0006156029994599521, + 0.000755852997826878, + 0.0008996239994303323, + 0.0004817359949811362, + 0.00045936699461890385, + 0.00046790899796178564, + 0.0004528659992502071, + 0.0005944310032646172, + 0.0006125299987616017, + 0.0004966300039086491, + 0.0005011229950468987, + 0.0005933669963269494, + 0.0005562060032389127, + 0.0005982839938951656, + 0.0004850650002481416, + 0.00046059300075285137, + 0.00046116800513118505, + 0.0004526619959506206, + 0.0012388960021780804, + 0.000720693999028299, + 0.0008003179973457009, + 0.0005146469993633218, + 0.0013740610011154786, + 0.0004795830027433112, + 0.0004601239998009987, + 0.00045142499584471807, + 0.0013201789988670498, + 0.0005980089990771376, + 0.0010146240019821562, + 0.0004935750039294362, + 0.00045657299779122695, + 0.00045747299736831337, + 0.00044641599379247054, + 0.0005357499976526015, + 0.0006545209980686195, + 0.0006090050010243431, + 0.000647281005512923, + 0.001055590997566469, + 0.00045608700020238757, + 0.00046341100096469745, + 0.0007175979990279302, + 0.001272019995667506, + 0.0005828319990541786, + 0.0011135039967484772, + 0.0005107640026835725, + 0.0004546489944914356, + 0.00045842299732612446, + 0.00045098899863660336, + 0.0004628100068657659, + 0.0004524930045590736, + 0.000448673999926541, + 0.0005966640019323677, + 0.0006104319982114248, + 0.0006599120024475269, + 0.0006137250020401552, + 0.0015644420054741204, + 0.0004906030007987283, + 0.00046466100320685655, + 0.00045555999531643465, + 0.0004532570019364357, + 0.0005253740018815733, + 0.0004682690050685778, + 0.0005839290024596266, + 0.0005633159962599166, + 0.0004700700010289438, + 0.0004555240011541173, + 0.00045166799827711657, + 0.000453672997537069, + 0.00045688899990636855, + 0.000582236003538128, + 0.0006326250004349276, + 0.001840765995439142, + 0.0004902220025542192, + 0.00047284699394367635, + 0.00047287900088122115, + 0.0007506149995606393, + 0.00048150400107260793, + 0.00047808400267967954, + 0.0005055190049461089, + 0.0006235870023374446, + 0.0006290809978963807, + 0.000635169999441132, + 0.00048320600035367534, + 0.0004720439974335022, + 0.00046979100443422794, + 0.00046394600212806836, + 0.00047045199607964605, + 0.0006214069944689982, + 0.0012972830008948222, + 0.000486422999529168, + 0.0004725699982373044, + 0.00046508899686159566, + 0.0005668640005751513, + 0.0006156580056995153, + 0.0006220450013643131, + 0.0005502500061993487, + 0.0004690019995905459, + 0.0005083819996798411, + 0.0015630869966116734, + 0.0005583319943980314, + 0.0009834200027398765, + 0.00046486999781336635, + 0.00047918899508658797, + 0.00046243600081652403, + 0.0005237900040810928, + 0.0006129769972176291, + 0.00126590600120835, + 0.0006551879996550269, + 0.00047279200225602835, + 0.0004659890037146397, + 0.0004895559977740049, + 0.0004669230038416572, + 0.0004997919968445785, + 0.0006306829964159988, + 0.0009138180030276999, + 0.0006361500054481439, + 0.0006368949980242178, + 0.0005388459976529703, + 0.0004681020000134595, + 0.0004690869973273948, + 0.0004626949958037585, + 0.00047341999743366614, + 0.000589079994824715, + 0.0009057560018845834, + 0.0016662599955452606, + 0.0009373240027343854, + 0.00048792699817568064, + 0.000476135995995719, + 0.0004647060050047003, + 0.0004679449994000606, + 0.00046185099927242845, + 0.001136767998104915, + 0.0006134670038591139, + 0.0005512200004886836, + 0.0004771390013047494, + 0.00046959100291132927, + 0.00046447299973806366, + 0.0004651359995477833, + 0.0004650190021493472, + 0.000457444999483414, + 0.001568929001223296, + 0.0014634149993071333, + 0.0004729770007543266, + 0.00046180199569789693, + 0.0016788499997346662, + 0.000574049998249393, + 0.0005017880030209199, + 0.0008842430033837445, + 0.0006492920001619495, + 0.000592088996199891, + 0.0005001769968657754, + 0.00047569599701091647, + 0.00046790099440841004, + 0.0004625310029950924, + 0.0005006809951737523, + 0.0004782550022355281, + 0.0005402069946285337, + 0.0020321309930295683, + 0.0007184369969763793, + 0.001078648005204741, + 0.0010502479999559, + 0.0005035440044593997, + 0.0004762050011777319, + 0.00046771000052103773, + 0.0005159979991731234, + 0.0019049249967793003, + 0.0006416779942810535, + 0.00048815600166562945, + 0.00048023199633462355, + 0.00048181800229940563, + 0.00046279699745355174, + 0.0004930510040139779, + 0.0006702399987261742, + 0.0012607229946297593, + 0.0007697119945078157, + 0.0006830679994891398, + 0.0004956390039296821, + 0.00047183599963318557, + 0.00046011099766474217, + 0.00048464699648320675, + 0.0004611280019162223, + 0.0011411580053390935, + 0.0006360129991662689, + 0.0005935799999861047, + 0.0012131949988543056, + 0.0004913450029562227, + 0.00047714399988763034, + 0.0004602449989761226, + 0.0005107030010549352, + 0.0004690179994213395, + 0.0006289719967753626, + 0.0006212190055521205, + 0.0008015819985303096, + 0.0005654030028381385, + 0.0006332490011118352, + 0.0005196240017539822, + 0.0004846619995078072, + 0.0004606220027199015, + 0.0005374969987315126, + 0.0016637409935356118, + 0.0004948209971189499, + 0.00046703599946340546, + 0.001051127001119312, + 0.00046971999836387113, + 0.00046019799628993496, + 0.0005699429966625758, + 0.000621778002823703, + 0.0005864290069439448, + 0.0004829419995076023, + 0.0005048460006946698, + 0.0004678069963119924, + 0.0004686440006480552, + 0.00046363400178961456, + 0.0004630429975804873, + 0.00046008000208530575, + 0.0006065999987185933, + 0.0006175689995870925, + 0.003885705998982303, + 0.0005725220034946688, + 0.0005135389947099611, + 0.0004890319978585467, + 0.000516353000421077, + 0.005891453001822811, + 0.0008573590021114796, + 0.0010659519975888543, + 0.000511504003952723, + 0.0004693410010077059, + 0.00048344099923269823, + 0.00046325500443344936, + 0.004565624003589619, + 0.0004866880044573918, + 0.0004950200018356554, + 0.00047235999954864383, + 0.0006303080008365214, + 0.0006015179969836026, + 0.0005492330019478686, + 0.0004904480010736734, + 0.00046262300020316616, + 0.0004705440005636774, + 0.00046715499775018543, + 0.00047231200005626306, + 0.0004604099958669394, + 0.005500633000337984, + 0.0007056729955365881, + 0.0006461609955294989, + 0.0006349230025080033, + 0.0005067640013294294, + 0.002516916996682994, + 0.0006675810000160709, + 0.0044176329975016415, + 0.0004988229993614368, + 0.0007506859983550385, + 0.0006512320032925345, + 0.0006270360026974231, + 0.002632404997712001, + 0.0005090620034025051, + 0.0011688469967339188, + 0.0006355970035656355, + 0.002547529998992104, + 0.0005258300006971695, + 0.0006109800015110523, + 0.0005682720002369024, + 0.0004987029969925061, + 0.004481914002099074, + 0.0006675810000160709, + 0.0016904520016396418, + 0.0004990090019418858, + 0.004648479000024963, + 0.0005099269983475097, + 0.000634174000879284, + 0.0008088930044323206, + 0.004645165005058516, + 0.0005526990062207915, + 0.000546125003893394, + 0.0006510370003525168, + 0.0006709269946441054, + 0.005549524001253303, + 0.0006862990048830397, + 0.0006893789977766573, + 0.0005220099992584437, + 0.0004798459995072335, + 0.0035872260050382465, + 0.0007604649945278652, + 0.0005512970019481145, + 0.00047639600234106183, + 0.0004754699984914623, + 0.0004620240069925785, + 0.00046799799747532234, + 0.0004772749962285161, + 0.003777366000576876, + 0.005516616998647805, + 0.0005585459948633797, + 0.0005009619999327697, + 0.00047780299792066216, + 0.00047248600458260626, + 0.00047365100181195885, + 0.0004693549999501556, + 0.0022570049986825325, + 0.0004894480007351376, + 0.0017360450001433492, + 0.001492070994572714, + 0.0005043229975854047, + 0.00047032199654495344, + 0.003057289999560453, + 0.004996011994080618, + 0.006004079004924279, + 0.00807432399597019, + 0.007910516993433703, + 0.007986378994246479, + 0.006994411000050604, + 0.007995280000614002, + 0.00599553400388686, + 0.007996601998456754, + 0.004002104004030116, + 0.0005571330038947053, + 0.0005325719976099208, + 0.0004763149991049431, + 0.0023455740010831505, + 0.0005355100001906976, + 0.0004977179996785708, + 0.0004643220017896965, + 0.00046233300236053765, + 0.00047549699957016855, + 0.0004622999986167997, + 0.006715913004882168, + 0.0005879349992028438, + 0.0005519410042325035, + 0.00047632600035285577, + 0.000496526001370512, + 0.0004667000030167401, + 0.0004622529959306121, + 0.007183898000221234, + 0.0005725030059693381, + 0.000539620996278245, + 0.0011271820039837621, + 0.00047512199671473354, + 0.00047153599734883755, + 0.0004681330028688535, + 0.00046372500219149515, + 0.00046883500181138515, + 0.0004574609993142076, + 0.0010540099974605255, + 0.0035883439995814115, + 0.0005941000054008327, + 0.0005581410005106591, + 0.0025823110045166686, + 0.0005407889984780923, + 0.0005379770009312779, + 0.0005132890000822954, + 0.0004885609960183501, + 0.0004633369971998036, + 0.00046817600377835333, + 0.0004757860006066039, + 0.005966798002191354, + 0.0005888410014449619, + 0.0005426049974630587, + 0.0004732469969894737, + 0.0004595680002239533, + 0.000489286998345051, + 0.0004642929998226464, + 0.003482008003629744, + 0.0005878800002392381, + 0.000593561002460774, + 0.0005038980016252026, + 0.00046722200204385445, + 0.00046598599874414504, + 0.0004782110045198351, + 0.0004719720018329099, + 0.005884987003810238, + 0.0005845569976372644, + 0.0005619720031972975, + 0.00046984200162114576, + 0.00046660199586767703, + 0.00046424200263572857, + 0.0004584669950418174, + 0.004655320000892971, + 0.0005807019988424145, + 0.0005772049989900552, + 0.0004766729980474338, + 0.00047907100088195875, + 0.00046637299965368584, + 0.00046107899834169075, + 0.00048514099762542173, + 0.00047137799992924556, + 0.0004621760017471388, + 0.005177481005375739, + 0.0005658309964928776, + 0.0004931250005029142, + 0.000464086995634716, + 0.0005388179997680709, + 0.00047498699859716, + 0.0004685989988502115, + 0.0004579480009851977, + 0.006200010000611655, + 0.0005441750035970472, + 0.0005064170036348514, + 0.00046854300308041275, + 0.00045479200343834236, + 0.00045194899576017633, + 0.00044494900066638365, + 0.005323228004272096, + 0.0005844220067956485, + 0.0004942500017932616, + 0.000454234002972953, + 0.0004557850043056533, + 0.00045878699893364683, + 0.0007123690011212602, + 0.002632263996929396, + 0.0008469349995721132, + 0.00045487099851015955, + 0.0004506550030782819, + 0.00046024400216992944, + 0.00063539799884893, + 0.0005996320032863878, + 0.0004776700006914325, + 0.0004685399981099181, + 0.0004488179984036833, + 0.0005307730025378987, + 0.0029234570029075257, + 0.0005612639943137765, + 0.0004840769979637116, + 0.0004632190029951744, + 0.0004994359987904318, + 0.0006500570016214624, + 0.0012717360004899092, + 0.0006119819954619743, + 0.0005157339983270504, + 0.00045906900049885735, + 0.0004613439959939569, + 0.0004488529957598075, + 0.00044837900350103155, + 0.0006105299980845302, + 0.0006367820024024695, + 0.0006571970006916672, + 0.0008518409958924167, + 0.00048280299961334094, + 0.0004508259953581728, + 0.0004630999974324368, + 0.0004571309982566163, + 0.0006398699988494627, + 0.0004581179964588955, + 0.0004645269946195185, + 0.0004598090017680079, + 0.0004515429973253049, + 0.00045233799755806103, + 0.00045407099969452247, + 0.00044852500286651775, + 0.00044468399573815987, + 0.000570497999433428, + 0.000608973998168949, + 0.0006806059973314404, + 0.0006356899975799024, + 0.0006623429944738746, + 0.0021488729980774224, + 0.00048638899897923693, + 0.0004932550000376068, + 0.0004755670015583746, + 0.0010095760007970966, + 0.0006286650022957474, + 0.0005990909994579852, + 0.0005801450024591759, + 0.0004807069999515079, + 0.00045587999920826405, + 0.00044703000457957387, + 0.00045682799827773124, + 0.0004500499999267049, + 0.0005511389972525649, + 0.0035541050019674003, + 0.0005987350014038384, + 0.00048350999713875353, + 0.00045938100083731115, + 0.0004552239988697693, + 0.0005815670010633767, + 0.000898990998393856, + 0.0006172319990582764, + 0.0005829690053360537, + 0.00047675300447735935, + 0.0004857599997194484, + 0.00047000400081742555, + 0.00046470599772874266, + 0.0004986730054952204, + 0.0009641380020184442, + 0.0009273040050175041, + 0.00046970199764473364, + 0.000465421995613724, + 0.0031967370014172047, + 0.000789368998084683, + 0.0007967700003064238, + 0.000483703006466385, + 0.00046457799908239394, + 0.0004668059991672635, + 0.00046412999654421583, + 0.0004659219994209707, + 0.00047003200597828254, + 0.004178594004770275, + 0.0015497150016017258, + 0.000505452997458633, + 0.0005599669966613874, + 0.0005196599959162995, + 0.00046867499622749165, + 0.00046138700417941436, + 0.0004747740022139624, + 0.0004724399987026118, + 0.0010164799969061278, + 0.004130403001909144, + 0.0005793180025648326, + 0.0004905829991912469, + 0.0004661249986384064, + 0.00046280300011858344, + 0.0004873020006925799, + 0.0004613130004145205, + 0.000469954997242894, + 0.0004606500006048009, + 0.0030915190000087023, + 0.0005576019975706004, + 0.0005242059996817261, + 0.0004694379967986606, + 0.0004586869981721975, + 0.00047882399667287245, + 0.00046909199591027573, + 0.005187303999264259, + 0.0005670099999406375, + 0.0005408129945863038, + 0.00044376999721862376, + 0.0004477760012377985, + 0.00044814199645770714, + 0.00044571900070877746, + 0.0004940180006087758, + 0.00044475699542090297, + 0.00569863899727352, + 0.000567408002098091, + 0.0005948920006630942, + 0.0004614380013663322, + 0.0004561870009638369, + 0.0004522769959294237, + 0.000449020997621119, + 0.002488326994352974, + 0.0026775799997267313, + 0.0005234920026850887, + 0.0005136039981152862, + 0.00045226699876366183, + 0.00045123599556973204, + 0.0004561339956126176, + 0.0004477869952097535, + 0.0035613470026873983, + 0.0005620299998554401, + 0.0005401270027505234, + 0.0005215080018388107, + 0.00046119200123939663, + 0.0004559270018944517, + 0.00045674500142922625, + 0.0005119749985169619, + 0.006031659999280237, + 0.0005491960037034005, + 0.0004960469959769398, + 0.00045324000529944897, + 0.0004510889993980527, + 0.0004560209999908693, + 0.0004552219979814254, + 0.0037363110022852197, + 0.001411837998603005, + 0.0004586020004353486, + 0.0004512779996730387, + 0.0004527589990175329, + 0.0004560629968182184, + 0.00045189200318418443, + 0.0004492110019782558, + 0.0004480839997995645, + 0.004959329002304003, + 0.0005632690008496866, + 0.00047443299990845844, + 0.0004971239977749065, + 0.0004527380006038584, + 0.00045376699563348666, + 0.0004492660009418614, + 0.001201238002977334, + 0.0016507019972777925, + 0.00046691799798281863, + 0.0004573679980239831, + 0.0014283440032158978, + 0.0004904040033579804, + 0.0004674989977502264, + 0.0004523899988271296, + 0.0004567279975162819, + 0.0005385289987316355, + 0.0005048540042480454, + 0.0004583120025927201, + 0.00045599300210596994, + 0.00045579300058307126, + 0.00044949300354346633, + 0.0010134350013686344, + 0.00047070300206542015, + 0.0004582350011332892, + 0.00047212500066962093, + 0.0005324559970176779, + 0.0004507289995672181, + 0.0004497580011957325, + 0.0004587069997796789, + 0.0004516039989539422, + 0.0004797299989149906, + 0.0004466619939194061, + 0.0004511090010055341, + 0.00045650600077351555, + 0.0006176079987199046, + 0.000470567996671889, + 0.0004629219984053634, + 0.0004704120001406409, + 0.00046943699999246746, + 0.000456911999208387, + 0.00044999200326856226, + 0.000451830004749354, + 0.0004450270062079653, + 0.000452606996987015, + 0.00045141600276110694, + 0.000475129003461916, + 0.0004561039968393743, + 0.00045787199633195996, + 0.00044905200047651306, + 0.0004476429967326112, + 0.0005435619968920946, + 0.0027471659996081144, + 0.0005204909975873306, + 0.000506595999468118, + 0.0005050650070188567, + 0.0012245699981576763, + 0.0010182469995925203, + 0.0004603080014931038, + 0.00044628999603446573, + 0.00045697600580751896, + 0.00045727600081590936, + 0.0004578849984682165, + 0.0004484449964365922, + 0.0005117250038892962, + 0.00047425500088138506, + 0.0029384430017671548, + 0.0007325469996430911, + 0.0005235410062596202, + 0.00044995999633101746, + 0.0004574489939841442, + 0.0011644690021057613, + 0.00047806000657146797, + 0.0005183700050110929, + 0.0007328819992835633, + 0.00047331699897767976, + 0.00044626599992625415, + 0.0004537470013019629, + 0.00045216400030767545, + 0.0004509510035859421, + 0.0004713140006060712, + 0.00045338100608205423, + 0.0020472030009841546, + 0.0005528520050575025, + 0.0005948290054220706, + 0.0005076789966551587, + 0.0004766269994433969, + 0.000463005002529826, + 0.00046749800094403327, + 0.0004591710021486506, + 0.0004992619942640886, + 0.0009475789993302897, + 0.0010109270006068982, + 0.001057032000971958, + 0.00046300200483528897, + 0.0004713710004580207, + 0.0004629460017895326, + 0.00046056000428507105, + 0.0004924289969494566, + 0.0004783480035257526, + 0.00046712699986528605, + 0.0004630250041373074, + 0.00046539200411643833, + 0.00046328999451361597, + 0.00048157299897866324, + 0.00048361100198235363, + 0.000519100998644717, + 0.0004900229978375137, + 0.00047472900041611865, + 0.0004683980005211197, + 0.00046612200094386935, + 0.0004629260001820512, + 0.00047083499521249905, + 0.00046422100422205403, + 0.0005040550022386014, + 0.00047786500363145024, + 0.0004824759962502867, + 0.000481041002785787, + 0.00047223099682014436, + 0.00048249700193991885, + 0.0004566749994410202, + 0.0005262070044409484, + 0.0010732279988587834, + 0.00046543700591428205, + 0.0004668930050684139, + 0.0004620789986802265, + 0.0004645249937311746, + 0.0004869800031883642, + 0.0010862670023925602, + 0.0004867489988100715, + 0.00048693999997340143, + 0.0014267639999161474, + 0.0004770969972014427, + 0.0004868120013270527, + 0.00046522600314347073, + 0.0004652169955079444, + 0.0004667250032071024, + 0.0015479319990845397, + 0.002575611993961502, + 0.0005191219970583916, + 0.00047790000098757446, + 0.00047786300274310634, + 0.00046357700193766505, + 0.0004664469961426221, + 0.0004687609980464913, + 0.0007434480066876858, + 0.000975096998445224, + 0.0011309530018479563, + 0.0011541400017449632, + 0.00047519500367343426, + 0.0004693449955084361, + 0.00047209699550876394, + 0.0004627680027624592, + 0.00046663299872307107, + 0.00046011200174689293, + 0.00046667500282637775, + 0.0029499299998860806, + 0.0005215229975874536, + 0.0004750360021716915, + 0.0004631470001186244, + 0.0005222619947744533, + 0.00046121099876472726, + 0.0004642749991035089, + 0.0005023060002713464, + 0.0004965870029991493, + 0.0008486220031045377, + 0.0007503529996029101, + 0.0005551349968300201, + 0.0014408610004466027, + 0.001329742997768335, + 0.0018722469976637512, + 0.0007306479965336621, + 0.0004718290001619607, + 0.0004908409973722883, + 0.00047290799557231367 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil", + "fullname": "TestTrulyHorizontalDiffusionNablaOfThetaOverSteepPoints", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.00043865200132131577, + "max": 0.01299809200281743, + "mean": 0.005433601697726129, + "stddev": 0.002916803648316478, + "rounds": 334, + "median": 0.006328357496386161, + "iqr": 0.004274203005479649, + "q1": 0.0035999009996885434, + "q3": 0.007874104005168192, + "iqr_outliers": 0, + "stddev_outliers": 94, + "outliers": "94;0", + "ld15iqr": 0.00043865200132131577, + "hd15iqr": 0.01299809200281743, + "ops": 184.0399896110315, + "total": 1.8148229670405271, + "data": [ + 0.001974742997845169, + 0.005991189005726483, + 0.009000550999189727, + 0.006991414004005492, + 0.007001578000199515, + 0.006994010000198614, + 0.006997163000050932, + 0.007995958003448322, + 0.006995802999881562, + 0.005996199994115159, + 0.006018956999469083, + 0.006973362003918737, + 0.0071339560017804615, + 0.007860489997256082, + 0.006996161995630246, + 0.005015518996515311, + 0.003978914006438572, + 0.007992282000486739, + 0.006995340998400934, + 0.006994711002334952, + 0.005994097999064252, + 0.00544539799739141, + 0.008554105996154249, + 0.007000958998105489, + 0.006986596999922767, + 0.005006983999919612, + 0.006985355998040177, + 0.010004787000070792, + 0.006988712004385889, + 0.004989676002878696, + 0.0035999009996885434, + 0.004390064001199789, + 0.005999424000037834, + 0.008001123002031818, + 0.007989593003003392, + 0.005003239995858166, + 0.004988310996850487, + 0.007001017998845782, + 0.006993556002271362, + 0.006996714000706561, + 0.006998297001700848, + 0.007992247999936808, + 0.008000565001566429, + 0.007993757993972395, + 0.007998884000699036, + 0.00498649100336479, + 0.005005370003345888, + 0.005989564000628889, + 0.006994236005994026, + 0.005601311000646092, + 0.0043902879988309, + 0.005650762999721337, + 0.010345394999603741, + 0.00800501800404163, + 0.0069852609958616085, + 0.010001710004871711, + 0.007991350001248065, + 0.007007741995039396, + 0.006987105000007432, + 0.006994623996433802, + 0.006999433004239108, + 0.007995441003004089, + 0.006994821997068357, + 0.006998480996116996, + 0.007997773995157331, + 0.004383501996926498, + 0.004607748000125866, + 0.006997113996476401, + 0.005997098000079859, + 0.006998528006079141, + 0.0069963889982318506, + 0.004002338006102946, + 0.0055955049974727444, + 0.007530669005063828, + 0.007885408005677164, + 0.007964916003402323, + 0.00799650899716653, + 0.007995707004738506, + 0.010001860006013885, + 0.005989666999084875, + 0.007998255001439247, + 0.006993564005824737, + 0.007995579995622393, + 0.01101143000414595, + 0.007987722994585056, + 0.002987391002534423, + 0.007005019004282076, + 0.005986613999994006, + 0.006008823998854496, + 0.010106360001373105, + 0.007874104005168192, + 0.005003666003176477, + 0.003263197999331169, + 0.0057198280046577565, + 0.005000937002478167, + 0.006996062002144754, + 0.006994974995905068, + 0.006996826996328309, + 0.007995678999577649, + 0.005997674998070579, + 0.00799658599862596, + 0.00599930300086271, + 0.007000842997513246, + 0.00599088599847164, + 0.006995800998993218, + 0.008003256996744312, + 0.005990116005705204, + 0.007996366999577731, + 0.004998793003323954, + 0.006996833995799534, + 0.006995094001467805, + 0.01299809200281743, + 0.006999208002525847, + 0.005994849998387508, + 0.006996675001573749, + 0.00800123299995903, + 0.004645459994208068, + 0.0063461239988100715, + 0.010003677998611238, + 0.00699224800337106, + 0.007001341000432149, + 0.007982658004038967, + 0.00856472400482744, + 0.006437834999815095, + 0.004790803002833854, + 0.009191071992972866, + 0.006994742005190346, + 0.004003381996881217, + 0.007991778002178762, + 0.007997699998668395, + 0.005990095000015572, + 0.005998540997097734, + 0.00799470700439997, + 0.008999022997159045, + 0.006711800997436512, + 0.008280941998236813, + 0.007005444997048471, + 0.007990449004864786, + 0.007995879001100548, + 0.005994004997774027, + 0.0069981410051696, + 0.00408851799875265, + 0.007906076003564522, + 0.007999010995263234, + 0.007994106999831274, + 0.0030264739980339073, + 0.005968646000837907, + 0.0059971549999318086, + 0.008009050994587597, + 0.007985228003235534, + 0.003996015999291558, + 0.003717268002219498, + 0.007280263002030551, + 0.005995504005113617, + 0.007997016000445, + 0.006997060998401139, + 0.006018002997734584, + 0.00797720999980811, + 0.00613597099436447, + 0.006861289999505971, + 0.005993583996314555, + 0.006001155001285952, + 0.0076487129990709946, + 0.0063404139946214855, + 0.004996001996914856, + 0.008015557999897283, + 0.007089963000908028, + 0.00788709899643436, + 0.007997032000275794, + 0.0049968439998338, + 0.007000988996878732, + 0.005991933998302557, + 0.006998119002673775, + 0.007995038999069948, + 0.00699873200210277, + 0.0059983889950672165, + 0.008290698002383579, + 0.005698961998859886, + 0.008002511000086088, + 0.007990065001649782, + 0.008000920999620575, + 0.007994004001375288, + 0.0060702460032189265, + 0.007926733000203967, + 0.007991951999429148, + 0.0020022589960717596, + 0.004993943999579642, + 0.007997232001798693, + 0.00556829300330719, + 0.007057952003378887, + 0.007364971999777481, + 0.00699617500504246, + 0.006997907999902964, + 0.006010372002492659, + 0.010984045002260245, + 0.008000554000318516, + 0.0069925749994581565, + 0.006997967000643257, + 0.006996511998295318, + 0.006996566000452731, + 0.006999626006290782, + 0.007996460997674149, + 0.005998517997795716, + 0.0059949010028503835, + 0.0069965720031177625, + 0.006995342002483085, + 0.008010443001694512, + 0.007983816001797095, + 0.00807205299497582, + 0.005924641001911368, + 0.007992898994416464, + 0.0049989170001936145, + 0.001994942002056632, + 0.006001251000270713, + 0.009996341999794822, + 0.006996201002039015, + 0.007009569999354426, + 0.005992614998831414, + 0.0029895109983044676, + 0.0049970100008067675, + 0.007996861000719946, + 0.005995997002173681, + 0.009997639994253404, + 0.00600160300382413, + 0.00799373300105799, + 0.007998348002729472, + 0.006991235997702461, + 0.005997046995616984, + 0.007999383997230325, + 0.0069934519997332245, + 0.007001720005064271, + 0.007999565998034086, + 0.007990493999386672, + 0.005494175995409023, + 0.007498842001950834, + 0.00599625900213141, + 0.005995519000862259, + 0.0069984400033717975, + 0.0069979149993741885, + 0.00799809299496701, + 0.0029960610045236535, + 0.005997669999487698, + 0.0059970380025333725, + 0.003995432001829613, + 0.005996744999720249, + 0.006019512002239935, + 0.006975092997890897, + 0.00800279800023418, + 0.007992077997187153, + 0.007003685997915454, + 0.006575943996722344, + 0.009417635999852791, + 0.006990684996708296, + 0.004999432996555697, + 0.0069948259988450445, + 0.007995234002009965, + 0.004678716999478638, + 0.006316300998150837, + 0.00404667900147615, + 0.0006232809973880649, + 0.0005330519998096861, + 0.0004933329983032309, + 0.0004446890015969984, + 0.0004407110027386807, + 0.0004930610011797398, + 0.0004519720023381524, + 0.0004728269996121526, + 0.0017756599991116673, + 0.0004546629934338853, + 0.0004396450021886267, + 0.0004416840019985102, + 0.0004401259939186275, + 0.00044349800009513274, + 0.0004404499995871447, + 0.00043865200132131577, + 0.0027661889980663545, + 0.0005029629974160343, + 0.0004405919971759431, + 0.0007621610056958161, + 0.002285353002662305, + 0.0008558879999327473, + 0.0006801280032959767, + 0.0012495810005930252, + 0.0004579379965434782, + 0.0004506350014708005, + 0.00044887799595016986, + 0.0010036720050266013, + 0.0013806799979647622, + 0.0005710850018658675, + 0.0004599349995260127, + 0.0004619920000550337, + 0.00046049900265643373, + 0.0004525660042418167, + 0.0005878629963262938, + 0.0008274319989141077, + 0.0011609160064836033, + 0.0008610100048827007, + 0.0005931089981459081, + 0.00046643800305901095, + 0.0004572990001179278, + 0.00045643899647984654, + 0.0004507630001171492, + 0.0005164570029592142, + 0.001232244998391252, + 0.0006568030003109016, + 0.0005342459990060888, + 0.0013320750003913417, + 0.0008081330015556887, + 0.0005203359978622757, + 0.0004650750051951036, + 0.0005771090000052936, + 0.0013719560010940768, + 0.0005489649993251078, + 0.0004547159987851046, + 0.0004655149969039485, + 0.00044797000009566545, + 0.00045815500197932124, + 0.0004518229980021715, + 0.0004522119998000562, + 0.0008975939999800175, + 0.0008318710024468601, + 0.0008007409996935166, + 0.00047246599569916725, + 0.00045867999870097265, + 0.0004511660008574836, + 0.00047887899563647807, + 0.0006941660030861385, + 0.0007910339991212822, + 0.0008062839988269843, + 0.0004884090012637898, + 0.0004576439969241619, + 0.000460267998278141, + 0.0004626869995263405, + 0.0004585720016621053 + ], + "iterations": 1 + } + }, + { + "group": null, + "name": "test_stencil", + "fullname": "TestUpdateThetaAndExner", + "params": null, + "param": null, + "extra_info": {}, + "options": { + "disable_gc": false, + "timer": "perf_counter", + "min_rounds": 5, + "max_time": 1.0, + "min_time": 5e-06, + "warmup": 30 + }, + "stats": { + "min": 0.0003912679967470467, + "max": 0.015237425999657717, + "mean": 0.0013409496156103144, + "stddev": 0.0021647318011531686, + "rounds": 419, + "median": 0.0004286800030968152, + "iqr": 0.00018946225281979423, + "q1": 0.0004108402463316452, + "q3": 0.0006003024991514394, + "iqr_outliers": 84, + "stddev_outliers": 53, + "outliers": "53;84", + "ld15iqr": 0.0003912679967470467, + "hd15iqr": 0.0008869759985827841, + "ops": 745.7401742457445, + "total": 0.5618578889407218, + "data": [ + 0.006992770999204367, + 0.006002845999319106, + 0.007988664998265449, + 0.007998503002454527, + 0.0069941710025887005, + 0.004002454996225424, + 0.00799194900173461, + 0.007994188003067393, + 0.006059844003175385, + 0.006943410000531003, + 0.0005575140021392144, + 0.00043153799924766645, + 0.00041680599679239094, + 0.00041543800034560263, + 0.00042569299694150686, + 0.0005926720041316003, + 0.00041415000305278227, + 0.0004103050014236942, + 0.0017139010014943779, + 0.001903217998915352, + 0.0004176619986537844, + 0.00042315600148867816, + 0.0004184680001344532, + 0.0006799550028517842, + 0.0006761240001651458, + 0.0009831689967541024, + 0.000668533997668419, + 0.00041593299829401076, + 0.00040881700260797516, + 0.0004080930011696182, + 0.00041492799937259406, + 0.00040742500277701765, + 0.00041404600051464513, + 0.00040775500383460894, + 0.0004060770006617531, + 0.0008869759985827841, + 0.0023436600022250786, + 0.000502120004966855, + 0.0004662139981519431, + 0.0004103460014448501, + 0.00042253600258845836, + 0.00042197899892926216, + 0.00041369199607288465, + 0.0004148919979343191, + 0.00040894099947763607, + 0.0008331249991897494, + 0.0009188339972752146, + 0.0011976000023423694, + 0.0005343790035112761, + 0.0005851380046806298, + 0.0021103570034028962, + 0.0008271670012618415, + 0.0004581070024869405, + 0.0004140880046179518, + 0.0004116240015719086, + 0.0004176039947196841, + 0.00042191799730062485, + 0.0004241760034346953, + 0.0004107369968551211, + 0.0008361369982594624, + 0.0007823870037100278, + 0.00041465400136075914, + 0.0004115510018891655, + 0.0004210299957776442, + 0.0004142849938943982, + 0.00041533300100127235, + 0.00040430899389320984, + 0.00042454199865460396, + 0.0004089290014235303, + 0.000406731000111904, + 0.003070404003665317, + 0.0007727080010226928, + 0.00042935399687848985, + 0.001190342998597771, + 0.0005357120026019402, + 0.0004272720034350641, + 0.000412878995120991, + 0.0004288260024623014, + 0.00041081499512074515, + 0.00041749299998627976, + 0.0004127149950363673, + 0.0004079899954376742, + 0.0014516169976559468, + 0.0005152920057298616, + 0.000620179001998622, + 0.0004467609978746623, + 0.0008768599946051836, + 0.0004120699959457852, + 0.00041091599996434525, + 0.00041525800043018535, + 0.00041545099520590156, + 0.0004060279970872216, + 0.0004238260007696226, + 0.0004105970001546666, + 0.0010547290003160015, + 0.0005210900053498335, + 0.0007050790009088814, + 0.0004593269986798987, + 0.0013161509996280074, + 0.0004172219996689819, + 0.00042333199962740764, + 0.00041422699723625556, + 0.0004144199992879294, + 0.0004133120019105263, + 0.00041848199907690287, + 0.00040770499617792666, + 0.0004991290043108165, + 0.003759169005206786, + 0.015237425999657717, + 0.007782090004184283, + 0.009160306995909195, + 0.0028064559955964796, + 0.00915346800320549, + 0.0005482709966599941, + 0.0004978110009687953, + 0.0004399819954414852, + 0.0004412439957377501, + 0.0004294829996069893, + 0.0004205899967928417, + 0.0004288300042389892, + 0.00042279899935238063, + 0.007910255997558124, + 0.0006028459974913858, + 0.0005315550006343983, + 0.0004304690010030754, + 0.00042214099812554196, + 0.00042611399840097874, + 0.0004679359990404919, + 0.005422180001914967, + 0.0005011620014556684, + 0.0004977249991497956, + 0.0004438879986992106, + 0.0004302079978515394, + 0.001989188000152353, + 0.0005351639993023127, + 0.000442086995462887, + 0.0004381239996291697, + 0.0004231730054016225, + 0.0016870550025487319, + 0.000426906997745391, + 0.0004465769961825572, + 0.0004446849998203106, + 0.0004286800030968152, + 0.0004445000013220124, + 0.000426254999183584, + 0.00041831999988062307, + 0.00041862199577735737, + 0.0032311420000041835, + 0.0005486490044859238, + 0.0018845430022338405, + 0.0006083530024625361, + 0.0005672050028806552, + 0.00047016999451443553, + 0.0004272270016372204, + 0.0004228180041536689, + 0.0004223380019539036, + 0.0004181379990768619, + 0.0033738900019670837, + 0.00047595299838576466, + 0.00047661099961260334, + 0.0004590819953591563, + 0.00046570500126108527, + 0.0004320980006013997, + 0.0004133690017624758, + 0.0004072520023328252, + 0.0004059830025653355, + 0.004549698001937941, + 0.0005017610019422136, + 0.0004148820007685572, + 0.00040707200241740793, + 0.00046060900058364496, + 0.0004153459958615713, + 0.00041548700392013416, + 0.0004113510003662668, + 0.00041347099613631144, + 0.005194863995711785, + 0.00056048299302347, + 0.000490767000883352, + 0.00042168100480921566, + 0.0004323640023358166, + 0.0004092390008736402, + 0.00041155199869535863, + 0.0004220320042804815, + 0.000409783999202773, + 0.004203453994705342, + 0.00044779200106859207, + 0.00040930099930847064, + 0.00041845300438581035, + 0.0004140390010434203, + 0.0004488760023377836, + 0.00042370900337118655, + 0.00041452699952060357, + 0.004747687999042682, + 0.0004903630033368245, + 0.0004480280040297657, + 0.00041536700155120343, + 0.0004188149978290312, + 0.000415644004533533, + 0.00041204899753211066, + 0.00040947799425339326, + 0.004354295997472946, + 0.0069930769968777895, + 0.00799689099949319, + 0.00699728200561367, + 0.006997250995482318, + 0.006999044999247417, + 0.005997407999529969, + 0.006723004000377841, + 0.007832219002011698, + 0.007435739993525203, + 0.006996334996074438, + 0.005997653002850711, + 0.006998229997407179, + 0.007995398998900782, + 0.006777542002964765, + 0.0072169339982792735, + 0.0070020260027376935, + 0.007032200002868194, + 0.007956777000799775, + 0.008005229996342678, + 0.00798193299851846, + 0.004838141998334322, + 0.006166530001792125, + 0.0005497320016729645, + 0.00048432099720230326, + 0.000535990999196656, + 0.00044581400288734585, + 0.00041610899643274024, + 0.00041372799751115963, + 0.0004242160066496581, + 0.00041348899685544893, + 0.008395297001698054, + 0.000534187005541753, + 0.00047266699402825907, + 0.00041214399971067905, + 0.0004114879993721843, + 0.0037913900014245883, + 0.0005592990055447444, + 0.0004859119944740087, + 0.0004600489992299117, + 0.00041225900349672884, + 0.00041697399865370244, + 0.00042154100083280355, + 0.00041317500290460885, + 0.007000172998232301, + 0.00061266800184967, + 0.0005187950009712949, + 0.00041748499643290415, + 0.00040107100358000025, + 0.0004037179969600402, + 0.00039999699947657064, + 0.0004119789955439046, + 0.0004213430001982488, + 0.00040602299850434065, + 0.0017026019995682873, + 0.0004711169967777096, + 0.00045805399713572115, + 0.0004541180023807101, + 0.0004138450021855533, + 0.00040274299681186676, + 0.0041299000004073605, + 0.0005063479984528385, + 0.00044109100417699665, + 0.0004021089989691973, + 0.00041661599971121177, + 0.0004082930026925169, + 0.000396371993701905, + 0.00039478400140069425, + 0.0004098959980183281, + 0.0004119810037082061, + 0.0018871270003728569, + 0.0008601959998486564, + 0.00046006400225451216, + 0.00040993999573402107, + 0.00040595400059828535, + 0.0004091480004717596, + 0.0004032259967061691, + 0.0004004889997304417, + 0.00040121500205714256, + 0.00040217499918071553, + 0.0004129799999645911, + 0.0004448200052138418, + 0.0023394300005747937, + 0.00046710899914614856, + 0.00045551900257123634, + 0.0004008690011687577, + 0.00039480599662056193, + 0.001220270998601336, + 0.0018803510029101744, + 0.00048549799976171926, + 0.00044783199700759724, + 0.000429797000833787, + 0.00040246099524665624, + 0.00040947300294646993, + 0.00039920500421430916, + 0.0003967489974456839, + 0.003729116993781645, + 0.0005104090014356188, + 0.000521317997481674, + 0.0027825069992104545, + 0.00047056699986569583, + 0.0004743430035887286, + 0.00046953000128269196, + 0.0004117360003874637, + 0.00040142399666365236, + 0.004321016996982507, + 0.0005177839993848465, + 0.0005053080021752976, + 0.00040503800119040534, + 0.00043105600343551487, + 0.0004105489933863282, + 0.00039833600021665916, + 0.0007295989998965524, + 0.00040976700256578624, + 0.0004144609993090853, + 0.0004002330024377443, + 0.0004049560011480935, + 0.0004000150001957081, + 0.00040201900264946744, + 0.003993055994214956, + 0.0007822560000931844, + 0.0004687049950007349, + 0.00039691799611318856, + 0.00039831799949752167, + 0.0020261640020180494, + 0.0005143110029166564, + 0.0004976519994670525, + 0.0004184119970886968, + 0.0004058939957758412, + 0.0004086799963261001, + 0.0003960879985243082, + 0.002382086997386068, + 0.0004926879992126487, + 0.00046877199929440394, + 0.00042050400224979967, + 0.00039525600004708394, + 0.00040530000114813447, + 0.0004025189991807565, + 0.000394793001760263, + 0.002435160000459291, + 0.0004606789952958934, + 0.00044785899808630347, + 0.00043066900252597407, + 0.0004034769954159856, + 0.00040459699812345207, + 0.0003950909958803095, + 0.004040186002384871, + 0.0004892569995718077, + 0.00042297199979657307, + 0.0004027780014439486, + 0.0004029180054203607, + 0.00040767400059849024, + 0.0003985169969382696, + 0.0003951129983761348, + 0.0003979539978899993, + 0.000400479999370873, + 0.00041372000123374164, + 0.00040516800072509795, + 0.0003912679967470467, + 0.0005104939991724677, + 0.00043200599611736834, + 0.0004090770016773604, + 0.0003951390026486479, + 0.000410091997764539, + 0.0003993089994764887, + 0.0003988310054410249, + 0.00040726899896981195, + 0.00039981699956115335, + 0.0008987980036181398, + 0.0005792240044684149, + 0.00047407400415977463, + 0.0033222220008610748, + 0.0004990490051568486, + 0.0004385530046420172, + 0.0004004930015071295, + 0.00039780899533070624, + 0.0004005599985248409, + 0.00040605899994261563, + 0.00040538900066167116, + 0.0006110469985287637, + 0.00047482999798376113, + 0.0004079020000062883, + 0.0004043180015287362, + 0.0008415639968006872, + 0.0007869309993111528, + 0.000415587994211819, + 0.00040972099668579176, + 0.0003951520047849044, + 0.0004081389997736551, + 0.00040210499719250947, + 0.00040502000047126785, + 0.0004125639970880002, + 0.0003981990012107417, + 0.004230504993756767, + 0.0005000940000172704, + 0.0004890950003755279, + 0.0004110769950784743, + 0.0004119040022487752, + 0.00041098099609371275, + 0.00039607299549970776, + 0.0004067860063514672, + 0.0004223889991408214, + 0.0003944570053135976, + 0.00041221699939342216, + 0.00040148200059775263, + 0.000415548995079007, + 0.0004016379971290007, + 0.0003998000029241666, + 0.0006599919943255372, + 0.0005283650025376119, + 0.00042095599928870797, + 0.00043135399755556136, + 0.0004228710022289306, + 0.0004098369972780347, + 0.0004152910041739233, + 0.0004080099970451556, + 0.0032610690032015555, + 0.0004780099989147857, + 0.000468849997560028 + ], + "iterations": 1 + } + } + ], + "datetime": "2025-08-21T16:44:50.947945+00:00", + "version": "5.1.0" +} \ No newline at end of file diff --git a/model/atmosphere/diffusion/src/icon4py/model/atmosphere/diffusion/config/__init__.py b/model/atmosphere/diffusion/src/icon4py/model/atmosphere/diffusion/config/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/model/atmosphere/diffusion/tests/diffusion/unit_tests/diffusion_default.yaml b/model/atmosphere/diffusion/tests/diffusion/unit_tests/diffusion_default.yaml new file mode 100644 index 0000000000..94b00021be --- /dev/null +++ b/model/atmosphere/diffusion/tests/diffusion/unit_tests/diffusion_default.yaml @@ -0,0 +1,21 @@ +diffusion: + diffusion_type: "SMAGORINSKY_4TH_ORDER" + apply_to_vertical_wind: True + apply_to_horizontal_wind: True + apply_to_temperature: True + type_vn_diffu: 1 + compute_3d_smag_coeff: False + type_t_diffu: 2 + hdiff_efdt_ratio: 36.0 + hdiff_w_efdt_ratio: 15.0 + smagorinski_scaling_factor: 0.015 + n_substeps: 5 + apply_zdiffusion_t: True + thslp_zdiffu: 0.025 + thhgtd_zdiffu: 200.0 + temperature_boundary_diffusion_denom: 135.0 + velocity_boundary_diffusion_denom: 200.0 + max_nudging_coefficient: 0.1 + nudging_decay_rate: 2.0 + shear_type: "VERTICAL_OF_HORIZONTAL_WIND" + ltkeshs: True diff --git a/model/atmosphere/dycore/tests/dycore/mpi_tests/test_parallel_solve_nonhydro.py b/model/atmosphere/dycore/tests/dycore/mpi_tests/test_parallel_solve_nonhydro.py index 6c09ec7ae3..1711bcea78 100644 --- a/model/atmosphere/dycore/tests/dycore/mpi_tests/test_parallel_solve_nonhydro.py +++ b/model/atmosphere/dycore/tests/dycore/mpi_tests/test_parallel_solve_nonhydro.py @@ -23,15 +23,23 @@ @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.datatest @pytest.mark.parametrize( - "istep_init, jstep_init, step_date_init,istep_exit, jstep_exit, step_date_exit", - [(1, 0, "2021-06-20T12:00:10.000", 2, 0, "2021-06-20T12:00:10.000")], + "experiment, istep_init, step_date_init, substep_init, istep_exit, step_date_exit, substep_exit", + [ + ( + test_defs.Experiments.MCH_CH_R04B09, + 1, + "2021-06-20T12:00:10.000", + 1, + 2, + "2021-06-20T12:00:10.000", + 1, + ) + ], ) @pytest.mark.mpi def test_run_solve_nonhydro_single_step( istep_init, istep_exit, - jstep_init, - jstep_exit, step_date_init, step_date_exit, experiment, @@ -43,7 +51,6 @@ def test_run_solve_nonhydro_single_step( stretch_factor, damping_height, grid_savepoint, - savepoint_velocity_init, metrics_savepoint, interpolation_savepoint, savepoint_nonhydro_exit, @@ -77,8 +84,6 @@ def test_run_solve_nonhydro_single_step( ) config = test_defs.construct_nonhydrostatic_config(experiment) - sp = savepoint_nonhydro_init - sp_step_exit = savepoint_nonhydro_step_final nonhydro_params = nh.NonHydrostaticParams(config) vertical_config = v_grid.VerticalGridConfig( icon_grid.num_levels, @@ -87,59 +92,28 @@ def test_run_solve_nonhydro_single_step( stretch_factor=stretch_factor, rayleigh_damping_height=damping_height, ) - vertical_params = v_grid.VerticalGrid( - config=vertical_config, - vct_a=grid_savepoint.vct_a(), - vct_b=grid_savepoint.vct_b(), - ) - sp_v = savepoint_velocity_init - dtime = sp_v.get_metadata("dtime").get("dtime") - lprep_adv = sp_v.get_metadata("prep_adv").get("prep_adv") + vertical_params = utils.create_vertical_params(vertical_config, grid_savepoint) + dtime = savepoint_nonhydro_init.get_metadata("dtime").get("dtime") + lprep_adv = savepoint_nonhydro_init.get_metadata("prep_adv").get("prep_adv") prep_adv = dycore_states.PrepAdvection( - vn_traj=sp.vn_traj(), - mass_flx_me=sp.mass_flx_me(), - dynamical_vertical_mass_flux_at_cells_on_half_levels=sp.mass_flx_ic(), + vn_traj=savepoint_nonhydro_init.vn_traj(), + mass_flx_me=savepoint_nonhydro_init.mass_flx_me(), + dynamical_vertical_mass_flux_at_cells_on_half_levels=savepoint_nonhydro_init.mass_flx_ic(), dynamical_vertical_volumetric_flux_at_cells_on_half_levels=data_alloc.zero_field( icon_grid, dims.CellDim, dims.KDim, allocator=backend ), ) - recompute = sp_v.get_metadata("recompute").get("recompute") - - diagnostic_state_nh = dycore_states.DiagnosticStateNonHydro( - max_vertical_cfl=0.0, - theta_v_at_cells_on_half_levels=sp.theta_v_ic(), - perturbed_exner_at_cells_on_model_levels=sp.exner_pr(), - rho_at_cells_on_half_levels=sp.rho_ic(), - exner_tendency_due_to_slow_physics=sp.ddt_exner_phy(), - grf_tend_rho=sp.grf_tend_rho(), - grf_tend_thv=sp.grf_tend_thv(), - grf_tend_w=sp.grf_tend_w(), - mass_flux_at_edges_on_model_levels=sp.mass_fl_e(), - normal_wind_tendency_due_to_slow_physics_process=sp.ddt_vn_phy(), - grf_tend_vn=sp.grf_tend_vn(), - normal_wind_advective_tendency=common_utils.PredictorCorrectorPair( - sp_v.ddt_vn_apc_pc(1), sp_v.ddt_vn_apc_pc(2) - ), - vertical_wind_advective_tendency=common_utils.PredictorCorrectorPair( - sp_v.ddt_w_adv_pc(1), sp_v.ddt_w_adv_pc(2) - ), - tangential_wind=sp_v.vt(), - vn_on_half_levels=sp_v.vn_ie(), - contravariant_correction_at_cells_on_half_levels=sp_v.w_concorr_c(), - rho_iau_increment=None, # sp.rho_incr(), - normal_wind_iau_increment=None, # sp.vn_incr(), - exner_iau_increment=None, # sp.exner_incr(), - exner_dynamical_increment=sp.exner_dyn_incr(), - ) - second_order_divdamp_factor = sp.divdamp_fac_o2() + diagnostic_state_nh = utils.construct_diagnostics(savepoint_nonhydro_init, icon_grid, backend) + interpolation_state = utils.construct_interpolation_state(interpolation_savepoint) metric_state_nonhydro = utils.construct_metric_state(metrics_savepoint, grid_savepoint) - + second_order_divdamp_factor = savepoint_nonhydro_init.divdamp_fac_o2() + at_initial_timestep = True cell_geometry: grid_states.CellParams = grid_savepoint.construct_cell_geometry() edge_geometry: grid_states.EdgeParams = grid_savepoint.construct_edge_geometry() - prognostic_states = utils.create_prognostic_states(sp) + prognostic_states = utils.create_prognostic_states(savepoint_nonhydro_init) exchange = definitions.create_exchange(processor_props, decomposition_info) @@ -168,20 +142,20 @@ def test_run_solve_nonhydro_single_step( second_order_divdamp_factor=second_order_divdamp_factor, dtime=dtime, ndyn_substeps_var=ndyn_substeps, - at_initial_timestep=recompute, + at_initial_timestep=at_initial_timestep, lprep_adv=lprep_adv, - at_first_substep=jstep_init == 0, - at_last_substep=jstep_init == (ndyn_substeps - 1), + at_first_substep=(substep_init == 1), + at_last_substep=(substep_init == ndyn_substeps), ) print(f"rank={processor_props.rank}/{processor_props.comm_size}: dycore step run ") - expected_theta_v = sp_step_exit.theta_v_new().asnumpy() + expected_theta_v = savepoint_nonhydro_step_final.theta_v_new().asnumpy() calculated_theta_v = prognostic_states.next.theta_v.asnumpy() assert test_utils.dallclose( expected_theta_v, calculated_theta_v, ) - expected_exner = sp_step_exit.exner_new().asnumpy() + expected_exner = savepoint_nonhydro_step_final.exner_new().asnumpy() calculated_exner = prognostic_states.next.exner.asnumpy() assert test_utils.dallclose( expected_exner, @@ -202,11 +176,6 @@ def test_run_solve_nonhydro_single_step( prognostic_states.next.rho.asnumpy(), ) - assert test_utils.dallclose( - savepoint_nonhydro_exit.rho_ic().asnumpy(), - diagnostic_state_nh.rho_ic.asnumpy(), - ) - assert test_utils.dallclose( savepoint_nonhydro_exit.theta_v_ic().asnumpy(), diagnostic_state_nh.theta_v_at_cells_on_half_levels.asnumpy(), diff --git a/pytest b/pytest new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pytest_benchmark_results_3.10.json b/pytest_benchmark_results_3.10.json new file mode 100644 index 0000000000..e69de29bb2 From ef3b4940526a4da3449d966261096c13a31c9fd7 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 14 Oct 2025 09:09:48 +0200 Subject: [PATCH 103/240] delete files unrelated to this PR --- .../atmosphere/diffusion/config/__init__.py | 0 .../unit_tests/diffusion_default.yaml | 21 ------------------- pytest_benchmark_results_3.10.json | 0 3 files changed, 21 deletions(-) delete mode 100644 model/atmosphere/diffusion/src/icon4py/model/atmosphere/diffusion/config/__init__.py delete mode 100644 model/atmosphere/diffusion/tests/diffusion/unit_tests/diffusion_default.yaml delete mode 100644 pytest_benchmark_results_3.10.json diff --git a/model/atmosphere/diffusion/src/icon4py/model/atmosphere/diffusion/config/__init__.py b/model/atmosphere/diffusion/src/icon4py/model/atmosphere/diffusion/config/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/model/atmosphere/diffusion/tests/diffusion/unit_tests/diffusion_default.yaml b/model/atmosphere/diffusion/tests/diffusion/unit_tests/diffusion_default.yaml deleted file mode 100644 index 94b00021be..0000000000 --- a/model/atmosphere/diffusion/tests/diffusion/unit_tests/diffusion_default.yaml +++ /dev/null @@ -1,21 +0,0 @@ -diffusion: - diffusion_type: "SMAGORINSKY_4TH_ORDER" - apply_to_vertical_wind: True - apply_to_horizontal_wind: True - apply_to_temperature: True - type_vn_diffu: 1 - compute_3d_smag_coeff: False - type_t_diffu: 2 - hdiff_efdt_ratio: 36.0 - hdiff_w_efdt_ratio: 15.0 - smagorinski_scaling_factor: 0.015 - n_substeps: 5 - apply_zdiffusion_t: True - thslp_zdiffu: 0.025 - thhgtd_zdiffu: 200.0 - temperature_boundary_diffusion_denom: 135.0 - velocity_boundary_diffusion_denom: 200.0 - max_nudging_coefficient: 0.1 - nudging_decay_rate: 2.0 - shear_type: "VERTICAL_OF_HORIZONTAL_WIND" - ltkeshs: True diff --git a/pytest_benchmark_results_3.10.json b/pytest_benchmark_results_3.10.json deleted file mode 100644 index e69de29bb2..0000000000 From c87800ca27e508bb73471024181437c85ac12a7b Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 14 Oct 2025 09:11:54 +0200 Subject: [PATCH 104/240] delete files unrelated to this PR --- .../pytest_benchmark_results_3.10.json | 23870 ---------------- 1 file changed, 23870 deletions(-) delete mode 100644 model/atmosphere/diffusion/pytest_benchmark_results_3.10.json diff --git a/model/atmosphere/diffusion/pytest_benchmark_results_3.10.json b/model/atmosphere/diffusion/pytest_benchmark_results_3.10.json deleted file mode 100644 index 0b27906f0f..0000000000 --- a/model/atmosphere/diffusion/pytest_benchmark_results_3.10.json +++ /dev/null @@ -1,23870 +0,0 @@ -{ - "machine_info": { - "node": "luzm", - "processor": "x86_64", - "machine": "x86_64", - "python_compiler": "Clang 20.1.4 ", - "python_implementation": "CPython", - "python_implementation_version": "3.10.18", - "python_version": "3.10.18", - "python_build": [ - "main", - "Jul 11 2025 22:43:29" - ], - "release": "6.14.0-27-generic", - "system": "Linux", - "cpu": { - "python_version": "3.10.18.final.0 (64 bit)", - "cpuinfo_version": [ - 9, - 0, - 0 - ], - "cpuinfo_version_string": "9.0.0", - "arch": "X86_64", - "bits": 64, - "count": 8, - "arch_string_raw": "x86_64", - "vendor_id_raw": "GenuineIntel", - "brand_raw": "11th Gen Intel(R) Core(TM) i7-1165G7 @ 2.80GHz", - "hz_advertised_friendly": "2.8000 GHz", - "hz_actual_friendly": "3.6964 GHz", - "hz_advertised": [ - 2800000000, - 0 - ], - "hz_actual": [ - 3696436000, - 0 - ], - "stepping": 1, - "model": 140, - "family": 6, - "flags": [ - "3dnowprefetch", - "abm", - "acpi", - "adx", - "aes", - "aperfmperf", - "apic", - "arat", - "arch_capabilities", - "arch_perfmon", - "art", - "avx", - "avx2", - "avx512_bitalg", - "avx512_vbmi2", - "avx512_vnni", - "avx512_vp2intersect", - "avx512_vpopcntdq", - "avx512bw", - "avx512cd", - "avx512dq", - "avx512f", - "avx512ifma", - "avx512vbmi", - "avx512vl", - "bmi1", - "bmi2", - "bts", - "cat_l2", - "cdp_l2", - "clflush", - "clflushopt", - "clwb", - "cmov", - "constant_tsc", - "cpuid", - "cpuid_fault", - "cx16", - "cx8", - "de", - "ds_cpl", - "dtes64", - "dtherm", - "dts", - "epb", - "ept", - "ept_ad", - "erms", - "est", - "f16c", - "flexpriority", - "flush_l1d", - "fma", - "fpu", - "fsgsbase", - "fsrm", - "fxsr", - "gfni", - "ht", - "hwp", - "hwp_act_window", - "hwp_epp", - "hwp_notify", - "hwp_pkg_req", - "ibpb", - "ibrs", - "ibrs_enhanced", - "ibt", - "ida", - "intel_pt", - "invpcid", - "lahf_lm", - "lm", - "mca", - "mce", - "md_clear", - "mmx", - "monitor", - "movbe", - "movdir64b", - "movdiri", - "msr", - "mtrr", - "nonstop_tsc", - "nopl", - "nx", - "ospke", - "pae", - "pat", - "pbe", - "pcid", - "pclmulqdq", - "pdcm", - "pdpe1gb", - "pebs", - "pge", - "pku", - "pln", - "pni", - "popcnt", - "pse", - "pse36", - "pts", - "rdpid", - "rdrand", - "rdseed", - "rdt_a", - "rdtscp", - "rep_good", - "sdbg", - "sep", - "sha_ni", - "smap", - "smep", - "split_lock_detect", - "ss", - "ssbd", - "sse", - "sse2", - "sse4_1", - "sse4_2", - "ssse3", - "stibp", - "syscall", - "tm", - "tm2", - "tpr_shadow", - "tsc", - "tsc_adjust", - "tsc_deadline_timer", - "tsc_known_freq", - "umip", - "user_shstk", - "vaes", - "vme", - "vmx", - "vnmi", - "vpclmulqdq", - "vpid", - "x2apic", - "xgetbv1", - "xsave", - "xsavec", - "xsaveopt", - "xsaves", - "xtopology", - "xtpr" - ], - "l3_cache_size": 12582912, - "l2_cache_size": 5242880, - "l1_data_cache_size": 196608, - "l1_instruction_cache_size": 131072 - } - }, - "commit_info": { - "id": "4701cc492a34ba6f88f368973e3338ba63e2ebde", - "time": "2025-08-15T18:27:55+02:00", - "author_time": "2025-08-15T18:27:55+02:00", - "dirty": true, - "project": "icon4py", - "branch": "main" - }, - "benchmarks": [ - { - "group": null, - "name": "test_stencil", - "fullname": "TestApplyDiffusionToVn", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.0006441649966291152, - "max": 0.0189960389980115, - "mean": 0.003602255105346275, - "stddev": 0.004880199014392282, - "rounds": 1236, - "median": 0.000828790998639306, - "iqr": 0.0031777689982845914, - "q1": 0.0006916990023455583, - "q3": 0.0038694680006301496, - "iqr_outliers": 213, - "stddev_outliers": 214, - "outliers": "214;213", - "ld15iqr": 0.0006441649966291152, - "hd15iqr": 0.00882603400532389, - "ops": 277.60388166730706, - "total": 4.452387310207996, - "data": [ - 0.0017910800015670247, - 0.001038121001329273, - 0.0019165579942637123, - 0.0010132710012840107, - 0.0009261520026484504, - 0.004629479000868741, - 0.0008771790016908199, - 0.0012659669955610298, - 0.005838680997840129, - 0.0009167329990305007, - 0.0017449129954911768, - 0.005366599005355965, - 0.006050699004845228, - 0.003211258001101669, - 0.004396117001306266, - 0.00084640400018543, - 0.0024797090009087697, - 0.0016823099940665998, - 0.0008128440022119321, - 0.0008048430026974529, - 0.0007994300030986778, - 0.0014144599990686402, - 0.003872152003168594, - 0.0008281319969682954, - 0.001688773998466786, - 0.0017891370007419027, - 0.000797337997937575, - 0.0008002229951671325, - 0.005781596999440808, - 0.0008682740008225664, - 0.000802238006144762, - 0.0007971419981913641, - 0.0007916649992694147, - 0.002750313993601594, - 0.0009287129942094907, - 0.0008001629976206459, - 0.0007929630010039546, - 0.0037936029984848574, - 0.0008390020011574961, - 0.0008065389993134886, - 0.0007898670010035858, - 0.0007860760015319102, - 0.003697043997817673, - 0.0012517420036601834, - 0.0007957960042404011, - 0.0007862189959269017, - 0.0008000800007721409, - 0.005445961003715638, - 0.0008278699970105663, - 0.0007592789988848381, - 0.0007590489985886961, - 0.0007524009997723624, - 0.005092090999823995, - 0.0008520120027242228, - 0.0007779300067340955, - 0.0007659469993086532, - 0.0007740900036878884, - 0.0007893439978943206, - 0.0008224100020015612, - 0.000788523000665009, - 0.0007705399984843098, - 0.0072099870012607425, - 0.0015989450039342046, - 0.0008028480006032623, - 0.0007706379983574152, - 0.0007822869956726208, - 0.005300292999891099, - 0.001071673999831546, - 0.0007753850004519336, - 0.0007566089989268221, - 0.012242778997460846, - 0.01343844900111435, - 0.010967854999762494, - 0.014991911004472058, - 0.015992868000466842, - 0.014988950999395456, - 0.015996636997442693, - 0.013996709996717982, - 0.01500271300028544, - 0.013999702998262364, - 0.011992769002972636, - 0.013994649998494424, - 0.012100498002837412, - 0.014886088996718172, - 0.014009182996232994, - 0.012987723996047862, - 0.014995753997936845, - 0.013992221996886656, - 0.0149990389982122, - 0.014988329996413086, - 0.016006876998289954, - 0.013994688000821043, - 0.01097399299760582, - 0.012134317999880295, - 0.01583950199710671, - 0.01399741999921389, - 0.015000230996520258, - 0.011995387001661584, - 0.015997483998944517, - 0.018990815995493904, - 0.013995096000144258, - 0.014473850998911075, - 0.014523929996357765, - 0.011994951004453469, - 0.010994398006005213, - 0.016998277998936828, - 0.014996942998550367, - 0.014010519000294153, - 0.014747959001397248, - 0.014230581997253466, - 0.01399532100185752, - 0.013998589005495887, - 0.014007051002408843, - 0.01298610599769745, - 0.0051392789973760955, - 0.0009202680012094788, - 0.0007703189985477366, - 0.0030886509994161315, - 0.0008120610000332817, - 0.0007868709944887087, - 0.0007098870046320371, - 0.0006741730030626059, - 0.0006771789994672872, - 0.0034376510011497885, - 0.0010097769991261885, - 0.0013776490013697185, - 0.0007482189976144582, - 0.0006907060014782473, - 0.0006935190031072125, - 0.0006853170052636415, - 0.0006840689966338687, - 0.0017324999935226515, - 0.0009010850044433028, - 0.0006875019971630536, - 0.0006748980013071559, - 0.0006843349983682856, - 0.0006826409953646362, - 0.004676359996665269, - 0.0009134450010606088, - 0.0008405240005231462, - 0.0020881609962088987, - 0.0007717850021435879, - 0.000826235998829361, - 0.0006955919961910695, - 0.0006876129991724156, - 0.004158816002018284, - 0.0008792500011622906, - 0.002929597001639195, - 0.000708334002410993, - 0.0006838389963377267, - 0.0006767490049242042, - 0.003451850003330037, - 0.000850248005008325, - 0.0006967220033402555, - 0.0006935470009921119, - 0.0007926910038804635, - 0.0021651139977620915, - 0.0007912029977887869, - 0.0006857030020910315, - 0.0006827429970144294, - 0.001303429999097716, - 0.0026995899970643222, - 0.0007826500004739501, - 0.0007142590038711205, - 0.0006970189933781512, - 0.0008166030020220205, - 0.0010894769948208705, - 0.0015481800000998192, - 0.0007005610023043118, - 0.0006860220019007102, - 0.0006968589950702153, - 0.0009087079961318523, - 0.0007494289966416545, - 0.000702844001352787, - 0.001902452000649646, - 0.0007230479968711734, - 0.0006832869985373691, - 0.0006783950011595152, - 0.00070327399589587, - 0.005190780997509137, - 0.0008920629989006557, - 0.0007052569999359548, - 0.0034616859993548132, - 0.0009027089981827885, - 0.0022201649990165606, - 0.0007084669996402226, - 0.0007376709982054308, - 0.0006867730044177733, - 0.0006797189998906106, - 0.005230715003563091, - 0.0009112719999393448, - 0.000815441002487205, - 0.0006780999974580482, - 0.0006866279945825227, - 0.0006723890037392266, - 0.005616223003016785, - 0.0008412009992753156, - 0.0007267809996847063, - 0.0006852669976069592, - 0.0006903789981151931, - 0.006091538998589385, - 0.0009737299988046288, - 0.0007210430048871785, - 0.0006902410023030825, - 0.0006961269973544404, - 0.005290292996505741, - 0.0008242100011557341, - 0.0008306529998662882, - 0.0006930720046511851, - 0.0006762459961464629, - 0.0006921829990460537, - 0.0006861619986011647, - 0.0006841010035714135, - 0.0006810150007368065, - 0.0006723330006934702, - 0.0006782819982618093, - 0.0006768980019842274, - 0.006596188999537844, - 0.000919655998586677, - 0.0007246750028571114, - 0.0006889689975650981, - 0.0006798319955123588, - 0.0006720399978803471, - 0.005219725004280917, - 0.0008396979974349961, - 0.0007689699996262789, - 0.0006852070000604726, - 0.0006828020050306804, - 0.0006788080063415691, - 0.004055716999573633, - 0.000846512004500255, - 0.0016434919962193817, - 0.0006885229959152639, - 0.0006782359996577725, - 0.0006871969962958246, - 0.0029133729985915124, - 0.0008294500003103167, - 0.001203034000354819, - 0.0007630440013599582, - 0.000751017993025016, - 0.0006791370033170097, - 0.0006889659998705611, - 0.0006898530045873486, - 0.004213459003949538, - 0.0010181969992117956, - 0.0008593070015194826, - 0.0006856960026198067, - 0.0006748019950464368, - 0.0007484379966626875, - 0.0007289580025826581, - 0.000738310998713132, - 0.0034082240017596632, - 0.0008494829962728545, - 0.000776590000896249, - 0.001259662996744737, - 0.0006905920017743483, - 0.0012289579972275533, - 0.000685144004819449, - 0.002149034000467509, - 0.0007475829988834448, - 0.0006964900021557696, - 0.0006933309996384196, - 0.0006761490003555082, - 0.0048670210017007776, - 0.0008394500036956742, - 0.0007035880043986253, - 0.0006794599976274185, - 0.0006767599988961592, - 0.006080268001824152, - 0.0009304570048698224, - 0.0006987810047576204, - 0.0006869270000606775, - 0.0006958629965083674, - 0.005236438999418169, - 0.0008326259994646534, - 0.0007064410019665956, - 0.0006727880027028732, - 0.0006837069959146902, - 0.005553811002755538, - 0.0020070940008736216, - 0.0007699159978074022, - 0.0007866730011301115, - 0.0006747890001861379, - 0.0035857870025211014, - 0.0008065969959716313, - 0.0007472510042134672, - 0.0006813910003984347, - 0.0006746689978172071, - 0.003838793003524188, - 0.0008327440009452403, - 0.0007404409989248961, - 0.0007590080058434978, - 0.0006722549951518886, - 0.0032885819964576513, - 0.0008321939967572689, - 0.0007692299986956641, - 0.0006855060055386275, - 0.0006880729997646995, - 0.0006753459965693764, - 0.013277963000291493, - 0.014996954996604472, - 0.01400588000251446, - 0.013997317997564096, - 0.014990302996011451, - 0.0119932469970081, - 0.015031267997983377, - 0.014960679000068922, - 0.014997362995927688, - 0.014998681006545667, - 0.009994018000725191, - 0.0139999489983893, - 0.014027012002770789, - 0.01402186400082428, - 0.014922634996764828, - 0.011998909998510499, - 0.015992255001037847, - 0.017012759002682287, - 0.015979450006852858, - 0.015995595000276808, - 0.01399729099648539, - 0.015997946997231338, - 0.00816186799784191, - 0.013839431005180813, - 0.011990033999609295, - 0.01399113599472912, - 0.015995378002116922, - 0.014997542995843105, - 0.012995884993870277, - 0.012996395998925436, - 0.014997926999058109, - 0.016995944002701435, - 0.012996407000173349, - 0.012142337996920105, - 0.0010720740028773434, - 0.0007739220018265769, - 0.00921738900069613, - 0.0010049550037365407, - 0.00835946699953638, - 0.001091660000383854, - 0.008205612997699063, - 0.00101242600067053, - 0.007878041004005354, - 0.001033660002576653, - 0.008108693997201044, - 0.0011257290025241673, - 0.0070560599997406825, - 0.0010155300042242743, - 0.0008834160034894012, - 0.007685147997108288, - 0.0009773570054676384, - 0.0073984570044558495, - 0.0010255140005028807, - 0.0008422340033575892, - 0.007080301998939831, - 0.0010176259966101497, - 0.0009207919938489795, - 0.003618520997406449, - 0.0008373159944312647, - 0.0006624589950661175, - 0.0006661289953626692, - 0.0006601030036108568, - 0.0053736840054625645, - 0.0009387800018885173, - 0.0008009119992493652, - 0.0006687670029350556, - 0.0006622039945796132, - 0.0006656810001004487, - 0.0006609379997826181, - 0.0006610670025111176, - 0.003530932001012843, - 0.0010325059993192554, - 0.000679892000334803, - 0.000677563002682291, - 0.002730662999965716, - 0.0008024830021895468, - 0.0006642020016442984, - 0.0006685269981971942, - 0.002612967000459321, - 0.0008180069999070838, - 0.0007221660052891821, - 0.0037159029961912893, - 0.0008379869977943599, - 0.0007986510026967153, - 0.0022608359940932132, - 0.0007860870027798228, - 0.000665643994580023, - 0.0006730829991283827, - 0.0006669259964837693, - 0.0006608140029129572, - 0.006249673999263905, - 0.000901301005796995, - 0.0007788880029693246, - 0.0006685859989374876, - 0.0006551970000145957, - 0.005126595999172423, - 0.0014320670015877113, - 0.0006652900046901777, - 0.0006679729995084926, - 0.0006613799996557645, - 0.0006604529989999719, - 0.0006652240044786595, - 0.000651909998850897, - 0.00351586800388759, - 0.0014690399984829128, - 0.0007013659997028299, - 0.0007391670005745254, - 0.0006547949960804544, - 0.0006645640023634769, - 0.0006578989996341988, - 0.000658456003293395, - 0.0027067019982496276, - 0.000782657996751368, - 0.0031238439987646416, - 0.000808352000603918, - 0.0006871520017739385, - 0.0006570369951077737, - 0.004035136000311468, - 0.0008187990024453029, - 0.0007005279985605739, - 0.0006751819964847527, - 0.0006659170030616224, - 0.00629470800049603, - 0.0008239150047302246, - 0.0007601379984407686, - 0.0006842779985163361, - 0.0006582579953828827, - 0.0006632799995713867, - 0.006981098995311186, - 0.0008957399986684322, - 0.0007338850045925938, - 0.000655679999908898, - 0.00065670700132614, - 0.005755550999310799, - 0.0008629369986010715, - 0.0007685009986744262, - 0.0006624219968216494, - 0.0006584259972441941, - 0.0006555549989570864, - 0.00433205500303302, - 0.0019576179984142072, - 0.0009757139996509068, - 0.001187513000331819, - 0.0006658679994870909, - 0.0012016670007142238, - 0.0006727420040988363, - 0.0017837729974417016, - 0.0006780079929740168, - 0.0006643629967584275, - 0.0006728230000589974, - 0.0006652550000580959, - 0.0006663370004389435, - 0.002592982003989164, - 0.0008373349992325529, - 0.0026158489999943413, - 0.0023612910008523613, - 0.0007161040048231371, - 0.00067721900268225, - 0.0006851099969935603, - 0.005526471002667677, - 0.0007175790015025996, - 0.0006839790003141388, - 0.001467125999624841, - 0.0020381970025482588, - 0.0010490800050320104, - 0.001983059002668597, - 0.0010545699988142587, - 0.000693579000653699, - 0.0006772519991500303, - 0.0006753790003131144, - 0.0006721170066157356, - 0.002581227003247477, - 0.0007170859971665777, - 0.0006907110000611283, - 0.002507149998564273, - 0.0006870400029583834, - 0.0019304599991301075, - 0.0006960670070839114, - 0.0006807949976064265, - 0.0006779269970138557, - 0.0006863909948151559, - 0.0011748649994842708, - 0.002878939005313441, - 0.0007106350021786056, - 0.0006847620024927892, - 0.0006700459998683073, - 0.0013088090054225177, - 0.0020193320015096106, - 0.0007081650037434883, - 0.0018973309997818433, - 0.0009361350021208636, - 0.0017246229981537908, - 0.0007085549950716086, - 0.0006753330017090775, - 0.0006766779988538474, - 0.0006891470038681291, - 0.0006870720026199706, - 0.005214709999563638, - 0.0007031560016912408, - 0.000689591994159855, - 0.0014511680055875331, - 0.0007158640000852756, - 0.0006776890004402958, - 0.0006778859969926998, - 0.000671159999910742, - 0.002394701004959643, - 0.0013668770043295808, - 0.0017691259999992326, - 0.0007009969995124266, - 0.0006871260047773831, - 0.002504557000065688, - 0.0010904869996011257, - 0.0006928770017111674, - 0.0006915250050951727, - 0.001246539999556262, - 0.0010909169941442087, - 0.000688129999616649, - 0.0022335239991662093, - 0.0006951329996809363, - 0.00223937799455598, - 0.0007098260030033998, - 0.0006957570003578439, - 0.0007053109948174097, - 0.0006876989937154576, - 0.002564862006693147, - 0.0006882289962959476, - 0.001398373999109026, - 0.0006695260017295368, - 0.0006807380050304346, - 0.0014034650012035854, - 0.0008949980037868954, - 0.0006754249989171512, - 0.0006810500053688884, - 0.001405041002726648, - 0.002864344001864083, - 0.0007108950012479909, - 0.0006898870051372796, - 0.000678028998663649, - 0.001494257005106192, - 0.0006964130006963387, - 0.0007005590014159679, - 0.0006871120058349334, - 0.0015778640008647926, - 0.00221269199391827, - 0.0006781219999538735, - 0.0006867629999760538, - 0.0027849890029756352, - 0.0007597730000270531, - 0.0006855060055386275, - 0.0007122509996406734, - 0.0020545329971355386, - 0.0011135120003018528, - 0.0006825899981777184, - 0.0006785109944758005, - 0.0006781219999538735, - 0.0031887820005067624, - 0.0007630219988641329, - 0.0007112840030458756, - 0.003957031003665179, - 0.000749092003388796, - 0.0006988620007177815, - 0.0006892059973324649, - 0.004495975997997448, - 0.0007085629986249842, - 0.001034615001117345, - 0.0017626250046305358, - 0.0006837410037405789, - 0.0006739859964000061, - 0.0006863849994260818, - 0.0037853359972359613, - 0.0009548950038151816, - 0.0006877539999550208, - 0.0006787080055801198, - 0.0006861049987492152, - 0.0022048329992685467, - 0.0007115670014172792, - 0.002972616995975841, - 0.0007278300035977736, - 0.0006941309984540567, - 0.0006766689984942786, - 0.004057909005496185, - 0.0007205699948826805, - 0.0006775490037398413, - 0.0006850369973108172, - 0.001393377999193035, - 0.00070643299841322, - 0.000680536002619192, - 0.0006866859985166229, - 0.0006690630034427159, - 0.0057137350013363175, - 0.0006828359983046539, - 0.000672313995892182, - 0.0038154860012582503, - 0.000689309999870602, - 0.0006728130028932355, - 0.0007057580005493946, - 0.0018902369993156753, - 0.0006944979977561161, - 0.0006762630000594072, - 0.0006833129955339245, - 0.003848602995276451, - 0.0006970559988985769, - 0.0006809239930589683, - 0.0006803099968237802, - 0.0008943220018409193, - 0.0033171359973493963, - 0.000700976001098752, - 0.000823524002043996, - 0.0006887830022606067, - 0.0006752499975846149, - 0.002297744002135005, - 0.0006994849973125383, - 0.0028659050003625453, - 0.0007681380011490546, - 0.003481259998807218, - 0.000715172995114699, - 0.0006863460002932698, - 0.000680360994010698, - 0.0011495290018501692, - 0.0006909040021128021, - 0.0030735059990547597, - 0.0006914670011610724, - 0.0006749940002919175, - 0.0006723510014126077, - 0.0009121270049945451, - 0.0019783509997068904, - 0.0007132620012271218, - 0.0006877160049043596, - 0.0006825110031059012, - 0.0006736649957019836, - 0.0031842519965721294, - 0.001590730003954377, - 0.0006900720036355779, - 0.000681030003761407, - 0.0006787180027458817, - 0.005444091999379452, - 0.0007174739948823117, - 0.0006827769975643605, - 0.000681227000313811, - 0.0006828810001024976, - 0.003960244001063984, - 0.0006969610039959662, - 0.0006755590002285317, - 0.0016811459936434403, - 0.0007047300023259595, - 0.0024264460007543676, - 0.001438552004401572, - 0.0007114920008461922, - 0.0007034739974187687, - 0.0006976500008022413, - 0.0010882770002353936, - 0.0010688300026231445, - 0.0006971140028326772, - 0.0006982889972277917, - 0.0007098669957485981, - 0.005251219998172019, - 0.0007156339997891337, - 0.0006982740014791489, - 0.0029132030031178147, - 0.0007328459978452884, - 0.0022250499969231896, - 0.0007178769956226461, - 0.0014682149994769134, - 0.0010176200012210757, - 0.0007034320005914196, - 0.0006974119969527237, - 0.0027817829977720976, - 0.002993792004417628, - 0.0007249809932545759, - 0.0013100010037305765, - 0.0017852689998107962, - 0.0007438800021191128, - 0.0006969719979679212, - 0.0007084440003382042, - 0.002715000999160111, - 0.0019542000009096228, - 0.0007005899969954044, - 0.0007002620041021146, - 0.0012344019996817224, - 0.0035043480020249262, - 0.0007569090012111701, - 0.0007075620014802553, - 0.0030635959992650896, - 0.0007067620026646182, - 0.0006918280050740577, - 0.0016197620061575435, - 0.002282294000906404, - 0.0014180979997036047, - 0.0007271309950738214, - 0.0025243920026696287, - 0.0007452229983755387, - 0.0007058269984554499, - 0.0006961140024941415, - 0.0006890700024086982, - 0.0018224250015919097, - 0.0031207190040731803, - 0.0007449479962815531, - 0.0008608069983893074, - 0.0020296660004532896, - 0.0007447489988408051, - 0.0006992649941821583, - 0.0006991369955358095, - 0.0032927349966485053, - 0.0007206199952634051, - 0.0016941339999902993, - 0.0013757519991486333, - 0.0008682800034875982, - 0.0007359400042332709, - 0.0007009140026639216, - 0.000693074005539529, - 0.003865333004796412, - 0.001095166000595782, - 0.0007157270010793582, - 0.0014535279988194816, - 0.0018351059989072382, - 0.000734460998501163, - 0.0007044180019875057, - 0.0006894530015415512, - 0.0030038519980735146, - 0.0013576779965660535, - 0.0007181300024967641, - 0.0007024599981377833, - 0.0012384410001686774, - 0.0022200050007086247, - 0.0007282540027517825, - 0.00417627199931303, - 0.0007483500012313016, - 0.0007085430042934604, - 0.005535593998502009, - 0.0007870209956308827, - 0.0011025339990737848, - 0.0017045849963324144, - 0.0034600380022311583, - 0.002302426000824198, - 0.0007047610051813535, - 0.0006993639981374145, - 0.0032406310056103393, - 0.001342467003269121, - 0.0053512369995587505, - 0.0011755170053220354, - 0.0007462610010406934, - 0.0007037510004010983, - 0.0012827310056309216, - 0.0014206860068952665, - 0.0007302849990082905, - 0.0007054640009300783, - 0.000703749006788712, - 0.00156170800619293, - 0.0019955909956479445, - 0.0007022179997875355, - 0.0007006819942034781, - 0.0009914190013660118, - 0.003422150999540463, - 0.000715813999704551, - 0.0006959899983485229, - 0.000692592999257613, - 0.00334922899492085, - 0.0007189200041466393, - 0.0007022279969532974, - 0.0006999600009294227, - 0.0011550870040082373, - 0.0016998030041577294, - 0.0006975960059207864, - 0.0007036309980321676, - 0.0027305390030960552, - 0.0007340619995375164, - 0.0018107630021404475, - 0.0007106850025593303, - 0.0007229940019897185, - 0.0014076110019232146, - 0.00201169300271431, - 0.0007083230011630803, - 0.0007144949995563366, - 0.0007097619964042678, - 0.0006997029995545745, - 0.0036892650023219176, - 0.0025281889975303784, - 0.0007388489975710399, - 0.0014548940016538836, - 0.0013216720035416074, - 0.0007065530007821508, - 0.0006958200028748251, - 0.0007154269987950101, - 0.003230203001294285, - 0.0015059009965625592, - 0.0010338199936086312, - 0.0006942510008229874, - 0.0006971030015847646, - 0.0025428749941056594, - 0.0007087889971444383, - 0.0006974169955356047, - 0.0006982269987929612, - 0.0015545970018138178, - 0.0014501300029223785, - 0.000706057995557785, - 0.0006919500010553747, - 0.0006922200045664795, - 0.004539338005997706, - 0.0007425009971484542, - 0.0007006849991739728, - 0.0024432509962935, - 0.0036128540014033206, - 0.0007056879985611886, - 0.0037446940041263588, - 0.0033494610033812933, - 0.0007768559953547083, - 0.0007169829987105913, - 0.0007041470016702078, - 0.0009051700035342947, - 0.003812253999058157, - 0.0020949229947291315, - 0.0007240609993459657, - 0.0007075110042933375, - 0.0007015649971435778, - 0.0007105870026862249, - 0.0032772599952295423, - 0.0007912270011729561, - 0.0007335129994316958, - 0.0007059319977997802, - 0.0006990180045249872, - 0.006193378001626115, - 0.0008627930001239292, - 0.0007589070009998977, - 0.0006915699996170588, - 0.0006897889979882166, - 0.003330450002977159, - 0.000906525005120784, - 0.000781043003371451, - 0.0006982650011195801, - 0.000694044996635057, - 0.0006898470019223168, - 0.004675897995184641, - 0.009329692999017425, - 0.015998779999790713, - 0.013996175002830569, - 0.017996565002249554, - 0.014997022000898141, - 0.013002581996261142, - 0.01399139500426827, - 0.017008114999043755, - 0.013989232000312768, - 0.015991132997442037, - 0.014999168000940699, - 0.012994498996704351, - 0.013993189000757411, - 0.013999613001942635, - 0.014055856001505163, - 0.014932019999832846, - 0.013996579000377096, - 0.010168239998165518, - 0.00882603400532389, - 0.0189960389980115, - 0.014001965995703358, - 0.015009688002464827, - 0.0008891190009308048, - 0.000827873001981061, - 0.0006866819967399351, - 0.0006826600001659244, - 0.0006736900031683035, - 0.0018607809979585, - 0.0007085880060913041, - 0.004029736999655142, - 0.0008575549945817329, - 0.0007179370004450902, - 0.0006744559941580519, - 0.0006819109985372052, - 0.004156554001383483, - 0.0020543280043057166, - 0.0006963719933992252, - 0.000680156001180876, - 0.0006794840010115877, - 0.0006738670053891838, - 0.000667729000269901, - 0.0026178709958912805, - 0.0008472560002701357, - 0.0006976689983275719, - 0.0006826259996159934, - 0.0006797639944124967, - 0.0057821380032692105, - 0.0008597550040576607, - 0.0007626489968970418, - 0.0022597330025746487, - 0.0007829230016795918, - 0.0007823560008546337, - 0.0006848449993412942, - 0.0006732479960191995, - 0.0006814540029154159, - 0.01721084000018891, - 0.013990293999086134, - 0.015001831998233683, - 0.014985650996095501, - 0.012996694997127634, - 0.014998753998952452, - 0.01199634500517277, - 0.013995281005918514, - 0.008998073004477192, - 0.01000490099977469, - 0.01698785299959127, - 0.013086255996313412, - 0.012908202996186446, - 0.010967248999804724, - 0.010027083000750281, - 0.016007022000849247, - 0.014995539997471496, - 0.012101795000489801, - 0.013976592999824788, - 0.013912536996940617, - 0.015983441000571474, - 0.011993097999948077, - 0.008997261997137684, - 0.010995410004397854, - 0.013998877999256365, - 0.010957386002701242, - 0.013034584997512866, - 0.01299632099835435, - 0.01399909100291552, - 0.017210156001965515, - 0.011780458000430372, - 0.015006030000222381, - 0.008204630998079665, - 0.013781535002635792, - 0.015449698999873362, - 0.008544001000700518, - 0.010570547005045228, - 0.011469051998574287, - 0.007524071996158455, - 0.007418832996336278, - 0.009999041001719888, - 0.0129947710011038, - 0.01499587600119412, - 0.01299915999697987, - 0.014997711004980374, - 0.013997406997077633, - 0.013463531999150291, - 0.010525315999984741, - 0.01399249200039776, - 0.014995074001490138, - 0.01299564799410291, - 0.014000293005665299, - 0.009335827999166213, - 0.014654366998001933, - 0.013995138004247565, - 0.01202050700521795, - 0.011971794003329705, - 0.01098796000587754, - 0.01317414699587971, - 0.011819847000879236, - 0.014992199998232536, - 0.015996132999134716, - 0.015996570997231174, - 0.00990859200101113, - 0.01108338100311812, - 0.013997317000757903, - 0.015999062998162117, - 0.012997612000617664, - 0.010843969997949898, - 0.011147671000799164, - 0.010996580997016281, - 0.01499785600026371, - 0.011997271001746412, - 0.014997639998910017, - 0.014997935999417678, - 0.011010803005774505, - 0.013004902000830043, - 0.016166642999451142, - 0.008060742002271581, - 0.015736809000372887, - 0.013995386005262844, - 0.0159973620029632, - 0.011679576004098635, - 0.008315186998515856, - 0.012249552004504949, - 0.009710174002975691, - 0.010061015003884677, - 0.008966413995949551, - 0.01699818899942329, - 0.010994519994710572, - 0.013997187998029403, - 0.013000286002352368, - 0.009993918996769935, - 0.01399671500257682, - 0.013998062000609934, - 0.012997527002880815, - 0.013992230000440031, - 0.012705887995252851, - 0.012662977998843417, - 0.011630150002019946, - 0.011756139996577986, - 0.000875347999681253, - 0.0006930349991307594, - 0.0006804830045439303, - 0.0006741949982824735, - 0.007205429006717168, - 0.0009332019981229678, - 0.0008181290031643584, - 0.0006819479967816733, - 0.006743777004885487, - 0.0008641840031486936, - 0.0007692069993936457, - 0.0006661229999735951, - 0.0006698919969494455, - 0.0006707959983032197, - 0.00899465899419738, - 0.0008899239983293228, - 0.0007529919967055321, - 0.0006819629998062737, - 0.0006668620044365525, - 0.007933629996841773, - 0.0008920649997889996, - 0.0007031060013105161, - 0.0006765350044588558, - 0.000675780996971298, - 0.0035401640052441508, - 0.000847084003908094, - 0.0007730670040473342, - 0.0006847350014140829, - 0.0006722700054524466, - 0.0006733939953846857, - 0.005794488999526948, - 0.0014429130023927428, - 0.0006988960012677126, - 0.0006843029987066984, - 0.0006741430042893626, - 0.0006623540029977448, - 0.0006736630020895973, - 0.004057046004163567, - 0.0007998669971129857, - 0.0006957369987503625, - 0.0006773969944333658, - 0.0006963589985389262, - 0.0049242930035688914, - 0.0008505189980496652, - 0.0007333890025620349, - 0.0006777140006306581, - 0.0006943820044398308, - 0.0056434940051985905, - 0.0008966950044850819, - 0.0008442089965683408, - 0.0006760719988960773, - 0.004272825004591141, - 0.0013183129995013587, - 0.0006868929995107464, - 0.0006716029965900816, - 0.0006728500011377037, - 0.0006724479972035624, - 0.0034345449967077, - 0.0008453020054730587, - 0.0007633400018676184, - 0.0006711539972457103, - 0.0006811790008214302, - 0.0006779220057069324, - 0.005257974000414833, - 0.0008752130015636794, - 0.0007913080044090748, - 0.0006692220049444586, - 0.0006719960001646541, - 0.0006756380025763065, - 0.005556652999075595, - 0.0008274159990833141, - 0.000739420996978879, - 0.0006862189984531142, - 0.0006738439988112077, - 0.0006695960037177429, - 0.005007834995922167, - 0.0009048799984157085, - 0.0008181640005204827, - 0.0006763519995729439, - 0.003470529001788236, - 0.0008065370057011023, - 0.0007425760049954988, - 0.0006795509980292991, - 0.0006654489989159629, - 0.014282055999501608, - 0.014998681996075902, - 0.013997670997923706, - 0.014995279998402111, - 0.013994829001603648, - 0.015999165996618103, - 0.013994423003168777, - 0.014996181002061348, - 0.013996458001201972, - 0.013998007001646329, - 0.014002175004861783, - 0.015000366998719983, - 0.014989848001278006, - 0.01301359900389798, - 0.013992136002343614, - 0.01115384699369315, - 0.007882895995862782, - 0.0009154349972959608, - 0.0007981029993970878, - 0.000671084999339655, - 0.0006660069993813522, - 0.0006613890000153333, - 0.0017708480008877814, - 0.0006780450057704002, - 0.0006609499978367239, - 0.0006655770048382692, - 0.0007110980004654266, - 0.0008024959970498458, - 0.0006791150008211844, - 0.0006587739990209229, - 0.0006562670023413375, - 0.0008436389980488457, - 0.0007920400021248497, - 0.0007625349971931428, - 0.0008127940018312074, - 0.0007400390022667125, - 0.0006790990009903908, - 0.0006680110018351115, - 0.0007353190012509003, - 0.0006654529934166931, - 0.000694153000949882, - 0.0033360279994667508, - 0.0007754809994366951, - 0.0007598259981023148, - 0.0006938399965292774, - 0.000667038002575282, - 0.0006820150010753423, - 0.0031000509989098646, - 0.0007829739988665096, - 0.000700103999406565, - 0.000666849002300296, - 0.004227871002512984, - 0.0008113330040941946, - 0.0008093090000329539, - 0.0036166319987387396, - 0.0008273060011561029, - 0.0007655360022909008, - 0.0007472560027963482, - 0.0006787840029573999, - 0.0006730309978593141, - 0.0006718459990224801, - 0.002792012004647404, - 0.001506660002632998, - 0.0007012249989202246, - 0.001143160006904509, - 0.0008142529986798763, - 0.0006782769996789284, - 0.0006730219974997453, - 0.0006832569997641258, - 0.0006713780021527782, - 0.000667309999698773, - 0.0008310639968840405, - 0.0006789430044591427, - 0.000673335998726543, - 0.0006651549992966466, - 0.0006756209986633621, - 0.009154687002592254, - 0.0008588060009060428, - 0.0006862579975859262, - 0.0006687319983029738, - 0.0006766899969079532, - 0.007199638006568421, - 0.00090012900182046, - 0.000712305998604279, - 0.000679506003507413, - 0.0006677390047116205, - 0.006892980003613047, - 0.0008698439996805973, - 0.000689045999024529, - 0.0006791509949835017, - 0.0006694670009892434, - 0.007896093004092108, - 0.0009740610039443709, - 0.0007165659990278073, - 0.0006809689948568121, - 0.000689997999870684, - 0.007670699997106567, - 0.0009882910017040558, - 0.0007867530002840795, - 0.000679896998917684, - 0.005662076000589877, - 0.002210350001405459, - 0.000745679993997328, - 0.0006926639980520122, - 0.0006751970067853108, - 0.0006753020061296411, - 0.0006773080021957867, - 0.005479168001329526, - 0.0008371859948965721, - 0.0007276300020748749, - 0.0006576129962923005, - 0.0006620429994654842, - 0.0006503039985545911, - 0.006897346000187099, - 0.0008572070000809617, - 0.0007476390019292012, - 0.0006576960004167631, - 0.0006486919955932535, - 0.0049372170033166185, - 0.0010415380020276643, - 0.0007120220034266822, - 0.0006639359999098815, - 0.0006542799965245649, - 0.0006517350047943182, - 0.00294785900041461, - 0.0008692920018802397, - 0.000693622998369392, - 0.0006526090000988916, - 0.0006565809962921776, - 0.0006441649966291152, - 0.006381900006090291, - 0.0008673100019223057, - 0.0007240010017994791, - 0.0006595669983653352, - 0.0006546989970956929, - 0.0006510179955512285, - 0.005161802997463383, - 0.0008239210001192987, - 0.0007591289977426641, - 0.0006559510002261959, - 0.0006503399999928661, - 0.0006628459959756583, - 0.006093924996093847, - 0.0008499609975842759, - 0.0006861099973320961, - 0.0006622689979849383, - 0.0006520099996123463, - 0.004953345000103582, - 0.0008464999991701916, - 0.0006593030047952197, - 0.0006519840026157908, - 0.0006503330005216412, - 0.0048150570000871085, - 0.0008382349988096394, - 0.008344993002538104, - 0.0008942290005506948, - 0.0007426170050166547, - 0.0006680410006083548, - 0.0006727979998686351, - 0.006251193000935018, - 0.0010079470011987723, - 0.0007276050018845126, - 0.0006617379985982552, - 0.0006751940018148161, - 0.003866783998091705 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil", - "fullname": "TestApplyDiffusionToWAndComputeHorizontalGradientsForTurbulence", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.0006457909985329024, - "max": 0.04000352900038706, - "mean": 0.00400752713244851, - "stddev": 0.008077417551711983, - "rounds": 1314, - "median": 0.0008514215005561709, - "iqr": 0.0011078840034315363, - "q1": 0.0007288289998541586, - "q3": 0.0018367130032856949, - "iqr_outliers": 231, - "stddev_outliers": 127, - "outliers": "127;231", - "ld15iqr": 0.0006457909985329024, - "hd15iqr": 0.0035382470014155842, - "ops": 249.5304378361182, - "total": 5.265890652037342, - "data": [ - 0.00077442799374694, - 0.0018532749963924289, - 0.000916031000087969, - 0.0014507240048260428, - 0.0007920850039226934, - 0.0009337419978692196, - 0.0007719889981672168, - 0.0007630020045326091, - 0.0017018709986587055, - 0.0007858419994590804, - 0.0007685279997531325, - 0.0007649699982721359, - 0.002698370997677557, - 0.0008242500043706968, - 0.0008363980014109984, - 0.000772204999520909, - 0.0011583710002014413, - 0.0008893339982023463, - 0.0011884140039910562, - 0.0008578499982832, - 0.0007884509977884591, - 0.0007802230029483326, - 0.0010829520033439621, - 0.0008356420003110543, - 0.0007971919985720888, - 0.002005868998821825, - 0.0007452229983755387, - 0.0007695200038142502, - 0.0018337970032007433, - 0.0007786970018059947, - 0.0007794650009600446, - 0.0007438609973178245, - 0.004853819998970721, - 0.0008506919984938577, - 0.0007837379962438717, - 0.0009050289954757318, - 0.0021129099986865185, - 0.0008086169982561842, - 0.0007708449993515387, - 0.0008593879974796437, - 0.0007667430036235601, - 0.0010544500037212856, - 0.0007558780052931979, - 0.0019881429980159737, - 0.0028000740057905205, - 0.0008560700007365085, - 0.0007617170049343258, - 0.0007421849950333126, - 0.0007415680011035874, - 0.0008956780002336018, - 0.0008545570017304271, - 0.001191261995700188, - 0.0007476600003428757, - 0.0009343130004708655, - 0.0016764739993959665, - 0.0008163669990608469, - 0.0009733849947224371, - 0.0007512420052080415, - 0.0007441779962391593, - 0.0007492419972550124, - 0.0008181570010492578, - 0.0008682680054334924, - 0.0009551260009175166, - 0.0009213390003424138, - 0.002339077000215184, - 0.0007688530022278428, - 0.0007785319976392202, - 0.0018367130032856949, - 0.0008795390021987259, - 0.0008219679948524572, - 0.0008295689985970967, - 0.0007565719934063964, - 0.0007433790015056729, - 0.0007411580008920282, - 0.0008488100065733306, - 0.0007455790037056431, - 0.0007494259989471175, - 0.0007503329979954287, - 0.0007426129959640093, - 0.0008702639970579185, - 0.0009202859946526587, - 0.0009092010004678741, - 0.000760117000027094, - 0.0007511299991165288, - 0.0007392469997284934, - 0.0007466070019290783, - 0.000848846000735648, - 0.0015564259956590831, - 0.0015150720064411871, - 0.0011640529992291704, - 0.000757408000936266, - 0.0007987050048541278, - 0.000750163002521731, - 0.0008558799963793717, - 0.0008651620009914041, - 0.0007520660001318902, - 0.0007478029947378673, - 0.0007411379992845468, - 0.0008190419976017438, - 0.0019483899959595874, - 0.0007787079957779497, - 0.002816832005919423, - 0.0019585240006563254, - 0.0008283209972432815, - 0.0007485649985028431, - 0.0008089760012808256, - 0.0008789220009930432, - 0.0009139450048678555, - 0.0011081039992859587, - 0.000762791998567991, - 0.0007479010018869303, - 0.0010326370029360987, - 0.0008040210013859905, - 0.000754628999857232, - 0.00093315100093605, - 0.000756075001845602, - 0.0007589550004922785, - 0.000741356001526583, - 0.0007411319966195151, - 0.0015166270022746176, - 0.0016136970007210039, - 0.002023695000389125, - 0.000800528003310319, - 0.0008215549969463609, - 0.0007185770009527914, - 0.0007120410009520128, - 0.0007149239972932264, - 0.0007175350037869066, - 0.0008745779996388592, - 0.000908551002794411, - 0.0007289029963430949, - 0.0007185400027083233, - 0.0008399680009461008, - 0.0009604419974493794, - 0.0008145789979607798, - 0.0007191789991338737, - 0.0007259370031533763, - 0.0007298319978872314, - 0.001548826003272552, - 0.0007808060036040843, - 0.0007403460040222853, - 0.0015896939949016087, - 0.0009177940009976737, - 0.002340906001336407, - 0.000743151998904068, - 0.0007209320028778166, - 0.0015914079995127395, - 0.0011069760003010742, - 0.0007263990046340041, - 0.0007231089984998107, - 0.000729876002878882, - 0.0007269080015248619, - 0.02465823999955319, - 0.03199898800085066, - 0.03202362199954223, - 0.02895602199714631, - 0.033997705999354366, - 0.029978872000356205, - 0.030989227001555264, - 0.028998112997214776, - 0.027002154994988814, - 0.03201166300277691, - 0.025977030003559776, - 0.030009289002919104, - 0.02198553700145567, - 0.02998564000154147, - 0.030999986003735103, - 0.031989271999918856, - 0.026994284002284985, - 0.03000758800044423, - 0.02898513300169725, - 0.028993122999963816, - 0.03200163899600739, - 0.029867540994018782, - 0.02811903799738502, - 0.02899829300440615, - 0.027995643999020103, - 0.02699330799805466, - 0.027995220996672288, - 0.029006511998886708, - 0.025989943998865783, - 0.02799480400426546, - 0.024036424998485018, - 0.027961279003648087, - 0.027989342997898348, - 0.028983084004721604, - 0.02800291400490096, - 0.027210609005123843, - 0.0239983739957097, - 0.02876254199509276, - 0.02793400800146628, - 0.027989852002065163, - 0.028001939004752785, - 0.01637267199839698, - 0.01802093100559432, - 0.030583466003008652, - 0.014996969999629073, - 0.026431767997564748, - 0.011537070000485983, - 0.027014577004592866, - 0.029001667993725277, - 0.031002421004814096, - 0.027971392999461386, - 0.02702135399886174, - 0.03154648100462509, - 0.03741034400445642, - 0.022994439001195133, - 0.04000352900038706, - 0.028225814996403642, - 0.03277729499677662, - 0.031003878997580614, - 0.03098092800064478, - 0.029967089998535812, - 0.031014061998575926, - 0.0310600919983699, - 0.02893962000234751, - 0.027963341999566182, - 0.02598815400415333, - 0.035991705997730605, - 0.028016821001074277, - 0.02896342799795093, - 0.027974069002084434, - 0.023002527996140998, - 0.023997733995201997, - 0.02398909500334412, - 0.025983822997659445, - 0.03004683699691668, - 0.030911196998204105, - 0.026990464997652452, - 0.02799783699447289, - 0.02698978199623525, - 0.02399986799719045, - 0.02399632900051074, - 0.0310335159956594, - 0.026946876998408698, - 0.029993367003044114, - 0.027999666999676265, - 0.026987857003405225, - 0.029017818000284024, - 0.028984150994801894, - 0.021976326002913993, - 0.02523916499922052, - 0.026750123004603665, - 0.015059836994623765, - 0.026929134997772053, - 0.027021095003874507, - 0.029976065998198465, - 0.02905906399973901, - 0.03192309199948795, - 0.030002460996911395, - 0.03205992400035029, - 0.0319276149966754, - 0.026703584000642877, - 0.03028732500388287, - 0.029992390002007596, - 0.030026241001905873, - 0.025957375000871252, - 0.03501569700165419, - 0.03096974900108762, - 0.027004753006622195, - 0.02698286299710162, - 0.026005093001003843, - 0.031033238003146835, - 0.035936677995778155, - 0.027973737996944692, - 0.026988376994268037, - 0.02620010799728334, - 0.03180055000120774, - 0.027470395994896535, - 0.025503179000224918, - 0.03500026700203307, - 0.031003217001853045, - 0.028974433997063898, - 0.029994186996191274, - 0.012385759000608232, - 0.0008500879994244315, - 0.0008262920018751174, - 0.000653845003398601, - 0.009072995999304112, - 0.00096381099865539, - 0.0007404780044453219, - 0.0006625099995289929, - 0.0006492940010502934, - 0.004101951002667192, - 0.0031682310000178404, - 0.0007739549982943572, - 0.0006719270022585988, - 0.0006531309991260059, - 0.0006457909985329024, - 0.0006527290024678223, - 0.0034462940020603128, - 0.0038571520053665154, - 0.0007666390010854229, - 0.004747505001432728, - 0.0007945039978949353, - 0.0006991639966145158, - 0.0006507280049845576, - 0.007521864004957024, - 0.00898761699500028, - 0.0011325230007059872, - 0.0006494389963336289, - 0.0037696390063501894, - 0.000824949995148927, - 0.0007568680011900142, - 0.0006689430010737851, - 0.0006701919992337935, - 0.005615424997813534, - 0.0008278160021291114, - 0.0020008289939141832, - 0.0007314149988815188, - 0.0007151530007831752, - 0.0030612359987571836, - 0.0007810360039002262, - 0.0007147600044845603, - 0.000674789996992331, - 0.000677301999530755, - 0.0006658189959125593, - 0.0029439330028253607, - 0.004803543000889476, - 0.0008768300031078979, - 0.000770033999287989, - 0.003814395000517834, - 0.0008620890002930537, - 0.00114027300151065, - 0.0006717399955959991, - 0.007097511996107642, - 0.0008315469967783429, - 0.009022151003591716, - 0.0008516849993611686, - 0.0007419820030918345, - 0.00979068799642846, - 0.0016720400017220527, - 0.004217012996377889, - 0.0009371859996463172, - 0.0036119430005783215, - 0.0008552729996154085, - 0.0007387130026472732, - 0.004114515002584085, - 0.002768260004813783, - 0.0008039930035010912, - 0.0007049949999782257, - 0.0008558030021958984, - 0.0006719719967804849, - 0.0006676409975625575, - 0.0025564250026945956, - 0.0008054219943005592, - 0.0032483109971508384, - 0.000822693997179158, - 0.0006907380011398345, - 0.0010807979997480288, - 0.0007646259982720949, - 0.0007137079956009984, - 0.0006826210010331124, - 0.0028439150046324357, - 0.0020209409995004535, - 0.0012242160009918734, - 0.0007009690016275272, - 0.0011031760004698299, - 0.0007243139989441261, - 0.0006771490006940439, - 0.0019919910046155564, - 0.0014774009978282265, - 0.0023540250040241517, - 0.0007795460041961633, - 0.0007646770027349703, - 0.0006833860024926253, - 0.0006997589953243732, - 0.0015452770021511242, - 0.0006883440000819974, - 0.0006649679999100044, - 0.00132683999981964, - 0.0029916959974798374, - 0.0008137109980452806, - 0.0009060409938683733, - 0.0027108759968541563, - 0.0008711340051377192, - 0.0006964610001887195, - 0.0007139139997889288, - 0.0018121770044672303, - 0.0012501399978646077, - 0.0006702359969494864, - 0.0007048719999147579, - 0.0006855030005681328, - 0.003468365001026541, - 0.0035382470014155842, - 0.0015514459955738857, - 0.0009069780062418431, - 0.000699462994816713, - 0.006473078996350523, - 0.0008490850013913587, - 0.0006795580047764815, - 0.0009018310011015274, - 0.007294235001609195, - 0.0008552720028092153, - 0.0007149770026444457, - 0.0006824739975854754, - 0.0007185420035966672, - 0.00671289800084196, - 0.0008571310027036816, - 0.0008019009983399883, - 0.0006720510064042173, - 0.0006756680013495497, - 0.00672603900602553, - 0.0008501609991071746, - 0.0008109460031846538, - 0.0010050950004369952, - 0.0007310869987122715, - 0.007393046995275654, - 0.0008725840016268194, - 0.0009661719959694892, - 0.0007280279969563708, - 0.0007310590008273721, - 0.006418038996343967, - 0.0008431449969066307, - 0.0007222629938041791, - 0.000673677001032047, - 0.0007347430000663735, - 0.0028638779986067675, - 0.0010597999935271218, - 0.001825813997129444, - 0.0008024890048545785, - 0.0007628160019521601, - 0.0021887519978918135, - 0.0007098460046108812, - 0.0007186380025814287, - 0.000673255002766382, - 0.0008141409998643212, - 0.0008022689944482408, - 0.0006926049973117188, - 0.0008435440031462349, - 0.0006710410016239621, - 0.0008000599991646595, - 0.0006897940038470551, - 0.0006844609961262904, - 0.0008024870039662346, - 0.0009236240002792329, - 0.0006943309999769554, - 0.0007156130013754591, - 0.0008733740032766946, - 0.0010337730054743588, - 0.0008166190018528141, - 0.0007355100024142303, - 0.0006769120009266771, - 0.000672187001327984, - 0.0006738290030625649, - 0.0006668099958915263, - 0.0006905459958943538, - 0.0006693209943477996, - 0.0007606759972986765, - 0.0007038240000838414, - 0.0014060169996810146, - 0.0008548030018573627, - 0.0008396789999096654, - 0.0007900049968156964, - 0.0006796320012654178, - 0.0006777779999538325, - 0.0006762160046491772, - 0.0006783749995520338, - 0.0006674169999314472, - 0.0006986759981373325, - 0.0006776590016670525, - 0.0008410810041823424, - 0.0007876339950598776, - 0.000722486998711247, - 0.0008444339982816018, - 0.0007917429975350387, - 0.0008771369975875132, - 0.0008271270053228363, - 0.0007604120037285611, - 0.0009025730032590218, - 0.0008444600025541149, - 0.0006691429953207262, - 0.0007202179986052215, - 0.0006945170025574043, - 0.0009141010013991036, - 0.0006722469988744706, - 0.0007094789980328642, - 0.0007085079996613786, - 0.000723860997823067, - 0.0006811779967392795, - 0.000666086001729127, - 0.0006798519971198402, - 0.0006757869996363297, - 0.0008435880008619279, - 0.000854636004078202, - 0.0008094639997580089, - 0.0008401279992540367, - 0.0007263980005518533, - 0.0007571899986942299, - 0.0007306740008061752, - 0.0007674869993934408, - 0.0008590919969719835, - 0.0007872499991208315, - 0.0020851259978371672, - 0.0007723400049144402, - 0.0007143900002120063, - 0.0014457810029853135, - 0.0009239110004273243, - 0.0009351949993288144, - 0.0017922349943546578, - 0.0008108339970931411, - 0.0009817879981710576, - 0.0008635880003566854, - 0.0008276080043287948, - 0.0010922590008703992, - 0.0012835169982281514, - 0.0007018690012046136, - 0.0006865629984531552, - 0.0006784240031265654, - 0.0028202160028740764, - 0.0009408549958607182, - 0.0008795690009719692, - 0.0006913940014783293, - 0.0006876369952806272, - 0.0013750749931205064, - 0.0006966190048842691, - 0.0007255360033013858, - 0.0007194299978436902, - 0.0006833149964222685, - 0.0008737300013308413, - 0.0009257659985451028, - 0.001368223995086737, - 0.0011142470029881224, - 0.0007002010024734773, - 0.0007906019964138977, - 0.0016154829936567694, - 0.0007204919966170564, - 0.0006856440013507381, - 0.004561460002150852, - 0.013781686000584159, - 0.006798492999223527, - 0.0008153369999490678, - 0.0007171539982664399, - 0.0006935890050954185, - 0.000684015998558607, - 0.005086684999696445, - 0.0008439770026598126, - 0.0007211839983938262, - 0.004094493997399695, - 0.000810331002867315, - 0.0007231420022435486, - 0.0028128020057920367, - 0.0007290110006579198, - 0.0038398079996113665, - 0.0007193860001279972, - 0.0008246270008385181, - 0.0038395849987864494, - 0.0007859469988034107, - 0.0009002890001283959, - 0.0026076520007336512, - 0.0014405779947992414, - 0.0007291340007213876, - 0.0007158829976106063, - 0.0006858979977550916, - 0.0018843819998437539, - 0.000990131993603427, - 0.0007168149968492799, - 0.000708637002389878, - 0.001376186999550555, - 0.0009516240024822764, - 0.0008474549977108836, - 0.001867973005573731, - 0.0009449829958612099, - 0.002326143003301695, - 0.0009362880009575747, - 0.0007374449996859767, - 0.0006842020011390559, - 0.0006877340056234971, - 0.0027068490016972646, - 0.0010856180015252903, - 0.0006958269950700924, - 0.0006932780015631579, - 0.0006900900043547153, - 0.0008438389995717444, - 0.0031610180012648925, - 0.0009981539988075383, - 0.0007709999990765937, - 0.0006939559971215203, - 0.0025323770023533143, - 0.0013072370056761429, - 0.0007238690013764426, - 0.0007042420038487762, - 0.0006864210008643568, - 0.0016603839976596646, - 0.0014686229988001287, - 0.0007098470014170744, - 0.0006920489977346733, - 0.0006939879967831075, - 0.0006908030045451596, - 0.0008484209974994883, - 0.0012906410047435202, - 0.0006874230020912364, - 0.0013672280038008466, - 0.0017948060049093328, - 0.0009365480000269599, - 0.0007239839978865348, - 0.0006984729989198968, - 0.0006940950042917393, - 0.0008328269977937452, - 0.0008339770065504126, - 0.002748899001744576, - 0.0008119760022964329, - 0.0006859909990453161, - 0.0006903399989823811, - 0.0014484089988400228, - 0.0008739609984331764, - 0.0014540809934260324, - 0.0012846740064560436, - 0.0008976910030469298, - 0.0026787699971464463, - 0.0007852300041122362, - 0.0007056539980112575, - 0.0006903489993419498, - 0.00121280099847354, - 0.0020452690005186014, - 0.000763513999118004, - 0.0008428700020886026, - 0.001010626001516357, - 0.0010719700003392063, - 0.0006866679977974854, - 0.0006867499978397973, - 0.0006951989998924546, - 0.0008368959970539436, - 0.0010836000001290813, - 0.0008347240000148304, - 0.0029221660006442107, - 0.0007767370043438859, - 0.0008350030038855039, - 0.0007767999995849095, - 0.0009147050004685298, - 0.0012513439942267723, - 0.0006867799966130406, - 0.00073648699617479, - 0.0010246259989799, - 0.0007003790015005507, - 0.004955730997608043, - 0.0009369319959660061, - 0.0008965160013758577, - 0.0007231119961943477, - 0.0007554409967269748, - 0.0007499610001104884, - 0.0010528450002311729, - 0.0010982360035995953, - 0.0008331499993801117, - 0.0008911289987736382, - 0.0010921459979726933, - 0.0007857089949538931, - 0.0007179769963840954, - 0.0017078879973269068, - 0.0012627809992409311, - 0.0009425539974472485, - 0.0026538279998931102, - 0.0008153600065270439, - 0.0007083079981384799, - 0.0007018589967628941, - 0.0006856550025986508, - 0.0006928710063220933, - 0.0008732490023248829, - 0.0012303570038056932, - 0.0009897370036924258, - 0.0014850289953756146, - 0.0007515539982705377, - 0.0006990319961914793, - 0.0006959399979677983, - 0.0008493630011798814, - 0.0018904239987023175, - 0.0017279859966947697, - 0.000731403000827413, - 0.0007479980049538426, - 0.0006919820007169619, - 0.0011058119998779148, - 0.0009460810033488087, - 0.0008534839944331907, - 0.000720758005627431, - 0.0008698729943716899, - 0.0009306759966420941, - 0.00225118800153723, - 0.0007155179991968907, - 0.0008288859971798956, - 0.0009812549978960305, - 0.0009355279980809428, - 0.0007898780022514984, - 0.0014333790022647008, - 0.0012921329980599694, - 0.0007262750004883856, - 0.0006972940027480945, - 0.0006972990013309754, - 0.0006917930004419759, - 0.0008204069963539951, - 0.0009127270022872835, - 0.0010387719958089292, - 0.0008496439986629412, - 0.0008050209944485687, - 0.002613420998386573, - 0.000713255001755897, - 0.0006884899994474836, - 0.0007342009994317777, - 0.0006923940018168651, - 0.0009317950025433674, - 0.0011943700010306202, - 0.0014227559950086288, - 0.000999190000584349, - 0.0007358579969150014, - 0.0012508159998105839, - 0.0007091739971656352, - 0.000971011002548039, - 0.0009018759956234135, - 0.0007267350010806695, - 0.0006817870016675442, - 0.000693118003255222, - 0.0007336970011238009, - 0.0008555949971196242, - 0.0009587859967723489, - 0.0008819470021990128, - 0.0008208860017475672, - 0.0015267910057445988, - 0.0008511580017511733, - 0.001031143001455348, - 0.0008268640012829565, - 0.0028379119976307265, - 0.0007245969973155297, - 0.0006852219958091155, - 0.0006873759994050488, - 0.0008686579967616126, - 0.0010540109942667186, - 0.0009378350005135871, - 0.0008122709987219423, - 0.0007022979989415035, - 0.0006966010041651316, - 0.0018847360042855144, - 0.000757858004362788, - 0.0008463679987471551, - 0.0007202030028565787, - 0.0010253689979435876, - 0.0008838909998303279, - 0.0009541539984638803, - 0.0017858409992186353, - 0.0013422349948086776, - 0.0012120129977120087, - 0.0009018429991556332, - 0.0013131340019754134, - 0.0007698830013396218, - 0.0006992150010773912, - 0.0006890700024086982, - 0.0012072299941792153, - 0.002002828005061019, - 0.0027390320028644055, - 0.0007650589977856725, - 0.00069922400143696, - 0.0025331659999210387, - 0.0018625539960339665, - 0.004182821998256259, - 0.0009857879995252006, - 0.0007201670014183037, - 0.0024559160010539927, - 0.0018555550050223246, - 0.0029957959995954297, - 0.0008103500003926456, - 0.0021406689993455075, - 0.0008594309983891435, - 0.0008087320020422339, - 0.0007160960012697615, - 0.0009191700009978376, - 0.0009682090021669865, - 0.0009652820008341223, - 0.0007935820030979812, - 0.0014979449988459237, - 0.0007325350015889853, - 0.0013049869958194904, - 0.0009682180025265552, - 0.0009240430008503608, - 0.0008910149990697391, - 0.0007233770011225715, - 0.0006961059989407659, - 0.0022790820003137924, - 0.000955151001107879, - 0.0007199660030892119, - 0.0007129820005502552, - 0.0007082939991960302, - 0.0007007849999354221, - 0.0010026429954450577, - 0.0035543319972930476, - 0.0010167639993596822, - 0.0007266659958986565, - 0.0007024760052445345, - 0.004081062004843261, - 0.000750287996197585, - 0.0017955129951587878, - 0.0009352570050396025, - 0.0007838229939807206, - 0.0007494590026908554, - 0.0007539319994975813, - 0.0009214160018018447, - 0.0021715159964514896, - 0.0007616349976160564, - 0.000715157002559863, - 0.0013081129945931025, - 0.001422964000084903, - 0.0009595129959052429, - 0.0009597570024197921, - 0.0008363839951925911, - 0.0007820810060366057, - 0.0007257550023496151, - 0.000711242995748762, - 0.0007894219961599447, - 0.0011505540023790672, - 0.0026476869970792904, - 0.0007328680003411137, - 0.0007209010000224225, - 0.0007031269997241907, - 0.0010612100013531744, - 0.0009605270024621859, - 0.0011899770033778623, - 0.0007333910034503788, - 0.0007484629968530498, - 0.0007203490022220649, - 0.0008196670023608021, - 0.0028265880027902313, - 0.0013895209995098412, - 0.0007324030011659488, - 0.0007170160024543293, - 0.0009334560018032789, - 0.001224457002535928, - 0.000776851004047785, - 0.0007301859950530343, - 0.0007032720022834837, - 0.0006997129967203364, - 0.0010891139972954988, - 0.0007254359952639788, - 0.0013274039956741035, - 0.002023815999564249, - 0.0007196370061137713, - 0.0015744839984108694, - 0.0007671170023968443, - 0.0007099529975675978, - 0.0011917689989786595, - 0.002952928000013344, - 0.000726792000932619, - 0.000709904998075217, - 0.0007173600024543703, - 0.0007037200048216619, - 0.004075222997926176, - 0.0011324750012136064, - 0.0007852990020182915, - 0.0011959359981119633, - 0.0007363049953710288, - 0.0007138489963836037, - 0.0007150950041250326, - 0.0007395919965347275, - 0.002008028001000639, - 0.0009501569948042743, - 0.0007663050055271015, - 0.0007254079973790795, - 0.0017345290034427308, - 0.0007211819975054823, - 0.0007209670002339408, - 0.0007065329991746694, - 0.0016835360002005473, - 0.0010446649976074696, - 0.001939629000844434, - 0.0007429730030708015, - 0.0007120000009308569, - 0.0008760869968682528, - 0.002441486998577602, - 0.0008032389960135333, - 0.0007194569989223965, - 0.0016674310027156025, - 0.0009984669959521852, - 0.000914972995815333, - 0.0007619160023750737, - 0.0007156790015869774, - 0.0007030519991531037, - 0.0006978829987929203, - 0.001249893000931479, - 0.0009818339967750944, - 0.0020837310003116727, - 0.0007234599979710765, - 0.0014628580029238947, - 0.0008923749992391095, - 0.0016695019949111156, - 0.0007661070048925467, - 0.0007314920003409497, - 0.0007084179960656911, - 0.0006977720040595159, - 0.0009548320012982003, - 0.0016550010041100904, - 0.0007370419989456423, - 0.0007148220029193908, - 0.001192542003991548, - 0.0008448830049019307, - 0.00073781399987638, - 0.0009209499985445291, - 0.0008498960014549084, - 0.0007226460002129897, - 0.0007002040001680143, - 0.0007461959976353683, - 0.0007139989975257777, - 0.0013083339945296757, - 0.0015681599979870953, - 0.0007680650014663115, - 0.000704593003320042, - 0.0008995249954750761, - 0.0009445569958188571, - 0.0012940730011905544, - 0.0007306269981199875, - 0.0007406750009977259, - 0.0007475359961972572, - 0.0007140430025174282, - 0.0009759489985299297, - 0.0009635839960537851, - 0.0009918539944919758, - 0.00110855299863033, - 0.0007317039999179542, - 0.0007158180014812388, - 0.0007032209978206083, - 0.0007337619972531684, - 0.000946087995544076, - 0.0008984290034277365, - 0.0007397540030069649, - 0.0027004120056517422, - 0.0012103650005883537, - 0.0018697160048759542, - 0.0007528269998147152, - 0.0007046890023048036, - 0.0007042200013529509, - 0.0008692520059412345, - 0.0009569889953127131, - 0.001015663001453504, - 0.0018405089940642938, - 0.0007621070035384037, - 0.0007878870019339956, - 0.00284192700200947, - 0.0007438510001520626, - 0.0007103480020305142, - 0.0010369130031904206, - 0.0009872129958239384, - 0.0009021730002132244, - 0.000980615004664287, - 0.0013278760015964508, - 0.0007523320018663071, - 0.0007658329996047541, - 0.0011902039987035096, - 0.0007308750064112246, - 0.001090128003852442, - 0.0009551379989716224, - 0.0007381980030913837, - 0.000938074997975491, - 0.0016550500004086643, - 0.0007390620012301952, - 0.0007611359978909604, - 0.0007093960011843592, - 0.0006962289990042336, - 0.0011938060051761568, - 0.001708595002128277, - 0.0007463759975507855, - 0.002191325998865068, - 0.0007341759992414154, - 0.0015523440015385859, - 0.0010155330019188114, - 0.0007288289998541586, - 0.0007086320038069971, - 0.0007266469983733259, - 0.0007881639976403676, - 0.004725930004497059, - 0.0007581209938507527, - 0.0007310419969144277, - 0.0007233340002130717, - 0.0011255139979766682, - 0.0010312919985153712, - 0.001136614999268204, - 0.0007356919959420338, - 0.0007069189960020594, - 0.000726419995771721, - 0.0008621430024504662, - 0.002753319997282233, - 0.0043371779975132085, - 0.0020292170011089183, - 0.0009098270020331256, - 0.0007424799987347797, - 0.000719082003342919, - 0.0009697240020614117, - 0.0025126799955614842, - 0.0010146970016648993, - 0.0007396500004688278, - 0.0007163289992604405, - 0.0007373370026471093, - 0.0009146060037892312, - 0.0015452820007340051, - 0.0007314489994314499, - 0.0010039439948741347, - 0.0008856659987941384, - 0.0007408769961330108, - 0.0014770810012123547, - 0.0008462889963993803, - 0.0007267159962793812, - 0.0007224910004879348, - 0.0007045449965517037, - 0.0009769419993972406, - 0.0009709189980640076, - 0.0009658449998823926, - 0.0007893430010881275, - 0.0007731769946985878, - 0.0007336179987760261, - 0.0007040529962978326, - 0.000937798002269119, - 0.0010175770003115758, - 0.0008500769981765188, - 0.000736481997591909, - 0.0007108940044417977, - 0.0006980640027904883, - 0.0014032760009285994, - 0.0009397760004503652, - 0.0007189300013124011, - 0.000978535994363483, - 0.0007544499967480078, - 0.001520047997473739, - 0.0007710510035394691, - 0.0007176130020525306, - 0.0007193969940999523, - 0.0008509119943482801, - 0.0009534839991829358, - 0.0009565010041114874, - 0.0009404849988641217, - 0.0007692129947827198, - 0.0007110369988367893, - 0.0009449639983358793, - 0.0016987679991871119, - 0.0007698569970671088, - 0.0013660870026797056, - 0.0007358020011452027, - 0.000767431003623642, - 0.0007910140047897585, - 0.0014746029992238618, - 0.0011054179994971491, - 0.0010499339987291023, - 0.0007905829988885671, - 0.0007107130004442297, - 0.0007080909999785945, - 0.002844568000000436, - 0.0007420779948006384, - 0.0007026409948593937, - 0.0007137760039768182, - 0.0007035050002741627, - 0.0007027100000414066, - 0.0009607050014892593, - 0.0008150880021275952, - 0.0007212069976958446, - 0.0008832200037431903, - 0.0010293689992977306, - 0.0009662260054028593, - 0.000742768999771215, - 0.0007103360039764084, - 0.005325322003045585, - 0.0008368019989575259, - 0.0010197529991273768, - 0.0020825850006076507, - 0.0007975609987624921, - 0.0007265570020535961, - 0.0008784770034253597, - 0.0013190209938329645, - 0.0007568220025859773, - 0.0007387850055238232, - 0.000724282996088732, - 0.0007049690029816702, - 0.001118161999329459, - 0.0009849910056800582, - 0.0009624550002627075, - 0.0009785730071598664, - 0.0008037009974941611, - 0.0007199550018412992, - 0.0007083589953253977, - 0.0006983409984968603, - 0.0012524720004876144, - 0.0007273649971466511, - 0.0007063700031721964, - 0.0008276180014945567, - 0.0009551059993100353, - 0.003471120995527599, - 0.005744520996813662, - 0.0008001320020412095, - 0.0007213979988591745, - 0.0008749190019443631, - 0.0007067830010782927, - 0.004123031998460647, - 0.0009197540057357401, - 0.0007578220029245131, - 0.0007228610047604889, - 0.0007540670048911124, - 0.0007079089991748333, - 0.005746051007008646, - 0.0007969220023369417, - 0.0008396929988521151, - 0.000716768998245243, - 0.0007608239975525066, - 0.005804171996715013, - 0.0007185639988165349, - 0.0006922820030013099, - 0.0006890939985169098, - 0.000698533003742341, - 0.0006800560004194267, - 0.0014689190065837465, - 0.003763412998523563, - 0.0009747400035848841, - 0.0006970799950067885, - 0.0006883969981572591, - 0.0055556890001753345, - 0.000879202998476103, - 0.0007026450039120391, - 0.0007566129934275523, - 0.00073654100560816, - 0.005306906998157501, - 0.0007336859998758882, - 0.0006917030041222461, - 0.0006936830031918362, - 0.0033833939960459247, - 0.000731480002286844, - 0.0006902399982209317, - 0.0006871220030006953, - 0.0007323480022023432, - 0.005878959003894124, - 0.0007563910039607435, - 0.0006928719958523288, - 0.0006915200065122917, - 0.0006842279981356114, - 0.0034819569991668686, - 0.0030541000014636666, - 0.0007018269971013069, - 0.0006877510022604838, - 0.0006855859974166378, - 0.0006799699986004271, - 0.0028819619983551092, - 0.003449570998782292, - 0.0007265359963639639, - 0.0006906790003995411, - 0.0006885559996590018, - 0.0006902439999976195, - 0.002728530998865608, - 0.0007454150036210194, - 0.0006941219980944879, - 0.0006929070004844107, - 0.0006915120029589161, - 0.00637327699951129, - 0.0007433789942297153, - 0.0007000170007813722, - 0.0006876889965496957, - 0.004923783002595883, - 0.0009263720057788305, - 0.0007018850010354072, - 0.0006889170035719872, - 0.000686109000525903, - 0.006429372995626181, - 0.0007313009991776198, - 0.0006926360001671128, - 0.0006917630016687326, - 0.0006874799946672283, - 0.0006852070000604726, - 0.0059736280018114485, - 0.0007316760020330548, - 0.0006957360019441694, - 0.0007219870021799579, - 0.0007029339976725169, - 0.006657090001681354, - 0.0007367370053543709, - 0.0006928029979462735, - 0.0007440459958161227, - 0.000696929004334379, - 0.0006809390033595264, - 0.008102243999019265, - 0.0007441569978254847, - 0.000695834998623468, - 0.0006965040010982193, - 0.0006819249974796548, - 0.003707506002683658, - 0.000767213998187799, - 0.0006956360011827201, - 0.0006936360005056486, - 0.0007550829977844842, - 0.0014138240003376268, - 0.0018184130021836609, - 0.0037673580009141006, - 0.0022450140022556297, - 0.002177049005695153, - 0.0010458250035298988, - 0.004621825995855033, - 0.0012424730011844076, - 0.0006899430009070784, - 0.0007234290023916401, - 0.0006819560003350489, - 0.012981266001588665, - 0.002339573999051936, - 0.0007116339984349906, - 0.004486628997256048, - 0.00858919500024058, - 0.002322679996723309, - 0.0007608349988004193, - 0.007961650997458491, - 0.001167192000139039, - 0.0006953979973332025, - 0.000682111996866297, - 0.015986179001629353, - 0.0041653549997136, - 0.007580409001093358, - 0.008830095997836906, - 0.0009391209969180636, - 0.006154263995995279, - 0.009526153997285292, - 0.007486882001103368, - 0.017018168997310568, - 0.0030574339980375953, - 0.006662430001597386, - 0.006317311999737285, - 0.002048975002253428, - 0.008564965006371494, - 0.004712835994723719, - 0.0018059979993267916, - 0.006183233002957422, - 0.0019254340004408732, - 0.004615721001755446, - 0.0017689859960228205, - 0.008569698002247605, - 0.000973596004769206, - 0.00622433700482361, - 0.02857370799756609, - 0.007461766996129882, - 0.0009095959976548329, - 0.0007384540003840812, - 0.005284945997118484, - 0.008932430995628238, - 0.004051603005791549, - 0.0007870100016589276, - 0.009412742001586594, - 0.001859782998508308, - 0.00145579400123097, - 0.005841094003699254, - 0.0008164259998011403, - 0.0007443069989676587, - 0.0007366549980361015, - 0.0007333940011449158, - 0.004903162996924948, - 0.0007809620001353323, - 0.0007291849979083054, - 0.0008379010032513179, - 0.0007863099963287823, - 0.004202891002933029, - 0.0009033629976329394, - 0.0007520520011894405, - 0.0007308330023079179, - 0.000724999001249671, - 0.005226975001278333, - 0.0016959460044745356, - 0.0007365350029431283, - 0.0007808149966876954, - 0.004738130002806429, - 0.0009204720045090653, - 0.0007457940009771846, - 0.000730882995412685, - 0.0007841959959478118, - 0.006482313001470175, - 0.0008167580017470755, - 0.000742279997211881, - 0.0007331350061576813, - 0.0006982259947108105, - 0.005489553994266316, - 0.0007790460003889166, - 0.0007190809992607683, - 0.0007137089996831492, - 0.0007133629987947643, - 0.005623031000141054, - 0.000781968999945093, - 0.000720691001333762, - 0.0007113289975677617, - 0.0007051150023471564, - 0.0034177320048911497, - 0.0007502709995605983, - 0.0007181580003816634, - 0.000842934001411777, - 0.000749785998777952, - 0.000711082000634633, - 0.005801979001262225, - 0.0011773939986596815, - 0.0007171789984568022, - 0.0007077879999997094, - 0.004271816003893036, - 0.0008932850032579154, - 0.005218831000092905, - 0.0008347619950654916, - 0.0007291469955816865, - 0.00732767899899045, - 0.0009357360031572171, - 0.0007425579970004037, - 0.005867056002898607, - 0.0023440669974661432, - 0.0008063249988481402, - 0.004294343001674861, - 0.004775016997882631, - 0.0007641699994564988, - 0.003430463999393396, - 0.005678002002241556, - 0.002249310004117433, - 0.00896358000318287, - 0.0025734869996085763, - 0.00622986099915579, - 0.008280789996206295, - 0.0008486010046908632, - 0.004929226997774094, - 0.0011874650008394383, - 0.006726987005094998, - 0.0011522309941938147, - 0.010131912000360899, - 0.011571277005714364, - 0.008810767001705244, - 0.00112060499668587, - 0.0062385150013142265, - 0.01173161299811909 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil", - "fullname": "TestApplyNabla2AndNabla4GlobalToVn", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.00039931200444698334, - "max": 0.019986325001809746, - "mean": 0.004566150372730821, - "stddev": 0.0033528803643517098, - "rounds": 2058, - "median": 0.005793093998363474, - "iqr": 0.006479188006778713, - "q1": 0.0005192009994061664, - "q3": 0.00699838900618488, - "iqr_outliers": 3, - "stddev_outliers": 1051, - "outliers": "1051;3", - "ld15iqr": 0.00039931200444698334, - "hd15iqr": 0.017988031002460048, - "ops": 219.00286201085893, - "total": 9.397137467080029, - "data": [ - 0.000849110001581721, - 0.0004843339993385598, - 0.0005138319975230843, - 0.00048041399713838473, - 0.0017207330020028166, - 0.0010303910021320917, - 0.0004963859973940998, - 0.0004971070011379197, - 0.0004960939986631274, - 0.00048297100147465244, - 0.0004886249953415245, - 0.0004909499984933063, - 0.0008418560028076172, - 0.0007908600018708967, - 0.0019391400055610575, - 0.0005233300034888089, - 0.0005507500027306378, - 0.0005142179943504743, - 0.0009753519989317283, - 0.0011250469979131594, - 0.0007332269960897975, - 0.0005424120026873425, - 0.0005133610029588453, - 0.0005126199976075441, - 0.0004992079993826337, - 0.0019838879961753264, - 0.0011630610024440102, - 0.0005152550002094358, - 0.0005082709976704791, - 0.0004990419984096661, - 0.0004722500016214326, - 0.0006321869950625114, - 0.0006683980027446523, - 0.0006922629982000217, - 0.00047247399925254285, - 0.0004763039978570305, - 0.0004927670015604235, - 0.0005604959951597266, - 0.0008943430002545938, - 0.0012978629965800792, - 0.000633553005172871, - 0.0004975740012014285, - 0.00047916000039549544, - 0.0008542229988961481, - 0.0006197859984240495, - 0.0005009369997424074, - 0.0004831510013900697, - 0.0004732410016003996, - 0.0005303249999997206, - 0.0004737690032925457, - 0.0005360009963624179, - 0.0004778139991685748, - 0.0004908580012852326, - 0.00047292200179072097, - 0.0004949380017933436, - 0.0005751360004069284, - 0.0006288899967330508, - 0.004128188003960531, - 0.0007412940030917525, - 0.0005539179983315989, - 0.0004914169985568151, - 0.0004834479987039231, - 0.0004816750006284565, - 0.0004788859951077029, - 0.007905243001005147, - 0.0006642090011155233, - 0.0005215659984969534, - 0.0004791920000570826, - 0.004186444995866623, - 0.0006161860001157038, - 0.0005121889989823103, - 0.004363692001788877, - 0.0005438579973997548, - 0.0005546430038521066, - 0.0005111370046506636, - 0.014544705001753755, - 0.0055251070007216185, - 0.010039668006356806, - 0.007494762001442723, - 0.008136989999911748, - 0.007109959995432291, - 0.010497048999241088, - 0.004138379001233261, - 0.007838704994355794, - 0.006994101997406688, - 0.007000021003477741, - 0.006990991998463869, - 0.005061910997028463, - 0.0059250259946566075, - 0.006003172995406203, - 0.006993337003223132, - 0.006995110998104792, - 0.005996217005304061, - 0.007994069004780613, - 0.00901813600648893, - 0.006978566001635045, - 0.0069876540001132526, - 0.006995332005317323, - 0.006996043004619423, - 0.0070079020006232895, - 0.007984349002072122, - 0.005998301996442024, - 0.007993671999429353, - 0.007995751002454199, - 0.0039924489974509925, - 0.005000384997401852, - 0.006999994002399035, - 0.004171204003796447, - 0.0038706579944118857, - 0.005943084004684351, - 0.00799853199714562, - 0.00599272800172912, - 0.006994673996814527, - 0.0069982669956516474, - 0.006994473995291628, - 0.0069957500018063, - 0.007995581996510737, - 0.007995464002306107, - 0.007995959997060709, - 0.003995327999291476, - 0.003993915001046844, - 0.00700024000252597, - 0.006996423006057739, - 0.007996641004865523, - 0.003992311998445075, - 0.006367202004184946, - 0.006630110998230521, - 0.005993984996166546, - 0.005991933001496363, - 0.01100147599936463, - 0.007995038002263755, - 0.007994608000444714, - 0.007996314998308662, - 0.007997417000296991, - 0.006997008000325877, - 0.007993692001036834, - 0.0069984760048100725, - 0.0076225510056247, - 0.006368378002662212, - 0.007045712001854554, - 0.007947363999846857, - 0.006992440001340583, - 0.006995649004238658, - 0.005992831000185106, - 0.008000699002877809, - 0.006994244002271444, - 0.007002917998761404, - 0.006016166000335943, - 0.007977039000252262, - 0.006988918001297861, - 0.0059931209980277345, - 0.004998654992959928, - 0.002992122004798148, - 0.0060016589995939285, - 0.007994334999239072, - 0.007999852998182178, - 0.007993121995241381, - 0.00799574200209463, - 0.007998831999429967, - 0.008029780998185743, - 0.007949246995849535, - 0.010004628995375242, - 0.006989313995291013, - 0.009995230000640731, - 0.007000244004302658, - 0.008991954993689433, - 0.007999133995326702, - 0.005992869999317918, - 0.008010204001038801, - 0.007985943004314322, - 0.006001389003358781, - 0.0059903650035266764, - 0.005995153995172586, - 0.005996899002639111, - 0.005995612998958677, - 0.007996061001904309, - 0.0059984199979226105, - 0.0069968909956514835, - 0.006993101000261959, - 0.007998752000276, - 0.006993580005655531, - 0.005994475002808031, - 0.00799333599570673, - 0.008997926001029555, - 0.006996628995693754, - 0.004995969000447076, - 0.006157789001008496, - 0.009842084000410978, - 0.001992356999835465, - 0.007050539999909233, - 0.007944697004859336, - 0.006428233995393384, - 0.006559198001923505, - 0.007998148998012766, - 0.006986050997511484, - 0.007993491999513935, - 0.006995100004132837, - 0.005994996005028952, - 0.006994859999394976, - 0.007996029002242722, - 0.004993978000129573, - 0.00599740900361212, - 0.006994413000938948, - 0.006999065000854898, - 0.007998916997166816, - 0.007991482998477295, - 0.009996452004997991, - 0.007996117004950065, - 0.006994544994086027, - 0.006006459996569902, - 0.007983970994246192, - 0.0060007489955751225, - 0.005994241997541394, - 0.00799658599862596, - 0.005002117999538314, - 0.005989424003928434, - 0.005997920001391321, - 0.010002351002185605, - 0.009992906998377293, - 0.00799847100279294, - 0.00809000699518947, - 0.006895548998727463, - 0.007999069996003527, - 0.007003170998359565, - 0.005988755998259876, - 0.0043344689984223805, - 0.003653756997664459, - 0.007083378994138911, - 0.007914240995887667, - 0.006190227999468334, - 0.006803352996939793, - 0.010993101997883059, - 0.011003597996023018, - 0.007991014994331636, - 0.009993595995183568, - 0.0033077410043915734, - 0.008035061000555288, - 0.008659335995616857, - 0.005984299001283944, - 0.005994604995066766, - 0.009999816997151356, - 0.009995153995987494, - 0.0067661950015462935, - 0.0032316400029230863, - 0.00399047699465882, - 0.005999396998959128, - 0.006995741001446731, - 0.006998247001320124, - 0.005995874998916406, - 0.012998354999581352, - 0.0069955500002834015, - 0.003998569998657331, - 0.003993348000221886, - 0.00800205000268761, - 0.007007412001257762, - 0.008983622996311169, - 0.003997029998572543, - 0.006994578005105723, - 0.006994708004640415, - 0.006998430006206036, - 0.00400463400001172, - 0.004028556999401189, - 0.006953383002837654, - 0.005996742998831905, - 0.005993999999191146, - 0.00699676999647636, - 0.006997648000833578, - 0.005996239997330122, - 0.007997742002771702, - 0.006992087000980973, - 0.008001145994057879, - 0.006993543000135105, - 0.006998764001764357, - 0.00799958199786488, - 0.00899337200098671, - 0.0069958169988240115, - 0.007998237997526303, - 0.007993548002559692, - 0.006997399999818299, - 0.005994172002829146, - 0.00699706200248329, - 0.0069930409954395145, - 0.00699880400497932, - 0.006996211006480735, - 0.006996984993747901, - 0.007582074002129957, - 0.007412163002300076, - 0.00799562699830858, - 0.008994338000775315, - 0.005995347004500218, - 0.006000218003464397, - 0.00799409100000048, - 0.004993611000827514, - 0.004993415997887496, - 0.006998071003181394, - 0.007995231004315428, - 0.010998607998772059, - 0.0069922280017635785, - 0.005997246000333689, - 0.006997203003265895, - 0.006997012998908758, - 0.005996754996886011, - 0.008006190997548401, - 0.005988430995785166, - 0.007996375003131106, - 0.00502524300100049, - 0.00998031900235219, - 0.0059808389996760525, - 0.019006259004527237, - 0.013993726999615319, - 0.012987242997041903, - 0.007078235001245048, - 0.003011445005540736, - 0.00588725799752865, - 0.007995707004738506, - 0.008005035000678618, - 0.0030250070049078204, - 0.006557344997418113, - 0.014878232002956793, - 0.0025042670022230595, - 0.005072002000815701, - 0.0070520319932256825, - 0.002656892997038085, - 0.005198156999540515, - 0.006034837002516724, - 0.00410852899949532, - 0.005006462000892498, - 0.004833617997064721, - 0.003240464997361414, - 0.002934969998023007, - 0.00481246599520091, - 0.010999842001183424, - 0.011995335997198708, - 0.00499272200249834, - 0.006998273995122872, - 0.006994798000960145, - 0.005697642998711672, - 0.008298884997202549, - 0.006996369003900327, - 0.0070067090055090375, - 0.007980879003298469, - 0.010998840996762738, - 0.005238402998656966, - 0.009755150000273716, - 0.004003544003353454, - 0.006952921001357026, - 0.003028459999768529, - 0.003156151004077401, - 0.002834762002748903, - 0.0051348599954508245, - 0.006845190000603907, - 0.007009376997302752, - 0.00900363500113599, - 0.007979055000760127, - 0.005993660000967793, - 0.0069956919978722, - 0.007997740001883358, - 0.006000278997817077, - 0.004996727999241557, - 0.009997993998695165, - 0.0051227250005467795, - 0.006867732998216525, - 0.000999298004899174, - 0.00399354499677429, - 0.006998397999268491, - 0.0069998969993321225, - 0.006995290998020209, - 0.004002330999355763, - 0.005996930005494505, - 0.006994704999669921, - 0.006994491996010765, - 0.006997379998210818, - 0.006995695999648888, - 0.006996919997618534, - 0.006998228003794793, - 0.00699722900026245, - 0.006997010998020414, - 0.008471597997413483, - 0.006521358998725191, - 0.006993751005211379, - 0.00600956600101199, - 0.0039829070010455325, - 0.004995339004381094, - 0.006998734002991114, - 0.00799665300291963, - 0.00699811200320255, - 0.007995075997314416, - 0.00599769100517733, - 0.006997448996116873, - 0.006998123993980698, - 0.00499832299828995, - 0.005997972999466583, - 0.007993769002496265, - 0.006997582997428253, - 0.00699831700330833, - 0.005583744998148177, - 0.0034033330011880025, - 0.007004557002801448, - 0.00800162100495072, - 0.006985244996030815, - 0.004665134998504072, - 0.005327797996869776, - 0.00800625300325919, - 0.007996583997737616, - 0.006991321002715267, - 0.008046481001656502, - 0.005940923001617193, - 0.007998249995580409, - 0.006690690002869815, - 0.00830275600310415, - 0.00699873200210277, - 0.004558861997793429, - 0.005434125996544026, - 0.003997584004537202, - 0.010998576995916665, - 0.0069980679982108995, - 0.006995394003752153, - 0.006998655997449532, - 0.007996963999175932, - 0.010996889999660198, - 0.006995496994932182, - 0.00799782700050855, - 0.005995877007080708, - 0.005311761997290887, - 0.005697638000128791, - 0.007984128998941742, - 0.006005096001899801, - 0.007991130994923878, - 0.007991657003003638, - 0.007998850000149105, - 0.007992773003934417, - 0.006002988004183862, - 0.006989169000007678, - 0.006997039999987464, - 0.007995041996764485, - 0.005997193999064621, - 0.00799779299995862, - 0.007996230997378007, - 0.005997318003210239, - 0.007997136999620125, - 0.007998145993042272, - 0.00600735199986957, - 0.005986074000247754, - 0.003135354992991779, - 0.002858774001651909, - 0.004996328003471717, - 0.0059989090004819445, - 0.007013695998466574, - 0.00698201400518883, - 0.004505798999161925, - 0.0009923360048560426, - 0.007487563998438418, - 0.008182507001038175, - 0.007810644994606264, - 0.006530799000756815, - 0.005464973000925966, - 0.004521216003922746, - 0.004472337997867726, - 0.0070019949998822995, - 0.0049817219987744465, - 0.006999399003689177, - 0.00700159899861319, - 0.006989093999436591, - 0.0089986540042446, - 0.0069929989986121655, - 0.006998262004344724, - 0.007993252998858225, - 0.007002407000982203, - 0.006991760004893877, - 0.007994403000338934, - 0.0040032329998211935, - 0.004396973003167659, - 0.004585763999784831, - 0.007999443005246576, - 0.007994829000381287, - 0.007999647001270205, - 0.008001630994840525, - 0.0079900450000423, - 0.007996207001269795, - 0.00786882499960484, - 0.006124159001046792, - 0.006989768000494223, - 0.011004571999364998, - 0.009058314994035754, - 0.007279208999534603, - 0.0076155650021974, - 0.006998182994720992, - 0.010151077003683895, - 0.008157409996783827, - 0.01067414200224448, - 0.0059916620011790656, - 0.006985524996707682, - 0.006997503005550243, - 0.007993694998731371, - 0.006995217998337466, - 0.006999344004725572, - 0.006989087996771559, - 0.007995387000846677, - 0.01300138799706474, - 0.005989359997329302, - 0.00799544600158697, - 0.006991708003624808, - 0.006994620998739265, - 0.005996689003950451, - 0.007998512999620289, - 0.007994512998266146, - 0.0160583909964771, - 0.006944617998669855, - 0.011011782997229602, - 0.00995819899981143, - 0.008012879996385891, - 0.0059468960025697015, - 0.006988583998463582, - 0.006990066001890227, - 0.007020331999228802, - 0.007662329000595491, - 0.003315163994557224, - 0.007378559996141121, - 0.003568171006918419, - 0.005999906999932136, - 0.005992750004224945, - 0.0059930539937340654, - 0.0049912249960470945, - 0.011001638995367102, - 0.003989103999629151, - 0.0049971210028161295, - 0.007004491999396123, - 0.00944667400472099, - 0.005540552003367338, - 0.016021512004954275, - 0.014986927999416366, - 0.008728100001462735, - 0.013225497998064384, - 0.010965923996991478, - 0.012021991002256982, - 0.004102111000975128, - 0.00296287899982417, - 0.008979072998045012, - 0.014894244995957706, - 0.019986325001809746, - 0.010994137002853677, - 0.017988031002460048, - 0.007578005002869759, - 0.008384647997445427, - 0.005978486005915329, - 0.0059958630008623, - 0.009003740997286513, - 0.006984812003793195, - 0.007101196999428794, - 0.01088699400133919, - 0.004991440997400787, - 0.007999164001375902, - 0.006993982999119908, - 0.007119155001419131, - 0.00787547099753283, - 0.006995650001044851, - 0.005999512002745178, - 0.010995882003044244, - 0.007995729996764567, - 0.006996287003858015, - 0.006998333003139123, - 0.0079996500062407, - 0.007016548006504308, - 0.007968739002535585, - 0.006988465000176802, - 0.008004461997188628, - 0.0069916669963276945, - 0.0019964369930676185, - 0.0019907310052076355, - 0.006132925002020784, - 0.006865878000098746, - 0.006986084998061415, - 0.008007221993466374, - 0.006985741994867567, - 0.007997007996891625, - 0.006997557997237891, - 0.0069975630030967295, - 0.005997010004648473, - 0.00799846200243337, - 0.006996864998654928, - 0.006997518998105079, - 0.007997533000889234, - 0.006996656993578654, - 0.0069957310042809695, - 0.007000087003689259, - 0.006994543000473641, - 0.0056128949945559725, - 0.002379534998908639, - 0.005998155000270344, - 0.00699978099873988, - 0.006999264995101839, - 0.006992369999352377, - 0.00699787699704757, - 0.007994395004061516, - 0.007999542001925875, - 0.00799696800095262, - 0.0059977729979436845, - 0.005992667000100482, - 0.006997305004915688, - 0.008001043999684043, - 0.0069953229976817966, - 0.0059946059991489165, - 0.006992820999585092, - 0.0069985649970476516, - 0.007236802004626952, - 0.00776060300268, - 0.00699898500170093, - 0.00799113199900603, - 0.004464675999770407, - 0.0065262839998467825, - 0.005999735003570095, - 0.00799478300177725, - 0.00899895799375372, - 0.006992424001509789, - 0.005997764994390309, - 0.006994399998802692, - 0.007000029007031117, - 0.0079939569986891, - 0.007170354001573287, - 0.0068210560057195835, - 0.005993826001940761, - 0.006001847003062721, - 0.009352156994282268, - 0.008636631995614152, - 0.006000381996273063, - 0.006989010995312128, - 0.007988474993908312, - 0.007000645993684884, - 0.00699149799766019, - 0.006670856004348025, - 0.0033278889968642034, - 0.006003848000545986, - 0.008989946996734943, - 0.0066554600052768365, - 0.00461040699883597, - 0.005717164000088815, - 0.007996877997356933, - 0.007995935004146304, - 0.005994619001285173, - 0.00599870099540567, - 0.006998612996540032, - 0.0059960500002489425, - 0.006996898002398666, - 0.006996525997237768, - 0.007999663001100998, - 0.00699830300436588, - 0.0059956250042887405, - 0.006995905998337548, - 0.006997295997280162, - 0.012000277994957287, - 0.00699378799617989, - 0.006997292002779432, - 0.0069985279988031834, - 0.006995120995270554, - 0.00699766800244106, - 0.006997113996476401, - 0.006997175994911231, - 0.005998877000820357, - 0.006001415000355337, - 0.00499180600309046, - 0.004997472002287395, - 0.007997863998753019, - 0.006993467002757825, - 0.00699911799893016, - 0.006995545998506714, - 0.00699676700605778, - 0.005985896998026874, - 0.004005695001978893, - 0.006001547000778373, - 0.006045624002581462, - 0.00795081500109518, - 0.006993581002461724, - 0.006997448996116873, - 0.007993568004167173, - 0.007997439002792817, - 0.005999950000841636, - 0.007995212996320333, - 0.008001665002666414, - 0.007994034000148531, - 0.007998692002729513, - 0.0069992169956094585, - 0.006988765999267343, - 0.0059906009992118925, - 0.007992890001332853, - 0.005999806999170687, - 0.00799643299978925, - 0.005997926993586589, - 0.007992050996108446, - 0.005997020998620428, - 0.0049996360030490905, - 0.007994820996827912, - 0.006996096002694685, - 0.006997526004852261, - 0.006996727999649011, - 0.007998065004358068, - 0.0010064899979624897, - 0.006988465996982995, - 0.00799710599676473, - 0.005996089006657712, - 0.005996683001285419, - 0.006999667995842174, - 0.006994783994741738, - 0.007997922002687119, - 0.006999047000135761, - 0.009995561995310709, - 0.0070020700004533865, - 0.007991113001480699, - 0.008010626996110659, - 0.006994212002609856, - 0.006976644006499555, - 0.006994431998464279, - 0.008000269997864962, - 0.007995740001206286, - 0.01099387599970214, - 0.009996741995564662, - 0.006398973004252184, - 0.0075934659980703145, - 0.005997877000481822, - 0.006995805000769906, - 0.007997901004273444, - 0.00300515800336143, - 0.0075731370016001165, - 0.007415344000037294, - 0.006996304000495002, - 0.007000915997195989, - 0.006993075003265403, - 0.006998058997851331, - 0.005996225001581479, - 0.007000254998274613, - 0.008996841999760363, - 0.008002433998626657, - 0.004994455004634801, - 0.0040149069973267615, - 0.005990271994960494, - 0.005980128000373952, - 0.006992953996814322, - 0.006994276998739224, - 0.005999269000312779, - 0.00799670100241201, - 0.005998590000672266, - 0.008000207999430131, - 0.006988047003687825, - 0.005000868004572112, - 0.007001211000897456, - 0.010989888003678061, - 0.003995166996901389, - 0.004690984002081677, - 0.006298350999713875, - 0.009003106999443844, - 0.006991535003180616, - 0.006993310002144426, - 0.006995682997512631, - 0.006994424998993054, - 0.0069999409970478155, - 0.009998093999456614, - 0.00599207900086185, - 0.008000517002074048, - 0.007993205996172037, - 0.0059956049954053015, - 0.0059965519976685755, - 0.005823913001222536, - 0.00616686099965591, - 0.005997504995320924, - 0.007996127002115827, - 0.00899599999684142, - 0.006996028998401016, - 0.008016465995751787, - 0.005850109002494719, - 0.006163383004604839, - 0.0029523960038204677, - 0.006001251000270713, - 0.006992063994402997, - 0.007995102001586929, - 0.00799560800078325, - 0.006996286996582057, - 0.00800099500338547, - 0.006991237998590805, - 0.006995622003159951, - 0.005096474997117184, - 0.004896112994174473, - 0.007019427001068834, - 0.005983653005387168, - 0.007992506994924042, - 0.00600953699904494, - 0.004976432996045332, - 0.009998187000746839, - 0.002983696002047509, - 0.004001575995062012, - 0.011006213004293386, - 0.007790689000103157, - 0.00434215600398602, - 0.007860343001084402, - 0.002965974999824539, - 0.004006602997833397, - 0.0069440419974853285, - 0.009045104998222087, - 0.00799199200264411, - 0.0060795919998781756, - 0.003906420002749655, - 0.007002710000961088, - 0.007995439998921938, - 0.006995712996285874, - 0.007002359998296015, - 0.010989804002747405, - 0.0099961070009158, - 0.0069958299936843105, - 0.008007624994206708, - 0.00698433599609416, - 0.009042127996508498, - 0.005951230996288359, - 0.0049911370006157085, - 0.005994434999593068, - 0.0070011989955673926, - 0.006995007999648806, - 0.00800293300562771, - 0.006989016001170967, - 0.006996035997872241, - 0.008994904994324315, - 0.0069925079951644875, - 0.0079954219982028, - 0.006996291005634703, - 0.0059976150005240925, - 0.004026660004456062, - 0.0029577570021501742, - 0.004006410999863874, - 0.00498894300108077, - 0.004993828006263357, - 0.004237565997755155, - 0.005750780001108069, - 0.008008550998056307, - 0.0019843310001306236, - 0.006999730001552962, - 0.006994766001298558, - 0.007996145999641158, - 0.006992198002990335, - 0.01311893000092823, - 0.005889463005587459, - 0.006000522000249475, - 0.010946695001621265, - 0.015030219998152461, - 0.008986264001578093, - 0.006991851005295757, - 0.00599592000071425, - 0.006996585005254019, - 0.007997216998774093, - 0.006997021002462134, - 0.008000333997188136, - 0.007994419000169728, - 0.002996041002916172, - 0.00699838900618488, - 0.008021446003112942, - 0.006968818997847848, - 0.006995451003604103, - 0.007003694998275023, - 0.007989818004716653, - 0.008996247997856699, - 0.007996633998118341, - 0.00800137199985329, - 0.007993231003638357, - 0.006000498004141264, - 0.006993104005232453, - 0.006994557996222284, - 0.006998891003604513, - 0.006997443000727799, - 0.0010011919948738068, - 0.007991692997165956, - 0.005997667998599354, - 0.00699650299793575, - 0.008007942000404, - 0.00600281500373967, - 0.006975397001951933, - 0.006992488997639157, - 0.006996989002800547, - 0.005078606998722535, - 0.00692008099576924, - 0.008996019001642708, - 0.006995701005507726, - 0.005003480000596028, - 0.007987114004208706, - 0.00599652600067202, - 0.007996610998816323, - 0.006997682998189703, - 0.007997635002539027, - 0.0069966239971108735, - 0.005996754000079818, - 0.008997700999316294, - 0.005198464001296088, - 0.00879780999821378, - 0.00499430200579809, - 0.0059986060005030595, - 0.006856273001176305, - 0.003138876003504265, - 0.00799762700626161, - 0.007996034997631796, - 0.007998027998837642, - 0.005998375003400724, - 0.007996724001714028, - 0.005996733998472337, - 0.007003572005487513, - 0.004993112997908611, - 0.006997090000368189, - 0.006997115000558551, - 0.007998910004971549, - 0.006989679000980686, - 0.00800188300490845, - 0.006992372997046914, - 0.006998573000601027, - 0.007996517997526098, - 0.006997083997703157, - 0.006997637996391859, - 0.007997590000741184, - 0.007010264998825733, - 0.005987825003103353, - 0.007001983998634387, - 0.004643552005290985, - 0.005336542999430094, - 0.007184264999523293, - 0.007170826000219677, - 0.0076351449970388785, - 0.0069994670047890395, - 0.007174590005888604, - 0.006816915003582835, - 0.004997515003196895, - 0.007005103005212732, - 0.003990133001934737, - 0.001621257993974723, - 0.003987235999375116, - 0.006086143999709748, - 0.003184383996995166, - 0.005109142999572214, - 0.005527868001081515, - 0.006305205999524333, - 0.0007609329986735247, - 0.006530760998430196, - 0.007991151003807317, - 0.008867884003848303, - 0.006997504002356436, - 0.005428686003142502, - 0.006566055002622306, - 0.003996697996626608, - 0.004994950999389403, - 0.006028099996910896, - 0.004968867004208732, - 0.005993733000650536, - 0.0049980119947576895, - 0.006995286996243522, - 0.006998299002589192, - 0.005006229002901819, - 0.007987496006535366, - 0.005997565996949561, - 0.005997438995109405, - 0.010999771999195218, - 0.006994604002102278, - 0.007997419997991528, - 0.006997572003456298, - 0.007007680003880523, - 0.006987022003158927, - 0.006996624993917067, - 0.007003272003203165, - 0.0079916390022845, - 0.008997543001896702, - 0.004995098999643233, - 0.005000983001082204, - 0.00799450499471277, - 0.005997635002131574, - 0.007998549997864757, - 0.007003020997217391, - 0.005989298995700665, - 0.006996520001848694, - 0.007997895001608413, - 0.007996758999070153, - 0.007998804998351261, - 0.006012664001900703, - 0.00806956400629133, - 0.005835635005496442, - 0.006998279997787904, - 0.007992983002623077, - 0.0059995999981765635, - 0.00699524900119286, - 0.007997289998456836, - 0.005998704000376165, - 0.0069957390005583875, - 0.005003175996534992, - 0.005997596999804955, - 0.003990768003859557, - 0.006998458004090935, - 0.006996716998401098, - 0.007002518999797758, - 0.006993580005655531, - 0.007997417997103184, - 0.007998358996701427, - 0.0029988559981575236, - 0.00599389300623443, - 0.006999065000854898, - 0.006240414004423656, - 0.006752481000148691, - 0.005996922998747323, - 0.008136125004966743, - 0.00685799300117651, - 0.007027095001831185, - 0.005034443995100446, - 0.0029318230008357204, - 0.004996873998607043, - 0.006008880998706445, - 0.007984422998561058, - 0.007997635002539027, - 0.007999101995665114, - 0.004771117004565895, - 0.00522198300313903, - 0.007998109998879954, - 0.010016232998168562, - 0.007971799997903872, - 0.006996322998020332, - 0.007997495995368809, - 0.006996759002504405, - 0.00799948199710343, - 0.007995406005647965, - 0.008996969998406712, - 0.00918188499781536, - 0.0038121170000522397, - 0.004007718001957983, - 0.004987986998457927, - 0.006997139993472956, - 0.006997248005063739, - 0.00699702299607452, - 0.006997580996539909, - 0.007997168999281712, - 0.006002406997140497, - 0.00699373099632794, - 0.008997284996439703, - 0.006997128999501001, - 0.007997551998414565, - 0.008005275994946714, - 0.005989187004161067, - 0.0079963329990278, - 0.007995423999091145, - 0.005997161999403033, - 0.004995950002921745, - 0.004661090002628043, - 0.0043344630030333064, - 0.005998471999191679, - 0.006996846001129597, - 0.003974986000685021, - 0.00601975300378399, - 0.0059979490033583716, - 0.007994446998054627, - 0.007997500004421454, - 0.006997154996497557, - 0.006007543997839093, - 0.008345663001819048, - 0.006647501002589706, - 0.006989705994783435, - 0.004524753996520303, - 0.007470452997949906, - 0.008993808995001018, - 0.01099551300285384, - 0.005996497995511163, - 0.008019020002393518, - 0.00797613200120395, - 0.009165846000541933, - 0.006835993001004681, - 0.0023255519990925677, - 0.004707434003648814, - 0.007026799001323525, - 0.007472432000213303, - 0.0064538809965597466, - 0.006190501997480169, - 0.0017912080002133735, - 0.0022847359941806644, - 0.0037102040005265735, - 0.004997447002097033, - 0.005484861001605168, - 0.006509115999506321, - 0.005998207001539413, - 0.006997695003519766, - 0.004217189001792576, - 0.005780649997177534, - 0.004994144001102541, - 0.007999164998182096, - 0.007998282999324147, - 0.006993478004005738, - 0.005996182000671979, - 0.004998393997084349, - 0.005111099002533592, - 0.002882534005038906, - 0.0051206990028731525, - 0.007875575000070967, - 0.005185031004657503, - 0.002808917000947986, - 0.005017625000618864, - 0.0019763590025831945, - 0.006000162000418641, - 0.005995731997245457, - 0.0059979649959132075, - 0.00799763599934522, - 0.005997923006361816, - 0.00799491599900648, - 0.003997066000010818, - 0.004997595999157056, - 0.00699700899713207, - 0.006998569006100297, - 0.008995060998131521, - 0.006999946999712847, - 0.007994725005119108, - 0.003997112005890813, - 0.008997734003060032, - 0.00799933000234887, - 0.006995129995630123, - 0.0069985980007913895, - 0.00701109799410915, - 0.0069815269962418824, - 0.005997877000481822, - 0.007002007005212363, - 0.007992620005097706, - 0.011004447005689144, - 0.007991924998350441, - 0.002995554001245182, - 0.005997996006044559, - 0.007999517998541705, - 0.006997441996645648, - 0.007995266001671553, - 0.008004431001609191, - 0.006989950998104177, - 0.006996565993176773, - 0.006997046999458689, - 0.008003224997082725, - 0.004988788001355715, - 0.007003669998084661, - 0.009989840000343975, - 0.0049967879967880435, - 0.007831731003534514, - 0.006161591001728084, - 0.005997672000376042, - 0.004355625998869073, - 0.001635426000575535, - 0.006299520005995873, - 0.010698707003029995, - 0.0060508050009957515, - 0.003684800998598803, - 0.002362560000619851, - 0.006890379001561087, - 0.004996458999812603, - 0.005535131000215188, - 0.006463096993684303, - 0.008094119002635125, - 0.006029448995832354, - 0.008867758006090298, - 0.0059937990008620545, - 0.009006332998978905, - 0.00558929200633429, - 0.0033978040009969845, - 0.004996073999791406, - 0.006992756003455725, - 0.009502616994723212, - 0.005489826995471958, - 0.004997684998670593, - 0.003996344996266998, - 0.007997795997653157, - 0.0060023889964213595, - 0.006992328002525028, - 0.006421691003197338, - 0.008573104998504277, - 0.005394908002926968, - 0.004680578000261448, - 0.009916000002704095, - 0.005350596002244856, - 0.00965098100277828, - 0.008001344001968391, - 0.007988223995198496, - 0.006994960996962618, - 0.005998551998345647, - 0.005996911000693217, - 0.005996986998070497, - 0.007995690997631755, - 0.005892374996619765, - 0.005102301998704206, - 0.011998688001767732, - 0.0069956590014044195, - 0.006997266995313112, - 0.006999773002462462, - 0.005996239997330122, - 0.006997068005148321, - 0.006758914998499677, - 0.009236830002919305, - 0.007993252998858225, - 0.00200333799875807, - 0.005425184004707262, - 0.0075634450040524825, - 0.00799651399574941, - 0.005001883000659291, - 0.006995522999204695, - 0.004994031995011028, - 0.005999439999868628, - 0.008014969003852457, - 0.006986940003116615, - 0.004539980996923987, - 0.0034432149986969307, - 0.006995725001615938, - 0.006881160996272229, - 0.0071130329961306415, - 0.006996335003350396, - 0.007998031003808137, - 0.006997356002102606, - 0.007996499000000767, - 0.006999197001277935, - 0.00299572399671888, - 0.0059989529981976375, - 0.00799689200357534, - 0.003996416999143548, - 0.009013955997943413, - 0.0030562070023734123, - 0.003919406997738406, - 0.007998103996214923, - 0.00599870099540567, - 0.0069890279992250726, - 0.006994482995651197, - 0.005998345004627481, - 0.006996677999268286, - 0.007005678999121301, - 0.006983597006183118, - 0.006997222000791226, - 0.005995880004775245, - 0.006998643002589233, - 0.0070147250007721595, - 0.0079815380013315, - 0.007186357004684396, - 0.0018104409973602742, - 0.003989706005086191, - 0.006991235000896268, - 0.006719748002069537, - 0.006292389996815473, - 0.006770739004423376, - 0.006208081002114341, - 0.0029879819994675927, - 0.006000028995913453, - 0.004996600997401401, - 0.006998210999881849, - 0.008996765995107125, - 0.005995250998239499, - 0.006332153003313579, - 0.0016534419992240146, - 0.006242811999982223, - 0.00575846300489502, - 0.006831651000538841, - 0.006162662997667212, - 0.005993325998133514, - 0.003992596997704823, - 0.0030102529999567196, - 0.003967095006373711, - 0.006528183999762405, - 0.010497154995391611, - 0.005958980000286829, - 0.004994289003661834, - 0.0066656880007940345, - 0.0033136659985757433, - 0.006004141003359109, - 0.005990507001115475, - 0.004718508003861643, - 0.008282165996206459, - 0.0069883499963907525, - 0.004443322999577504, - 0.0035471359951770864, - 0.0069961170011083595, - 0.006995241994445678, - 0.006997853997745551, - 0.007334743997489568, - 0.007652222004253417, - 0.00799389500025427, - 0.007002913000178523, - 0.00599494599737227, - 0.009026473999256268, - 0.007962283001688775, - 0.004602645996783394, - 0.005408981000073254, - 0.004964742998708971, - 0.007566604996100068, - 0.007427676995575894, - 0.00599004200194031, - 0.006999328004894778, - 0.006000444998790044, - 0.007996278000064194, - 0.006999826000537723, - 0.007991462996869814, - 0.005995084997266531, - 0.006985897998674773, - 0.0079932029984775, - 0.006994638002652209, - 0.007011982997937594, - 0.00697993300127564, - 0.005991565005388111, - 0.0026889430009759963, - 0.0053084090031916276, - 0.0069981710039428435, - 0.006990904999838676, - 0.0069976929953554645, - 0.005715606006560847, - 0.004272741003660485, - 0.002109450004354585, - 0.0038834880033391528, - 0.004177262999291997, - 0.005835606003529392, - 0.00698286199622089, - 0.0069926479991409, - 0.006993545997829642, - 0.002807953998853918, - 0.0005893279958399944, - 0.00045897599920863286, - 0.0004213880019960925, - 0.00041237700497731566, - 0.00042335699981777, - 0.0004186550067970529, - 0.00041174699435941875, - 0.005773154000053182, - 0.0005311920031090267, - 0.0004833470011362806, - 0.00044719599827658385, - 0.0004231450002407655, - 0.003905874997144565, - 0.0005994129969622009, - 0.0005052939959568903, - 0.002591693999420386, - 0.0005083590003778227, - 0.002468833001330495, - 0.0005090209961053915, - 0.0004951070004608482, - 0.00046980100159998983, - 0.00041821200284175575, - 0.0039387159995385446, - 0.0009322629994130693, - 0.00051651599642355, - 0.0005038300005253404, - 0.00043583599472185597, - 0.0004144790000282228, - 0.00041921700176317245, - 0.0004099220022908412, - 0.00041240300197387114, - 0.00048096100363181904, - 0.007387561003270093, - 0.0005983299997751601, - 0.0005283080026856624, - 0.001963545997568872, - 0.0005741909990319982, - 0.007141993999539409, - 0.0006277529973885976, - 0.00045301299542188644, - 0.00043289300083415583, - 0.0019188540027244017, - 0.0005726160015910864, - 0.0004732939996756613, - 0.00042412400216562673, - 0.00040811399958329275, - 0.0004625000001396984, - 0.004709800996351987, - 0.0005536259996006265, - 0.0005343070006347261, - 0.00043224999535596, - 0.0004659269980038516, - 0.000408599000365939, - 0.00040659200021764264, - 0.00040993700531544164, - 0.006057279999367893, - 0.0005822349994559772, - 0.0004609039970091544, - 0.00040949799586087465, - 0.0004043419976369478, - 0.00039931200444698334, - 0.0004072720039403066, - 0.0004003180001745932, - 0.006708965003781486, - 0.0006573680002475157, - 0.0004917670012218878, - 0.00041548500303179026, - 0.00046903300244593993, - 0.0004074559983564541, - 0.0004152370020165108, - 0.00040100800106301904, - 0.006694408002658747, - 0.000575268997636158, - 0.00185005600360455, - 0.0029108989983797073, - 0.000552738994883839, - 0.0004911389987682924, - 0.002028867995250039, - 0.0005613219982478768, - 0.0004457079994608648, - 0.0004344279950601049, - 0.002309224000782706, - 0.0005044799981988035, - 0.0004493369997362606, - 0.0003997289968538098, - 0.0017714400019031018, - 0.0004474279994610697, - 0.00041977400542236865, - 0.0009727130018291064, - 0.0013309179994394071, - 0.0035502489990903996, - 0.0006200239949976094, - 0.0005075209992355667, - 0.0016360040026484057, - 0.0005403649993240833, - 0.0005343800003174692, - 0.0004323570028645918, - 0.00040687400178285316, - 0.0004025980015285313, - 0.0005558319971896708, - 0.0009049720028997399, - 0.0004906250032945536, - 0.0004142539983149618, - 0.0004192689957562834, - 0.00041974399937316775, - 0.00041306600178359076, - 0.0004032020005979575, - 0.006082920997869223, - 0.003009138999914285, - 0.005415140003606211, - 0.0005527059984160587, - 0.0005361250005080365, - 0.0004406360021675937, - 0.0004052619988215156, - 0.00040309900214197114, - 0.0004023200017400086, - 0.0004039550040033646, - 0.000408480002079159, - 0.005514217002200894, - 0.0006598239997401834, - 0.0005578930067713372, - 0.00042913299694191664, - 0.0004306230039219372, - 0.0004104819963686168, - 0.0004157679941272363, - 0.0004003670037491247, - 0.002798542001983151, - 0.0005323520017554983, - 0.00046168299741111696, - 0.0004285969989723526, - 0.0004116160052944906, - 0.00040595600148662925, - 0.0004068890048074536, - 0.00039932900108397007, - 0.0003995480001321994, - 0.00490952000109246, - 0.0005246939981589094, - 0.00044496900227386504, - 0.00041309999505756423, - 0.00041541999962646514, - 0.00040013300167629495, - 0.0004841129994019866, - 0.0004067629997734912, - 0.00423031700483989, - 0.0005893959969398566, - 0.00042528899939497933, - 0.0004067390036652796, - 0.0004058439953951165, - 0.00040919399907579646, - 0.0004030510026495904, - 0.0004124789993511513, - 0.00040713699854677543, - 0.00039996900159167126, - 0.004566217998217326, - 0.0021361300023272634, - 0.0005244850035523996, - 0.00043509600072866306, - 0.00042977100383723155, - 0.0004565010021906346, - 0.00040546200034441426, - 0.00042258499888703227, - 0.0005974849991616793, - 0.005589348002104089, - 0.0005876380018889904, - 0.0005339369963621721, - 0.00042372300231363624, - 0.0004199780014459975, - 0.0004294870013836771, - 0.00042015699727926403, - 0.0004175809954176657, - 0.00043020700104534626, - 0.00046261899842647836, - 0.0014627279961132444, - 0.0006034440011717379, - 0.0004575849961838685, - 0.0004265340030542575, - 0.0004136499992455356, - 0.0004167829974903725, - 0.0004192460037302226, - 0.0004908119954052381, - 0.00042356699850643054, - 0.00043177499901503325, - 0.0004836980006075464, - 0.00043944199569523335, - 0.00042740299977594987, - 0.00046874599502189085, - 0.0004710020002676174, - 0.0005300800039549358, - 0.0004579660017043352, - 0.00042324300011387095, - 0.00042522900184849277, - 0.0004357349971542135, - 0.0004092860035598278, - 0.0004111070011276752, - 0.0004155430069658905, - 0.0004346470013842918, - 0.0004123960025026463, - 0.00040875199920265004, - 0.00042596999992383644, - 0.0004170680040260777, - 0.0004282329973648302, - 0.00042309099808335304, - 0.00042325899994466454, - 0.00043371600622776896, - 0.0004238980036461726, - 0.00047582700062775984, - 0.00044597400119528174, - 0.0004477719994611107, - 0.0005402380047598854, - 0.0004628220049198717, - 0.00044023599912179634, - 0.00042534199747024104, - 0.0004449990010471083, - 0.0004429280015756376, - 0.00046746800217079, - 0.00043287099833833054, - 0.00042421800026204437, - 0.00043831099901581183, - 0.0004215750013827346, - 0.0004366450011730194, - 0.00043000800360459834, - 0.0004246130047249608, - 0.0004930419963784516, - 0.0004677909964811988, - 0.0004474520028452389, - 0.0004917740006931126, - 0.00044018599874107167, - 0.0004221680064802058, - 0.00042257699533365667, - 0.00041337999573443085, - 0.0004219599941279739, - 0.0004477719994611107, - 0.0004418989992700517, - 0.00043986899981973693, - 0.00043716800428228453, - 0.00044093500036979094, - 0.0004216570014250465, - 0.0004331460004323162, - 0.00042302600195398554, - 0.000426780003181193, - 0.00044893199810758233, - 0.00046882099559297785, - 0.0004257589971530251, - 0.000429085994255729, - 0.0004194310022285208, - 0.0004827090015169233, - 0.000490660997456871, - 0.00045354800386121497, - 0.00045328000123845413, - 0.00043312699563102797, - 0.00043986999662593007, - 0.00043444300536066294, - 0.000439362003817223, - 0.00042303800000809133, - 0.00042762900557136163, - 0.0004586790018947795, - 0.00042069800110766664, - 0.0004173199995420873, - 0.0005182169988984242, - 0.0004604069981724024, - 0.00045862099796067923, - 0.0004428590036695823, - 0.0005070309998700395, - 0.0004571080062305555, - 0.00041501499799778685, - 0.00040785000601317734, - 0.0004170319953118451, - 0.0004073210002388805, - 0.003108046999841463, - 0.0005600000004051253, - 0.0013571099989349023, - 0.0004399539975565858, - 0.00041789199894992635, - 0.00041015399619936943, - 0.00046636300248792395, - 0.00046564999502152205, - 0.005148782998730894, - 0.0006061519961804152, - 0.0004445410013431683, - 0.0004168310042587109, - 0.00045888699969509616, - 0.00047143400297500193, - 0.0004239950067130849, - 0.006852272003015969, - 0.0007195939979283139, - 0.0004272370060789399, - 0.0004960359947290272, - 0.0004103210012544878, - 0.0004125740015297197, - 0.0004160110038355924, - 0.00041015399619936943, - 0.00041398200119147077, - 0.00040564400114817545, - 0.005503396001586225, - 0.0005785959947388619, - 0.0004816899963770993, - 0.0004449460029718466, - 0.00041417800093768165, - 0.000410963999456726, - 0.00040774299850454554, - 0.000411877001170069, - 0.00041647600301075727, - 0.007862619000661653, - 0.0006997770033194683, - 0.000733927998226136, - 0.000498016997880768, - 0.0010593979968689382, - 0.0004357619982329197, - 0.00042146000487264246, - 0.0004097150012967177, - 0.0019253489954280667, - 0.0004374920026748441, - 0.0004236780005157925, - 0.0004242489958414808, - 0.00042395800119265914, - 0.0008056680017034523, - 0.0031756120006321, - 0.0005129989949637093, - 0.000457558999187313, - 0.00041165600123349577, - 0.001211383001646027, - 0.0005407130011008121, - 0.005232711002463475, - 0.0006116380027378909, - 0.000691451998136472, - 0.0004932489973725751, - 0.00042578799912007526, - 0.00041931799933081493, - 0.004649107999284752, - 0.0005397220011218451, - 0.0011894779963768087, - 0.00041530499584041536, - 0.0013364710030145943, - 0.0005554640010814182, - 0.0004286760013201274, - 0.0005394399995566346, - 0.0015245600006892346, - 0.0005748359981225803, - 0.00045709999540122226, - 0.0017841480002971366, - 0.00047188200551318005, - 0.000500303998705931, - 0.0004452759967534803, - 0.0008081949999905191, - 0.00044076699850847945, - 0.000550598997506313, - 0.0004410559995449148, - 0.0005076040033600293, - 0.0004271970028639771, - 0.0004131560053792782, - 0.000422930002969224, - 0.00041467000119155273, - 0.0004165639984421432, - 0.0016139729996211827, - 0.000792326005466748, - 0.00042574300459818915, - 0.00041465800313744694, - 0.0012701240048045292, - 0.00243062200024724, - 0.0005630469968309626, - 0.0004313159952289425, - 0.0005132180012878962, - 0.0006384670050465502, - 0.00047497400373686105, - 0.00043742600246332586, - 0.00041409899858990684, - 0.00041179900290444493, - 0.0004848489988944493, - 0.0004216999950585887, - 0.00041995300125563517, - 0.00041185799636878073, - 0.004139583994401619, - 0.0005771170035586692, - 0.00043779899715445936, - 0.00042371499876026064, - 0.00041168200550600886, - 0.0004135109993512742, - 0.00041185400186805055, - 0.000410842003475409, - 0.0005442370020318776, - 0.0005098039982840419, - 0.0005975079984636977, - 0.0008429139998042956, - 0.0008573239974793978, - 0.0010511469954508357, - 0.00044152699410915375, - 0.0011690129977068864, - 0.0004389570021885447, - 0.0004228960024192929, - 0.0004189140017842874, - 0.0004119100049138069, - 0.0004766490019392222, - 0.0004474620000110008, - 0.00041926799895009026, - 0.0025811990053625777, - 0.0015877949990681373, - 0.0004263030059519224, - 0.0004859320033574477, - 0.00048793099995236844, - 0.0004360899984021671, - 0.0004131219975533895, - 0.0004081899969605729, - 0.0011239289960940368, - 0.0016837509974720888, - 0.000965792998613324, - 0.0005848669970873743, - 0.0004203689968562685, - 0.00041159099782817066, - 0.00042084400047315285, - 0.0004135269991820678, - 0.00041655100358184427, - 0.00041046199476113543, - 0.00044154100032756105, - 0.00042176999704679474, - 0.000409067994041834, - 0.0008330809941980988, - 0.0019419130039750598, - 0.000836181003251113, - 0.0004225899974699132, - 0.0004155139977228828, - 0.0004099569996469654, - 0.0004184019999229349, - 0.00041122599941445515, - 0.00041153000347549096, - 0.000413905996538233, - 0.0004082430023117922, - 0.0004658920006477274, - 0.00045406699791783467, - 0.0006168530017021112, - 0.00247904600109905, - 0.0007618520030518994, - 0.003098316003161017, - 0.0005031140026403591, - 0.00042403700354043394, - 0.00041875900205923244, - 0.00041210299968952313, - 0.00040988700493471697, - 0.004289761003747117, - 0.0005467480004881509, - 0.0005185249974601902, - 0.0019665469953906722, - 0.0004568610020214692, - 0.0004811890030396171, - 0.0004582210021908395, - 0.0013440179973258637, - 0.005050351996032987, - 0.004520132999459747, - 0.0019623889966169372, - 0.0017506660005892627, - 0.0014627009950345382, - 0.00070852800126886, - 0.001436575003026519, - 0.001302479999139905, - 0.0005258340024738573, - 0.0005060590046923608, - 0.0006119859972386621, - 0.0004202249983791262, - 0.0009955200002877973, - 0.00046713499614270404, - 0.00048732299910625443, - 0.0004473879962461069, - 0.001970242999959737, - 0.0011343219957780093, - 0.0004310020012781024, - 0.0004224009971949272, - 0.0004336580022936687, - 0.00042989700159523636, - 0.005107106000650674, - 0.005266679996566381, - 0.014999858998635318, - 0.0029539069946622476, - 0.0009996759981731884, - 0.0030267940019257367, - 0.0006770089967176318, - 0.0004419860051712021, - 0.00043163999362150207, - 0.00043454099795781076, - 0.0004373539995867759, - 0.00042621100146789104, - 0.005196728998271283, - 0.0018891189974965528, - 0.0005192009994061664, - 0.00043393299711169675, - 0.0005100010021124035, - 0.00042821300303330645, - 0.006168899999465793, - 0.0079924489982659, - 0.007993428996996954, - 0.007995692998520099, - 0.005022416000429075, - 0.0006168350009829737, - 0.001215545999002643, - 0.0004399110039230436, - 0.0004652360003092326, - 0.0007317639974644408, - 0.0007758569990983233, - 0.005462615998112597, - 0.0005935439985478297, - 0.0009405299933860078, - 0.0028292229981161654, - 0.0005396740016294643, - 0.0004423849968588911, - 0.0005223219995968975, - 0.0004360759994597174, - 0.0005175870028324425, - 0.00043631799780996516, - 0.0004322159948060289, - 0.00042726699757622555, - 0.009150302001216915, - 0.004959778001648374, - 0.0008896770013961941, - 0.0011620540026342496, - 0.00494571500166785, - 0.0022475539954029955, - 0.0005130279969307594, - 0.0023446569975931197, - 0.005256322998320684, - 0.000707118000718765, - 0.0004922619991702959, - 0.0004870949996984564, - 0.000437201997556258, - 0.0005126800024299882, - 0.00042497899994486943, - 0.0004316600025049411, - 0.0013469730038195848, - 0.00047297600394813344, - 0.0004373219999251887, - 0.00042451500485185534, - 0.0004248770055710338, - 0.0004241939968778752, - 0.00042387299617985263, - 0.0004257260006852448, - 0.00042174999543931335, - 0.002728508996369783, - 0.0024525349945179187, - 0.0005246069995337166, - 0.0004461719945538789, - 0.0005158869971637614, - 0.0004650850023608655, - 0.0004528130011749454, - 0.0004327349961386062, - 0.004358765996585134, - 0.0005702009948436171, - 0.0005147740012034774, - 0.0004647050009225495, - 0.000429525003710296, - 0.0004256890024407767, - 0.0004269579949323088, - 0.00042387300345581025, - 0.0004279229979147203, - 0.0026965510041918606, - 0.0027819280003313906, - 0.000564293994102627, - 0.0005910079999011941, - 0.0004541550006251782, - 0.00043634000030579045, - 0.00045075200614519417, - 0.0004393560011521913, - 0.0004264289964339696, - 0.005805537999549415, - 0.0005808950008940883, - 0.0005109639969305135, - 0.0004316600025049411, - 0.0005748839976149611, - 0.0017935489959199913, - 0.001604575001692865, - 0.000501141999848187, - 0.0017770000049495138, - 0.00048509699990972877, - 0.0005314759982866235, - 0.00043602000368991867, - 0.00046626199764432386, - 0.0004497539994190447, - 0.000437351998698432, - 0.00042340700019849464, - 0.0012587379969772883, - 0.0018455810059094802, - 0.0012248679995536804, - 0.00043808000191347674, - 0.0005278749958961271, - 0.0012545339995995164, - 0.00044431499554775655, - 0.00047083299432415515, - 0.00046792899956926703, - 0.0004235050000716001, - 0.00043595900206128135, - 0.0015799789980519563, - 0.0032255670012091286, - 0.0005111659993417561, - 0.0006115830037742853, - 0.0004489229977480136, - 0.00045907699677627534, - 0.0004376779979793355, - 0.0004336929996497929, - 0.0004965469997841865, - 0.0007007320018601604, - 0.001592954999068752, - 0.00044435499876271933, - 0.00042419700184836984, - 0.0004236370004946366, - 0.0004242590002832003, - 0.0004256890024407767, - 0.000417760995333083, - 0.00041889099520631135, - 0.0017392859954270534, - 0.0016034910004236735, - 0.0018899360002251342, - 0.0004939849968650378, - 0.0004367609944893047, - 0.0004321079977671616, - 0.00042632899567252025, - 0.0005060929979663342, - 0.00043147899850737303, - 0.006581768000614829, - 0.0005895180001971312, - 0.0005091959974379279, - 0.0004254300001775846, - 0.00042234900320181623, - 0.00042507699981797487, - 0.00042648900125641376, - 0.00041591099579818547, - 0.006594184000277892, - 0.0006180630007293075, - 0.0005730819975724444, - 0.0004413620044942945, - 0.0004597890001605265, - 0.0004380430036690086, - 0.0004287710034986958, - 0.0004818259985768236, - 0.007139991997973993, - 0.0005716890009352937, - 0.0004581850007525645, - 0.00042797799687832594, - 0.0018880520001403056, - 0.0004938209967804141, - 0.0004627670059562661, - 0.0004350059971329756, - 0.0010306309995939955, - 0.0005094929947517812, - 0.00045845900604035705, - 0.0004341280000517145, - 0.0004233590007061139, - 0.00044363200140651315, - 0.0005052990018157288, - 0.00046369300252990797, - 0.004448856998351403, - 0.0011135280001326464, - 0.00042310099524911493, - 0.0004238260007696226, - 0.0004244290030328557, - 0.0004268500051693991, - 0.0004598829982569441, - 0.0004429479959071614, - 0.00042992499948013574, - 0.004180626005108934, - 0.0006696060008835047, - 0.0004779860028065741, - 0.00045028800377622247, - 0.0004427979947649874, - 0.00052683700050693, - 0.0006816519962740131, - 0.0006394559968612157, - 0.0005079769980511628, - 0.0004631420015357435, - 0.0004276670006220229, - 0.0004263130031176843, - 0.000422443998104427, - 0.0019549459975678474, - 0.00046013700193725526, - 0.0004442950012162328, - 0.0004495590037549846, - 0.0004328479990363121, - 0.00042523500451352447, - 0.0019179650043952279, - 0.002973447997646872, - 0.0005370959988795221, - 0.002821308000420686, - 0.0005983769951853901, - 0.0009628959960537031, - 0.0006365369990817271, - 0.0005064509969088249, - 0.00044259899732423946, - 0.007883489997766446, - 0.0007038080002530478, - 0.00043556199671002105, - 0.00041051099833566695, - 0.0004167380029684864, - 0.00041293600224889815, - 0.00042051399941556156, - 0.0004096630000276491, - 0.008301637004478835, - 0.0006101530016167089, - 0.0005058809983893298, - 0.00048303200310328975, - 0.00043103999632876366, - 0.0004183710043434985, - 0.00041793299897108227, - 0.005567845997575205, - 0.0005826709966640919, - 0.0005374090033001266, - 0.0004286519979359582, - 0.0004947119959979318, - 0.00042985400068573654, - 0.0004251869977451861, - 0.006088528003601823, - 0.000671276000502985, - 0.0004980300000170246, - 0.00042566599586280063, - 0.005146460003743414, - 0.0014868569996906444, - 0.005325068996171467, - 0.005994006001856178, - 0.008004033996257931, - 0.00412221800070256, - 0.007868110995332245, - 0.008021148998523131, - 0.008315480001328979, - 0.0076290029974188656, - 0.007966858000145294, - 0.008115434997307602, - 0.004877728002611548, - 0.006960826998692937, - 0.012390636002237443, - 0.009891097004583571, - 0.006405741994967684, - 0.007270049005455803, - 0.007976983004482463, - 0.006981789003475569, - 0.005996892999974079, - 0.004609898998751305, - 0.006953416006581392, - 0.006693666000501253, - 0.0026925159982056357, - 0.006106196000473574, - 0.0007108600038918667, - 0.0005514509975910187, - 0.0004326539929024875, - 0.00044223000440979376, - 0.0006158249962027185, - 0.0005129519995534793, - 0.0004297129999031313, - 0.005739988999266643, - 0.0005379510039347224, - 0.0006950210008653812, - 0.00046390500210691243, - 0.0004238290057401173, - 0.00042304800444981083, - 0.0005066000012448058, - 0.005497856000147294, - 0.0008519760012859479, - 0.0010510209976928309, - 0.000511669997649733, - 0.0005121729991515167 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil", - "fullname": "TestApplyNabla2AndNabla4ToVn", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.0004973409959347919, - "max": 0.058552956994390115, - "mean": 0.004031990086174464, - "stddev": 0.0048726557169137584, - "rounds": 743, - "median": 0.0020961110058124177, - "iqr": 0.0064202395024040015, - "q1": 0.0005695842482964508, - "q3": 0.006989823750700452, - "iqr_outliers": 15, - "stddev_outliers": 56, - "outliers": "56;15", - "ld15iqr": 0.0004973409959347919, - "hd15iqr": 0.016996696998830885, - "ops": 248.0164828353524, - "total": 2.995768634027627, - "data": [ - 0.006994562005274929, - 0.006991027003095951, - 0.005631523999909405, - 0.0063559890040778555, - 0.002378231001785025, - 0.003608762999647297, - 0.006002092995913699, - 0.005542909995710943, - 0.00445655300427461, - 0.007015672999841627, - 0.007958791000419296, - 0.00699589800206013, - 0.007017026997345965, - 0.004994367998733651, - 0.007973642997967545, - 0.007005079998634756, - 0.002881583000998944, - 0.006096122997405473, - 0.0059965159962303005, - 0.00500260699482169, - 0.005992222002532799, - 0.005994234001263976, - 0.007348270999500528, - 0.00664360399969155, - 0.004607791997841559, - 0.004972246999386698, - 0.005358853995858226, - 0.0050204250001115724, - 0.011986496996541973, - 0.006678450001345482, - 0.004364954998891335, - 0.0029140600017854013, - 0.0019970120047219098, - 0.005040560004999861, - 0.007959331000165548, - 0.008996851996926125, - 0.006984968997130636, - 0.00613956899906043, - 0.008656292004161514, - 0.01590454700635746, - 0.007063761004246771, - 0.004177809001703281, - 0.009028174994455185, - 0.005219322003540583, - 0.036127849998592865, - 0.058552956994390115, - 0.008960809995187446, - 0.004985585997928865, - 0.0029832810032530688, - 0.0006678110003122129, - 0.0005811189985251985, - 0.0005671550024999306, - 0.003948980993300211, - 0.0006183750010677613, - 0.0005771709984401241, - 0.000567188995773904, - 0.0005966619937680662, - 0.0005660599999828264, - 0.0066457790017011575, - 0.0006662870000582188, - 0.0006052800017641857, - 0.0005676740038325079, - 0.0005677000008290634, - 0.0005664149939548224, - 0.0005797109988634475, - 0.008043335998081602, - 0.005994635001115967, - 0.006998886994551867, - 0.006059131002984941, - 0.006931538999197073, - 0.0059968519999529235, - 0.0080002320028143, - 0.0039951189974090084, - 0.006998056000156794, - 0.005812134004372638, - 0.0051825179980369285, - 0.011007768000126816, - 0.00798626100004185, - 0.004740664997370914, - 0.004680704005295411, - 0.0075858799973502755, - 0.00797694699576823, - 0.006997620999754872, - 0.006096311997680459, - 0.00795734699931927, - 0.007936092995805666, - 0.00699226300639566, - 0.006988197004829999, - 0.01100306100124726, - 0.005990965000819415, - 0.006876409002870787, - 0.03130758999759564, - 0.017450975996325724, - 0.006330295997031499, - 0.0072219940047943965, - 0.01576943699910771, - 0.007984333002241328, - 0.00500780400034273, - 0.006980900994676631, - 0.007997835004061926, - 0.007999840003321879, - 0.006988398003159091, - 0.006997960001172032, - 0.007996105996426195, - 0.015007540001533926, - 0.007004330000199843, - 0.0069741999977850355, - 0.007993500999873504, - 0.006993415001488756, - 0.008032603000174277, - 0.005945443001110107, - 0.007989359000930563, - 0.006994648996624164, - 0.007137139997212216, - 0.006857950997073203, - 0.00799287900008494, - 0.006998343000304885, - 0.0069964459980838, - 0.0060244589985813946, - 0.007980123999004718, - 0.006968109999434091, - 0.006996915995841846, - 0.007995526997547131, - 0.00658665900118649, - 0.011409502993046772, - 0.007994179999514017, - 0.00800040900503518, - 0.0059939850034425035, - 0.005995601997710764, - 0.006131230002210941, - 0.006865515999379568, - 0.006994287999987137, - 0.007999980000022333, - 0.006993959999817889, - 0.006380777995218523, - 0.002612044001580216, - 0.005008995998650789, - 0.008997354001621716, - 0.007989359997736756, - 0.006991103000473231, - 0.00800332299695583, - 0.007990974001586437, - 0.005995570005325135, - 0.00802095299877692, - 0.0069759410034748726, - 0.006990785004745703, - 0.005993061997287441, - 0.00599350099946605, - 0.010996153003361542, - 0.006990886002313346, - 0.006996061994868796, - 0.006997555996349547, - 0.006999411001743283, - 0.006006217001413461, - 0.0069850560030317865, - 0.005997750995447859, - 0.005997414999001194, - 0.007997170003363863, - 0.0069976489976397716, - 0.006995525996899232, - 0.007998587003385182, - 0.006996823998633772, - 0.006998470998951234, - 0.007995636995474342, - 0.008006392999959644, - 0.00799943000311032, - 0.00800268400053028, - 0.00998302500374848, - 0.006992570997681469, - 0.0066425340046407655, - 0.00535209499503253, - 0.003996389001258649, - 0.0042673719945014454, - 0.0037253039990901016, - 0.007004428000072949, - 0.006990298999880906, - 0.0070012160067562945, - 0.0069918370063533075, - 0.005489588998898398, - 0.005503510001290124, - 0.005997133004711941, - 0.00526943699514959, - 0.007725327006482985, - 0.008000112997251563, - 0.007995791995199397, - 0.006011361998389475, - 0.005982627997582313, - 0.005993833001411986, - 0.006999349003308453, - 0.00799556100537302, - 0.006998526005190797, - 0.007998672997928225, - 0.008005513001990039, - 0.002831731006153859, - 0.00615058599942131, - 0.006996809002885129, - 0.006998087003012188, - 0.0069961970002623275, - 0.005270785994071048, - 0.006732321999152191, - 0.00799919600103749, - 0.008002187001693528, - 0.007976642998983152, - 0.007000614998105448, - 0.00800067499949364, - 0.0059865930015803315, - 0.007996555999852717, - 0.005996566993417218, - 0.007997609995072708, - 0.008006119001947809, - 0.006980546997510828, - 0.005989805002172943, - 0.010621886001899838, - 0.018283220997545868, - 0.007090318002155982, - 0.005040611002186779, - 0.010044545000710059, - 0.004366619003121741, - 0.007521368999732658, - 0.008138336001138669, - 0.005855260002135765, - 0.006993369999690913, - 0.0014571010033250786, - 0.0005440029999590479, - 0.0007443780050380155, - 0.0050039880006806925, - 0.0006875909966765903, - 0.0008669029994052835, - 0.0005267070009722374, - 0.0005272459966363385, - 0.0005005029961466789, - 0.006421720994694624, - 0.0006751140026608482, - 0.0006437669944716617, - 0.0005111139980726875, - 0.0005005890052416362, - 0.0005044490026193671, - 0.0004973409959347919, - 0.007596393006679136, - 0.0006808690013713203, - 0.0014914749990566634, - 0.0015149629980442114, - 0.0050402800043229945, - 0.008014885002921801, - 0.008029242002521642, - 0.007952508000016678, - 0.005990621000819374, - 0.005988559001707472, - 0.007995218998985365, - 0.005999528999382164, - 0.007991838996531442, - 0.007998150002094917, - 0.00799506000475958, - 0.005997803993523121, - 0.005999279004754499, - 0.008001417001651134, - 0.007991824997588992, - 0.0059952500014333054, - 0.007997194996278267, - 0.010159366000152659, - 0.00785266899765702, - 0.002970162000565324, - 0.008014334001927637, - 0.0070014209995861165, - 0.016996696998830885, - 0.007963983000081498, - 0.0250037260047975, - 0.0179822550053359, - 0.014230025000870228, - 0.012733201001537964, - 0.013998372000060044, - 0.008658115999423899, - 0.01431455399870174, - 0.006975834003242198, - 0.01076965299580479, - 0.01320597300218651, - 0.014987090005888604, - 0.004994940994947683, - 0.01099947199691087, - 0.0070905309985391796, - 0.007609396998304874, - 0.0020978449974791147, - 0.0021085639964439906, - 0.0047299939978984185, - 0.003269556000304874, - 0.0057940119950217195, - 0.0018774370037135668, - 0.002108307999151293, - 0.004217319001327269, - 0.0021085219996166416, - 0.0021071439987281337, - 0.0029124450011295266, - 0.003418195999984164, - 0.004219526002998464, - 0.0031654839986003935, - 0.002716859002248384, - 0.003598735995183233, - 0.0021049779970780946, - 0.002108905006025452, - 0.004224109994538594, - 0.002102717997331638, - 0.002225634998467285, - 0.006212686996150296, - 0.002106374995491933, - 0.0045747160038445145, - 0.001389523000398185, - 0.002471509993483778, - 0.005290199005685281, - 0.0020961110058124177, - 0.008443619000900071, - 0.0019817809952655807, - 0.007179264001024421, - 0.0012037829947075807, - 0.0028717370005324483, - 0.005573643000388984, - 0.00210044900450157, - 0.002103441998769995, - 0.00903404699784005, - 0.0014559760020347312, - 0.008427533000940457, - 0.0010524840035941452, - 0.0077359850038192235, - 0.0018589860046631657, - 0.0016758120036683977, - 0.012041800997394603, - 0.01899078700080281, - 0.027720194004359655, - 0.006334364996291697, - 0.00549884200154338, - 0.02540032000251813, - 0.009992733001126908, - 0.020010331005323678, - 0.023079421000147704, - 0.023889434000011533, - 0.01039563099766383, - 0.013558276994444896, - 0.009949986000719946, - 0.01197211100225104, - 0.0116800410032738, - 0.012587748002260923, - 0.011650362001091707, - 0.0037321749987313524, - 0.0063158340053632855, - 0.011234031000640243, - 0.022748417999537196, - 0.006130968999059405, - 0.005073749001894612, - 0.00676139400457032, - 0.004774894994625356, - 0.004190997999103274, - 0.0069922309994581155, - 0.008450638997601345, - 0.004524202005995903, - 0.013016716002312023, - 0.006959984006243758, - 0.007378674003120977, - 0.002133362999302335, - 0.0010484700032975525, - 0.0005984800009173341, - 0.0005823030005558394, - 0.0006238639980438165, - 0.011739699999452569, - 0.007019801996648312, - 0.000615723998635076, - 0.0006097299992688932, - 0.002353111995034851, - 0.0077823500032536685, - 0.0006231859952094965, - 0.0005966989992884919, - 0.0005898099989281036, - 0.0005809009962831624, - 0.0005857979995198548, - 0.007754672995361034, - 0.01105301600182429, - 0.0007138000000850298, - 0.0005811480004922487, - 0.010033489001216367, - 0.0007318249990930781, - 0.0005879629970877431, - 0.0005682699993485585, - 0.000573734003410209, - 0.000574616999074351, - 0.00828221499978099, - 0.0006667429988738149, - 0.007793806005793158, - 0.0006403600054909475, - 0.0005945459997747093, - 0.0005695209983969107, - 0.0005720370027120225, - 0.0007742450034129433, - 0.0059562939932220615, - 0.0006976700024097227, - 0.0005966149983578362, - 0.000633537994872313, - 0.0005661239993060008, - 0.0005700960027752444, - 0.0005626999991363846, - 0.0005665680000674911, - 0.0031778640041011386, - 0.0006143259961390868, - 0.0005940670016570948, - 0.0006538660018122755, - 0.0006792499989387579, - 0.0020138120016781613, - 0.0022927110039745457, - 0.0006754889982403256, - 0.0005820570004289038, - 0.0005660550013999455, - 0.0005681380062014796, - 0.0005618919967673719, - 0.0061545460048364475, - 0.0006998890021350235, - 0.0007126310010789894, - 0.000561273998755496, - 0.0005676210057572462, - 0.0005472560005728155, - 0.0005428419972304255, - 0.008189828993636183, - 0.0007093989988788962, - 0.0005465979993459769, - 0.0005408459983300418, - 0.0005459189997054636, - 0.0005398880020948127, - 0.006925421999767423, - 0.000613666998106055, - 0.0005708460012101568, - 0.0009373069988214411, - 0.0005472150005516596, - 0.0005479539977386594, - 0.006598647996725049, - 0.0006391880015144125, - 0.00058548100059852, - 0.0005513629948836751, - 0.0005429589946288615, - 0.0005500049956026487, - 0.0005850629968335852, - 0.004457117000129074, - 0.0010175209972658195, - 0.0006670750008197501, - 0.0005732969948439859, - 0.0005665279968525283, - 0.0006016670013195835, - 0.0005905950019950978, - 0.0005795839970232919, - 0.0005622050011879764, - 0.0005670800019288436, - 0.0005563439990510233, - 0.0005734639998991042, - 0.0005666049983119592, - 0.0005569849963649176, - 0.0005592350062215701, - 0.0005963369985693134, - 0.000571995995414909, - 0.0005791600051452406, - 0.0005754220037488267, - 0.0005458380037453026, - 0.0005829950023326091, - 0.0005747769973822869, - 0.0008038329979171976, - 0.0005504259970621206, - 0.000569775998883415, - 0.0005600990043603815, - 0.0005866020001121797, - 0.0005705139992642216, - 0.0005792010051663965, - 0.001112601996283047, - 0.0005519160040421411, - 0.0005645689961966127, - 0.0005772889999207109, - 0.000580451000132598, - 0.0005598409989033826, - 0.0005970389975118451, - 0.0005689669997082092, - 0.00058239400095772, - 0.0005537799952435307, - 0.0005516979945241474, - 0.0005590139990090393, - 0.0005790340001112781, - 0.0005579560020123608, - 0.0005595019974862225, - 0.0005473679993883707, - 0.0005720300032407977, - 0.0005738750041928142, - 0.0005804840038763359, - 0.0005865550047019497, - 0.0005598990028374828, - 0.0005811490045743994, - 0.0005468309973366559, - 0.0005725630035158247, - 0.0005667579971486703, - 0.0005642510004690848, - 0.0005782189982710406, - 0.0005630140003631823, - 0.0005764699963037856, - 0.0006167579995235428, - 0.0005710750047001056, - 0.0005739500047639012, - 0.0005819819998578168, - 0.0005841440070071258, - 0.0005941530034760945, - 0.0005810450020362623, - 0.0006465109981945716, - 0.0005530780035769567, - 0.0005706199954147451, - 0.000555638995137997, - 0.000571333002881147, - 0.0005607689963653684, - 0.000577224993321579, - 0.0005833290051668882, - 0.0005777830010629259, - 0.0005567760017584078, - 0.0005862840043846518, - 0.0005670050013577566, - 0.0006121469996287487, - 0.0005589540014625527, - 0.0005697739979950711, - 0.0005540339989238419, - 0.0005742729990743101, - 0.000587424001423642, - 0.0005680999965989031, - 0.0005539590056287125, - 0.0005508340036612935, - 0.002512952996767126, - 0.0011879960002261214, - 0.000574616999074351, - 0.0005649919985444285, - 0.0005428099975688383, - 0.0005506500019691885, - 0.008185247002984397, - 0.0023080129976733588, - 0.003732064003997948, - 0.0024327330029336736, - 0.0006784640063415281, - 0.0006507860016427003, - 0.0015392209970741533, - 0.0005915770016144961, - 0.007828200999938417, - 0.0007310460059670731, - 0.0006030410004314035, - 0.0005648840015055612, - 0.0005373909953050315, - 0.0005352199950721115, - 0.006533519997901749, - 0.0006881549998070113, - 0.0006167860046843998, - 0.0005911920015932992, - 0.0005435569983092137, - 0.0005340079951565713, - 0.0005288040047162212, - 0.005173612000362482, - 0.0006295589992078021, - 0.0005571460060309619, - 0.0005405549964052625, - 0.0005386359989643097, - 0.0005292050045682117, - 0.004466076999960933, - 0.0009522429973003455, - 0.0005489799950737506, - 0.0005440199965960346, - 0.0005310299966367893, - 0.0005304969963617623, - 0.0005311539935064502, - 0.004294058999221306, - 0.0006288680015131831, - 0.0005898320014239289, - 0.0005580760043812916, - 0.0005312690045684576, - 0.0005295009977999143, - 0.003407015996344853, - 0.001492406998295337, - 0.0005473939963849261, - 0.0005345239987946115, - 0.000538278000021819, - 0.0005288079992169514, - 0.0005324430021573789, - 0.0005279130054987036, - 0.005060841001977678, - 0.0022791139999753796, - 0.0005615479967673309, - 0.0032696449998184107, - 0.000597351994656492, - 0.0005979449997539632, - 0.0005365049946703948, - 0.0005214369957684539, - 0.0005113189981784672, - 0.0005689059980795719, - 0.0007209920004243031, - 0.0022765729954699054, - 0.003544889004842844, - 0.0005884040001546964, - 0.000580479005293455, - 0.0010554420005064458, - 0.0005115059975651093, - 0.0034466939978301525, - 0.000599785998929292, - 0.0006419359997380525, - 0.0005199489969527349, - 0.0005203740001888946, - 0.0005246590008027852, - 0.0005164270041859709, - 0.0007886159946792759, - 0.0012780550023308024, - 0.0032144219949259423, - 0.0006139380056993105, - 0.0005228170048212633, - 0.0005968320037936792, - 0.004073264004546218, - 0.0006728480002493598, - 0.0005487369999173097, - 0.00054050099424785, - 0.0005325090023688972, - 0.0007560360027127899, - 0.00285581099888077, - 0.0011344500017003156, - 0.000529780998476781, - 0.0005301130004227161, - 0.000522490001458209, - 0.000579009996727109, - 0.0005660330061800778, - 0.0005225670029176399, - 0.005106713004352059, - 0.0006233530002646148, - 0.0006123880011728033, - 0.0005885780046810396, - 0.0005560319987125695, - 0.0005223069965722971, - 0.0013014659998589195, - 0.0005878099982510321, - 0.0005331490028765984, - 0.0005280760015011765, - 0.0005347190017346293, - 0.0005376740009523928, - 0.0005419689987320453, - 0.0005330630010575987, - 0.0005364969983929768, - 0.0005500949991983362, - 0.0005612540044239722, - 0.0005780719948234037, - 0.0005270370020298287, - 0.0005593349997070618, - 0.0005318109979270957, - 0.0005475869984366, - 0.0005340900024748407, - 0.0006253120009205304, - 0.0005328859988367185, - 0.0005410969970398583, - 0.0007931200016173534, - 0.000539191001735162, - 0.000554875994566828, - 0.01413342599698808, - 0.004224557000270579, - 0.005993480001052376, - 0.004059097002027556, - 0.004934497999784071, - 0.003993958998762537, - 0.007999226996616926, - 0.00600529999792343, - 0.004268120996130165, - 0.0006563770002685487, - 0.0006273759936448187, - 0.0031590729995514266, - 0.0005753940058639273, - 0.0005240039972704835, - 0.0005334689994924702, - 0.0005131510042701848, - 0.000524314003996551, - 0.00134149199584499, - 0.005637961003230885, - 0.0006635730023845099, - 0.0005832020033267327, - 0.0005257099983282387, - 0.0005277900054352358, - 0.0005267879969323985, - 0.0005181279993848875, - 0.007770302996505052, - 0.0012983019987586886, - 0.009059942000021692, - 0.0006469839936471544, - 0.0006381600032909773, - 0.00114534000022104, - 0.0005323860023054294, - 0.006631276999542024, - 0.0007586640058434568, - 0.0005989049968775362, - 0.000520156005222816, - 0.0005092880019219592, - 0.0007743919995846227, - 0.005243039995548315, - 0.0006580219996976666, - 0.0005906359947402962, - 0.0005489839968504384, - 0.0005430390010587871, - 0.0006195390014909208, - 0.0005389050056692213, - 0.0012339259992586449, - 0.0006620950007345527, - 0.0022131020014057867, - 0.001033159001963213, - 0.0005568870037677698, - 0.0005224140040809289, - 0.0005422760004876181, - 0.0006367109963321127, - 0.0005685759970219806, - 0.0005445510032586753, - 0.0005472199991345406, - 0.0005523649961105548, - 0.0006577230014954694, - 0.0005653769985656254, - 0.0005568039996433072, - 0.0005376879998948425, - 0.0005222899999353103, - 0.0005616639973595738, - 0.0005504540022229776, - 0.0005298140022205189, - 0.0005377979978220537, - 0.0005177830025786534, - 0.0005518839971045963, - 0.0005474729987327009, - 0.0014396709957509302, - 0.000622326995653566, - 0.0010748510030680336, - 0.002318990998901427, - 0.0005807880006614141, - 0.0006406839966075495, - 0.0005502560015884228, - 0.0005239230013103224, - 0.0005453390040202066, - 0.0005319429983501323, - 0.0005269520042929798, - 0.0005280719997244887, - 0.0005554559975280426, - 0.000549183001567144, - 0.0005682109986082651, - 0.0005418770015239716, - 0.0005502429994521663, - 0.0005435720013338141, - 0.0005524000007426366, - 0.0005597569979727268, - 0.0005749510019086301, - 0.0005359470014809631, - 0.0005312889988999814, - 0.000556057995709125, - 0.0005327730032149702 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil", - "fullname": "TestApplyNabla2ToVnInLateralBoundary", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.0014347019969136454, - "max": 0.06811486399965361, - "mean": 0.007994707482992896, - "stddev": 0.0056226463559909965, - "rounds": 238, - "median": 0.00699729499683599, - "iqr": 0.0019951159993070178, - "q1": 0.006001934998494107, - "q3": 0.007997050997801125, - "iqr_outliers": 23, - "stddev_outliers": 12, - "outliers": "12;23", - "ld15iqr": 0.003037618997041136, - "hd15iqr": 0.010994338001182768, - "ops": 125.08275032292242, - "total": 1.9027403809523094, - "data": [ - 0.007985402997292113, - 0.009993523002776783, - 0.006993151000642683, - 0.011007699999026954, - 0.007988115001353435, - 0.010104262000822928, - 0.00588547299412312, - 0.017939395002031233, - 0.004087536995939445, - 0.009966363999410532, - 0.007978166002430953, - 0.008001551999768708, - 0.01699503899726551, - 0.0070393279966083355, - 0.007955715002026409, - 0.010978432001138572, - 0.006075367004086729, - 0.006911040996783413, - 0.006999073004408274, - 0.008991479997348506, - 0.007994654995854944, - 0.006995906995143741, - 0.004193683002085891, - 0.007797225996910129, - 0.005993325998133514, - 0.006994330004090443, - 0.005498847000126261, - 0.006495075002021622, - 0.0037655419946531765, - 0.013229272000899073, - 0.006997770004090853, - 0.008002462003787514, - 0.010994338001182768, - 0.005994423001538962, - 0.017082939004467335, - 0.006908132003445644, - 0.005992974998662248, - 0.006996201002039015, - 0.00699418100703042, - 0.0069998969993321225, - 0.004995508999854792, - 0.005997168002068065, - 0.007998900997336023, - 0.009996009001042694, - 0.011038135999115184, - 0.007950437000545207, - 0.005932811000093352, - 0.0070504219984286465, - 0.007995973996003158, - 0.003260641999077052, - 0.0037316320012905635, - 0.0059993389950250275, - 0.0059965469990856946, - 0.005659041002218146, - 0.008349645999260247, - 0.004736792005132884, - 0.006713797993143089, - 0.006535526998050045, - 0.011990901999524795, - 0.006977835997531656, - 0.009313091999501921, - 0.012312163002206944, - 0.005355577006412204, - 0.005107101998873986, - 0.006863142996735405, - 0.006682758998067584, - 0.004310420001274906, - 0.005995553998218384, - 0.007003457998507656, - 0.05503793800016865, - 0.06811486399965361, - 0.008852195998770185, - 0.010968752998451237, - 0.008400856000662316, - 0.005572589005168993, - 0.003037618997041136, - 0.02095632700365968, - 0.007947438003611751, - 0.009032950998516753, - 0.006941204999748152, - 0.007013998998445459, - 0.008982072002254426, - 0.008060260995989665, - 0.007960192997416016, - 0.005961650997051038, - 0.003999249995104037, - 0.016008259997761343, - 0.01999303000047803, - 0.013994057997479104, - 0.007978702000400517, - 0.006037437997292727, - 0.008980914004496299, - 0.007952906998980325, - 0.007976846995006781, - 0.008294725994346663, - 0.00470622000284493, - 0.005990377001580782, - 0.007997360000445042, - 0.006000409004627727, - 0.007992175000254065, - 0.006002628004353028, - 0.011148650999530219, - 0.0068407799990382046, - 0.007990664002136327, - 0.0079964190008468, - 0.005442189001769293, - 0.006555268999363761, - 0.00699679500394268, - 0.006994550996751059, - 0.0069899149966659024, - 0.007992620994627941, - 0.006999113997153472, - 0.006006253999657929, - 0.007142312002542894, - 0.007843214996682946, - 0.007996499000000767, - 0.007000899000559002, - 0.007993958002771251, - 0.006207585000083782, - 0.007787083995935973, - 0.0069963549976819195, - 0.006996185002208222, - 0.005993177997879684, - 0.008004693998373114, - 0.007014021997747477, - 0.01798247799888486, - 0.007984834999660961, - 0.007990982994670048, - 0.0060132090002298355, - 0.005519532001926564, - 0.0014347019969136454, - 0.0049982310010818765, - 0.01505067000107374, - 0.00695005899615353, - 0.005983446004393045, - 0.008039501997700427, - 0.007955818000482395, - 0.007983948002220131, - 0.006989805006014649, - 0.006997409000177868, - 0.009184009999444243, - 0.0068138200003886595, - 0.005995150997478049, - 0.006991264002863318, - 0.010997356002917513, - 0.009987599994929042, - 0.007001787998888176, - 0.004988920998584945, - 0.009997007997299079, - 0.006017835999955423, - 0.007984791998751462, - 0.007003424005233683, - 0.005977941997116432, - 0.007310572000278626, - 0.006682944993372075, - 0.006009391996485647, - 0.007973220002895687, - 0.00799451499915449, - 0.008037209998292383, - 0.004954550000547897, - 0.007997738997801207, - 0.005996311992930714, - 0.006001934998494107, - 0.00599336699815467, - 0.006997180993494112, - 0.0069964270005584694, - 0.006997755001066253, - 0.007002410005952697, - 0.005993118000333197, - 0.004996421994292177, - 0.006911958000273444, - 0.0060834390023956075, - 0.006996290998358745, - 0.007994961000804324, - 0.005997373998980038, - 0.007997118998900987, - 0.006999441000516526, - 0.006008676995406859, - 0.0065792899986263365, - 0.005404704003012739, - 0.005996188003337011, - 0.007998098997632042, - 0.007992544000444468, - 0.0059919799969065934, - 0.005996715997753199, - 0.006997909003985114, - 0.0069976430022506975, - 0.006997632997808978, - 0.00499684799433453, - 0.007997050997801125, - 0.006882517001940869, - 0.005112959006510209, - 0.00893375500163529, - 0.010060048000013921, - 0.006996613003138918, - 0.007996640000783373, - 0.005996913998387754, - 0.007997116998012643, - 0.005998016997182276, - 0.00499765499989735, - 0.006014407998009119, - 0.006984005005506333, - 0.007983249000972137, - 0.006995992996962741, - 0.01254787200014107, - 0.005462999004521407, - 0.006959405000088736, - 0.0070300520019372925, - 0.007980376998602878, - 0.005969108002318535, - 0.006987649998336565, - 0.006986774998949841, - 0.007999034998647403, - 0.006994611001573503, - 0.006995044001087081, - 0.006994526003836654, - 0.00699555900064297, - 0.007997565000550821, - 0.0069943339985911734, - 0.006995138006459456, - 0.006012899000779726, - 0.0069876639972790144, - 0.007989933001226746, - 0.00799363699479727, - 0.006994294002652168, - 0.006990702997427434, - 0.00801062900427496, - 0.010997422999935225, - 0.00597184900107095, - 0.004974215000402182, - 0.010446911001054104, - 0.01055090400041081, - 0.007102935000148136, - 0.003163291999953799, - 0.00771810000151163, - 0.005989331999444403, - 0.002988249994814396, - 0.008007942000404 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil", - "fullname": "TestMoApplyNabla2ToW", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.0009703689938760363, - "max": 0.021006240000133403, - "mean": 0.006861762038649382, - "stddev": 0.0020891229171211413, - "rounds": 334, - "median": 0.006995611500315135, - "iqr": 0.002003421999688726, - "q1": 0.005990964004013222, - "q3": 0.007994386003701948, - "iqr_outliers": 23, - "stddev_outliers": 68, - "outliers": "68;23", - "ld15iqr": 0.0030023319995962083, - "hd15iqr": 0.01100079799653031, - "ops": 145.73516166364064, - "total": 2.2918285209088936, - "data": [ - 0.0020699829983641393, - 0.005125960997247603, - 0.01103598700137809, - 0.00597243600350339, - 0.00495679400046356, - 0.008002058006240986, - 0.006991759000811726, - 0.0061907429990242235, - 0.0057985839957837015, - 0.007993293002073187, - 0.007000350997259375, - 0.0059933929951512255, - 0.007997305001481436, - 0.00600098699942464, - 0.010864024996408261, - 0.006125586005509831, - 0.0049870590009959415, - 0.001910188999318052, - 0.0051974180023535155, - 0.006882449000841007, - 0.009987735000322573, - 0.00799688199913362, - 0.007008653003140353, - 0.008781401003943756, - 0.006183436002174858, - 0.00394815700565232, - 0.00503611599560827, - 0.004990233006537892, - 0.008008311000594404, - 0.007987794997461606, - 0.008993139999802224, - 0.0069913589977659285, - 0.006989465000515338, - 0.008002759001101367, - 0.007987589000549633, - 0.008991048998723272, - 0.005990917998133227, - 0.007984951000253204, - 0.008012913996935822, - 0.007966632998432033, - 0.007977642999321688, - 0.007990518999577034, - 0.007998100001714192, - 0.0069951670011505485, - 0.007011786998191383, - 0.0072359580008196644, - 0.012399139995977748, - 0.0023287209987756796, - 0.003989844000898302, - 0.008650486000988167, - 0.007358834001934156, - 0.007991131002199836, - 0.00697333000425715, - 0.00698862099670805, - 0.006994228999246843, - 0.006994686998950783, - 0.008002663998922799, - 0.004993379996449221, - 0.007995403997483663, - 0.006999040000664536, - 0.007408530000247993, - 0.007607285006088205, - 0.006998376004048623, - 0.006953311996767297, - 0.007209711999166757, - 0.008777991999522783, - 0.00813713399838889, - 0.00784561299951747, - 0.006994453004153911, - 0.006994637995376252, - 0.01108317499893019, - 0.00790803999552736, - 0.0069804040031158365, - 0.006941988998732995, - 0.008074920995568391, - 0.006966371998714749, - 0.006756334005331155, - 0.00824371600174345, - 0.005978256005619187, - 0.008788608000031672, - 0.0037989010015735403, - 0.0053950529982103035, - 0.004982058999303263, - 0.006995610005105846, - 0.007996211003046483, - 0.0069969579999451526, - 0.0070170199978747405, - 0.009392781001224648, - 0.006496939000498969, - 0.006994739000219852, - 0.006978861994866747, - 0.00699431799876038, - 0.006996819000050891, - 0.006997039003181271, - 0.00740686799690593, - 0.0036245709998183884, - 0.006951355004275683, - 0.00699691400222946, - 0.007994386003701948, - 0.008000608999282122, - 0.005993888997181784, - 0.013136133005900774, - 0.007883478996518534, - 0.006951939001737628, - 0.004985977000615094, - 0.007673785999941174, - 0.0015678319978178479, - 0.009760387001733761, - 0.007994431995030027, - 0.005974911000521388, - 0.008007639000425115, - 0.005014747002860531, - 0.007946099001856055, - 0.008746201994654257, - 0.004560171000775881, - 0.00666371999977855, - 0.00599760399927618, - 0.00699539700144669, - 0.0070236469982774, - 0.005001117999199778, - 0.00796632599667646, - 0.014001057999848854, - 0.0012907709970022552, - 0.006676580997009296, - 0.006997039003181271, - 0.007994020001206081, - 0.005064021999714896, - 0.007877782001742162, - 0.0066687829967122525, - 0.0073690680001163855, - 0.01100079799653031, - 0.010984288004692644, - 0.010088067996548489, - 0.008876530002453364, - 0.0069759510006406344, - 0.011997080997389276, - 0.007981704002304468, - 0.009971826999390032, - 0.001979239998036064, - 0.007014899994828738, - 0.009160591995168943, - 0.005819821002660319, - 0.021006240000133403, - 0.0009703689938760363, - 0.010017872999014799, - 0.010981346997141372, - 0.004967597997165285, - 0.004140518001804594, - 0.0028372080050758086, - 0.008011613004782703, - 0.00498796500323806, - 0.005990964004013222, - 0.009640801996283699, - 0.009660000003350433, - 0.004676138996728696, - 0.007950093000545166, - 0.006024508002155926, - 0.006196510003064759, - 0.0027794580018962733, - 0.007006004998402204, - 0.005988871998852119, - 0.003986639996583108, - 0.0039700580018688925, - 0.007030469001620077, - 0.007989574005478062, - 0.00798488599684788, - 0.007995965002919547, - 0.004991765999875497, - 0.004995356997824274, - 0.005131623001943808, - 0.0068727379984920844, - 0.0060019259981345385, - 0.005989239005430136, - 0.0049831789947347715, - 0.006993396003963426, - 0.005209337003179826, - 0.007787030997860711, - 0.008326301998749841, - 0.007666151002922561, - 0.004993659000319894, - 0.007009936998656485, - 0.005983711998851504, - 0.010998246994859073, - 0.0059859050015802495, - 0.009996537999541033, - 0.006996609998168424, - 0.008024490998650435, - 0.006971932998567354, - 0.008023210997635033, - 0.007961825001984835, - 0.005995570005325135, - 0.005004102997190785, - 0.006992170005105436, - 0.008002368995221332, - 0.007001261998084374, - 0.005986887998005841, - 0.005999547000101302, - 0.006993709001108073, - 0.007998448003490921, - 0.006999278994044289, - 0.008997057993838098, - 0.00999754799704533, - 0.007996139996976126, - 0.008006613999896217, - 0.006691567999951076, - 0.008279287998448126, - 0.007994034996954724, - 0.007018549003987573, - 0.0069783080034540035, - 0.00599385799432639, - 0.005005571998481173, - 0.006987779001065064, - 0.007000140998570714, - 0.005990746998577379, - 0.00821197599725565, - 0.006777450995286927, - 0.006985437998082489, - 0.002811417005432304, - 0.005181141001230571, - 0.006000426998070907, - 0.005990994002786465, - 0.004995759001758415, - 0.006997755001066253, - 0.008001039001101162, - 0.006993538001552224, - 0.0069967329982318915, - 0.007997317996341735, - 0.004904822999378666, - 0.004093683994142339, - 0.005994745006319135, - 0.006998697994276881, - 0.007004311999480706, - 0.006990019996010233, - 0.006996629999775905, - 0.007996273998287506, - 0.005996804997266736, - 0.0089968729953398, - 0.006995612995524425, - 0.00699684199935291, - 0.00699750999774551, - 0.007999221998034045, - 0.005995888997858856, - 0.007996856002137065, - 0.005997382999339607, - 0.006996486001298763, - 0.005998203996568918, - 0.004000594002718572, - 0.0049947619991144165, - 0.006995849005761556, - 0.00799906699830899, - 0.00699681799596874, - 0.007016727002337575, - 0.007976310000231024, - 0.00799772099708207, - 0.0039946670003701, - 0.005995392995828297, - 0.0039975589970708825, - 0.005999269000312779, - 0.0069945180002832785, - 0.007002158999966923, - 0.005999843000608962, - 0.008004167997569311, - 0.006983069994021207, - 0.007014406000962481, - 0.007979402995260898, - 0.005997790998662822, - 0.008003725000889972, - 0.0048404460030724294, - 0.007379562004643958, - 0.006764685997040942, - 0.008000237998203374, - 0.0030023319995962083, - 0.003801975995884277, - 0.001857725997979287, - 0.00632067299738992, - 0.00802090499928454, - 0.007988252000359353, - 0.008981498998764437, - 0.007996735999768134, - 0.00599702700128546, - 0.004998782002076041, - 0.0019948970002587885, - 0.007997983004315756, - 0.0059978040007990785, - 0.008003659997484647, - 0.007991311998921447, - 0.007996530999662355, - 0.005998098000418395, - 0.00800464899657527, - 0.0017265799979213625, - 0.0042590509983710945, - 0.005994473001919687, - 0.007008109001617413, - 0.005237605000729673, - 0.003742715998669155, - 0.00801198699628003, - 0.007985470998391975, - 0.005994323997583706, - 0.006096229997638147, - 0.007896581999375485, - 0.006995060000917874, - 0.00699703699501697, - 0.007993651997821871, - 0.006990312998823356, - 0.008003432994883042, - 0.006994115996349137, - 0.008009438002773095, - 0.006398858000466134, - 0.0025703540013637394, - 0.005995221006742213, - 0.004995772003894672, - 0.003977878004661761, - 0.006010031000187155, - 0.007031674998870585, - 0.010950953997962642, - 0.01041783800610574, - 0.0075691789970733225, - 0.006994262999796774, - 0.00699684199935291, - 0.0034278210005140863, - 0.0035554559945012443, - 0.006008786003803834, - 0.007727452997642104, - 0.0031647169962525368, - 0.0030798450025031343, - 0.006997014999797102, - 0.007008478001807816, - 0.004407599997648504, - 0.006615455000428483, - 0.007940900999528822, - 0.01201732299523428, - 0.004988779000996146, - 0.004989982997358311, - 0.005593310001131613 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil", - "fullname": "TestApplyNabla2ToWInUpperDampingLayer", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.00034094200236722827, - "max": 0.06015052899601869, - "mean": 0.0021926930991410095, - "stddev": 0.0031025214623593444, - "rounds": 2441, - "median": 0.00039833600021665916, - "iqr": 0.0036502199982351158, - "q1": 0.0003543577495292993, - "q3": 0.004004577747764415, - "iqr_outliers": 32, - "stddev_outliers": 490, - "outliers": "490;32", - "ld15iqr": 0.00034094200236722827, - "hd15iqr": 0.009606883002561517, - "ops": 456.0601756770025, - "total": 5.352363855003205, - "data": [ - 0.0005411559977801517, - 0.0004379070014692843, - 0.0004158369993092492, - 0.00041172000055667013, - 0.00040860399894881994, - 0.0003909480001311749, - 0.003540161000273656, - 0.0004282590016373433, - 0.00040564400114817545, - 0.0003995379956904799, - 0.00040284600254381076, - 0.00039712600118946284, - 0.0003914660046575591, - 0.00040854999679140747, - 0.0003998690008302219, - 0.0007878050018916838, - 0.0010405959983472712, - 0.0008402099992963485, - 0.0007076710026012734, - 0.0014143730004434474, - 0.0004935909964842722, - 0.001058183996065054, - 0.0004827469965675846, - 0.0004115449992241338, - 0.0004097259952686727, - 0.00039707300311420113, - 0.0014068599994061515, - 0.0009175329978461377, - 0.0013126159974490292, - 0.001836062001530081, - 0.0004544020048342645, - 0.00040832500235410407, - 0.0004041939973831177, - 0.0003986140000051819, - 0.00040151300345314667, - 0.0003956849977839738, - 0.00168190999829676, - 0.0007965400000102818, - 0.0011193160025868565, - 0.0005526740060304292, - 0.0006730120003339835, - 0.0004512049999902956, - 0.00040472399996360764, - 0.0003939259986509569, - 0.0004101550002815202, - 0.0003995200022473, - 0.0006469809959526174, - 0.000494239000545349, - 0.0004215649969410151, - 0.00039630199898965657, - 0.0003989080069004558, - 0.000402379002480302, - 0.0003975430008722469, - 0.00040086000080918893, - 0.00039493200165452436, - 0.0012439359998097643, - 0.0013363360048970208, - 0.0007080860013957135, - 0.003115052000794094, - 0.006996263997280039, - 0.007997303997399285, - 0.010999758000252768, - 0.006995121999352705, - 0.005995307001285255, - 0.007994952000444755, - 0.006996322001214139, - 0.009997727000154555, - 0.007996635999006685, - 0.009996517997933552, - 0.006998046999797225, - 0.007995448002475314, - 0.01100158200279111, - 0.00899280900193844, - 0.00799754499894334, - 0.014000007999129593, - 0.007995938998647034, - 0.005001682999136392, - 0.003989017001003958, - 0.01009269300266169, - 0.008902678004233167, - 0.009033501999510918, - 0.007963422998727765, - 0.009019022996653803, - 0.006980575999477878, - 0.007991559003130533, - 0.003989208002167288, - 0.005995318999339361, - 0.004068549002113286, - 0.007993854000233114, - 0.007929022998723667, - 0.0029945130008854903, - 0.00535256299917819, - 0.006648150003456976, - 0.003984513001341838, - 0.005998374996124767, - 0.007002559999818914, - 0.007990427999175154, - 0.007171101002313662, - 0.007823662999726366, - 0.006995711002673488, - 0.00599547599995276, - 0.006005564995575696, - 0.007991526996192988, - 0.00678035899909446, - 0.008212870998249855, - 0.007996029002242722, - 0.007998510998731945, - 0.006995755997195374, - 0.005997521002427675, - 0.0070277670020004734, - 0.006961165003303904, - 0.00865759199950844, - 0.0031672359982621856, - 0.004165427999396343, - 0.0059918219994870014, - 0.006996108000748791, - 0.005001224999432452, - 0.00799044499581214, - 0.0060003550024703145, - 0.00513321199832717, - 0.005854861999978311, - 0.005999951004923787, - 0.006994654999289196, - 0.004460541997104883, - 0.007548732006398495, - 0.004988501998013817, - 0.0029897300046286546, - 0.005995811996399425, - 0.005188317001739051, - 0.007810492999851704, - 0.007644536999578122, - 0.007341137999901548, - 0.007998154003871605, - 0.006002164001984056, - 0.009996644999773707, - 0.006125032996351365, - 0.007866895000915974, - 0.004077960002177861, - 0.007920418996945955, - 0.008203846999094822, - 0.007010852998064365, - 0.004775208995852154, - 0.00713431800249964, - 0.0068605670021497644, - 0.007997358996362891, - 0.007998571003554389, - 0.004997371994249988, - 0.0060000860030413605, - 0.00431641899922397, - 0.0066762860005837865, - 0.006064502005756367, - 0.010934410995105281, - 0.0010408800008008257, - 0.0003596460010157898, - 0.00035353000566828996, - 0.00034915399737656116, - 0.0003478279977571219, - 0.0003532939954311587, - 0.00038146600127220154, - 0.00048712499847169966, - 0.0004194110006210394, - 0.002181835996452719, - 0.00036446299782255664, - 0.0003473730030236766, - 0.0003479129954939708, - 0.0003474299955996685, - 0.0031911740006762557, - 0.000382637997972779, - 0.00035884699900634587, - 0.0003529169989633374, - 0.0003487069989205338, - 0.0003454289981164038, - 0.0038692029993399046, - 0.00037083300412632525, - 0.00036748599814018235, - 0.0003495980054140091, - 0.00034943499485962093, - 0.0003483569962554611, - 0.005068569000286516, - 0.00042650900286389515, - 0.0012085149937774986, - 0.0006144520011730492, - 0.000364705003448762, - 0.0003457400016486645, - 0.00034406000486342236, - 0.0010237000024062581, - 0.00035291600215714425, - 0.0003429490025155246, - 0.00034330800554016605, - 0.0005819299985887483, - 0.0025864359995466657, - 0.003001933997438755, - 0.003425938994041644, - 0.0065645270005916245, - 0.009996055006922688, - 0.00700181900174357, - 0.007993805993464775, - 0.006997368000156712, - 0.0059971549999318086, - 0.003997384003014304, - 0.0069973819990991615, - 0.004995205999875907, - 0.005997371998091694, - 0.007998293993296102, - 0.005997522996040061, - 0.007997817003342789, - 0.00999705700087361, - 0.004006598006526474, - 0.0011149040001328103, - 0.00038720699376426637, - 0.00039803999970899895, - 0.00040459899901179597, - 0.00039600199670530856, - 0.00038597000093432143, - 0.00039331500011030585, - 0.00038381400372600183, - 0.0004003430003649555, - 0.0003850450011668727, - 0.00038521099486388266, - 0.00040283400448970497, - 0.0004040200001327321, - 0.00038252599915722385, - 0.0004163960038567893, - 0.0003902630051015876, - 0.000394492999475915, - 0.00038967200089246035, - 0.0003867229970637709, - 0.0004117310018045828, - 0.00037303799763321877, - 0.0003984099967055954, - 0.0003947410004911944, - 0.00038214699452510104, - 0.0003968599994550459, - 0.0003808960027527064, - 0.0004099810030311346, - 0.0004013560028397478, - 0.0003786469969782047, - 0.0003845769970212132, - 0.000407414001529105, - 0.00039802800165489316, - 0.00040605599497212097, - 0.00037617799534928054, - 0.0003878499992424622, - 0.0004045200039399788, - 0.0004031289936392568, - 0.00038764400233048946, - 0.0003906049969373271, - 0.00039203499909490347, - 0.0004051529977004975, - 0.00039316499896813184, - 0.00038950800080783665, - 0.0003888569990522228, - 0.0003933409971068613, - 0.00040060700121102855, - 0.0003855460017803125, - 0.00038752200634917244, - 0.0003885360056301579, - 0.0003854349997709505, - 0.0003897260030498728, - 0.0004013249999843538, - 0.0003970789985032752, - 0.00037880199670325965, - 0.00038197499816305935, - 0.00038887400296516716, - 0.00038614200457232073, - 0.00037863000034121796, - 0.0003721030007000081, - 0.0003821119971689768, - 0.0003813399962382391, - 0.0004298730054870248, - 0.0003942969997297041, - 0.00040110300324158743, - 0.0003861639997921884, - 0.0003982400012318976, - 0.0004034449957543984, - 0.00039730000571580604, - 0.0003892110034939833, - 0.0003765769943129271, - 0.000376800999219995, - 0.0003884410034515895, - 0.0003850919965771027, - 0.0003800839986070059, - 0.0003845120008918457, - 0.00037778299883939326, - 0.0003924569973605685, - 0.00040352500218432397, - 0.00039630800165468827, - 0.0003993900027126074, - 0.00040568800613982603, - 0.00039173500408651307, - 0.00039788000140106305, - 0.00038676500116707757, - 0.00039416200161213055, - 0.0007804769993526861, - 0.0007944090029923245, - 0.00038848300027893856, - 0.00037695599894504994, - 0.00037714800419053063, - 0.00039572800596943125, - 0.0004095030017197132, - 0.0004036129976157099, - 0.0003903110045939684, - 0.0005734999940614216, - 0.00039346500125247985, - 0.0004175079957349226, - 0.0004208270038361661, - 0.0004120009980397299, - 0.00041327899816678837, - 0.00040647100104251876, - 0.0003931899991584942, - 0.00040074500429909676, - 0.0004176090005785227, - 0.00040841200097929686, - 0.00039833600021665916, - 0.0004955990007147193, - 0.0004165819991612807, - 0.00039370199374388903, - 0.00039974199899006635, - 0.000413970003137365, - 0.0004187659942544997, - 0.0004949479989591055, - 0.0004994260016246699, - 0.0004582840047078207, - 0.00039359099901048467, - 0.00038795400178059936, - 0.0004031650023534894, - 0.0003933850020985119, - 0.00039114499668357894, - 0.0003886580016114749, - 0.0003934400010621175, - 0.0004162969999015331, - 0.00039913799992064014, - 0.00039103200106183067, - 0.00040920800529420376, - 0.00041564700222807005, - 0.00039710000419290736, - 0.0025135270043392666, - 0.0055895879995659925, - 0.005997521999233868, - 0.005998327993438579, - 0.004997351999918465, - 0.007002502003160771, - 0.006995870004175231, - 0.007996983993507456, - 0.004994830000214279, - 0.005997405998641625, - 0.0036900529958074912, - 0.006305380993580911, - 0.007020491000730544, - 0.008992547998786904, - 0.005993673999910243, - 0.006985343999986071, - 0.007981522001500707, - 0.005997611006023362, - 0.004993902002752293, - 0.00999731000047177, - 0.007001680998655502, - 0.005992992999381386, - 0.0050062439986504614, - 0.0035631330028991215, - 0.007420472000376321, - 0.004992929003492463, - 0.004994408001948614, - 0.006000332999974489, - 0.006991811998886988, - 0.008017946005566046, - 0.006977693003136665, - 0.008011865997104906, - 0.006407318003766704, - 0.007569207999040373, - 0.0010017390013672411, - 0.007992227998329327, - 0.0059973559982609, - 0.00799736400222173, - 0.004997377000108827, - 0.006999835000897292, - 0.009996278000471648, - 0.008069071998761501, - 0.006918756997038145, - 0.007997790999070276, - 0.006415179996110965, - 0.006576339001185261, - 0.005806455999845639, - 0.004188886996416841, - 0.004501281000557356, - 0.005492238997248933, - 0.007997355998668354, - 0.007999670000572223, - 0.006995502000791021, - 0.01116645799629623, - 0.003637169997091405, - 0.004880289998254739, - 0.005302905003190972, - 0.006999067998549435, - 0.007238728998345323, - 0.007746676994429436, - 0.005995544001052622, - 0.006998229997407179, - 0.004494775006605778, - 0.007141464993765112, - 0.00935454000136815, - 0.006999337994784582, - 0.007993307001015637, - 0.0070009269984439015, - 0.00699446599901421, - 0.007998390996363014, - 0.006994274001044687, - 0.006996156997047365, - 0.006997825003054459, - 0.00599722199694952, - 0.004051689000334591, - 0.003943875999539159, - 0.005997849002596922, - 0.006996585005254019, - 0.00799813200137578, - 0.00799615200230619, - 0.006995946998358704, - 0.002999290998559445, - 0.007995854000910185, - 0.008001563997822814, - 0.007993553001142573, - 0.005823287996463478, - 0.006169626998598687, - 0.005998186003125738, - 0.00799630399706075, - 0.005998742002702784, - 0.003995577993919142, - 0.007021277000603732, - 0.002973866998218, - 0.0027280549984425306, - 0.004268028002115898, - 0.005993921004119329, - 0.005999795001116581, - 0.007994076004251838, - 0.006998180004302412, - 0.005997364998620469, - 0.005998619999445509, - 0.0079962469972088, - 0.005997631000354886, - 0.006996746000368148, - 0.00800720199913485, - 0.006986793006944936, - 0.0036289919953560457, - 0.0073658029941725545, - 0.007633384993823711, - 0.009363031000248156, - 0.0070058919955044985, - 0.007986076998349745, - 0.00699637000070652, - 0.00799719199858373, - 0.006005777002428658, - 0.005995751998852938, - 0.007985322998138145, - 0.006005907998769544, - 0.005986224001389928, - 0.006996495001658332, - 0.006751615997927729, - 0.007241160994453821, - 0.007770397001877427, - 0.006357594000292011, - 0.00786371999856783, - 0.0069961659974069335, - 0.008034922000661027, - 0.007960393995745108, - 0.005155968996405136, - 0.007672373001696542, - 0.009166341995296534, - 0.006993841001531109, - 0.0023040259984554723, - 0.0004017989995190874, - 0.0004234410007484257, - 0.0004078150013810955, - 0.00036882900167256594, - 0.0003522240003803745, - 0.00034786900505423546, - 0.00034724699798971415, - 0.002994342998135835, - 0.00042870300239883363, - 0.000405013001000043, - 0.00039059999835444614, - 0.00039049900078680366, - 0.0003449589930824004, - 0.00035683299938682467, - 0.0003462149979895912, - 0.00034702000266406685, - 0.002323899003386032, - 0.0003802409992204048, - 0.00042605099588399753, - 0.00040007800271268934, - 0.0003514019990689121, - 0.00035565199505072087, - 0.00034301800042157993, - 0.000343592997523956, - 0.000350327005435247, - 0.0003431369987083599, - 0.00034452199906809255, - 0.005287301006319467, - 0.00043911200191359967, - 0.00045501499698730186, - 0.0004360029997769743, - 0.0003466560010565445, - 0.00035493399627739564, - 0.00034573900484247133, - 0.000396482995711267, - 0.0003442539964453317, - 0.00034626200067577884, - 0.0030752980019315146, - 0.0005936890011071227, - 0.00040792799700284377, - 0.0003646650002337992, - 0.00035766999644692987, - 0.00036494700179900974, - 0.00036370899761095643, - 0.00036709900450659916, - 0.0003595569942262955, - 0.00036177199945086613, - 0.00034111800050595775, - 0.00042384500557091087, - 0.000344956002663821, - 0.00034673399932216853, - 0.00037649600562872365, - 0.0003565379956853576, - 0.00034820700238924474, - 0.00336743699881481, - 0.0004436409944901243, - 0.0005548799963435158, - 0.0004041279971715994, - 0.00034740300179691985, - 0.00037268899905029684, - 0.0003507049987092614, - 0.0003744179994100705, - 0.00034496799344196916, - 0.0003461939995759167, - 0.00397121599962702, - 0.0004572359976009466, - 0.00041921899537555873, - 0.00035355400177650154, - 0.0003472900061751716, - 0.00035031200241064653, - 0.00034784900344675407, - 0.0003448110001045279, - 0.00035048699646722525, - 0.0047341750032501295, - 0.0004463139994186349, - 0.00043781899876194075, - 0.00038937200588406995, - 0.00034778400004142895, - 0.00034243899426655844, - 0.00034849499934352934, - 0.0003516189972287975, - 0.00034389999927952886, - 0.0003426039984333329, - 0.0024780789972282946, - 0.000424956000642851, - 0.0005385540061979555, - 0.000388355998438783, - 0.0003568869942682795, - 0.00034956700255861506, - 0.0003452540040598251, - 0.000354718002199661, - 0.00034375800169073045, - 0.0003474440018180758, - 0.005440210996312089, - 0.0004405559957376681, - 0.001454500001273118, - 0.0003557460004230961, - 0.00036306100082583725, - 0.00038152699562488124, - 0.0003638159978436306, - 0.00034944300568895414, - 0.00035004299570573494, - 0.000341184000717476, - 0.003178544000547845, - 0.0004274599996278994, - 0.0004237629982526414, - 0.0003531070033204742, - 0.00034434199915267527, - 0.00035250300425104797, - 0.00035838299663737416, - 0.0003454730031080544, - 0.0003465980043984018, - 0.005109472003823612, - 0.006997370001045056, - 0.004394299001432955, - 0.009606883002561517, - 0.006990574998781085, - 0.005003409001801629, - 0.005992379999952391, - 0.007038807001663372, - 0.007952207000926137, - 0.006993542003328912, - 0.007040165000944398, - 0.0069560030024149455, - 0.008003101000213064, - 0.0011558459955267608, - 0.0038307380018522963, - 0.007008574997598771, - 0.006986152999161277, - 0.006997581003815867, - 0.004425916005857289, - 0.002568606003478635, - 0.00800048399833031, - 0.008001451002201065, - 0.007996745000127703, - 0.005988961005641613, - 0.0059950120048597455, - 0.012001211005554069, - 0.005988532997434959, - 0.006997633005084936, - 0.007002948994340841, - 0.006994180002948269, - 0.007997446999070235, - 0.004995920993678737, - 0.00399580699740909, - 0.007130680998670869, - 0.006863559996418189, - 0.008997309996630065, - 0.0069970710028428584, - 0.005983934999676421, - 0.004006732000561897, - 0.0044237620022613555, - 0.0035724209956242703, - 0.009998718000133522, - 0.007995097003004048, - 0.0069974520010873675, - 0.006999293000262696, - 0.006006417999742553, - 0.008023389003938064, - 0.003958369001338724, - 0.00667396900098538, - 0.003319380004541017, - 0.006009513002936728, - 0.0059854159990209155, - 0.006996182004513685, - 0.007999716995982453, - 0.00737704800121719, - 0.009615152004698757, - 0.006998081000347156, - 0.007995833999302704, - 0.004583747999276966, - 0.003271884001151193, - 0.008153147005941719, - 0.003124859002127778, - 0.0038668780034640804, - 0.009985073003917933, - 0.004820357004064135, - 0.007170414995925967, - 0.0059921610009041615, - 0.00799648099928163, - 0.005002184996556025, - 0.004991385001630988, - 0.006065948000468779, - 0.0059292559963068925, - 0.006881704000988975, - 0.0021090239970362745, - 0.0046743979983148165, - 0.006324607995338738, - 0.0069962720008334145, - 0.006997625998337753, - 0.006994462994043715, - 0.007006824998825323, - 0.005987220996757969, - 0.008999552999739535, - 0.001993098994717002, - 0.006133136994321831, - 0.0004518890054896474, - 0.00043085499783046544, - 0.00042491899512242526, - 0.0004000150001957081, - 0.0003650699945865199, - 0.0003523519990267232, - 0.00035310200473759323, - 0.0003618650007410906, - 0.00036024899600306526, - 0.005580191995250061, - 0.00044255800457904115, - 0.0004565880008158274, - 0.00046446899796137586, - 0.0003674199979286641, - 0.0003620530042098835, - 0.00035582800046540797, - 0.00035820199991576374, - 0.00035802600177703425, - 0.0003532269984134473, - 0.0017929239984368905, - 0.00042419700184836984, - 0.00042547599878162146, - 0.0003955919964937493, - 0.00036374999763211235, - 0.00039010999898891896, - 0.00037231799797154963, - 0.00037354599771788344, - 0.00035838899930240586, - 0.005507160996785387, - 0.0004940329963574186, - 0.0004547709977487102, - 0.0004360800012364052, - 0.0003805270025623031, - 0.00036263599758967757, - 0.0003571560009731911, - 0.0003672110033221543, - 0.00047480600187554955, - 0.00035441700310911983, - 0.005857090000063181, - 0.010243620003166143, - 0.0004296409970265813, - 0.0003627359983511269, - 0.0010647279996192083, - 0.00036237199674360454, - 0.0003528259985614568, - 0.00034937600139528513, - 0.00035595100052887574, - 0.00035364799987291917, - 0.005927468002482783, - 0.009461659996304661, - 0.00040736600203672424, - 0.00035765099892159924, - 0.0003631770014180802, - 0.00035422899964032695, - 0.0003500789971440099, - 0.0003532780028763227, - 0.0003875519978464581, - 0.0003444340036367066, - 0.0052875990004395135, - 0.00047127200377872214, - 0.00042693100112956017, - 0.0004112230017199181, - 0.00034818899439414963, - 0.0003582559947972186, - 0.00034602900268509984, - 0.0003543839993653819, - 0.000347825996868778, - 0.0003662870003608987, - 0.00384993899933761, - 0.0016182649997062981, - 0.00040943500061985105, - 0.00040655000339029357, - 0.00041290799708804116, - 0.0003509500020300038, - 0.0003499329977785237, - 0.00034335500095039606, - 0.0003517320001265034, - 0.0003564080034266226, - 0.0003539320023264736, - 0.00035036299959756434, - 0.004852256002777722, - 0.0004870350021519698, - 0.0004206340017844923, - 0.0011270610048086382, - 0.000434485002188012, - 0.000348544999724254, - 0.0003488989968900569, - 0.00035183400177629665, - 0.00034409599902573973, - 0.00034968699765158817, - 0.0030024350053281523, - 0.000455147004686296, - 0.00040127800457412377, - 0.000404826998419594, - 0.0003813619987340644, - 0.00037784300366183743, - 0.00035166599991498515, - 0.0003481050007394515, - 0.004910960997221991, - 0.0004697010008385405, - 0.00044213499495526776, - 0.0003745380017790012, - 0.000353587995050475, - 0.0003461539963609539, - 0.0003469050061539747, - 0.0003540879988577217, - 0.0003446400005486794, - 0.00437142800365109, - 0.0005184050023672171, - 0.00040789099875837564, - 0.00039000800461508334, - 0.00039538599958177656, - 0.00034812000376405194, - 0.00035016900073969737, - 0.0003532209957484156, - 0.0003558069947757758, - 0.0003451469965511933, - 0.0003433820020291023, - 0.00443311600247398, - 0.0004617020022124052, - 0.0004408050008350983, - 0.0004308940042392351, - 0.00040417700074613094, - 0.00037748499744338915, - 0.00036097800330026075, - 0.0003535189971444197, - 0.00034462500480003655, - 0.00034860499727074057, - 0.0003526110012899153, - 0.0003429339994909242, - 0.00035014199966099113, - 0.0023958349993336014, - 0.0004335150006227195, - 0.0004658289981307462, - 0.0004133399997954257, - 0.0003651730003184639, - 0.0003587970059015788, - 0.0003657749984995462, - 0.0003642800002126023, - 0.0003626500038080849, - 0.0003889360013999976, - 0.0003986200026702136, - 0.0003698250002344139, - 0.0003736399958143011, - 0.00040476099820807576, - 0.0003720480017364025, - 0.00036885000008624047, - 0.0003712700054165907, - 0.0003707529976963997, - 0.0003714159975061193, - 0.00036423799610929564, - 0.00036914300289936364, - 0.00036321300285635516, - 0.0003641069997684099, - 0.0003657180059235543, - 0.00036271200224291533, - 0.00036181700124870986, - 0.0003719669985002838, - 0.0003598219991545193, - 0.0003623099983087741, - 0.0003823910010396503, - 0.00036407200241228566, - 0.0003581970013328828, - 0.00036350300069898367, - 0.0003878779971273616, - 0.0003816820026258938, - 0.0003777129968511872, - 0.000376689997210633, - 0.0003776480007218197, - 0.0003717860017786734, - 0.00035789600224234164, - 0.00037041599716758355, - 0.0003559119941201061, - 0.0003723699992406182, - 0.0003632529987953603, - 0.000364061001164373, - 0.00036538299900712445, - 0.0003784969958360307, - 0.00037696800427511334, - 0.0003755309953703545, - 0.0003941529939766042, - 0.0003849019994959235, - 0.00037604800309054554, - 0.00038371499977074564, - 0.00037608700222335756, - 0.00036936400283593684, - 0.00037347900070017204, - 0.00039223900239448994, - 0.0003685650008264929, - 0.00038559599488507956, - 0.0003736889993888326, - 0.0003738449959200807, - 0.0003828129993053153, - 0.00037491200055228546, - 0.00046991900308057666, - 0.00036897899553878233, - 0.0003558329990482889, - 0.00037672300095437095, - 0.0003696120038512163, - 0.00039104800089262426, - 0.00036884399742120877, - 0.00037162299850024283, - 0.00036382700636750087, - 0.0003695249979500659, - 0.00037713000347139314, - 0.00038962300459388644, - 0.0003746619986486621, - 0.00037715899816248566, - 0.0003723879999597557, - 0.0003734799975063652, - 0.0003764650027733296, - 0.0003729599993675947, - 0.00036929799534846097, - 0.0030292260053101927, - 0.006998454002314247, - 0.009999071000493132, - 0.007996173000719864, - 0.006997062999289483, - 0.006996711003012024, - 0.0045772909943480045, - 0.0064184860020759515, - 0.007998873996257316, - 0.0058943860058207065, - 0.0005018080046284012, - 0.0004613619967130944, - 0.0003953399937017821, - 0.0003866810002364218, - 0.0003981650006608106, - 0.00036631099646911025, - 0.0003725829956238158, - 0.0012002410003333353, - 0.003074294996622484, - 0.00043340399861335754, - 0.00042503599979681894, - 0.00040599500061944127, - 0.0003964709976571612, - 0.0003705769995576702, - 0.0003674850013339892, - 0.0003690170051413588, - 0.00037510399852180853, - 0.0003665750045911409, - 0.0003681700036395341, - 0.00037601600342895836, - 0.0003715880011441186, - 0.00037426199560286477, - 0.00037335800152504817, - 0.0003713999976753257, - 0.00037112300196895376, - 0.00040142999932868406, - 0.0003837969998130575, - 0.0003768579990719445, - 0.00036426400038180873, - 0.0003689400036819279, - 0.002868434996344149, - 0.0031273470012820326, - 0.00043459100561449304, - 0.0004088649948243983, - 0.00041952099854825065, - 0.0004156499999226071, - 0.0003824559971690178, - 0.0003642980009317398, - 0.00037572200380964205, - 0.0003758530001505278, - 0.0003855300019495189, - 0.0003872800007229671, - 0.00038684199535055086, - 0.000373980998119805, - 0.00039920100243762136, - 0.0037322040006984025, - 0.006997190001129638, - 0.011002063001797069, - 0.006993913004407659, - 0.006995525000093039, - 0.006998081000347156, - 0.007998350003617816, - 0.007996893997187726, - 0.006181636999826878, - 0.001115904000471346, - 0.00047382699995068833, - 0.0003929469967260957, - 0.00036359400110086426, - 0.0003575040027499199, - 0.0037430320007842965, - 0.0010616610015858896, - 0.00044414700096240267, - 0.0005075660010334104, - 0.0003782009953283705, - 0.00036871200427412987, - 0.0003886900012730621, - 0.006801101000746712, - 0.00047611299669370055, - 0.00044110700400779024, - 0.0003885909973178059, - 0.0003576140006771311, - 0.0003632489970186725, - 0.0003571979978005402, - 0.0003604849989642389, - 0.00035719099832931533, - 0.006923174005351029, - 0.00046294899948406965, - 0.0004379050005809404, - 0.000388505999580957, - 0.00036389799788594246, - 0.0003534779971232638, - 0.00035677199775818735, - 0.00036032199568580836, - 0.00035484699765220284, - 0.003240810998249799, - 0.0028161660011392087, - 0.0004753559987875633, - 0.00044094599434174597, - 0.0003479709994280711, - 0.0003556299998308532, - 0.0003453819954302162, - 0.00035086799471173435, - 0.00034971500281244516, - 0.00034959500044351444, - 0.004440151002199855, - 0.0004752059976453893, - 0.0004422999991220422, - 0.00042529999336693436, - 0.0003582559947972186, - 0.00034687399602262303, - 0.0003472039970802143, - 0.0003503620027913712, - 0.0003473010001471266, - 0.0003674239997053519, - 0.002546517993323505, - 0.0004449170010047965, - 0.0004669109985115938, - 0.0025972029980039224, - 0.0004175649955868721, - 0.00039304199890466407, - 0.00036611600080505013, - 0.0003475940029602498, - 0.00034653599868761376, - 0.003421304005314596, - 0.00044187300227349624, - 0.00041875099850585684, - 0.00041012799920281395, - 0.00038863300142111257, - 0.001983553003810812, - 0.0004215709996060468, - 0.0004045049936394207, - 0.00035585599835030735, - 0.00035423900408204645, - 0.0003530989997670986, - 0.0034694889982347377, - 0.0004879670013906434, - 0.00045573600073112175, - 0.0004508869969868101, - 0.00035375500010559335, - 0.0003607239996199496, - 0.0003548579989001155, - 0.0003484900007606484, - 0.0003490860035526566, - 0.0003450430012890138, - 0.0015972409964888357, - 0.004261293994204607, - 0.00045084899466019124, - 0.00043867000204045326, - 0.0003564319995348342, - 0.00034825399779947475, - 0.00034812199737643823, - 0.0003481519961496815, - 0.00034811400109902024, - 0.0003420409993850626, - 0.0003454369943938218, - 0.0031853420005063526, - 0.00044514399633044377, - 0.00042730599670903757, - 0.00043337600072845817, - 0.0003885019978042692, - 0.00036247100069886073, - 0.00034535099985077977, - 0.00034291300107724965, - 0.0003486540008452721, - 0.0003423420057515614, - 0.00034289299946976826, - 0.0034286250011064112, - 0.00044195800001034513, - 0.0004385639986139722, - 0.00043247800203971565, - 0.0004369410016806796, - 0.0003446909977355972, - 0.003443157998844981, - 0.00045208300434751436, - 0.0004328920040279627, - 0.0017826249968493357, - 0.0004037280014017597, - 0.00042364600085420534, - 0.00039682299393462017, - 0.0003721370012499392, - 0.00034653200418688357, - 0.0003488079964881763, - 0.0003555329967639409, - 0.0003486459972918965, - 0.005273877999570686, - 0.0005746660026488826, - 0.00041888099804054946, - 0.00043725599971367046, - 0.0003491160023258999, - 0.0003458240025793202, - 0.0003489179944153875, - 0.00035371199919609353, - 0.00035645800380734727, - 0.0003748780000023544, - 0.0003541730038705282, - 0.004487320999032818, - 0.0004573900005198084, - 0.0004383019986562431, - 0.0004234259977238253, - 0.00035190700145903975, - 0.0003482120009721257, - 0.0003566049999790266, - 0.00034716800291789696, - 0.0003519320016494021, - 0.0003454180041444488, - 0.004113961003895383, - 0.0005982020011288114, - 0.0004068609996465966, - 0.0003628609993029386, - 0.0003463589964667335, - 0.0003510839960654266, - 0.00035450499854050577, - 0.00034390200016787276, - 0.0003441620065132156, - 0.00035608300095191225, - 0.0003483990003587678, - 0.0008867490032571368, - 0.0003494320044410415, - 0.00035419099731370807, - 0.00034461899485904723, - 0.00034302299900446087, - 0.0003490830058581196, - 0.000344883999787271, - 0.00034881299507105723, - 0.00034805299947038293, - 0.00034806100302375853, - 0.005201321000640746, - 0.0004937720004818402, - 0.0003864419995807111, - 0.00036304300010669976, - 0.00036026100133312866, - 0.00035200999991502613, - 0.00035367200325708836, - 0.0003459330037003383, - 0.00034370500361546874, - 0.00035071999445790425, - 0.00034705200232565403, - 0.0003484370026853867, - 0.0005076090019429103, - 0.0039458879982703365, - 0.0004442389981704764, - 0.00045635800051968545, - 0.0003838159973383881, - 0.00034816900006262586, - 0.00034672800393309444, - 0.0003597000031732023, - 0.0003568819956853986, - 0.00034319899714319035, - 0.00034250599856022745, - 0.005088250996777788, - 0.0005157560008228756, - 0.0004067740010214038, - 0.0012687820053542964, - 0.0005523860018001869, - 0.0051356619951548055, - 0.0005009379965486005, - 0.0004315209953347221, - 0.00035857100010616705, - 0.000359874997229781, - 0.0003507609944790602, - 0.00035408300027484074, - 0.00035267299972474575, - 0.00034909100213553756, - 0.004458387003978714, - 0.0004990960005670786, - 0.0004882840003119782, - 0.0004553679973469116, - 0.0003587829996831715, - 0.00048274200526066124, - 0.00035870300052920356, - 0.00035094100167043507, - 0.00035307399957673624, - 0.005851180001627654, - 0.0005008719963370822, - 0.0004520049988059327, - 0.00039210700197145343, - 0.00035436100006336346, - 0.00044547199649969116, - 0.0003866909974021837, - 0.0003498540027067065, - 0.00035503700200933963, - 0.005460904001665767, - 0.0004962780003552325, - 0.0004938619968015701, - 0.0004611080003087409, - 0.00035359500179765746, - 0.0003458279970800504, - 0.0003436779952608049, - 0.0005905930011067539, - 0.0003425510003580712, - 0.002307360999111552, - 0.000459770999441389, - 0.0004086030021426268, - 0.00041504699765937403, - 0.00039060800190782174, - 0.00035160900006303564, - 0.00034634899930097163, - 0.000494854997668881, - 0.0003487860012683086, - 0.00034348600456723943, - 0.00034572799631860107, - 0.004510152997681871, - 0.0004873100042459555, - 0.0004418079988681711, - 0.00037175299803493544, - 0.000510247002239339, - 0.00038887000118847936, - 0.00036911299685016274, - 0.00036575400008587167, - 0.000349346999428235, - 0.001215306001540739, - 0.004194789005850907, - 0.0004986730054952204, - 0.0004425939987413585, - 0.00042539299465715885, - 0.0003501210012473166, - 0.0003531949987518601, - 0.0003525379943312146, - 0.00035002100048586726, - 0.0003503909974824637, - 0.0003620909992605448, - 0.005746041999373119, - 0.0004662529972847551, - 0.0005185569971217774, - 0.0003829970009974204, - 0.0003556140000000596, - 0.0003431919976719655, - 0.000345571999787353, - 0.00034777399559970945, - 0.0003484090048004873, - 0.00034275899815838784, - 0.0016799779987195507, - 0.003556411000317894, - 0.0004382939951028675, - 0.0003938000008929521, - 0.00037324800359783694, - 0.00036629099486162886, - 0.00035245200706413016, - 0.00034510099794715643, - 0.0003453680037637241, - 0.0003486660061753355, - 0.0003504089982016012, - 0.00034896899887826294, - 0.0004290200013201684, - 0.00516378599422751, - 0.00042619699524948373, - 0.0004289759963285178, - 0.00039900000410852954, - 0.0003503719999571331, - 0.00034994200541405007, - 0.00034476899600122124, - 0.00035057999775744975, - 0.0003577309980755672, - 0.0003432600060477853, - 0.0035823180005536415, - 0.00044810200051870197, - 0.00044810799590777606, - 0.00037862599856453016, - 0.00034666500141611323, - 0.00035463600215734914, - 0.00034951199631905183, - 0.00034794599923770875, - 0.00034918200253741816, - 0.0003485330016701482, - 0.0003737439983524382, - 0.004216084998915903, - 0.00046174300223356113, - 0.00045301699719857424, - 0.00043179199565202, - 0.00035260000004200265, - 0.000343095998687204, - 0.00034250599856022745, - 0.000356894001015462, - 0.000347944995155558, - 0.0003476490019238554, - 0.003555257004336454, - 0.00043294599890941754, - 0.0004487059995881282, - 0.00045555199903901666, - 0.00036384700069902465, - 0.0003748680028365925, - 0.0003502999970805831, - 0.00035140499676344916, - 0.0003456659978837706, - 0.00034094200236722827, - 0.005819464000524022, - 0.00046226000267779455, - 0.00043093800195492804, - 0.000346601998899132, - 0.0003512919938657433, - 0.0003441680019022897, - 0.0003874239992001094, - 0.0003493839976727031, - 0.00035160100378561765, - 0.004781338997418061, - 0.00045905300066806376, - 0.000440818999777548, - 0.00038293900433927774, - 0.0003577330062398687, - 0.00034899399906862527, - 0.0004399429963086732, - 0.0003924890042981133, - 0.00035639099951367825, - 0.003045880002900958, - 0.0007339059957303107, - 0.00037434000114444643, - 0.00036381300014909357, - 0.0003506900029606186, - 0.0003472180032986216, - 0.00035735099663725123, - 0.00034928500099340454, - 0.00034724899887805805, - 0.00035352000122657046, - 0.00034941299963975325, - 0.0003493170006549917, - 0.00035080099769402295, - 0.00034379800490569323, - 0.003228415000194218, - 0.00043634000030579045, - 0.0003948949961340986, - 0.00036731200088979676, - 0.000363745006325189, - 0.0003571399938664399, - 0.0003546439984347671, - 0.00035226999898441136, - 0.00034711299667833373, - 0.0003453980025369674, - 0.0003499809972709045, - 0.004300770997360814, - 0.0004795019995071925, - 0.00039923900476424024, - 0.0003518859957694076, - 0.00035113900230498984, - 0.00034489900281187147, - 0.000356467004166916, - 0.00034590999712236226, - 0.0025961569990613498, - 0.002474524997523986, - 0.00043173499580007046, - 0.0004153420013608411, - 0.0003605070014600642, - 0.00035415199818089604, - 0.0003453899989835918, - 0.0003460040024947375, - 0.00035228799970354885, - 0.00034953000431414694, - 0.005231980998360086, - 0.0004727180057670921, - 0.00039224500505952165, - 0.00040308899770025164, - 0.0003528750021359883, - 0.0003557359959813766, - 0.00045178100117482245, - 0.0003656979970401153, - 0.0047066150000318885, - 0.0005890839966014028, - 0.0003696749990922399, - 0.0003589269981603138, - 0.0004151780012762174, - 0.0003579699987312779, - 0.00034560999483801425, - 0.0003505259956000373, - 0.0003445619950070977, - 0.0003451879965723492, - 0.005456300998048391, - 0.0005220700040808879, - 0.000974212001892738, - 0.0003763779968721792, - 0.0010469849949004129, - 0.0003916630012099631, - 0.005273361995932646, - 0.000671157002216205, - 0.0003598969997256063, - 0.00036385099519975483, - 0.0003478840008028783, - 0.00035717299761017784, - 0.0003570179978851229, - 0.00034748200414469466, - 0.004572891994030215, - 0.0005083350042696111, - 0.00048402699758298695, - 0.00040358799742534757, - 0.0009431029975530691, - 0.000392280999221839, - 0.0003596009992179461, - 0.000350878995959647, - 0.005537527002161369, - 0.000459530005173292, - 0.00044581000111065805, - 0.00042526500328676775, - 0.00034967400279128924, - 0.000346143999195192, - 0.0003454929974395782, - 0.0003543479979271069, - 0.0003492600008030422, - 0.0003499390004435554, - 0.0037718960011261515, - 0.0005431190002127551, - 0.0005040749965701252, - 0.00036624200583901256, - 0.00034676899667829275, - 0.0003864329992211424, - 0.0003649889986263588, - 0.000348888999724295, - 0.0003437409977777861, - 0.0003504960041027516, - 0.0003429670032346621, - 0.00037791200156789273, - 0.0056998619984369725, - 0.00046591100544901565, - 0.00043255700438749045, - 0.00034765000600600615, - 0.00035537000076146796, - 0.0003467229980742559, - 0.0003456539998296648, - 0.0003544159990269691, - 0.0003459010040387511, - 0.00034756500099319965, - 0.00034469400270609185, - 0.00034420099837007, - 0.005315588001394644, - 0.004996349998691585, - 0.004997251002350822, - 0.007998917004442774, - 0.005996653002512176, - 0.006998085998930037, - 0.006001257999741938, - 0.0069910090023768134, - 0.007623971003340557, - 0.008370530005777255, - 0.008007043004909065, - 0.007989651996467728, - 0.006999657998676412, - 0.003989552002167329, - 0.006998178003414068, - 0.006995936004386749, - 0.007996116000867914, - 0.007997624998097308, - 0.007997212996997405, - 0.006999334997090045, - 0.0089954380018753, - 0.006997379998210818, - 0.00514904799638316, - 0.005845216997840907, - 0.00599653999961447, - 0.0069981690030545, - 0.007001518999459222, - 0.007003512997471262, - 0.005991712998365983, - 0.006989823006733786, - 0.00800303700088989, - 0.06015052899601869, - 0.0011747509997803718, - 0.0005344919991330244, - 0.0004157579969614744, - 0.00040030300442595035, - 0.01573724399349885, - 0.00902320699970005, - 0.005525321001186967, - 0.0049930879977182485, - 0.0037250020031933673, - 0.0006687630011583678, - 0.0004244049996486865, - 0.002125696002622135, - 0.0034122900033253245, - 0.005605016005574726, - 0.023001296001893934, - 0.006943124004465062, - 0.0069836179973208345, - 0.006995459996687714, - 0.006999037999776192, - 0.007007986001553945, - 0.006982910002989229, - 0.006997631004196592, - 0.007022757003142033, - 0.005967395998595748, - 0.007994289997441228, - 0.00699754900415428, - 0.006997536002018023, - 0.00703717400028836, - 0.006964133004657924, - 0.006990957997913938, - 0.00699771100335056, - 0.00699743400036823, - 0.006996705997153185, - 0.0069946009971317835, - 0.007998328997928184, - 0.006997668999247253, - 0.005999458000587765, - 0.006995821000600699, - 0.007997340995643754, - 0.006998497003223747, - 0.006997235002927482, - 0.005997571999614593, - 0.00699759599956451, - 0.00799744499818189, - 0.007998312001291197, - 0.00800401399465045, - 0.007991099999344442, - 0.007997339002031367, - 0.006997550000960473, - 0.003999715998361353, - 0.007996913998795208, - 0.006997911994403694, - 0.010997682002198417, - 0.005019973999878857, - 0.003990425000665709, - 0.005006247003620956, - 0.006379394995747134, - 0.007505581001169048, - 0.006076632002077531, - 0.007995478001248557, - 0.008002057002158836, - 0.007991880993358791, - 0.0029960839965497144, - 0.005246001994237304, - 0.008749403001274914, - 0.004004340997198597, - 0.005990069999825209, - 0.005500637002114672, - 0.0035002950025955215, - 0.006993916002102196, - 0.010995024000294507, - 0.011997187000815757, - 0.006997709002462216, - 0.004997042000468355, - 0.005998159002047032, - 0.007998271998076234, - 0.005997877000481822, - 0.007997021995834075, - 0.007997945001989137, - 0.006999908000580035, - 0.005994793995341752, - 0.008027972005947959, - 0.006969757996557746, - 0.007995141000719741, - 0.0039948420017026365, - 0.00699607400019886, - 0.0059975010008201934, - 0.007997494998562615, - 0.006998114004090894, - 0.005000051998649724, - 0.009129835001658648, - 0.010862681003345642, - 0.005997240004944615, - 0.004998411997803487, - 0.004997134004952386, - 0.007997383996553253, - 0.006999377001193352, - 0.005995402003463823, - 0.0060028469961252995, - 0.005993050996039528, - 0.006998728997132275, - 0.005002920996048488, - 0.005991150996123906, - 0.008438719996775035, - 0.005553677001444157, - 0.009021169003972318, - 0.006975192998652346, - 0.0029976029982208274, - 0.0017196529952343553, - 0.001283607998630032, - 0.007239344005938619, - 0.008747038002184127, - 0.006993623996095266, - 0.007996363005077, - 0.006998813994869124, - 0.0059981559970765375, - 0.006996327996603213, - 0.00699854100093944, - 0.004089843001565896, - 0.003904517005139496, - 0.00599989000329515, - 0.007996806998562533, - 0.00699917600286426, - 0.005996766994940117, - 0.005006426996260416, - 0.004992938003852032, - 0.003649113998108078, - 0.007411229002173059, - 0.007454081001924351, - 0.007467604002158623, - 0.005997244996251538, - 0.007130179001251236, - 0.00886481899942737, - 0.006997576005232986, - 0.008008168995729648, - 0.00698767599533312, - 0.0020029760053148493, - 0.00499205900268862, - 0.006001094996463507, - 0.00039874499634606764, - 0.0003760620020329952, - 0.00035622300492832437, - 0.0003476190031506121, - 0.001362692994007375, - 0.0003576500021154061, - 0.0003485159977572039, - 0.00034273700293852016, - 0.0008817869966151193, - 0.002374112998950295, - 0.00038098399818409234, - 0.00037663200055249035, - 0.0003878479983541183, - 0.00037552999856416136, - 0.0003539199969964102, - 0.0003493839976727031, - 0.0003470800002105534, - 0.003409217002626974, - 0.0004247600008966401, - 0.0003586770035326481, - 0.0003513109986670315, - 0.0003446159971645102, - 0.0004375529970275238, - 0.0007732980011496693, - 0.00039301299693761393, - 0.0003610920030041598, - 0.00034524299553595483, - 0.0009586520027369261, - 0.00041370400140294805, - 0.00035194199881516397, - 0.0003532490009092726, - 0.0003451349984970875, - 0.0003454309990047477, - 0.0003545420040609315, - 0.00037095099833095446, - 0.0003511440008878708, - 0.00035199199919588864, - 0.0003433660021983087, - 0.00034751699422486126, - 0.005370618004235439, - 0.0003779840044444427, - 0.00039317899791058153, - 0.000385116996767465, - 0.0003657480046967976, - 0.00035407899849815294, - 0.00034767499892041087, - 0.00034920599864562973, - 0.0003466929993010126, - 0.0003485120032564737, - 0.00034343999868724495, - 0.0031203320031636395, - 0.0030477260006591678, - 0.004938631995173637, - 0.002283912996063009, - 0.005978671004413627, - 0.004723981997813098, - 0.006997279000643175, - 0.006998349002969917, - 0.0070848070026841015, - 0.006920244995853864, - 0.00799449099577032, - 0.0069888750003883615, - 0.007994261002750136, - 0.006996798001637217, - 0.0069982820059522055, - 0.0069968099996913224, - 0.006996906995482277, - 0.006997901000431739, - 0.005999682005494833, - 0.005996713000058662, - 0.00452927200240083, - 0.004467904000193812, - 0.005301665994920768, - 0.007689646998187527, - 0.00118368099356303, - 0.0003581389973987825, - 0.0003521799953887239, - 0.0003531339971232228, - 0.00034633299947017804, - 0.00034535099985077977, - 0.00035817000025417656, - 0.00034322699502808973, - 0.0005421140012913384, - 0.0004237340035615489, - 0.00037502100167330354, - 0.00038602200220339, - 0.00041108400182565674, - 0.0003550749970600009, - 0.00034637199860299006, - 0.00035359100002096966, - 0.0003518639932735823, - 0.0003461130036157556, - 0.00034347600012551993, - 0.00034934999712277204, - 0.0003422149966354482, - 0.0003421740038902499, - 0.0050298770001973026, - 0.00048744100058684126, - 0.0003531010006554425, - 0.00035545900027500466, - 0.00034624399995664135, - 0.0003438910061959177, - 0.0003495580021990463, - 0.00034273700293852016, - 0.00034817300183931366, - 0.0003517989971442148, - 0.00035043399839196354, - 0.00034510299883550033, - 0.005136842002684716, - 0.002729939005803317, - 0.0003851120054605417, - 0.00036683600046671927, - 0.00035416199534665793, - 0.000353683004505001, - 0.00034610399598022923, - 0.0026217780032311566, - 0.0003789260008488782, - 0.00035887300327885896, - 0.0003524190033203922, - 0.00034850699739763513, - 0.00035367600503377616, - 0.00034982400393346325, - 0.0003427839983487502, - 0.00034617300116224214, - 0.00034564700035844, - 0.0003485140041448176, - 0.005211850002524443, - 0.00047646299935877323, - 0.00037726799928350374, - 0.0020925939970766194, - 0.00042126799962716177, - 0.0003989529941463843, - 0.004828848999750335, - 0.000382555001124274, - 0.00037518500175792724, - 0.0003585779995773919, - 0.00035169699549442157, - 0.00034644099650904536, - 0.000347396002325695, - 0.0017175920002046041, - 0.0044415320007828996, - 0.0008022069960134104, - 0.00036669399560196325, - 0.0003556720039341599, - 0.000353540999640245, - 0.0013199020031606779, - 0.003786143999604974, - 0.0003789360052905977, - 0.0003679450019262731, - 0.00035772399860434234, - 0.00035738600126933306, - 0.0003493569965939969, - 0.0003547039959812537, - 0.0003434829995967448, - 0.00034955899900523946, - 0.0035050150036113337, - 0.0016290290004690178, - 0.0003551280024112202, - 0.0003690919984364882, - 0.00037985800008755177, - 0.0003779849939746782, - 0.0003602560027502477, - 0.0011717750021489337, - 0.005202868997002952, - 0.00043906499922741205, - 0.000383929000236094, - 0.0003571619963622652, - 0.000350420996255707, - 0.0003472829994279891, - 0.0003448209972702898, - 0.00034348400367889553, - 0.00035150199983036146, - 0.0003511889954097569, - 0.000342715997248888, - 0.0003603809964261018, - 0.0007935440007713623, - 0.0003621380019467324, - 0.00034293300268473104, - 0.0003553580027073622, - 0.0020675019986811094, - 0.00035119400126859546, - 0.00035253300302429125, - 0.0003487680005491711, - 0.00035386600211495534, - 0.0003422350055188872, - 0.0022612119937548414, - 0.0005957540051895194, - 0.0003632700027083047, - 0.0003536790027283132, - 0.0003440380023675971, - 0.0003544020000845194, - 0.00034594800672493875, - 0.000346526998328045, - 0.00035882199881598353, - 0.0003437160048633814, - 0.0003449220021138899, - 0.003910218001692556, - 0.0003796879973378964, - 0.0010044849987025373, - 0.0003475220000836998, - 0.00035025599936489016, - 0.00035026599653065205, - 0.0003474729965091683, - 0.000347621004038956, - 0.00035023999953409657, - 0.0003428859999985434, - 0.004277639003703371, - 0.00038739499723305926, - 0.00037532499845838174, - 0.0003529800014803186, - 0.0003495010023470968, - 0.0003575290029402822, - 0.00034863100154325366, - 0.00034380899887764826, - 0.00034469799720682204, - 0.005185319001611788, - 0.000884729997778777, - 0.0003496770004858263, - 0.0003471610034466721, - 0.00034678300289670005, - 0.0003600060008466244, - 0.00034845200570998713, - 0.0003484839980956167, - 0.0003536369986250065, - 0.002076431999739725, - 0.003450901996984612, - 0.0003881410011672415, - 0.0003646749973995611, - 0.00034999399940716103, - 0.00034603299718583, - 0.0017996839960687794, - 0.0003576720046112314, - 0.0003714649938046932, - 0.00037333100044634193, - 0.0003651490042102523, - 0.002373426999838557, - 0.0003779699982260354, - 0.0003630880019045435, - 0.0003796969976974651, - 0.00037088199314894155, - 0.00034579999919515103, - 0.0003597449976950884, - 0.0003433439997024834, - 0.0003433199963183142, - 0.0024236219978774898, - 0.00037959399924147874, - 0.0017520239998702891, - 0.0003809449990512803, - 0.00037313500070013106, - 0.00036258299951441586, - 0.0003515270000207238, - 0.0003486580026219599, - 0.0003466439957264811, - 0.00034826800401788205, - 0.00034528200194472447, - 0.0003518760058796033, - 0.0046955150028225034, - 0.0004445770027814433, - 0.0004226910023135133, - 0.0004064210006617941, - 0.00038629000482615083, - 0.00038375800068024546, - 0.00036536000698106363, - 0.0003522550032357685, - 0.000348692003171891, - 0.00034688199957599863, - 0.00035079699591733515, - 0.00035053500323556364, - 0.0003420229986659251, - 0.004130320005060639, - 0.000372824004443828, - 0.00138408099883236, - 0.00036719600029755384, - 0.00034396600676700473, - 0.00034733799839159474, - 0.0003490590024739504, - 0.00035016699985135347, - 0.00034377800329821184, - 0.0003494400007184595, - 0.002701166995393578, - 0.00037889899977017194, - 0.00035258999560028315, - 0.0003508809968479909, - 0.00034641199454199523, - 0.0003458240025793202, - 0.0004300189975765534, - 0.0011867720022564754, - 0.0008420670055784285, - 0.0003601200005505234, - 0.0003500070015434176, - 0.00034511100238887593, - 0.002045878005446866, - 0.0003666019983938895, - 0.0012420439961715601, - 0.000354531999619212, - 0.0003522750048432499, - 0.00034754499938571826, - 0.00035684099566424266, - 0.00034644099650904536, - 0.00035200400452595204, - 0.00034668800071813166, - 0.00034501399932196364, - 0.0003530710018821992, - 0.00034371799847576767, - 0.0003522280021570623, - 0.004192470005364157, - 0.0004128200016566552, - 0.0003528080051182769, - 0.0003550710025592707, - 0.00034339899866608903, - 0.0003433339952607639, - 0.00035152200143784285, - 0.00035265299811726436, - 0.00034368699562037364, - 0.00035206900065531954, - 0.00034346399479545653, - 0.0003496850040392019, - 0.005426871997769922, - 0.0004028349940199405, - 0.00038537500222446397, - 0.0003766369991353713, - 0.00034911499824374914, - 0.0003481739986455068, - 0.0003444399990257807, - 0.00034255899663548917, - 0.00035046499397139996, - 0.0003444889953243546, - 0.004522561997873709, - 0.0003713510013767518, - 0.0009452189988223836, - 0.0003621809955802746, - 0.000353880001057405, - 0.0003588190011214465, - 0.0003451290031080134, - 0.0003500890015857294, - 0.00034650100133148953, - 0.0020395800020196475, - 0.0018920229995273985, - 0.0003768159949686378, - 0.00037125000380910933, - 0.0003624190067057498, - 0.00035508399741956964, - 0.00034928099921671674, - 0.00034525500086601824, - 0.00034952200076077133, - 0.0003462800013949163, - 0.00034678699739743024, - 0.004832562000956386, - 0.0003768919996218756, - 0.0003675900006783195, - 0.0003522460028761998, - 0.00034782600414473563, - 0.0003452019955147989, - 0.00034356999822193757, - 0.0003559160031727515, - 0.00034667200088733807, - 0.00035160100378561765, - 0.0042818010042537935, - 0.00037126700044609606, - 0.0003562140045687556, - 0.0022729519987478852, - 0.008450995999737643, - 0.00037620000512106344, - 0.0003665749973151833, - 0.0003606490063248202, - 0.00035012800071854144, - 0.00034433900145813823, - 0.0003453289973549545, - 0.0003509969974402338, - 0.00034942500496981665, - 0.000343936997523997, - 0.004389365996757988, - 0.000402919999032747, - 0.0003492739997454919, - 0.0003514220006763935, - 0.0003466160051175393, - 0.0003510410024318844, - 0.0003446869959589094, - 0.0003439520005485974, - 0.0005811679948237725, - 0.0007390480022877455, - 0.0008017629952519201, - 0.00044303399772616103, - 0.0009749480013852008, - 0.0003524290004861541, - 0.00035364500217838213, - 0.00035272999957669526, - 0.00035144899447914213, - 0.00034655600029509515, - 0.000352048002241645, - 0.0003451869997661561, - 0.0012898299974040128, - 0.00036546599585562944, - 0.0003521690014167689, - 0.00035669699718710035, - 0.00034842600143747404, - 0.0003447720009717159, - 0.0004931049988954328, - 0.000389271997846663, - 0.00039841100078774616, - 0.0003969809986301698, - 0.0003965819996665232, - 0.0003961379989050329, - 0.0003720259992405772, - 0.00036926900065736845, - 0.00038690600194968283, - 0.0006103039995650761, - 0.00034963800135301426, - 0.00034660699748201296, - 0.0003524489948176779, - 0.00034240699460497126, - 0.0003452649980317801, - 0.00034831099765142426, - 0.00034506899828556925, - 0.00034428099752403796, - 0.00038916100311325863, - 0.0009363839999423362, - 0.0004981469974154606, - 0.0003522989936755039, - 0.00035520600067684427, - 0.00034839200088754296, - 0.0003497259967844002, - 0.00034953000431414694, - 0.0003451069933362305, - 0.0004198649985482916, - 0.00038661500002490357, - 0.0003962179980590008, - 0.0003950139944208786, - 0.0003886699996655807, - 0.0003867720006383024, - 0.00038400499761337414, - 0.0003915880006388761, - 0.0003740519969142042, - 0.0003453760000411421, - 0.00035319599555805326, - 0.0003437870036577806, - 0.0003419339991523884, - 0.000350456997693982, - 0.0003473519973340444, - 0.0003877420022035949, - 0.0003952350016334094, - 0.00038265599869191647, - 0.00038993900670902804, - 0.0003686479976749979, - 0.00034834699908969924, - 0.00034729199978755787, - 0.0003498510050121695, - 0.00034759799746097997, - 0.0009731289974297397, - 0.000439258998085279, - 0.00038134300120873377, - 0.0003767460002563894, - 0.00039412399928551167, - 0.0009269450019928627, - 0.0005220950042712502, - 0.0006620819985982962, - 0.0003509579983074218, - 0.0004803710035048425, - 0.00035427600232651457, - 0.00034964399674208835, - 0.00034772099752444774, - 0.00034476699511287734, - 0.0003472989992587827, - 0.00034715999936452135, - 0.00034774700179696083, - 0.000358994999260176, - 0.0003826870015473105, - 0.00039339000068139285, - 0.0003905379999196157, - 0.00041912800224963576, - 0.000855906997458078, - 0.00042866700096055865, - 0.00039692599966656417, - 0.000406040002417285, - 0.00035940000088885427, - 0.0003518129960866645, - 0.0003510410024318844, - 0.0004669600020861253, - 0.000558130006538704, - 0.00035339400346856564, - 0.0007596650029881857, - 0.00036287299735704437, - 0.00034964400401804596, - 0.00034903400228358805, - 0.00034724899887805805, - 0.0003497119978419505, - 0.0003483319960650988, - 0.0003494100019452162, - 0.0003489090013317764, - 0.00035735999699682, - 0.00034352699731243774, - 0.00035044999822275713, - 0.00040169100248022005, - 0.00040745700243860483, - 0.00039304500387515873, - 0.00038683900493197143, - 0.00037890899693593383, - 0.0016270269989036024, - 0.0014240100063034333, - 0.00046234500041464344, - 0.0005016840004827827, - 0.000375608004105743, - 0.00034486599906813353, - 0.0003441200024099089, - 0.0006362960048136301, - 0.0004112820024602115, - 0.00041528499423293397, - 0.00036040099803358316, - 0.0003540580000844784, - 0.000357051998435054, - 0.0003488570000627078, - 0.0003580109987524338, - 0.0003986449955846183, - 0.00035584400029620156, - 0.00035468900023261085, - 0.0003485039997030981, - 0.00039825099520385265, - 0.00039409100281773135, - 0.0005634840053971857, - 0.0004975769988959655, - 0.0003481819949229248, - 0.0003647330013336614, - 0.0003435399994486943, - 0.00034351900103501976, - 0.0007206610025605187, - 0.0004216129964333959, - 0.00039817200013203546, - 0.00040239800000563264, - 0.00038337300065904856, - 0.00039680399640928954, - 0.00038745900383219123, - 0.0003920879971701652, - 0.0003547040032572113, - 0.00044896899635205045, - 0.0007419859975925647, - 0.00035441500222077593, - 0.0003539219978847541, - 0.00035357299930183217, - 0.0003492620016913861, - 0.00034354699891991913, - 0.000595729004999157, - 0.0011242149994359352, - 0.0007576600037282333, - 0.0005193410033825785, - 0.0003637650006567128, - 0.00034793999657267705, - 0.00035692799428943545, - 0.00034754700027406216, - 0.00035131799813825637, - 0.00034385699837002903, - 0.0013719459966523573, - 0.0004014809965156019, - 0.00039372400351567194, - 0.0011433510007918812, - 0.00035190700145903975, - 0.0003492129981168546, - 0.0003510230017127469, - 0.0003436440019868314, - 0.00034911699913209304, - 0.00035324200143804774, - 0.0003519020028761588, - 0.000364705003448762, - 0.0005056629961472936, - 0.0004931929943268187, - 0.00039966099575394765, - 0.0004115569972782396, - 0.0013985080004204065, - 0.0007867870008340105, - 0.0003683070026454516, - 0.00036798400105908513, - 0.00035547500010579824, - 0.00035493400355335325, - 0.00035686299816006795, - 0.0027418610043241642, - 0.00037664399860659614, - 0.00037269099993864074, - 0.00036389799788594246, - 0.0003614859961089678, - 0.00035524000122677535, - 0.0003570319968275726, - 0.00036375399940880015, - 0.0003541069963830523, - 0.00035597399983089417, - 0.004625460001989268, - 0.0004120540033909492, - 0.0003609240011428483, - 0.00035436300095170736, - 0.00035126700095133856, - 0.0003595360030885786, - 0.00036133299727225676, - 0.0003564729995559901, - 0.00035482500243233517, - 0.00035566900623962283, - 0.004808443001820706, - 0.0003951799953938462, - 0.0003656720000435598, - 0.00036626200017053634, - 0.00035811900306725875, - 0.0003585090016713366, - 0.002688232001673896, - 0.0003786939996643923, - 0.00035653300437843427, - 0.00035827900137519464, - 0.0003548110034898855, - 0.00035307399957673624, - 0.0016703059955034405, - 0.00037207300192676485, - 0.001108358999772463, - 0.000371916001313366, - 0.00035670500074047595, - 0.0003629580023698509, - 0.0003521249964251183, - 0.00035266100167063996, - 0.0003589370026020333, - 0.0003603899967856705, - 0.0003527189983287826, - 0.0029468209977494553, - 0.00037783999869134277, - 0.00036942500446457416, - 0.0003611150023061782, - 0.0003614060042309575, - 0.00035567399754654616, - 0.00035885499528376386, - 0.00035589699837146327, - 0.0003515719945426099, - 0.005974225998215843, - 0.00037939399771858007, - 0.00036838799860561267, - 0.00034271099866600707, - 0.0003569960026652552, - 0.00034526100353104994, - 0.00034599700302351266, - 0.0003517200020723976, - 0.0003477769932942465, - 0.0049267439972027205, - 0.0003803389990935102, - 0.0008794440000201575, - 0.0003604770026868209, - 0.00034770499769365415, - 0.00034431899985065684, - 0.00036136899871053174, - 0.0003511690010782331, - 0.00035609299811767414, - 0.0003522469996823929, - 0.0003433559977565892, - 0.00629315800324548, - 0.006998878998274449, - 0.007995260006282479, - 0.006998752003710251, - 0.006997886004683096, - 0.007996660999197047, - 0.005998475004162174, - 0.004783037999004591, - 0.005219225000473671, - 0.001990837001358159, - 0.003002227997058071, - 0.005219598999246955, - 0.006771094995201565, - 0.006998560995270964, - 0.007072665001032874, - 0.00770352799736429, - 0.006219178998435382, - 0.008994142997835297, - 0.007995375999598764, - 0.00399810700037051, - 0.0030086569968261756, - 0.002008953997574281, - 0.006989667002926581, - 0.007983859999512788, - 0.005125508003402501, - 0.005868061998626217, - 0.007999335997737944, - 0.007996318992809393, - 0.005996665000566281, - 0.004005287999461871, - 0.00399398899753578, - 0.005994056999043096, - 0.0060001689998898655, - 0.007994535000761971, - 0.008118224002828356, - 0.0038777959998697042, - 0.004146761006268207, - 0.005931380997935776, - 0.003916581998055335, - 0.005121836999023799, - 0.007871347996115219, - 0.005241163999016862, - 0.004744315003335942, - 0.001999906002311036, - 0.006990253001276869, - 0.0049970599939115345, - 0.006999189005000517, - 0.010998690995620564, - 0.002093479997711256, - 0.003899923001881689, - 0.004998723998141941, - 0.006996990996412933, - 0.005060047005827073, - 0.002169924002373591, - 0.009762969995790627, - 0.006993637995037716, - 0.006992490998527501, - 0.007999437999387737, - 0.0060095509979873896, - 0.009984916003304534, - 0.006996554999204818, - 0.005993383005261421, - 0.008125872001983225, - 0.006943092004803475, - 0.006918535997101571, - 0.010002603004977573, - 0.006992476002778858, - 0.0032332019982277416, - 0.001035249006235972, - 0.0009619839984225109, - 0.0009822989959502593, - 0.0009145680014626123, - 0.0009780079999472946, - 0.0009408230034750886, - 0.0009674520042608492, - 0.0009048000065376982, - 0.0009326329964096658, - 0.000928841000131797, - 0.0009524240012979135, - 0.0009618330004741438, - 0.0008908520030672662, - 0.000939710000238847, - 0.0009331459950772114, - 0.0009477950006839819, - 0.000928918001591228, - 0.0009271779999835417, - 0.0009444359966437332, - 0.0009536430006846786, - 0.0009483940011705272, - 0.0009329690001322888, - 0.0030504870010190643, - 0.001385980001941789, - 0.0009434930034331046, - 0.0009319899982074276, - 0.0018788850065902807, - 0.0011495660000946373, - 0.0009688499994808808, - 0.0010448059983900748, - 0.0009556439981679432, - 0.0009573439965606667, - 0.0009274459953303449, - 0.001166312002169434, - 0.000901004001207184, - 0.002307315000507515, - 0.0011955809968640096, - 0.0010645930015016347, - 0.0009988960009650327, - 0.0009744909984874539, - 0.0008885779971024022, - 0.0006409790003090166, - 0.00039191500400193036, - 0.0005283520004013553, - 0.00035715099511435255, - 0.00035721799940802157, - 0.0003506180000840686, - 0.00034807599877240136, - 0.0058122990012634546, - 0.0003746980000869371, - 0.0003580009943107143, - 0.000354953997884877, - 0.0003702180038089864, - 0.0003510250026010908, - 0.00034905899519799277, - 0.0003549270040821284, - 0.00034507900272728875, - 0.00035331700200913474, - 0.005999223001708742, - 0.00037824099854333326, - 0.0003573360008886084, - 0.0003619779963628389, - 0.0003563270001905039, - 0.0003515170028549619, - 0.0003519239980960265, - 0.00035073800245299935, - 0.0003476109995972365, - 0.004438700001628604, - 0.0004221889976179227, - 0.0003700969973579049, - 0.0003572490022634156, - 0.0003466140042291954, - 0.000351178998243995, - 0.0003461219966993667, - 0.00034421199961798266, - 0.001099280001653824, - 0.0003853169982903637, - 0.0004004809961770661, - 0.00035433699667919427, - 0.0003514830023050308, - 0.0003426059993216768, - 0.002938441000878811, - 0.0005741380009567365, - 0.0003616499961935915, - 0.00034824600152205676, - 0.0003580150005291216, - 0.00034735599911073223, - 0.0003519180027069524, - 0.000350330003129784, - 0.00034451999817974865, - 0.0020151620046817698, - 0.0019474229993647896, - 0.00038740099989809096, - 0.0003531139955157414, - 0.0003476410056464374, - 0.0019390650049899705, - 0.0018300759984413162, - 0.0003741510008694604, - 0.001386174997605849, - 0.00036797000211663544, - 0.00035827299871016294, - 0.00034868300281232223, - 0.00035912000021198764, - 0.00035803399805445224, - 0.0003462149979895912, - 0.0003534930001478642, - 0.0038442750010290183, - 0.0007552159950137138, - 0.00035167600435670465, - 0.0003532449991325848, - 0.0003523079940350726, - 0.0003495760029181838, - 0.0003524610001477413, - 0.0003439579959376715, - 0.00035027400008402765, - 0.005429645003459882, - 0.007205285000964068, - 0.0006001650035614148, - 0.000674711998726707, - 0.00035391699930187315, - 0.0003464600013103336, - 0.0003478879953036085, - 0.00037680199602618814, - 0.0003633960004663095, - 0.005084693999378942, - 0.0004189280007267371, - 0.00035347299854038283, - 0.0003496999997878447, - 0.0003557839954737574, - 0.0003520500031299889, - 0.0004199329996481538, - 0.00035215199750382453, - 0.0003508650042931549, - 0.005503874002897646, - 0.0004082369996467605, - 0.0003624099990702234, - 0.00035431000287644565, - 0.0003480200030026026, - 0.0003663969982881099, - 0.00035602600109996274, - 0.0003530300018610433, - 0.00036074499803362414, - 0.000349654997990001, - 0.0003434870013734326, - 0.0061460520009859465, - 0.0003772990021388978, - 0.00035667000338435173, - 0.00036287499824538827, - 0.0003656370026874356, - 0.00035073199978796765, - 0.0003518899975460954, - 0.0003526619984768331, - 0.00034533200232544914, - 0.005917376998695545, - 0.00036764000105904415, - 0.00035172500065527856, - 0.00034838999999919906, - 0.0003521780017763376, - 0.000347747998603154 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil", - "fullname": "TestCalculateDiagnosticsForTurbulence", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.0008620799999334849, - "max": 0.01892632200178923, - "mean": 0.006499208834429737, - "stddev": 0.0020825534092384075, - "rounds": 465, - "median": 0.006990721994952764, - "iqr": 0.002739232999374508, - "q1": 0.005250037251244066, - "q3": 0.007989270250618574, - "iqr_outliers": 7, - "stddev_outliers": 116, - "outliers": "116;7", - "ld15iqr": 0.0014779140037717298, - "hd15iqr": 0.012571820996527094, - "ops": 153.86488193800952, - "total": 3.022132108009828, - "data": [ - 0.008456655996269546, - 0.00575521400605794, - 0.0037555399976554327, - 0.0035256189948995598, - 0.007968473000801168, - 0.007834352996724192, - 0.006606032999115996, - 0.009988350997446105, - 0.005999191002047155, - 0.006990850997681264, - 0.010103467000590172, - 0.00500432999979239, - 0.0028689920000033453, - 0.004192112995951902, - 0.0027951689990004525, - 0.00726114799908828, - 0.004808696998225059, - 0.002918116995715536, - 0.005103912997583393, - 0.002242561000457499, - 0.005041762997279875, - 0.0015944830010994337, - 0.006001943998853676, - 0.002512462997401599, - 0.0034750569975585677, - 0.0024270159992738627, - 0.0041683429953991435, - 0.007396637003694195, - 0.008000283000001218, - 0.005983488001220394, - 0.007079926996084396, - 0.006909409006766509, - 0.006002722002449445, - 0.008053311001276597, - 0.004905912996036932, - 0.010000529000535607, - 0.007999601999472361, - 0.006992700000409968, - 0.004995529001462273, - 0.003703932001371868, - 0.004288075004296843, - 0.002831216996128205, - 0.005348997998225968, - 0.012571820996527094, - 0.01892632200178923, - 0.006084300999646075, - 0.004648907997761853, - 0.005540612997720018, - 0.006000995999784209, - 0.009997960005421191, - 0.005989419005345553, - 0.00698322600510437, - 0.007995870000740979, - 0.008992641000077128, - 0.006132797003374435, - 0.007851493995985948, - 0.0042734289963846095, - 0.007706620999670122, - 0.00299807199917268, - 0.005776345999038313, - 0.007215574005385861, - 0.005520858998352196, - 0.006472859000496101, - 0.005004192003980279, - 0.004990725006791763, - 0.00599564299773192, - 0.006996023999818135, - 0.006997240001510363, - 0.0059911700009251945, - 0.006002120004268363, - 0.0018822220008587465, - 0.005968072000541724, - 0.0031356649997178465, - 0.003236573000322096, - 0.007794547003868502, - 0.003959066998504568, - 0.005055234003521036, - 0.0083563750013127, - 0.008643282002594788, - 0.005930769999395125, - 0.008997031996841542, - 0.006991223999648355, - 0.00699718100077007, - 0.007004084000072908, - 0.0059898690014961176, - 0.006993548995524179, - 0.007003953003732022, - 0.007988403005583677, - 0.006996575997618493, - 0.008997721000923775, - 0.006063041000743397, - 0.008809468999970704, - 0.005498006998095661, - 0.00456358099472709, - 0.0022579040014534257, - 0.0027918870036955923, - 0.0019865430003846996, - 0.004011655000795145, - 0.007989149002241902, - 0.011002008002833463, - 0.0059954189928248525, - 0.007995191997906659, - 0.010991284994815942, - 0.008005602001503576, - 0.0070865430025151, - 0.006893239995406475, - 0.006996751995757222, - 0.004995552000764292, - 0.0049941070028580725, - 0.003994480997789651, - 0.005012159002944827, - 0.006983241000853013, - 0.009994988999096677, - 0.004566083996905945, - 0.0034225010022055358, - 0.0030020870035514235, - 0.0039906040037749335, - 0.006998903998464812, - 0.007998095999937505, - 0.0076106170017737895, - 0.010385476998635568, - 0.007990137004526332, - 0.005998016000376083, - 0.007993224004167132, - 0.005996941996272653, - 0.0053068979977979325, - 0.007686237004236318, - 0.002027750000706874, - 0.004960065001796465, - 0.007003771999734454, - 0.006991175003349781, - 0.0074976000032620504, - 0.005818284997076262, - 0.006668226000329014, - 0.0020043240001541562, - 0.0030159349989844486, - 0.0049734529966372065, - 0.008996413998829667, - 0.005996210005832836, - 0.0089975129958475, - 0.0070023009975557216, - 0.005993787002807949, - 0.006994372000917792, - 0.007144226998207159, - 0.006851893005659804, - 0.00499157200101763, - 0.005254340001556557, - 0.003733215999091044, - 0.007000024997978471, - 0.0070257109982776456, - 0.00797537800099235, - 0.006990721994952764, - 0.0049891770031536, - 0.005237129000306595, - 0.003753422999579925, - 0.00701208200189285, - 0.005993008002405986, - 0.006997620999754872, - 0.00698680299683474, - 0.005992040998535231, - 0.006998517994361464, - 0.007993191000423394, - 0.006997890995990019, - 0.007993355000508018, - 0.007996843996807002, - 0.002993649999552872, - 0.004999337004846893, - 0.006994547002250329, - 0.0079948249986046, - 0.006995256000664085, - 0.006998197997745592, - 0.0054239609962678514, - 0.007573875998787116, - 0.005989814002532512, - 0.006002521004120354, - 0.002994107002450619, - 0.006994502000452485, - 0.013723189003940206, - 0.0032692359964130446, - 0.007003182006883435, - 0.00599466499988921, - 0.003998631000285968, - 0.007015703995421063, - 0.007973895000759512, - 0.009802556000067852, - 0.005186170994420536, - 0.007990977996087167, - 0.0029948160008643754, - 0.00599432599847205, - 0.008001709000382107, - 0.008001091999176424, - 0.006991140005993657, - 0.0069948729942552745, - 0.0071151279989862815, - 0.005876147995877545, - 0.006999650999205187, - 0.007997003995114937, - 0.007005008999840356, - 0.0059943119995296, - 0.006996979005634785, - 0.007987700999365188, - 0.007992732003913261, - 0.008994775998871773, - 0.005690530000720173, - 0.009306259998993482, - 0.00699671299662441, - 0.004003442001703661, - 0.0020582960059982724, - 0.003924323995306622, - 0.004752230997837614, - 0.004238811001414433, - 0.0033186649961862713, - 0.00367616000585258, - 0.005994920000375714, - 0.004727677995106205, - 0.00726813299843343, - 0.006940040002518799, - 0.01108098999975482, - 0.00901667399739381, - 0.006934067001566291, - 0.00580048199481098, - 0.006182806995639112, - 0.0029987819943926297, - 0.006167077001009602, - 0.00769057399884332, - 0.013147637000656687, - 0.004355270000814926, - 0.0015945600025588647, - 0.006022359004418831, - 0.011000746999343392, - 0.006992285001615528, - 0.007009196997387335, - 0.005516746001376305, - 0.006464359998062719, - 0.007993475002876949, - 0.003985802999523003, - 0.007001725003647152, - 0.008000554000318516, - 0.006989606001297943, - 0.006999195000389591, - 0.0069936149957356974, - 0.004001803004939575, - 0.001993203994061332, - 0.007998171000508592, - 0.007993944003828801, - 0.009998184003052302, - 0.006990697998844553, - 0.006592619996808935, - 0.010407622998172883, - 0.007994338004209567, - 0.001989917000173591, - 0.0039984699978958815, - 0.013001639999856707, - 0.0060951630002819, - 0.003893924003932625, - 0.0021353949996409938, - 0.005292094996548258, - 0.008568548997573089, - 0.0059959640057059005, - 0.006931965996045619, - 0.006058838000171818, - 0.004994098999304697, - 0.004999960998247843, - 0.0069998269973439164, - 0.004999313001462724, - 0.00999652300379239, - 0.004015632002847269, - 0.0014779140037717298, - 0.005662201998347882, - 0.007821659004548565, - 0.007995626001502387, - 0.006997751996095758, - 0.00799775100313127, - 0.006992194001213647, - 0.006567900003574323, - 0.005804642001749016, - 0.007627726998180151, - 0.007986320997588336, - 0.007995531996130012, - 0.007997988002898637, - 0.005999517998134252, - 0.0049969300016527995, - 0.005991309000819456, - 0.008001804002560675, - 0.00799426800222136, - 0.005010163004044443, - 0.007980820999364369, - 0.008004223003808875, - 0.00798963399574859, - 0.005171823999262415, - 0.006816487999458332, - 0.00901342999713961, - 0.010977947000355925, - 0.00899583900172729, - 0.006994627001404297, - 0.01099719599733362, - 0.0070561089960392565, - 0.008354186000360642, - 0.004587465999065898, - 0.0049189950004802085, - 0.003075200002058409, - 0.0069846810001763515, - 0.001991404002183117, - 0.006002235000778455, - 0.007999174995347857, - 0.007991288002813235, - 0.006001359994115774, - 0.0059946809997200035, - 0.008001303998753428, - 0.006999707002250943, - 0.007982686001923867, - 0.0059958479978377, - 0.0059988729990436696, - 0.00599826800316805, - 0.005993510996631812, - 0.006998863005719613, - 0.00799509699572809, - 0.006996440999500919, - 0.007995854000910185, - 0.006996854004682973, - 0.007998553999641445, - 0.00799399899551645, - 0.007995444000698626, - 0.007997409004019573, - 0.009002516999316867, - 0.007995512998604681, - 0.007990192003489938, - 0.006999488003202714, - 0.006993793998844922, - 0.005005414001061581, - 0.00798590300109936, - 0.006995334995735902, - 0.0059968599962303415, - 0.003018668998265639, - 0.0019680770055856556, - 0.004997422998712864, - 0.006997521995799616, - 0.007011310997768305, - 0.006646972993621603, - 0.00634390000050189, - 0.006986496002355125, - 0.008004315997823142, - 0.005990035002469085, - 0.005995542996970471, - 0.007996466003532987, - 0.006998583994572982, - 0.005998212000122294, - 0.00699610300216591, - 0.0069995189987821504, - 0.00799548900249647, - 0.004000387001724448, - 0.005996564999804832, - 0.005994804007059429, - 0.006998013996053487, - 0.008004657996934839, - 0.008554842999728862, - 0.006440402998123318, - 0.002986177998536732, - 0.005045960999268573, - 0.007946771998831537, - 0.007997448003152385, - 0.00799515100516146, - 0.006997303004027344, - 0.004999599004804622, - 0.006997274998866487, - 0.006995105999521911, - 0.00600043900340097, - 0.008995992997370195, - 0.007996936998097226, - 0.007996845000889152, - 0.006181951997859869, - 0.00781260499934433, - 0.007346473001234699, - 0.009648993000155315, - 0.007998387998668477, - 0.007997107000846881, - 0.005996181003865786, - 0.006249837999348529, - 0.005978959001367912, - 0.0037095289953867905, - 0.007056169997667894, - 0.008991504000732675, - 0.008000251000339631, - 0.006996592994255479, - 0.00625100800243672, - 0.0008620799999334849, - 0.00631148400134407, - 0.0036872959972242825, - 0.004386145999887958, - 0.002768272999674082, - 0.0009268690046155825, - 0.003771850999328308, - 0.006083134001528379, - 0.009916185001202393, - 0.007007115003943909, - 0.00798581699928036, - 0.00699505699594738, - 0.005996331994538195, - 0.0070000470004742965, - 0.004994178998458665, - 0.004162385994277429, - 0.008829997001157608, - 0.00507799399929354, - 0.006917627004440874, - 0.005994450002617668, - 0.010998650002875365, - 0.008002556998690125, - 0.007993277002242394, - 0.0069958500025677495, - 0.007007243999396451, - 0.006987291002587881, - 0.007995620006113313, - 0.007000259000051301, - 0.006999755001743324, - 0.006544903000758495, - 0.00845501200092258, - 0.0059894599980907515, - 0.00699310599884484, - 0.007535876000474673, - 0.0074594479956431314, - 0.006996627998887561, - 0.008998826000606641, - 0.0037371809958131053, - 0.004257284999766853, - 0.008002225993550383, - 0.007993241000804119, - 0.005997106003633235, - 0.005997214997478295, - 0.005998426000587642, - 0.007997979999345262, - 0.0059964170068269596, - 0.008003647999430541, - 0.006989806999627035, - 0.00625215800391743, - 0.00774121200083755, - 0.0059972710005240515, - 0.005010865002986975, - 0.004985420004231855, - 0.007400396003504284, - 0.007604655998875387, - 0.006985125997744035, - 0.007998436005436815, - 0.007996884996828157, - 0.006997133001277689, - 0.005004163998819422, - 0.007993102000909857, - 0.00754887599759968, - 0.0025271889971918426, - 0.0059132690003025346, - 0.007341365999309346, - 0.007651440995687153, - 0.005999214998155367, - 0.007996439999260474, - 0.007998786997632124, - 0.007999884001037572, - 0.006994915995164774, - 0.005994260005536489, - 0.003996416002337355, - 0.005996132000291254, - 0.005999282999255229, - 0.007999143999768421, - 0.00699604600231396, - 0.006994585994107183, - 0.007999351000762545, - 0.007996865999302827 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil", - "fullname": "TestCalculateHorizontalGradientsForTurbulence", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.00037598999915644526, - "max": 0.056945527001516894, - "mean": 0.0024951280792298884, - "stddev": 0.0032387813717274596, - "rounds": 2159, - "median": 0.0005633919936371967, - "iqr": 0.004570067998429295, - "q1": 0.000420880498495535, - "q3": 0.00499094849692483, - "iqr_outliers": 8, - "stddev_outliers": 494, - "outliers": "494;8", - "ld15iqr": 0.00037598999915644526, - "hd15iqr": 0.011923240999749396, - "ops": 400.7810293684988, - "total": 5.3869815230573295, - "data": [ - 0.0005756730024586432, - 0.0005950170016149059, - 0.0005938870017416775, - 0.0010072239965666085, - 0.0005097789980936795, - 0.0004728400017484091, - 0.0004727520063170232, - 0.0004641939958673902, - 0.0005057979942648672, - 0.00047229800111381337, - 0.0005490599942277186, - 0.0007633290006197058, - 0.0009610060005798005, - 0.001501126003859099, - 0.0006464219986810349, - 0.0011828780043288134, - 0.00048573499952908605, - 0.0004719119970104657, - 0.00045843099360354245, - 0.0005684079951606691, - 0.0005894790010643192, - 0.0005989979981677607, - 0.0005510199989657849, - 0.00047035900206537917, - 0.0004717470001196489, - 0.0004630980038200505, - 0.00056561800010968, - 0.0004613530036294833, - 0.0005238659941824153, - 0.0016906689997995272, - 0.0004875390004599467, - 0.00046130800183163956, - 0.0005517259996850044, - 0.000586733003729023, - 0.0006025170005159453, - 0.0009102449985221028, - 0.0004888959956588224, - 0.0008171919980668463, - 0.000691790999553632, - 0.0004884949958068319, - 0.0007162100009736605, - 0.0005035390058765188, - 0.0004728279964183457, - 0.0005049499959568493, - 0.000585602996579837, - 0.0005825339976581745, - 0.0005823620012961328, - 0.0005639359951601364, - 0.0005687739976565354, - 0.0007960350048961118, - 0.00047613100468879566, - 0.0004675939999287948, - 0.0005452089972095564, - 0.00045906299783382565, - 0.0009447929987800308, - 0.000617742000031285, - 0.0005947089957771823, - 0.0005831229937030002, - 0.0005512799980351701, - 0.0004735059992526658, - 0.0004638190002879128, - 0.0004638089958461933, - 0.0004646779998438433, - 0.0004633429998648353, - 0.0004548490032902919, - 0.0010793319961521775, - 0.002214510001067538, - 0.0007744020040263422, - 0.0005413050021161325, - 0.00048417499783681706, - 0.0004941320003126748, - 0.0005037800001446158, - 0.0011787610055762343, - 0.00048109100316651165, - 0.0006782999989809468, - 0.000661213998682797, - 0.00048272000276483595, - 0.0004714900060207583, - 0.0004695899988291785, - 0.0005719929977203719, - 0.0006001720030326396, - 0.0013380689997575246, - 0.0010002599956351332, - 0.00048013299965532497, - 0.0004590880053001456, - 0.00046679499791935086, - 0.0004453799992916174, - 0.0021024069937993772, - 0.0005035360009060241, - 0.0004487529949983582, - 0.0005413059989223257, - 0.00044666499888990074, - 0.00045144899922888726, - 0.0004464519952307455, - 0.00045266400411492214, - 0.00044659799459623173, - 0.0005625560006592423, - 0.0010849259997485206, - 0.0038005039968993515, - 0.0005039270035922527, - 0.00045960200077388436, - 0.0004503460004343651, - 0.0004537149943644181, - 0.00045053900248603895, - 0.0004460679992916994, - 0.00045309399865800515, - 0.0004450089982128702, - 0.0009807559981709346, - 0.0006724630002281629, - 0.0014480399986496195, - 0.0009822650026762858, - 0.0014364199960255064, - 0.0006057189966668375, - 0.00045415999920805916, - 0.0004603099951054901, - 0.00045567699999082834, - 0.0004458269977476448, - 0.0005633919936371967, - 0.000460836999991443, - 0.0007753099998808466, - 0.0007192589982878417, - 0.0010773980029625818, - 0.002680634999705944, - 0.0005013040063204244, - 0.00046943299821577966, - 0.0004541729940683581, - 0.00045101100113242865, - 0.0005683419949491508, - 0.0004509470018092543, - 0.0004516539993346669, - 0.0004426300001796335, - 0.0006121150072431192, - 0.0034514150029281154, - 0.000499007997859735, - 0.00045828700240235776, - 0.00044558499939739704, - 0.0004436719973455183, - 0.0005507950045284815, - 0.0004661900020437315, - 0.0004566060015349649, - 0.00044446500396588817, - 0.0015073499962454662, - 0.001214894997247029, - 0.0020864139951299876, - 0.0006260060035856441, - 0.0005062810014351271, - 0.00046049199590925127, - 0.0005603440004051663, - 0.00046489600208587945, - 0.00045131899969419464, - 0.0004853430000366643, - 0.0004549609948298894, - 0.0006885839975439012, - 0.0006869279968668707, - 0.0029719029989792034, - 0.0010514929963392206, - 0.0005796629993710667, - 0.00047710399667266756, - 0.0004496969995670952, - 0.0004542880051303655, - 0.0004549389996100217, - 0.00045249400136526674, - 0.00044596200314117596, - 0.0004450369960977696, - 0.0018379289977019653, - 0.003497565005091019, - 0.0007114030013326555, - 0.0004978100041626021, - 0.0004345029956311919, - 0.00044081800297135487, - 0.00042941699939547107, - 0.0004371070026536472, - 0.000428086997999344, - 0.0005283100035740063, - 0.0007735480030532926, - 0.000961409998126328, - 0.0007630769978277385, - 0.001327062003838364, - 0.0004868280011578463, - 0.00043330100015737116, - 0.00043709199962904677, - 0.0004344139961176552, - 0.000440361000073608, - 0.0004304190006223507, - 0.0007266039974638261, - 0.003655302003608085, - 0.00048070600314531475, - 0.0007453680009348318, - 0.0004570099990814924, - 0.0004455129965208471, - 0.00043683800322469324, - 0.00043329899926902726, - 0.0004432679997989908, - 0.0004347950016381219, - 0.0007867199965403415, - 0.004334860997914802, - 0.0009557949961163104, - 0.0005060540061094798, - 0.0004381789985927753, - 0.000452868000138551, - 0.0004377799996291287, - 0.00043865800398634747, - 0.0004294460013625212, - 0.00044102199899498373, - 0.0004356199933681637, - 0.0011492470002849586, - 0.0035833099973388016, - 0.0004952059971401468, - 0.0004541320013231598, - 0.0004378500016173348, - 0.00043493600242072716, - 0.00043388400081312284, - 0.0004377820005174726, - 0.00043681199895218015, - 0.00043579900375334546, - 0.0007152729958761483, - 0.000910431997908745, - 0.0008407339992118068, - 0.002822721995471511, - 0.0005302880017552525, - 0.00044935799814993516, - 0.0004403649945743382, - 0.00044441399950301275, - 0.0004369889938971028, - 0.0004363309999462217, - 0.0004755469999508932, - 0.00043522399937501177, - 0.0009965599965653382, - 0.001178376995085273, - 0.00204862800455885, - 0.00048152700037462637, - 0.00044958299986319616, - 0.0004401789992698468, - 0.00043519299651961774, - 0.000437276998127345, - 0.0004283540038159117, - 0.00043764700239989907, - 0.00043416400148998946, - 0.0005403780014603399, - 0.0008028149968595244, - 0.005926193996856455, - 0.0028862820036010817, - 0.0004869580006925389, - 0.00043842499871971086, - 0.0005008989974157885, - 0.00046001099690329283, - 0.00044283999886829406, - 0.0004332530006649904, - 0.004315772996051237, - 0.0004686879983637482, - 0.00044431100104702637, - 0.0004479709969018586, - 0.00043420499423518777, - 0.0004432319983607158, - 0.00048047299787867814, - 0.0030627190062659793, - 0.0012850419952883385, - 0.0004478289993130602, - 0.0004368480003904551, - 0.0004424780054250732, - 0.0004325659974711016, - 0.0004377840014058165, - 0.00043251700117252767, - 0.00047909100248944014, - 0.00048711500130593777, - 0.0058515740020084195, - 0.00048721799976192415, - 0.00044147399603389204, - 0.00042334299359936267, - 0.0004307670023990795, - 0.00045747600233880803, - 0.00043532900599529967, - 0.0004211209961795248, - 0.0036507990007521585, - 0.0011625490005826578, - 0.00042498399852775037, - 0.0012411529969540425, - 0.00043145099334651604, - 0.0005037500013713725, - 0.00044372100092004985, - 0.00043958199967164546, - 0.0004314300022087991, - 0.0004205010045552626, - 0.004545490999589674, - 0.00047085500409593806, - 0.000464078999357298, - 0.0004313690005801618, - 0.00046749199827900156, - 0.000491261002025567, - 0.00046866300544934347, - 0.0015938900032779202, - 0.0025566870026523247, - 0.0032178690016735345, - 0.0007935489993542433, - 0.0027084160028607585, - 0.003466874004516285, - 0.0007571460009785369, - 0.00048016300570452586, - 0.00043403600284364074, - 0.0004258780027157627, - 0.0004275289975339547, - 0.0004660669947043061, - 0.0038265360053628683, - 0.00048479500401299447, - 0.0004253799997968599, - 0.00041766500362427905, - 0.001110883997171186, - 0.000430148997111246, - 0.0004226010059937835, - 0.006010097000398673, - 0.0005255009964457713, - 0.00043780100531876087, - 0.0004425469960551709, - 0.00043176900362595916, - 0.005453423997096252, - 0.0009214210003847256, - 0.00044217299728188664, - 0.0004243020011927001, - 0.00042867699812632054, - 0.000422666002123151, - 0.00046320800174726173, - 0.0004303309979150072, - 0.005404210998676717, - 0.008997502998681739, - 0.010026304997154512, - 0.007963564996316563, - 0.0040001849993132055, - 0.0005100300040794536, - 0.0007521660008933395, - 0.00046559900511056185, - 0.00042895099613815546, - 0.0004237550019752234, - 0.003400457004318014, - 0.006024887996318284, - 0.0007576479983981699, - 0.0005513010037248023, - 0.0004583240006468259, - 0.00043519299651961774, - 0.00042079599370481446, - 0.0004258620028849691, - 0.0004270409990567714, - 0.00041636899550212547, - 0.00640832199860597, - 0.0005824539985042065, - 0.003130951998173259, - 0.0008678160011186264, - 0.000456109999504406, - 0.002596115999040194, - 0.006003661997965537, - 0.0079901740027708, - 0.006989854999119416, - 0.010997864999808371, - 0.007991721002326813, - 0.0021462299991981126, - 0.005844336999871302, - 0.0029964329951326363, - 0.0060183709938428365, - 0.006981901002291124, - 0.007997654996870551, - 0.006984231004025787, - 0.01100198199856095, - 0.005992620004690252, - 0.006992980997893028, - 0.006996984004217666, - 0.008995653006422799, - 0.011037987998861354, - 0.027100560000690166, - 0.0007916690010461025, - 0.056945527001516894, - 0.010118543999851681, - 0.01399031600158196, - 0.006884558002639096, - 0.02415811399987433, - 0.00812597000185633, - 0.003766355999687221, - 0.006962367006053682, - 0.007995025996933691, - 0.009980376999010332, - 0.007997271000931505, - 0.0059994360053678975, - 0.003984771996329073, - 0.006998657001531683, - 0.006997090997174382, - 0.007004205006523989, - 0.0006077010039007291, - 0.0005850629968335852, - 0.000522383998031728, - 0.00045880999823566526, - 0.00044504800462163985, - 0.0005122130023664795, - 0.0022267250024015084, - 0.0005096269960631616, - 0.00046078699961071834, - 0.0004438839969225228, - 0.0005771659998572432, - 0.0027939509964198805, - 0.0005109650010126643, - 0.001718090003123507, - 0.0006100070022512227, - 0.000523478003742639, - 0.0004600890024448745, - 0.0004559110020636581, - 0.0004597149963956326, - 0.0004597750012180768, - 0.00044694699317915365, - 0.0014180279977153987, - 0.0036684860024251975, - 0.0005837169956066646, - 0.0004531640006462112, - 0.00044540899398270994, - 0.00043664000259013847, - 0.00043584999366430566, - 0.0004323940011090599, - 0.003774019001866691, - 0.0007788530056132004, - 0.0004353600015747361, - 0.0004434080037754029, - 0.00043892499525099993, - 0.00046502600162057206, - 0.0004752910026581958, - 0.000444637997134123, - 0.0004383060004329309, - 0.0004411509944475256, - 0.004485476005356759, - 0.0005937640016782098, - 0.00048055799561552703, - 0.0004366080029285513, - 0.0005072499989182688, - 0.00045144899922888726, - 0.004493786000239197, - 0.0005089320038678125, - 0.0007230569972307421, - 0.0004652839998016134, - 0.00046394400123972446, - 0.0004584139969665557, - 0.0005042139964643866, - 0.0004463820005184971, - 0.0004541180023807101, - 0.00043412100058048964, - 0.003478943996015005, - 0.0013198369997553527, - 0.0004461009957594797, - 0.0004341309977462515, - 0.00044370699470164254, - 0.00169189200096298, - 0.0004800099995918572, - 0.00044462500227382407, - 0.0004381119942991063, - 0.00043796699901577085, - 0.000838476000353694, - 0.0017399919961462729, - 0.0009961600007954985, - 0.0005334039960871451, - 0.00045909899927210063, - 0.00048486800369573757, - 0.00045516699901781976, - 0.00043881600140593946, - 0.00044439200428314507, - 0.00044052200246369466, - 0.00043337600072845817, - 0.005968885001493618, - 0.0005593409950961359, - 0.0004406360021675937, - 0.002089241999783553, - 0.0004935300021315925, - 0.0004534270046860911, - 0.0013978290007798932, - 0.00044523299584398046, - 0.001334818996838294, - 0.00045156000123824924, - 0.00048410399904241785, - 0.0004647289970307611, - 0.000429411004006397, - 0.0004362210020190105, - 0.0004346330024418421, - 0.00044624199654208496, - 0.00043541600462049246, - 0.00626232499780599, - 0.0005418919972726144, - 0.0005081420022179373, - 0.0021242759976303205, - 0.00043460799497552216, - 0.0004206140001770109, - 0.0004360710008768365, - 0.00042363199463579804, - 0.0007962139934534207, - 0.002287324001372326, - 0.0006290599994827062, - 0.00048786800471134484, - 0.0004957499986630864, - 0.00044308900396572426, - 0.00043161900248378515, - 0.00042660000326577574, - 0.0017291139956796542, - 0.0030009939946467057, - 0.0005425749986898154, - 0.000938804994802922, - 0.00045687500096391886, - 0.00042826600110856816, - 0.0004344729968579486, - 0.00042194200068479404, - 0.00042991100053768605, - 0.00042362400563433766, - 0.0033467929970356636, - 0.0009570729962433688, - 0.00048107300244737417, - 0.00046473000111291185, - 0.0005189980001887307, - 0.0004457420000107959, - 0.00043208400165895, - 0.00042060700070578605, - 0.002643505999003537, - 0.0004937490011798218, - 0.00048038199747679755, - 0.00042242000199621543, - 0.0004321000014897436, - 0.00042336899787187576, - 0.000427088001742959, - 0.0004190619947621599, - 0.0004210599945508875, - 0.00524283800041303, - 0.000520128000061959, - 0.0004873529978794977, - 0.0004758240029332228, - 0.0004288709969841875, - 0.00042636899888748303, - 0.00042411400499986485, - 0.00042750599823193625, - 0.0004679359990404919, - 0.0057430520027992316, - 0.0005414610059233382, - 0.00046274399937829003, - 0.00042635099816834554, - 0.0004218989997752942, - 0.00046536000445485115, - 0.00043567099783103913, - 0.00042483599827392027, - 0.005306622006173711, - 0.000542205001693219, - 0.0005118730041431263, - 0.000888654998561833, - 0.0005693700004485436, - 0.0007091910010785796, - 0.0004346610003267415, - 0.005436273000668734, - 0.0005406159980338998, - 0.001201318002131302, - 0.0005955939996056259, - 0.00046576099703088403, - 0.00046007199853193015, - 0.00042628200026229024, - 0.0004293370002415031, - 0.0004229380065225996, - 0.0031140400024014525, - 0.0004920919964206405, - 0.0004619430037564598, - 0.0004914840028504841, - 0.00045808000140823424, - 0.00043264800478937104, - 0.0004217259993311018, - 0.004359174003184307, - 0.0005712810016120784, - 0.0006300789973465726, - 0.0004269299970474094, - 0.00041071300074690953, - 0.0004358060032245703, - 0.00041548500303179026, - 0.0008874870036379434, - 0.0015596580051351339, - 0.0021395739968284033, - 0.0005832440001540817, - 0.000490272999741137, - 0.00041946199780795723, - 0.00041496800258755684, - 0.00041010399581864476, - 0.0004131419991608709, - 0.0005867780055268668, - 0.003913713000656571, - 0.0010132489987881854, - 0.0004266359974280931, - 0.00043076599831692874, - 0.00041501499799778685, - 0.0004160820026299916, - 0.000427965002018027, - 0.0004145829952904023, - 0.00044410199916455895, - 0.00262374500016449, - 0.0022298900003079325, - 0.0005073060019640252, - 0.0004843980059376918, - 0.0004458699986571446, - 0.00043078000453533605, - 0.00041795500146690756, - 0.0004216899978928268, - 0.0004475620007724501, - 0.004387435001262929, - 0.0005301729979692027, - 0.0005063619973952882, - 0.00045655699796043336, - 0.000415923997934442, - 0.00041065100231207907, - 0.00041027599945664406, - 0.00041547100408934057, - 0.0004145580023759976, - 0.005404311006714124, - 0.0006763229976058938, - 0.0004925020039081573, - 0.0004521669980022125, - 0.0004135920025873929, - 0.00040613400051370263, - 0.00044318599975667894, - 0.0004304590038373135, - 0.00041235899698222056, - 0.005606316000921652, - 0.0005079300026409328, - 0.0004676630051108077, - 0.00041082499956246465, - 0.003464200002781581, - 0.000494656000228133, - 0.0004731190056190826, - 0.0004094679970876314, - 0.003171929005475249, - 0.0005408540018834174, - 0.00047457699838560075, - 0.00048788400454213843, - 0.00041372200212208554, - 0.000410573004046455, - 0.00040862699825083837, - 0.006175032001920044, - 0.0005734090009354986, - 0.00048807099665282294, - 0.0004806990036740899, - 0.00041606200102251023, - 0.00042168700019828975, - 0.00041144699935102835, - 0.005235445998550858, - 0.0005408619981608354, - 0.00048330199933843687, - 0.0004703309969045222, - 0.0004223119976813905, - 0.00045728400436928496, - 0.0004248140030540526, - 0.005216727004153654, - 0.0005475299985846505, - 0.0005068379978183657, - 0.0004790640014107339, - 0.0004239779955241829, - 0.0004170820029685274, - 0.002695392002351582, - 0.0008292409984278493, - 0.0004773470063810237, - 0.000452943000709638, - 0.0011082259952672757, - 0.0004331460004323162, - 0.00042663299973355606, - 0.00040986499516293406, - 0.0003959480018238537, - 0.00040045299829216674, - 0.0003990539989899844, - 0.000396557996282354, - 0.000402229001338128, - 0.00039701700006844476, - 0.006404580999515019, - 0.0005517440004041418, - 0.00048241399781545624, - 0.00043370600178604946, - 0.0004008840041933581, - 0.0004019089974462986, - 0.00039991800440475345, - 0.000405740000132937, - 0.0004017489991383627, - 0.00040006200288189575, - 0.002699898002902046, - 0.002087072003632784, - 0.0009810660048970021, - 0.0004236250024405308, - 0.00040537400491302833, - 0.000398156997107435, - 0.0013588869987870567, - 0.00041081100062001497, - 0.00040201900264946744, - 0.0004028739931527525, - 0.00039987999480217695, - 0.0019054579970543273, - 0.0004580230015562847, - 0.00040629299473948777, - 0.00040498199814464897, - 0.00040088099922286347, - 0.00040569900011178106, - 0.00040977300523081794, - 0.0004005619994131848, - 0.003620543997385539, - 0.00046585899690398946, - 0.000513416001922451, - 0.0004512589948717505, - 0.00040464699850417674, - 0.0003988539974670857, - 0.0003962069968110882, - 0.004284390000975691, - 0.0004399779936647974, - 0.0005932280037086457, - 0.00042006799776572734, - 0.0013205770010245033, - 0.00046071199903963134, - 0.00040353700023842975, - 0.001184467000712175, - 0.000832910998724401, - 0.0004138629956287332, - 0.00040177800110541284, - 0.0004061479994561523, - 0.00040392799564870074, - 0.00040385499596595764, - 0.00040718199306866154, - 0.0008553640000172891, - 0.0017520830006105825, - 0.002336140001716558, - 0.0004078210040461272, - 0.0004062430016347207, - 0.0004011839992017485, - 0.0004067710033268668, - 0.001025730001856573, - 0.0013024680010857992, - 0.000577087004785426, - 0.00040701199759496376, - 0.0004915600002277642, - 0.00045371799933491275, - 0.0004057810001540929, - 0.0020362669965834357, - 0.0015786320000188425, - 0.0004191330008325167, - 0.0004061500003444962, - 0.0004082039959030226, - 0.0004075760007253848, - 0.0004220319970045239, - 0.0004143469996051863, - 0.0012633250007638708, - 0.0024676710017956793, - 0.0012569829996209592, - 0.0004311379962018691, - 0.00042399499943712726, - 0.0008126050015562214, - 0.0013847589943907224, - 0.0004181459953542799, - 0.00047202599671436474, - 0.0004552529935608618, - 0.00040479499875800684, - 0.00040775499655865133, - 0.00040382699808105826, - 0.0003987580057582818, - 0.004201559000648558, - 0.0018218519981019199, - 0.00044732799869962037, - 0.0004042810032842681, - 0.0009958920054486953, - 0.001468877999286633, - 0.0004258189946995117, - 0.00040515799628337845, - 0.00040177199844038114, - 0.00040160000207833946, - 0.0004029459960293025, - 0.0004053049997310154, - 0.0007949169958010316, - 0.003425710994633846, - 0.0010665849986253306, - 0.0004159090021857992, - 0.00040248900040751323, - 0.000422358003561385, - 0.0004043549997732043, - 0.006421999001759104, - 0.006914705001690891, - 0.006649780996667687, - 0.0023468439976568334, - 0.000448560000222642, - 0.00041852999856928363, - 0.0005151029981789179, - 0.0004311579978093505, - 0.0007740380024188198, - 0.0004100340011063963, - 0.0005440230015665293, - 0.0004107080021640286, - 0.00040907200309447944, - 0.0003989020042354241, - 0.006964713000343181, - 0.00048273600259562954, - 0.0004132409958401695, - 0.0021623779975925572, - 0.0013699120027013123, - 0.00043791999632958323, - 0.0004179660027148202, - 0.0004128560030949302, - 0.0004082360028405674, - 0.0041747129944269545, - 0.0004515789987635799, - 0.0004074680036865175, - 0.0016735300014261156, - 0.0004216719971736893, - 0.0019393850016058423, - 0.0014136700046947226, - 0.00044575500214705244, - 0.0004142299949307926, - 0.0004100459991605021, - 0.0033867310048663057, - 0.00044149200402898714, - 0.0004065750035806559, - 0.0004128099972149357, - 0.0004103800019947812, - 0.00041436699393671006, - 0.0036954120005248114, - 0.0004208350001135841, - 0.0004141340032219887, - 0.0032067790016299114, - 0.00044189300388097763, - 0.0014247119979700074, - 0.00043324400030542165, - 0.0019478690010146238, - 0.0012837369940825738, - 0.0004188719976809807, - 0.00040791100036585703, - 0.00040077499579638243, - 0.0032333279959857464, - 0.00045303499791771173, - 0.00040624800021760166, - 0.0004183830023976043, - 0.0004029769988846965, - 0.0004016640014015138, - 0.0003988879980170168, - 0.00040573500155005604, - 0.004882916000497062, - 0.000418278003053274, - 0.00040160999924410135, - 0.00040245099808089435, - 0.0004325159970903769, - 0.00039797899808036163, - 0.00041354399581905454, - 0.00039898999966681004, - 0.0004027210015919991, - 0.0005032779954490252, - 0.00041842499922495335, - 0.0004417360032675788, - 0.00041015599708771333, - 0.0004152920009801164, - 0.00041011600114870816, - 0.00040556100429967046, - 0.0004003549984190613, - 0.00039933100197231397, - 0.008607367999502458, - 0.00042405500425957143, - 0.0004201600022497587, - 0.0004036940008518286, - 0.0004089700014446862, - 0.004814815001736861, - 0.0004237810062477365, - 0.0004004160000476986, - 0.0034392619927530177, - 0.0004166810031165369, - 0.0004059100028825924, - 0.00040480699681211263, - 0.00040032399556366727, - 0.0004081710067111999, - 0.0003988859971286729, - 0.005959334994258825, - 0.00044900100328959525, - 0.0004063399974256754, - 0.0003958059969590977, - 0.00040747800085227937, - 0.00039903699507704005, - 0.00040150499989977106, - 0.000401080003939569, - 0.00039928800106281415, - 0.005741594002756756, - 0.00041592700290493667, - 0.00040588900446891785, - 0.00040843799797585234, - 0.0004048590053571388, - 0.00039972599915927276, - 0.000400736003939528, - 0.00040322700078831986, - 0.000396363997424487, - 0.007569845001853537, - 0.00039887100138003007, - 0.00039996499981498346, - 0.0003877139970427379, - 0.0003993989957962185, - 0.000392889000067953, - 0.00039296699833357707, - 0.00039407899748766795, - 0.00038850700366310775, - 0.006873178004752845, - 0.0015027449990157038, - 0.00041503499960526824, - 0.0003943030023947358, - 0.0003977469968958758, - 0.0004352309988462366, - 0.0003951879989472218, - 0.0003935389977414161, - 0.0003884110046783462, - 0.0072743900018394925, - 0.00040059700404526666, - 0.0003927149955416098, - 0.00039322800148511305, - 0.0003923460026271641, - 0.0003891239975928329, - 0.0003933789994334802, - 0.00038519600639119744, - 0.0070788619996164925, - 0.000399910997657571, - 0.00039018299867166206, - 0.0003973810016759671, - 0.0003899259972968139, - 0.0003936260036425665, - 0.0003937840010621585, - 0.0003928280057152733, - 0.004688815002737101, - 0.000432438995630946, - 0.00039533499511890113, - 0.0003968009987147525, - 0.0003925339988199994, - 0.000387124004191719, - 0.0003932270046789199, - 0.0009395250017405488, - 0.0008064619978540577, - 0.0007260179991135374, - 0.001220004000060726, - 0.0014512649941025302, - 0.0004082710001966916, - 0.0006296589999692515, - 0.0011328450054861605, - 0.0003981509944424033, - 0.0004006999952252954, - 0.0003982899943366647, - 0.0003951780017814599, - 0.0003976480002165772, - 0.00039423300040652975, - 0.0004414120048750192, - 0.002478063994203694, - 0.001592683001945261, - 0.00117573299939977, - 0.00039105199539335445, - 0.00040134300070349127, - 0.000448899001639802, - 0.00039633599953958765, - 0.002138174997526221, - 0.0008866139978636056, - 0.00040978999459184706, - 0.00039418999949702993, - 0.00039671699778409675, - 0.0003908829967258498, - 0.0004035780002595857, - 0.0003910390005330555, - 0.0003942850016755983, - 0.0007818430021870881, - 0.0008274380015791394, - 0.00040075700235320255, - 0.0004002760033472441, - 0.0003921370007446967, - 0.00039470100455218926, - 0.0017000689986161888, - 0.0023826879987609573, - 0.00043314499635016546, - 0.00039890499465400353, - 0.00040270200406666845, - 0.00039677200402366, - 0.000411225002608262, - 0.0003955040010623634, - 0.0004689659981522709, - 0.004077457000676077, - 0.00042598599975463003, - 0.00039751300209900364, - 0.0004479139970499091, - 0.0003921489987988025, - 0.001141404005466029, - 0.00046265100536402315, - 0.00038808799581602216, - 0.00040183700184570625, - 0.00039213600393850356, - 0.0003980449982918799, - 0.00039039100374793634, - 0.0007997030043043196, - 0.002204286996857263, - 0.0019187470024917275, - 0.0004387910012155771, - 0.00040185699617723003, - 0.0003912449974450283, - 0.0015053859970066696, - 0.0009663840028224513, - 0.00046542500058421865, - 0.0003941950053558685, - 0.00039451899647247046, - 0.00038852899888297543, - 0.000437856993812602, - 0.0003962099945056252, - 0.0003901980016962625, - 0.0016954940001596697, - 0.0024404020005022176, - 0.0003933650004910305, - 0.0005946660021436401, - 0.000401427001634147, - 0.0003875190013786778, - 0.0006772170017939061, - 0.0005089209953439422, - 0.0004990640009054914, - 0.0005071050036349334, - 0.0004948259957018308, - 0.0005018290030420758, - 0.0005033619963796809, - 0.0004955859985784627, - 0.00040689999877940863, - 0.00039326000114670023, - 0.00039853400085121393, - 0.0003879270007018931, - 0.003514994001307059, - 0.0004960950027452782, - 0.0004075030010426417, - 0.0003927220022887923, - 0.0003958630040870048, - 0.00041254299867432564, - 0.00038918400241527706, - 0.0012264339966350235, - 0.002537617001507897, - 0.00046048700460232794, - 0.00039700700290268287, - 0.00039020500116748735, - 0.00044634100049734116, - 0.0003958099987357855, - 0.0005968120021861978, - 0.0037100609988556243, - 0.0018390329933026806, - 0.001293483997869771, - 0.0004239750051056035, - 0.001440997002646327, - 0.0005102259965497069, - 0.00041281099402112886, - 0.003323324999655597, - 0.00245696899946779, - 0.0004486309990170412, - 0.0004095999975106679, - 0.0015531210010522045, - 0.0010156749995076098, - 0.00042962300358340144, - 0.0004065989996888675, - 0.00041650400089565665, - 0.00039934900269145146, - 0.00041899200004991144, - 0.00039970799844013527, - 0.0005395320040406659, - 0.0007534370015491731, - 0.000964988001214806, - 0.0019006680013262667, - 0.001501561993791256, - 0.0009537389996694401, - 0.0004176560032647103, - 0.0008847479984979145, - 0.001103272006730549, - 0.0006105739958002232, - 0.0004102540042367764, - 0.00041328400402562693, - 0.00040128800173988566, - 0.0004001990018878132, - 0.00044689200149150565, - 0.0004010679986095056, - 0.001096685002266895, - 0.001084631003323011, - 0.00043642700620694086, - 0.00040494099812349305, - 0.0025166940031340346, - 0.001182861000415869, - 0.0004177029986749403, - 0.0015504830007557757, - 0.0004113039976800792, - 0.00041487300040898845, - 0.00040111199632519856, - 0.0004098110002814792, - 0.0003996360028395429, - 0.0003981909976573661, - 0.0007758100036880933, - 0.0006246789998840541, - 0.0032420880015706643, - 0.00045408100413624197, - 0.000407768995501101, - 0.00040123499638866633, - 0.0014534160000039265, - 0.0004046809990541078, - 0.0012740959937218577, - 0.00047276599798351526, - 0.000408037994930055, - 0.0005612460008705966, - 0.0004397750017233193, - 0.001302445998589974, - 0.0014783360020373948, - 0.00044882500515086576, - 0.0008114529991871677, - 0.001058212001225911, - 0.00041233999945688993, - 0.0003980020046583377, - 0.0007838689998607151, - 0.0004156970026087947, - 0.0020318070019129664, - 0.000463356998807285, - 0.0004134539994993247, - 0.0004356970021035522, - 0.00041345100180478767, - 0.0013061630015727133, - 0.0014812279987381771, - 0.00044929300202056766, - 0.00040635099867358804, - 0.0003995709994342178, - 0.00044709700159728527, - 0.0004073370000696741, - 0.0034253230041940697, - 0.0005323040022631176, - 0.00042576999840093777, - 0.0004050630013807677, - 0.0003982440030085854, - 0.0004131280002184212, - 0.00041465600224910304, - 0.0013240839980426244, - 0.002301294996868819, - 0.0004311699958634563, - 0.00041468699782853946, - 0.0004030679992865771, - 0.00040826600161381066, - 0.0003985559960710816, - 0.00040312399505637586, - 0.0027332750032655895, - 0.0013652269990416244, - 0.00042101699364138767, - 0.00039845900028012693, - 0.00041092999890679494, - 0.00041081900417339057, - 0.0005988249977235682, - 0.00044029700075043365, - 0.0004339809966040775, - 0.0004151009998167865, - 0.00040437599818687886, - 0.00040227700083050877, - 0.000401545999920927, - 0.0004042350046802312, - 0.0004033479999634437, - 0.000490427999466192, - 0.0018201380007667467, - 0.0024773569966782816, - 0.00043392299994593486, - 0.001866084996436257, - 0.0009626219980418682, - 0.0005480140025611036, - 0.0004093140014447272, - 0.00040239399822894484, - 0.0004082430023117922, - 0.0003988110038335435, - 0.00040849300421541557, - 0.0004024019945063628, - 0.0007454549995600246, - 0.0025954700031434186, - 0.0005506200031959452, - 0.0004182840057183057, - 0.00040600300417281687, - 0.000398231997678522, - 0.00044711700320476666, - 0.00040598299528937787, - 0.0008788139966782182, - 0.0035742570034926757, - 0.00044434700248530135, - 0.00040754899964667857, - 0.0005255590003798716, - 0.0008869250013958663, - 0.0007643600038136356, - 0.0008087860041996464, - 0.0005641270035994239, - 0.0004118939978070557, - 0.00045734299783362076, - 0.0004090410002390854, - 0.0004060399951413274, - 0.0004127389984205365, - 0.0017337780009256676, - 0.0029056749990559183, - 0.0004123309990973212, - 0.0004146049977862276, - 0.0004052540025440976, - 0.0015890659997239709, - 0.001527218999399338, - 0.00044831600098405033, - 0.0004078759957337752, - 0.00040766600432107225, - 0.0004009199983556755, - 0.0004047859983984381, - 0.0036825179995503277, - 0.0013523279994842596, - 0.0004636409939848818, - 0.0004091829978278838, - 0.0009723640032461844, - 0.0007563670005765744, - 0.0015534510021097958, - 0.0004410190013004467, - 0.0004065380053361878, - 0.00041386600059922785, - 0.00039759799983585253, - 0.0013180489986552857, - 0.002608397997391876, - 0.003967023003497161, - 0.005539988000236917, - 0.0004870260017924011, - 0.0004424360013217665, - 0.00040873199759516865, - 0.0007263900042744353, - 0.0004197960006422363, - 0.00040236399945570156, - 0.004161176999332383, - 0.00046819300041534007, - 0.0004680279962485656, - 0.00042151899833697826, - 0.00041333900298923254, - 0.0004083539970451966, - 0.000402115001634229, - 0.00040972699935082346, - 0.004222749004838988, - 0.000482153998746071, - 0.00042382000538054854, - 0.0004053150041727349, - 0.0004106359992874786, - 0.00040811399958329275, - 0.0004320719963288866, - 0.00042915900121442974, - 0.000400017001084052, - 0.004126453000935726, - 0.005998824002745096, - 0.00699350699869683, - 0.005991404999804217, - 0.007996314998308662, - 0.008009664001292549, - 0.005980863999866415, - 0.006995581999944989, - 0.006992411996179726, - 0.007000527999480255, - 0.00599915100610815, - 0.007991259000846185, - 0.002992608002386987, - 0.005004447993997019, - 0.0059900679989368655, - 0.005993288999889046, - 0.00699885499489028, - 0.007996191001439001, - 0.007996870001079515, - 0.006993662995228078, - 0.008999361001770012, - 0.005991865000396501, - 0.005995362997055054, - 0.006995121999352705, - 0.005995733001327608, - 0.006995903997449204, - 0.007002058999205474, - 0.006989301000430714, - 0.006995171999733429, - 0.007995537998795044, - 0.007996003005246166, - 0.0059945309985778295, - 0.005993537000904325, - 0.008001894006156363, - 0.009993270999984816, - 0.010996154000167735, - 0.00799727900448488, - 0.005994579005346168, - 0.0069952060002833605, - 0.0069973300051060505, - 0.0069891829989501275, - 0.008996299002319574, - 0.006994496005063411, - 0.00800233100017067, - 0.007995201005542185, - 0.005995239000185393, - 0.00498672899993835, - 0.006998230004683137, - 0.007009052002103999, - 0.006963070001802407, - 0.006334932004392613, - 0.005677234999893699, - 0.005972897000901867, - 0.00891102400055388, - 0.007009437998931389, - 0.007005706000200007, - 0.007998622000741307, - 0.007969917001901194, - 0.007009286004176829, - 0.005983376999211032, - 0.005992712998704519, - 0.010020137997344136, - 0.00703495999914594, - 0.006932001000677701, - 0.006027567993442062, - 0.006964191001316067, - 0.005975414002023172, - 0.0070129419982549734, - 0.007971107996127103, - 0.005128007003804669, - 0.003546228996128775, - 0.004781026000273414, - 0.007293271999515127, - 0.006235248001758009, - 0.007042101999104489, - 0.006944815002498217, - 0.007405812000797596, - 0.003598512994358316, - 0.00698281500081066, - 0.007001648998993915, - 0.003995184997620527, - 0.0030336920026456937, - 0.00592959899950074, - 0.006997936005063821, - 0.008007267999346368, - 0.005954817999736406, - 0.0052043550022062846, - 0.007796976002282463, - 0.007964544995047618, - 0.0059896489983657375, - 0.006032174998836126, - 0.006942368003365118, - 0.007014861002971884, - 0.00594765799905872, - 0.012994236996746622, - 0.00803839300351683, - 0.006952898998861201, - 0.005990576995827723, - 0.006154871996841393, - 0.006835795997176319, - 0.007995640000444837, - 0.006993168994085863, - 0.007995371001015883, - 0.005994402999931481, - 0.00799512099911226, - 0.006993518996750936, - 0.007994118001079187, - 0.005998395005008206, - 0.004977495998900849, - 0.007004897001024801, - 0.006997577998845372, - 0.005995439998514485, - 0.005988318000163417, - 0.006005722003465053, - 0.007005447994743008, - 0.006980231999477837, - 0.005991625002934597, - 0.006992617003561463, - 0.0100035739960731, - 0.006995080002525356, - 0.006992896000156179, - 0.00799565600027563, - 0.004995612005586736, - 0.009986148004827555, - 0.0059921200008830056, - 0.0069930250028846785, - 0.007993957005965058, - 0.006998850003583357, - 0.004992354995920323, - 0.006996912001341116, - 0.010996525001246482, - 0.007997223998245317, - 0.010016015003202483, - 0.007972660998348147, - 0.0060008830041624606, - 0.00898876699648099, - 0.007993409999471623, - 0.007994961000804324, - 0.005280110002786387, - 0.00770890100102406, - 0.0069985930022085086, - 0.007996057000127621, - 0.007987985001818743, - 0.005000574004952796, - 0.009989225000026636, - 0.0068030280017410405, - 0.008185936996596865, - 0.0069951650002622046, - 0.002238475004560314, - 0.007753814999887254, - 0.005992181002511643, - 0.00799335099873133, - 0.008395384000323247, - 0.007047566999972332, - 0.007544057996710762, - 0.007995093998033553, - 0.007992610000655986, - 0.008005853000213392, - 0.010998516001563985, - 0.007985676005773712, - 0.00798899399524089, - 0.007990847996552475, - 0.006995751995418686, - 0.005995968997012824, - 0.0069969910036888905, - 0.0080166210027528, - 0.007980298003531061, - 0.006980794001719914, - 0.006553148996317759, - 0.0074314919984317385, - 0.007999991001270246, - 0.006997416996455286, - 0.004521432005276438, - 0.004466347003472038, - 0.008099369006231427, - 0.007889190994319506, - 0.005992755002807826, - 0.007002455000474583, - 0.007995156003744341, - 0.007995456995558925, - 0.005995924999297131, - 0.00799791699682828, - 0.006993663999310229, - 0.0031716689991299063, - 0.0048216080031124875, - 0.006995512994762976, - 0.005992672995489556, - 0.007994419996975921, - 0.007997838001756463, - 0.004006094997748733, - 0.005987527001707349, - 0.005991908998112194, - 0.006993675997364335, - 0.01000739099981729, - 0.007016681993263774, - 0.008986421998997685, - 0.005974126004730351, - 0.007994239997060504, - 0.006995179996010847, - 0.008994106996397022, - 0.006995137002377305, - 0.009235154997440986, - 0.0067557450020103715, - 0.007995084000867791, - 0.002567445997556206, - 0.0040000409935601056, - 0.0054189209986361675, - 0.005995207000523806, - 0.004300392000004649, - 0.005691539998224471, - 0.003953706000174861, - 0.007004932995187119, - 0.003983882997999899, - 0.007000724996032659, - 0.00799837499653222, - 0.00500071400165325, - 0.0079893189977156, - 0.008160663994203787, - 0.00782920299388934, - 0.00599240799783729, - 0.008006609001313336, - 0.007997796004929114, - 0.005985607000184245, - 0.011018978999345563, - 0.007019681004749145, - 0.007947623002110049, - 0.006000972003675997, - 0.007005642997683026, - 0.00800482100021327, - 0.005975457999738865, - 0.005992787999275606, - 0.005133518003276549, - 0.003855989998555742, - 0.007993580999027472, - 0.007992365994141437, - 0.005990675002976786, - 0.007997777000127826, - 0.007998200002475642, - 0.005998123000608757, - 0.006990367000980768, - 0.00800003999756882, - 0.006998910001129843, - 0.006835410000348929, - 0.010153837996767834, - 0.005007621002732776, - 0.006975127005716786, - 0.011001517996191978, - 0.006995005001954269, - 0.007002876998740248, - 0.008004242001334205, - 0.008013147999008652, - 0.005956027001957409, - 0.005018577998271212, - 0.003980574998422526, - 0.005988388002151623, - 0.0069914809937472455, - 0.006062506996386219, - 0.006926581001607701, - 0.00711687799775973, - 0.009873374001472257, - 0.008698573998117354, - 0.007294766997802071, - 0.005986310003208928, - 0.0049960579999606125, - 0.0049968989987974055, - 0.00899145400035195, - 0.010008983001171146, - 0.01106339800026035, - 0.011923240999749396, - 0.004782791998877656, - 0.0065520200005266815, - 0.007646218000445515, - 0.0069967300005373545, - 0.00508691200229805, - 0.007031823006400373, - 0.008308721997309476, - 0.007744173999526538, - 0.006801688999985345, - 0.0099898039989057, - 0.006066383000870701, - 0.005926056997850537, - 0.0029972940028528683, - 0.00699052100389963, - 0.008007504999113735, - 0.007993020997673739, - 0.006983186001889408, - 0.006001530004141387, - 0.007989398996869568, - 0.004922191001242027, - 0.0020545349980238825, - 0.005993566999677569, - 0.009367801998450886, - 0.006625866000831593, - 0.005993295002554078, - 0.003303602003143169, - 0.0026921859971480444, - 0.010996680000971537, - 0.007001462996413466, - 0.005995581996103283, - 0.0069950979959685355, - 0.007994874002179131, - 0.006467020000854973, - 0.007515839002735447, - 0.006997374999627937, - 0.007994728002813645, - 0.007995405998372007, - 0.007992552003997844, - 0.006990904999838676, - 0.006999926998105366, - 0.006998070995905437, - 0.005148944997927174, - 0.005833444003656041, - 0.0059984999970765784, - 0.007994546002009884, - 0.006567005999386311, - 0.0034154299937654287, - 0.006998349002969917, - 0.004993381000531372, - 0.006997698001214303, - 0.006993065006099641, - 0.007998651002708357, - 0.006993791997956578, - 0.005001965997507796, - 0.0018735720004769973, - 0.006113468996773008, - 0.007996855994861107, - 0.007994316001713742, - 0.005992458995024208, - 0.005996975996822584, - 0.007995300002221484, - 0.005997396001475863, - 0.007996104002813809, - 0.005998831999022514, - 0.004993942995497491, - 0.006987515000218991, - 0.006995251998887397, - 0.00500505499803694, - 0.003982493995863479, - 0.006001467001624405, - 0.007995781001227442, - 0.007988126999407541, - 0.0060039099989808165, - 0.0060835580006823875, - 0.00686624700028915, - 0.006986155996855814, - 0.008077147998847067, - 0.0059137270000064746, - 0.003988043994468171, - 0.00699578199419193, - 0.014002515003085136, - 0.00799335099873133, - 0.006995522002398502, - 0.005993660997773986, - 0.007003407001320738, - 0.0059445670049171895, - 0.006996551004704088, - 0.006295320999925025, - 0.00571922300150618, - 0.007975859996804502, - 0.005988746001094114, - 0.00490743899717927, - 0.006085494998842478, - 0.007990484999027103, - 0.007999208995897789, - 0.0069934490020386875, - 0.006991941001615487, - 0.0070105359991430305, - 0.007990887999767438, - 0.007987568002135959, - 0.00699343100131955, - 0.006991714995820075, - 0.007000192999839783, - 0.005996086998493411, - 0.005989785997371655, - 0.0070058080018498, - 0.007990553000126965, - 0.006998280005063862, - 0.007995167004992254, - 0.00699520499620121, - 0.006999217999691609, - 0.006994991999818012, - 0.00699110199639108, - 0.010999500002071727, - 0.006018410000251606, - 0.006972025003051385, - 0.006994264003878925, - 0.005996419000439346, - 0.006997165997745469, - 0.006996440002694726, - 0.007996280000952538, - 0.007992762002686504, - 0.00799611399997957, - 0.00799301199731417, - 0.006997194002906326, - 0.0069933380000293255, - 0.006108576999395154, - 0.002886042006139178, - 0.007996446998731699, - 0.007996792999620084, - 0.005994074999762233, - 0.007989808997081127, - 0.005998738000926096, - 0.005459214000438806, - 0.005522196996025741, - 0.0039987359996302985, - 0.007995113999641035, - 0.006995292998908553, - 0.006992992995947134, - 0.007999444998858962, - 0.002987745996506419, - 0.006000998997478746, - 0.007997007000085432, - 0.006995670999458525, - 0.007997009000973776, - 0.005993489001411945, - 0.005995690000418108, - 0.00801628200133564, - 0.007979252004588488, - 0.007997733999218326, - 0.005996334002702497, - 0.007996864995220676, - 0.007992685998033267, - 0.006003613001666963, - 0.007986884003912564, - 0.004404502993565984, - 0.005584159000136424, - 0.013995777997479308, - 0.0069984119982109405, - 0.009830859999055974, - 0.004148111998802051, - 0.007989079000253696, - 0.009192869001708459, - 0.007560451005701907, - 0.004253338003763929, - 0.0056050789935397916, - 0.005369929996959399, - 0.008000717003596947, - 0.007987410004716367, - 0.010993777999829035, - 0.006991472000663634, - 0.006005828996421769, - 0.005982693997793831, - 0.007036924005660694, - 0.005944297998212278, - 0.008001902999239974, - 0.009992225997848436, - 0.007870579000154976, - 0.011303425999358296, - 0.007810023002093658, - 0.00799194400315173, - 0.005146547002368607, - 0.0043506260044523515, - 0.003487244001007639, - 0.00499673900048947, - 0.006021335000696126, - 0.007967106001160573, - 0.007998405999387614, - 0.006993290000536945, - 0.006986298998526763, - 0.00699121700017713, - 0.007002552003541496, - 0.005992434998915996, - 0.006995369003561791, - 0.007988238998223096, - 0.005998132000968326, - 0.006620700005441904, - 0.005367798003135249, - 0.009993328996642958, - 0.004685043997596949, - 0.003335943001729902, - 0.006960108003113419, - 0.0048086509996210225, - 0.009184419999655802, - 0.007004658000369091, - 0.0069664199982071295, - 0.010997762001352385, - 0.008998083001642954, - 0.007999800996913109, - 0.006991725000261795, - 0.006995800998993218, - 0.007996057996933814, - 0.0057716520022950135, - 0.005212623000261374, - 0.007995273001142778, - 0.005996509004035033, - 0.008042089000809938, - 0.006948620000912342, - 0.007993504004843999, - 0.004996228002710268, - 0.008003001996257808, - 0.0079923779994715, - 0.006993968003371265, - 0.0069992010030546226, - 0.0069994310033507645, - 0.001998690000618808, - 0.003991363002569415, - 0.009890187997370958, - 0.005098171001009177, - 0.007999599998584017, - 0.005993000006128568, - 0.005996646999847144, - 0.005999382003210485, - 0.007987142998899799, - 0.006000045999826398, - 0.0069959749962436035, - 0.0069958110034349374, - 0.0069929219971527345, - 0.006994957999268081, - 0.0069972600031178445, - 0.007993478997377679, - 0.007002275997365359, - 0.006991056005063001, - 0.006996333999268245, - 0.004998586999136023, - 0.002817979002429638, - 0.0005455499995150603, - 0.00045913799840491265, - 0.00044558100489666685, - 0.0003830299974652007, - 0.001604386001417879, - 0.003369992999068927, - 0.0009661130025051534, - 0.0004125279956497252, - 0.00038614899676758796, - 0.00037865799822611734, - 0.00041377700108569115, - 0.00038582699926337227, - 0.00038275900442386046, - 0.0005593149980995804, - 0.0003845550018013455, - 0.0008731260022614151, - 0.0007199190004030243, - 0.00041350300307385623, - 0.0003902480020769872, - 0.00038674600364174694, - 0.0003906150013790466, - 0.0004331460004323162, - 0.00039565299812238663, - 0.00039269699482247233, - 0.0003844599996227771, - 0.00073899699782487, - 0.0009413520019734278, - 0.0022623469994869083, - 0.0005260930047370493, - 0.0004698149932664819, - 0.0019888789975084364, - 0.0004251199934515171, - 0.0003890490042977035, - 0.00038752800173824653, - 0.000388925000152085, - 0.0004407359956530854, - 0.00039066500175977126, - 0.00040431899833492935, - 0.0003845779938274063, - 0.0014760330013814382, - 0.0009552190022077411, - 0.0004240670023136772, - 0.00039712800207780674, - 0.0003829990018857643, - 0.0003867009945679456, - 0.00039287300023715943, - 0.00038422199577325955, - 0.0007529959984822199, - 0.0009270279988413677, - 0.001469936003559269, - 0.0005772709992015734, - 0.00048444200365338475, - 0.00039990799996303394, - 0.00038482200034195557, - 0.0006199999988893978, - 0.0010122140010935254, - 0.0018921779992524534, - 0.0004178399976808578, - 0.0003960470057791099, - 0.00037948499812046066, - 0.00039491499774158, - 0.00042322700028307736, - 0.0003850860011880286, - 0.0012211950015625916, - 0.0006862469963380136, - 0.0004086940025445074, - 0.0003828140033874661, - 0.0004342760003055446, - 0.00039330100116785616, - 0.0003780570041271858, - 0.000751154002500698, - 0.0008366100009880029, - 0.0018893340020440519, - 0.0004206580051686615, - 0.00038285000482574105, - 0.00038701999437762424, - 0.00047062600060598925, - 0.00040197499765781686, - 0.00038239199784584343, - 0.00038978399970801547, - 0.00037922900082776323, - 0.0003802999999606982, - 0.0010083230008604005, - 0.0004336120036896318, - 0.0003830640052910894, - 0.000876758000231348, - 0.0008774809975875542, - 0.0007384259952232242, - 0.00040567899850429967, - 0.0003837999975075945, - 0.00038621100247837603, - 0.00038349000533344224, - 0.0003869279971695505, - 0.0003810119960689917, - 0.00037970599805703387, - 0.0009342619960079901, - 0.0014427529968088493, - 0.0014594460008083843, - 0.0004154540001763962, - 0.0003845270039164461, - 0.0004200890034553595, - 0.0003953179984819144, - 0.0003778500031330623, - 0.0006782249984098598, - 0.0007082960000843741, - 0.0004084210013388656, - 0.00038443099765572697, - 0.0005021350007154979, - 0.0003964019997511059, - 0.00039655499858781695, - 0.0003761810003197752, - 0.0003776950034080073, - 0.0005946400051470846, - 0.0004842490016017109, - 0.0006799019975005649, - 0.0007790710005792789, - 0.0009596870004315861, - 0.0004051669966429472, - 0.00038724100159015507, - 0.00038191599742276594, - 0.0004263840019120835, - 0.0003949550009565428, - 0.00040386899490840733, - 0.00048244000208796933, - 0.0004144990016357042, - 0.0005825429980177432, - 0.0008750999986659735, - 0.0006769199972040951, - 0.0004095840049558319, - 0.00038019999919924885, - 0.0015767699951538816, - 0.0006154099974082783, - 0.0004084409956703894, - 0.000384995000786148, - 0.00038390199915738776, - 0.0003771580013562925, - 0.000380425997718703, - 0.0003859330026898533, - 0.0006577000021934509, - 0.0006165989980218001, - 0.000406358994951006, - 0.0003912469983333722, - 0.00038286899507511407, - 0.0003880970034515485, - 0.00038345799839589745, - 0.00038189200131455436, - 0.0010577680004644208, - 0.0012506490020314232, - 0.0028419490045052953, - 0.0005633409964502789, - 0.00038819899782538414, - 0.0009013179951580241, - 0.001025195000693202, - 0.0003979129978688434, - 0.0003819539997493848, - 0.0005975870008114725, - 0.0008177249983418733, - 0.0003996089944848791, - 0.0003830190034932457, - 0.00038447100087068975, - 0.00037823699676664546, - 0.0005240820028120652, - 0.0013162510003894567, - 0.0007158070002333261, - 0.00040965600055642426, - 0.0003902160024154, - 0.00041898000199580565, - 0.0003771440024138428, - 0.0003858959971694276, - 0.00038168900209711865, - 0.0005043409983045422, - 0.0006269150035222992, - 0.00041435199818806723, - 0.0003864699974656105, - 0.00037988399708410725, - 0.0009562289997120388, - 0.0004354600023361854, - 0.0003837870026472956, - 0.0004891040007350966, - 0.0003958529996452853, - 0.0003874219983117655, - 0.0004064879976795055, - 0.0003866120023303665, - 0.0007288029955816455, - 0.002179894996515941, - 0.00041129500459646806, - 0.0003845289975288324, - 0.0003772690033656545, - 0.0014278219969128259, - 0.0007850390029489063, - 0.00041776199941523373, - 0.0004575969942379743, - 0.0003844480015686713, - 0.000383967999368906, - 0.0003848990018013865, - 0.00037991200224496424, - 0.00037598999915644526, - 0.0006038600040483288, - 0.0006712300018989481, - 0.0015986559956218116, - 0.00042319800559198484, - 0.00038770700484747067, - 0.00038436199974967167, - 0.00037862199678784236, - 0.0003833310038316995, - 0.0007941980002215132, - 0.00076980100129731, - 0.0006277530046645552, - 0.0003989610049757175, - 0.0003880420044879429, - 0.000384340004529804, - 0.0003780350016313605, - 0.0010447120002936572, - 0.0007114629988791421, - 0.0004114300027140416, - 0.0003945869975723326, - 0.0003834959934465587, - 0.00038302000029943883, - 0.00075131199992029, - 0.0008204279947676696, - 0.00046258200018201023, - 0.00039680599729763344, - 0.0003798600009758957, - 0.00252178300434025, - 0.0004470169951673597, - 0.0003958340021199547, - 0.0003856389957945794, - 0.00038023099477868527, - 0.00038426600076491013, - 0.0003788699978031218, - 0.00041906499973265454, - 0.0006865079994895495, - 0.00045643900375580415, - 0.0004655779994209297, - 0.000390635002986528, - 0.0003896060006809421, - 0.00038487400161102414, - 0.0008581210058764555, - 0.002230179998150561, - 0.0007464999944204465, - 0.0004263459995854646, - 0.00039233899587998167, - 0.00039117899723351, - 0.00039205900247907266, - 0.00039358499634545296, - 0.0004661509956349619, - 0.0006501009993371554, - 0.0014095799997448921, - 0.00043298499804222956, - 0.0007416970038320869, - 0.0012381980050122365, - 0.00042197299626423046, - 0.00039075200038496405, - 0.0003902809985447675, - 0.0003949250021832995, - 0.0003947349978261627, - 0.0011612230009632185, - 0.004145375998632517, - 0.00046493300033034757, - 0.0004042090004077181, - 0.0004034240046166815, - 0.0003879950018017553, - 0.0003982639973401092, - 0.000392273002944421, - 0.0003937319997930899, - 0.004972427006578073, - 0.000452943000709638, - 0.00040443600300932303, - 0.00039858699892647564, - 0.0003964369971072301, - 0.0003904469995177351, - 0.00039191100222524256, - 0.0003905279954778962, - 0.006588568998267874, - 0.00045723700168309733, - 0.00040241600072477013, - 0.0003930160019081086, - 0.0003923229960491881, - 0.0003938669979106635, - 0.00038910700095584616, - 0.00039063899748725817, - 0.000389840999559965, - 0.0003869050051434897, - 0.006048770002962556, - 0.00046122899948386475, - 0.00040064399945549667, - 0.0004007940005976707, - 0.0003933340049115941, - 0.0003900070005329326, - 0.00039217000448843464, - 0.00039281099452637136, - 0.006339722000120673, - 0.0004383749983389862, - 0.00038608799513895065, - 0.000388198001019191, - 0.00038159000541782007, - 0.0003891520027536899, - 0.00038458000199170783, - 0.0004699140044976957, - 0.00039550500514451414, - 0.0003830619971267879, - 0.008514553999702912, - 0.00045094399683875963, - 0.0003931379978894256, - 0.00040889400406740606, - 0.0003821709979092702, - 0.000382901998818852, - 0.00038539899833267555, - 0.0003853470043395646, - 0.000388159001886379, - 0.00037891599640715867, - 0.006577979002031498, - 0.0015313900003093295, - 0.0014572769941878505, - 0.0014833759996690787, - 0.001661787995544728, - 0.0015343580016633496, - 0.0015185439988272265, - 0.0014757870012545027, - 0.001483025997004006, - 0.0016154949989868328, - 0.0014793019945500419, - 0.0014527449966408312, - 0.0014986640017013997, - 0.001488362999225501, - 0.0015800020046299323, - 0.0015035770047688857, - 0.0014951670018490404, - 0.0014875599954393692, - 0.0014538050018018112, - 0.0014852540043648332, - 0.0016012429987313226, - 0.0014863609976600856, - 0.001502880004409235, - 0.001453051998396404, - 0.0014367519979714416, - 0.001645798998652026, - 0.0014953860008972697, - 0.001473948999773711, - 0.0014846449994365685, - 0.0014744079962838441, - 0.0016374850019929, - 0.0015246849943650886, - 0.0014700870015076362, - 0.0014717930025653914, - 0.0015344490020652302, - 0.0014587920013582334, - 0.0016304050004691817, - 0.0014925719951861538, - 0.0015299700025934726, - 0.0014278890012064949, - 0.001487434004957322, - 0.0015646290048607625, - 0.001515012001618743, - 0.001448397000785917, - 0.0014745480002602562, - 0.001508352994278539, - 0.0015512530007981695, - 0.0018497549972380511, - 0.0014560740019078366, - 0.001445729001716245, - 0.001426281000021845, - 0.001416869999957271, - 0.0016469350011902861, - 0.0014797070034546778, - 0.0014765899977646768, - 0.0015042049999465235, - 0.0014932759950170293, - 0.001662686001509428, - 0.001544710001326166, - 0.0015145700017455965, - 0.001589022002008278, - 0.001564352998684626, - 0.0016054020015872084, - 0.001531178000732325, - 0.001562295998155605, - 0.001550730004964862, - 0.0014788490007049404, - 0.0015430209969053976, - 0.0015883500018389896, - 0.0015110859967535362, - 0.0015203729999484494, - 0.0015118009978323244, - 0.0006338070015772246, - 0.0003910329978680238, - 0.0022013289999449626, - 0.0004408859967952594, - 0.00038687799678882584, - 0.0003950000027543865, - 0.0003833090013358742, - 0.0003789809998124838, - 0.0003888909996021539, - 0.00038298800063785166, - 0.005101198999909684, - 0.00043782099965028465, - 0.00038635700184386224, - 0.0003870979999192059, - 0.0003830920031759888, - 0.0003774049982894212, - 0.00038738900184398517, - 0.0003778550017159432, - 0.00038105499697849154, - 0.00624294799490599, - 0.0021134050039108843, - 0.0005960010021226481, - 0.0009335919967270456, - 0.0005966180033283308, - 0.004289306001737714, - 0.0004533749961410649, - 0.000386412997613661, - 0.0003810440030065365, - 0.000410573004046455, - 0.0003792830029851757, - 0.00046597199980169535, - 0.00042245499935233966, - 0.0010808029983309098, - 0.0007647140009794384, - 0.000779764995968435, - 0.0010729259956860915, - 0.0013970830041216686, - 0.0004318689971114509 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil", - "fullname": "TestCalculateNabla2AndSmagCoefficientsForVn", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.0006627990005654283, - "max": 0.01599517700378783, - "mean": 0.004580611106477362, - "stddev": 0.003070203786632812, - "rounds": 1127, - "median": 0.005403256000136025, - "iqr": 0.006111218497608206, - "q1": 0.0008873767510522157, - "q3": 0.006998595248660422, - "iqr_outliers": 0, - "stddev_outliers": 564, - "outliers": "564;0", - "ld15iqr": 0.0006627990005654283, - "hd15iqr": 0.01599517700378783, - "ops": 218.31148219195418, - "total": 5.162348716999986, - "data": [ - 0.005989123004837893, - 0.006119521000073291, - 0.006877184001496062, - 0.00799317000200972, - 0.007985330004885327, - 0.004990490997442976, - 0.00787348199810367, - 0.011124377000669483, - 0.0032321839971700683, - 0.004760497002280317, - 0.003995115002908278, - 0.005245815002126619, - 0.008750633998715784, - 0.01034987500315765, - 0.007647395999811124, - 0.006367273999785539, - 0.005687756005499978, - 0.00396536599873798, - 0.007327580999117345, - 0.005614620997221209, - 0.006554630999744404, - 0.0022770329960621893, - 0.001999681997403968, - 0.001998607003770303, - 0.0019248159951530397, - 0.0019477250025374815, - 0.0019209290039725602, - 0.001953696002601646, - 0.0018841609999071807, - 0.0018528110012994148, - 0.0018586669975775294, - 0.0019637820005300455, - 0.0019084230007138103, - 0.0019235300060245208, - 0.0018943020040751435, - 0.0019504800002323464, - 0.0019282009961898439, - 0.0025947260000975803, - 0.0018799789977492765, - 0.0019325929970364086, - 0.0019132910019834526, - 0.0018529219960328192, - 0.0018935490006697364, - 0.0018632650026120245, - 0.0018111569952452555, - 0.0018339599992032163, - 0.0019470499973976985, - 0.002004021000175271, - 0.001923192998219747, - 0.0018439900013618171, - 0.006870559001981746, - 0.005996510000841226, - 0.007996078995347489, - 0.007998351997230202, - 0.007996608997927979, - 0.008007604999875184, - 0.007987211996805854, - 0.00799562300380785, - 0.005996220999804791, - 0.006997753000177909, - 0.00601841200113995, - 0.007975404005264863, - 0.006998864002525806, - 0.00508273300511064, - 0.00791084200318437, - 0.008995979995233938, - 0.005995018997055013, - 0.006997356002102606, - 0.0069973330028005876, - 0.007998099004908, - 0.006996837000770029, - 0.005999615998007357, - 0.006992994996835478, - 0.00800204500410473, - 0.006985899999563117, - 0.007996984997589607, - 0.006020132001140155, - 0.007967411998833995, - 0.007994591003807727, - 0.010998828998708632, - 0.007995579995622393, - 0.006999811994319316, - 0.006994160998146981, - 0.006987225999182556, - 0.00799564200133318, - 0.005999572997097857, - 0.0060687580044032075, - 0.007925167999928817, - 0.007999105000635609, - 0.0061225640019983985, - 0.006866688003356103, - 0.006571669000550173, - 0.007409140998788644, - 0.002001392000238411, - 0.006016226994688623, - 0.007987520002643578, - 0.005982100999972317, - 0.004993124995962717, - 0.006997839001996908, - 0.006376997000188567, - 0.00460881299659377, - 0.009846916000242345, - 0.004157669005508069, - 0.007985557000210974, - 0.0099958919963683, - 0.007996624000952579, - 0.008119492005789652, - 0.006872439000289887, - 0.004877322993706912, - 0.006115021998994052, - 0.005998150001687463, - 0.01000697600102285, - 0.007986151002114639, - 0.008017770000151359, - 0.007975751002959441, - 0.0070015270030125976, - 0.004992243004380725, - 0.0031881719987723045, - 0.0038023189990781248, - 0.0049972790002357215, - 0.0052597960020648316, - 0.005738180996559095, - 0.007994669998879544, - 0.007999558998562861, - 0.005998452994390391, - 0.008007661999727134, - 0.014984157001890708, - 0.007984926996869035, - 0.007003992002864834, - 0.002990339999087155, - 0.004978362005203962, - 0.0010891210040426813, - 0.000906988003407605, - 0.0006927200010977685, - 0.0006911500022397377, - 0.0006862720038043335, - 0.0006835410022176802, - 0.004930620001687203, - 0.0008340760032297112, - 0.0007372109976131469, - 0.0006846770047559403, - 0.0006933730037417263, - 0.0015191540005616844, - 0.0006858389970147982, - 0.0006938860024092719, - 0.0006850169957033359, - 0.0006745030041201971, - 0.0006828289988334291, - 0.0020868560022790916, - 0.0015941430028760806, - 0.00106733400025405, - 0.0007480409985873848, - 0.0006800490009482019, - 0.0006917439968674444, - 0.0029656519982381724, - 0.0013292360017658211, - 0.0006927239955984987, - 0.0010652630007825792, - 0.0006867210031487048, - 0.0006681829981971532, - 0.0020061390023329295, - 0.0007864880026318133, - 0.0013054359951638617, - 0.0007095059991115704, - 0.0006907630013301969, - 0.0006886840055813082, - 0.0006746819999534637, - 0.0006858199994894676, - 0.001028398000926245, - 0.0006932700052857399, - 0.0006822560026193969, - 0.0006811090061091818, - 0.0006760609976481646, - 0.0006784959987271577, - 0.0012402779975673184, - 0.0017064089988707565, - 0.0031851450039539486, - 0.0008536800014553592, - 0.0007499150015064515, - 0.0006893559984746389, - 0.006924499997694511, - 0.0008380939980270341, - 0.000701283999660518, - 0.0007308479980565608, - 0.000703294004779309, - 0.0006889590003993362, - 0.007305384999199305, - 0.0009059079966391437, - 0.0007004639992373995, - 0.0006900619991938584, - 0.0006863589951535687, - 0.0006721939935232513, - 0.006488027000159491, - 0.0020606210018740967, - 0.0008312569989357144, - 0.005559928998991381, - 0.0019873070050380193, - 0.0008741979982005432, - 0.0035777919983956963, - 0.0011749070035875775, - 0.0007153499973355792, - 0.0007222870044643059, - 0.0007019190015853383, - 0.005821788996399846, - 0.0008740470002521761, - 0.0007605989958392456, - 0.0006998469980317168, - 0.0006843669980298728, - 0.0049499599990667775, - 0.0008517989990650676, - 0.000680414006637875, - 0.0007233229989651591, - 0.0007075160028762184, - 0.005908403996727429, - 0.0008372500014957041, - 0.0006793679931433871, - 0.0006850569989182986, - 0.0006863360031275079, - 0.0006770779946236871, - 0.005266471998766065, - 0.0008978610057965852, - 0.0007138850050978363, - 0.0006869739954709075, - 0.000685793005686719, - 0.0006789690014556982, - 0.0007166249997681007, - 0.003799879996222444, - 0.0008579339992138557, - 0.0007313850001082756, - 0.0007110470032785088, - 0.0008469399981549941, - 0.005130307996296324, - 0.0008616770064691082, - 0.0008109120026347227, - 0.0007013140057097189, - 0.000705490994732827, - 0.004868854994128924, - 0.0008525369994458742, - 0.0007376659996225499, - 0.0006774809953640215, - 0.0006885729962959886, - 0.005687210999894887, - 0.0009417050023330376, - 0.0006866149997222237, - 0.0006953440024517477, - 0.0006957150035304949, - 0.0044015680032316595, - 0.000928561006730888, - 0.0008888400043360889, - 0.0010050540004158393, - 0.0007408639939967543, - 0.0007102000017766841, - 0.0007248460024129599, - 0.0032853030061232857, - 0.0008294240033137612, - 0.0007213339995360002, - 0.0006905100017320365, - 0.0006856980035081506, - 0.0006828510013292544, - 0.006183035999129061, - 0.0009275709962821566, - 0.0007907629988039844, - 0.0006961760009289719, - 0.0007288979977602139, - 0.0027808220038423315, - 0.0008201499949791469, - 0.0007404260031762533, - 0.0006889249998494051, - 0.0007085460019879974, - 0.0007014720031293109, - 0.005137446001754142, - 0.007371861000137869, - 0.00799684799858369, - 0.006566984004166443, - 0.007433452999975998, - 0.006987598993873689, - 0.006996434000029694, - 0.006674950003798585, - 0.007329972002480645, - 0.008983446001366246, - 0.007989891004399396, - 0.011012698996637482, - 0.0049794260048656724, - 0.006999714998528361, - 0.007991757993295323, - 0.004995546994905453, - 0.0069970049953553826, - 0.005997969004965853, - 0.0069961609988240525, - 0.00799824199930299, - 0.006001187000947539, - 0.006991383001150098, - 0.007996257998456713, - 0.006996210999204777, - 0.0059994030016241595, - 0.009594451003067661, - 0.005403256000136025, - 0.003997687999799382, - 0.005897534996620379, - 0.008604294998804107, - 0.007485718000680208, - 0.0069925049974699505, - 0.008003823000763077, - 0.007990537000296172, - 0.007997979999345262, - 0.00799788299627835, - 0.006990144000155851, - 0.007999314002518076, - 0.005993717000819743, - 0.004996159004804213, - 0.007002819002082106, - 0.00799252599972533, - 0.008998923993203789, - 0.005993355000100564, - 0.005996874002448749, - 0.005995542000164278, - 0.006997334996412974, - 0.007996443004230969, - 0.007998763001523912, - 0.008001611000509001, - 0.010001562994148117, - 0.006986227999732364, - 0.005988974997308105, - 0.006000387002131902, - 0.005997981003019959, - 0.006994801995460875, - 0.008139039004163351, - 0.007853729999624193, - 0.008001381997019053, - 0.006991080997977406, - 0.006995171999733429, - 0.0070037499972386286, - 0.0069929579985910095, - 0.002210898994235322, - 0.004781445000844542, - 0.006002982001518831, - 0.007991311002115253, - 0.006999084995186422, - 0.005995259001792874, - 0.007004357998084743, - 0.005987259995890781, - 0.0070042139996076, - 0.00798952500190353, - 0.0061885200047981925, - 0.007804644003044814, - 0.005996225001581479, - 0.008057514001848176, - 0.007941453994135372, - 0.007992738006578293, - 0.0029943190020276234, - 0.005997658001433592, - 0.005000750999897718, - 0.00799547399947187, - 0.00650745999882929, - 0.0034827609997591935, - 0.0039041389973135665, - 0.0040898170045693405, - 0.005999720997351687, - 0.0069958500025677495, - 0.002996627998072654, - 0.005997947999276221, - 0.00799554099648958, - 0.005998589003866073, - 0.005996408995997626, - 0.007995776002644561, - 0.007003140999586321, - 0.007991730999492574, - 0.006996061994868796, - 0.00799729300342733, - 0.004993820999516174, - 0.010014149003836792, - 0.006979447003686801, - 0.009172791993478313, - 0.006815344997448847, - 0.006003332993714139, - 0.006991116999415681, - 0.007996652995643672, - 0.010998253994330298, - 0.005038864997914061, - 0.004954169999109581, - 0.004997493997507263, - 0.006119188001321163, - 0.006881957997393329, - 0.0069891129969619215, - 0.007037568997475319, - 0.0029619359993375838, - 0.0049991420019068755, - 0.00732265500118956, - 0.007660900999326259, - 0.006994606002990622, - 0.00599699799931841, - 0.010012078004365321, - 0.00699076900491491, - 0.0069808280022698455, - 0.007002131998888217, - 0.0069881919989711605, - 0.0030040849960641935, - 0.0029903899994678795, - 0.005997653002850711, - 0.006999379002081696, - 0.007999162000487559, - 0.006996559997787699, - 0.006993920003878884, - 0.00799656700110063, - 0.004771261003043037, - 0.006221375995664857, - 0.007005058003414888, - 0.006987318993196823, - 0.006575352999789175, - 0.0074221079994458705, - 0.010994842996296939, - 0.007264738000230864, - 0.006712864000292029, - 0.007996489999641199, - 0.007999212997674476, - 0.003993343998445198, - 0.003968921999330632, - 0.005280657998810057, - 0.004740284995932598, - 0.005809777001559269, - 0.009186235998640768, - 0.003069840000534896, - 0.0059216230001766235, - 0.005995807994622737, - 0.005997911001031753, - 0.005999656998028513, - 0.0033075129977078177, - 0.004684046994952951, - 0.00800343699665973, - 0.006993550996412523, - 0.009007095002743881, - 0.006013368998537771, - 0.00796387500304263, - 0.005967242999759037, - 0.009026546002132818, - 0.00799614300194662, - 0.007996029999048915, - 0.0020039619994349778, - 0.007997756001714151, - 0.003992204001406208, - 0.004997668998839799, - 0.007999216002644971, - 0.0069968159950803965, - 0.0051526830065995455, - 0.00684083100350108, - 0.005997075000777841, - 0.007998275999852922, - 0.005995029998302925, - 0.005998408996674698, - 0.007999098997970577, - 0.006000599001708906, - 0.00599066599534126, - 0.008001359994523227, - 0.005991833997541107, - 0.006470795000495855, - 0.006656364996160846, - 0.007867692998843268, - 0.007993580999027472, - 0.007998524000868201, - 0.007995113999641035, - 0.007003147002251353, - 0.006990713998675346, - 0.00799491700308863, - 0.005997621999995317, - 0.008994383002573159, - 0.006995161005761474, - 0.00699777699628612, - 0.006996787997195497, - 0.005997248001222033, - 0.007997141001396812, - 0.005591925997578073, - 0.006402992003131658, - 0.007996496999112424, - 0.006997604999924079, - 0.0030086159968050197, - 0.006990308997046668, - 0.009999663001508452, - 0.005003081998438574, - 0.006989799003349617, - 0.007996441003342625, - 0.007996979999006726, - 0.007997230997716542, - 0.009995989996241406, - 0.0069994310033507645, - 0.006994825002038851, - 0.007998291999683715, - 0.008000456997251604, - 0.005992983999021817, - 0.006994053997914307, - 0.0069978839965187944, - 0.00999315400258638, - 0.009998342997278087, - 0.006996819996857084, - 0.005997613996441942, - 0.00699649299349403, - 0.006997331998718437, - 0.00800400099979015, - 0.0069956870065652765, - 0.007008753003901802, - 0.005993446997308638, - 0.007978351997735444, - 0.0070097369971335866, - 0.005012936999264639, - 0.007963902004121337, - 0.004994948998501059, - 0.008003126000403427, - 0.005988411001453642, - 0.005057695001596585, - 0.004933146999974269, - 0.007997770000656601, - 0.007997265995072667, - 0.006000674002279993, - 0.007993610997800715, - 0.003994599996076431, - 0.004996796997147612, - 0.008007794996956363, - 0.008998487995995674, - 0.007990991005499382, - 0.010991306000505574, - 0.007247626002936158, - 0.0057506459997966886, - 0.007169099000748247, - 0.010814032000780571, - 0.006995862000621855, - 0.005998322005325463, - 0.007036012997559737, - 0.006957753001188394, - 0.006994887000473682, - 0.006999919001827948, - 0.006994093993853312, - 0.007996979002200533, - 0.006997086005867459, - 0.00799894300143933, - 0.005368220998207107, - 0.00462209199758945, - 0.008003535003808793, - 0.008010110999748576, - 0.009986796998418868, - 0.00498851799784461, - 0.004861718996835407, - 0.00520598900038749, - 0.001093063001462724, - 0.00782143300602911, - 0.008193408997613005, - 0.00779820499883499, - 0.0070005849993322045, - 0.0069931050020386465, - 0.005244690997642465, - 0.001742804997775238, - 0.005238309997366741, - 0.0037584399979095906, - 0.003995972998382058, - 0.007002288999501616, - 0.008994877003715374, - 0.007994450999831315, - 0.00641019699833123, - 0.004013607001979835, - 0.0011376269976608455, - 0.00842317299975548, - 0.00798893400497036, - 0.0064728439974715, - 0.007521834995714016, - 0.0049962159973802045, - 0.008997707998787519, - 0.005850552995980252, - 0.005883746998733841, - 0.009256940000341274, - 0.006996757998422254, - 0.005997397005558014, - 0.0069958189997123554, - 0.00699653599440353, - 0.006998441000177991, - 0.006998995995672885, - 0.0036758109999937005, - 0.00431486799789127, - 0.005999941997288261, - 0.006998898999881931, - 0.0069930640020174906, - 0.006998203003604431, - 0.004997110001568217, - 0.006998641001700889, - 0.005001869998523034, - 0.009989304999180604, - 0.0044903779999003746, - 0.006506375997560099, - 0.00799294099851977, - 0.007997428001544904, - 0.004850844998145476, - 0.006075199002225418, - 0.007071109001117293, - 0.005762765002145898, - 0.0052214469978935085, - 0.007995171996299177, - 0.009248007998394314, - 0.008279264999146108, - 0.00646749600127805, - 0.007985905998793896, - 0.0039959549976629205, - 0.005995880004775245, - 0.004995230003260076, - 0.008002217000466771, - 0.006990431000303943, - 0.0070015039964346215, - 0.007997214001079556, - 0.007991790000232868, - 0.0110019809944788, - 0.0010059580017696135, - 0.007985310003277846, - 0.004995334005798213, - 0.004909102004603483, - 0.003109222001512535, - 0.00821706800343236, - 0.007753521997074131, - 0.006739515003573615, - 0.0012537369984784164, - 0.007000464000157081, - 0.006992272996285465, - 0.007997553002496716, - 0.007003309001447633, - 0.003766014997381717, - 0.006218141999852378, - 0.00455583800066961, - 0.005238124998868443, - 0.004195837005681824, - 0.007998203000170179, - 0.005997514999762643, - 0.00499721599771874, - 0.007001676000072621, - 0.0079894350055838, - 0.0059976959964842536, - 0.006996068004809786, - 0.006997433003562037, - 0.006996616997639649, - 0.0059970430011162534, - 0.007005001003562938, - 0.006991943002503831, - 0.005246386004728265, - 0.007748728996375576, - 0.0069944539936841466, - 0.007993664999958128, - 0.0059978819990647025, - 0.005131504003657028, - 0.0024345160054508597, - 0.006747658000676893, - 0.0036730979991261847, - 0.003997766994871199, - 0.007997522996447515, - 0.002001927998207975, - 0.005992853999487124, - 0.005996201995003503, - 0.005996827996568754, - 0.00800016900029732, - 0.00799474900122732, - 0.007996875996468589, - 0.010997215998941101, - 0.006996438998612575, - 0.006998577999183908, - 0.003999173997726757, - 0.005054139997810125, - 0.0019449829997029155, - 0.005999954999424517, - 0.003990295001131017, - 0.006998153003223706, - 0.007996034997631796, - 0.006997894997766707, - 0.005995206993247848, - 0.005997730004310142, - 0.006998356999247335, - 0.0060011910027242266, - 0.006990781002969015, - 0.007997998000064399, - 0.005998070999339689, - 0.008000989000720438, - 0.007990824997250456, - 0.005998291999276262, - 0.007993054001417477, - 0.007990391997736879, - 0.0060009550070390105, - 0.003993421996710822, - 0.0079969149956014, - 0.00599683600012213, - 0.00800184299441753, - 0.009128210003837012, - 0.00786213600076735, - 0.006993227005295921, - 0.006184696998388972, - 0.007806801004335284, - 0.006998167002166156, - 0.006999297002039384, - 0.006995201998506673, - 0.005995158993755467, - 0.006996732001425698, - 0.008190981003281195, - 0.007818810998287518, - 0.007978269997693133, - 0.0070002020001993515, - 0.006234624997887295, - 0.002745050995144993, - 0.007000881996646058, - 0.006995046998781618, - 0.007028025996987708, - 0.007964094998897053, - 0.006997031996434089, - 0.006999342003837228, - 0.005997554995701648, - 0.006997481999860611, - 0.0069964270005584694, - 0.005991878002532758, - 0.0069988200048101135, - 0.005994566003209911, - 0.006005846000334714, - 0.005988051998429, - 0.005996187996061053, - 0.005997681997541804, - 0.0069966520022717305, - 0.006997616997978184, - 0.0069986009984859265, - 0.006993964001594577, - 0.006999391996941995, - 0.006996661999437492, - 0.0069957349987817, - 0.006477725000877399, - 0.007516912002756726, - 0.006996750002144836, - 0.005996974003210198, - 0.006259547000809107, - 0.008731253001315054, - 0.00799748300050851, - 0.007996054999239277, - 0.00699688600434456, - 0.006995403004111722, - 0.006996559000981506, - 0.007000279001658782, - 0.005994571001792792, - 0.007995892003236804, - 0.0026531839976087213, - 0.0007937429982121103, - 0.0006810609993408434, - 0.0006823540024925023, - 0.0006730979948770255, - 0.003378611996595282, - 0.0015208729964797385, - 0.0010897059983108193, - 0.0006967939989408478, - 0.0006708329965476878, - 0.0006791640043957159, - 0.0006916080019436777, - 0.0006634820019826293, - 0.0018997029983438551, - 0.002047467998636421, - 0.0006956159995752387, - 0.000685382001393009, - 0.0006627990005654283, - 0.0006701059974147938, - 0.0020140089982305653, - 0.0019894919969374314, - 0.000864364999870304, - 0.0007255880045704544, - 0.000685542996507138, - 0.000681878998875618, - 0.0006954700002097525, - 0.0006854789971839637, - 0.0023022949972073548, - 0.0017205289987032302, - 0.0007158399967011064, - 0.0006992540002102032, - 0.0007504280001739971, - 0.0009250119983335026, - 0.0008707239976502024, - 0.0008071159973042086, - 0.0010564109979895875, - 0.0007047449980746023, - 0.0006883999958517961, - 0.0006940640014363453, - 0.0009024199971463531, - 0.001194379001390189, - 0.0012556719957501628, - 0.000689488006173633, - 0.0007003149949014187, - 0.000704362006217707, - 0.0007797979997121729, - 0.0009107420046348125, - 0.0017559059997438453, - 0.000729421000869479, - 0.004131326000788249, - 0.0008240669994847849, - 0.0009412350045749918, - 0.0025551680009812117, - 0.0011639929944067262, - 0.002341471001273021, - 0.0007805749992257915, - 0.0016852660046424717, - 0.0027746549967559986, - 0.0007088510028552264, - 0.0006812310020904988, - 0.0009327040024800226, - 0.0015418800030602142, - 0.0007332090026466176, - 0.00070570800016867, - 0.000692442998115439, - 0.0008628590003354475, - 0.0023525549986516126, - 0.0009450209981878288, - 0.0007734890023129992, - 0.0006922510001459159, - 0.0007590429959236644, - 0.0009140120018855669, - 0.0008940589978010394, - 0.001588210005138535, - 0.0007124309995560907, - 0.0006812930005253293, - 0.0006792929998482578, - 0.0023364150038105436, - 0.0010486780010978691, - 0.0008777399998507462, - 0.0008124150044750422, - 0.0007770299998810515, - 0.0013841079999110661, - 0.001439093000954017, - 0.0007601300021633506, - 0.0006899629952386022, - 0.0009069750012713484, - 0.0008993840019684285, - 0.000849798001581803, - 0.0023211070001707412, - 0.000718825998774264, - 0.0006841000067652203, - 0.0006756220027455129, - 0.001104467002733145, - 0.000923464001971297, - 0.0008142809965647757, - 0.0007253139992826618, - 0.000709451996954158, - 0.0006873660022392869, - 0.0008620430016890168, - 0.0012466519983718172, - 0.0017834679965744726, - 0.0007387590012513101, - 0.0007098889982444234, - 0.0006965560023672879, - 0.0008131590002449229, - 0.0008865249983500689, - 0.0014990429990575649, - 0.0010298409979441203, - 0.000716763999662362, - 0.0007067219994496554, - 0.0014031200043973513, - 0.0016864470017026179, - 0.000788598001236096, - 0.0007176130020525306, - 0.0006958269950700924, - 0.0009237769991159439, - 0.0011459480010671541, - 0.0019969109998783097, - 0.002965740997751709, - 0.0008053130004554987, - 0.0007883609941927716, - 0.0007271349968505092, - 0.0009632770015741698, - 0.0009996630033128895, - 0.0009410639977431856, - 0.0007395009961328469, - 0.0007128379947971553, - 0.0017383630038239062, - 0.0033309469945379533, - 0.000828716998512391, - 0.0007670550039620139, - 0.0007181149994721636, - 0.0007317089985008352, - 0.0016493439979967661, - 0.0028708120007649995, - 0.0007422769995173439, - 0.0007058449991745874, - 0.0007119529982446693, - 0.0008922810011426918, - 0.0009674220054876059, - 0.0015645590028725564, - 0.0007401969996863045, - 0.0021431040004245006, - 0.0007442630012519658, - 0.002052839001407847, - 0.0007436700034304522, - 0.0020427639974514022, - 0.0013894100047764368, - 0.0007489159979741089, - 0.0007083579985192046, - 0.0006936660065548494, - 0.0008497079979861155, - 0.002275284001370892, - 0.0007317950003198348, - 0.0010989709990099072, - 0.0007376790017588064, - 0.0020292000044719316, - 0.0007394750064122491, - 0.0007045519960229285, - 0.0006962789993849583, - 0.0007074239983921871, - 0.004503689000557642, - 0.0007709909987170249, - 0.0007119130023056641, - 0.0007101070004864596, - 0.0006995219955570064, - 0.0009669739956734702, - 0.0007993519975570962, - 0.0007374839988187887, - 0.004512156003329437, - 0.0024536630007787608, - 0.0007549450019723736, - 0.0007033559959381819, - 0.0007045920065138489, - 0.005060479998064693, - 0.0007486629983759485, - 0.0007118229987099767, - 0.0006954079944989644, - 0.004104948995518498, - 0.0008718570024939254, - 0.0007225359950098209, - 0.002587686998595018, - 0.0007392019979306497, - 0.0007011420020717196, - 0.0007079690039972775, - 0.0006974630014155991, - 0.000695214002917055, - 0.01128958600020269, - 0.007000094003160484, - 0.008010076002392452, - 0.007015157003479544, - 0.006946350004000124, - 0.007991521000803914, - 0.006000348999805283, - 0.005992491001961753, - 0.007997663000423927, - 0.006044319998181891, - 0.007955182998557575, - 0.007989197998540476, - 0.008000903006177396, - 0.0069941290057613514, - 0.007993860999704339, - 0.0059972300005028956, - 0.008461620003799908, - 0.006532315994263627, - 0.001000481002847664, - 0.005994112005282659, - 0.00799785499839345, - 0.006996853000600822, - 0.006005554998409934, - 0.006990704998315778, - 0.008996580996608827, - 0.00799915699462872, - 0.006995681003900245, - 0.0059968519999529235, - 0.006995504998485558, - 0.0059968830028083175, - 0.006999408004048746, - 0.006995284995355178, - 0.006997272997978143, - 0.00699708100000862, - 0.006997766999120358, - 0.006996251999225933, - 0.00699746300233528, - 0.006996994998189621, - 0.005997906999255065, - 0.007996514003025368, - 0.006998566001129802, - 0.007997512999281753, - 0.00799828000162961, - 0.005995129002258182, - 0.005997387001116294, - 0.007997417000296991, - 0.007998671004315838, - 0.007994788997166324, - 0.005999458000587765, - 0.007997861001058482, - 0.007993760002136696, - 0.005998590997478459, - 0.0059888909963774495, - 0.006998705997830257, - 0.00799559500592295, - 0.005000690995075274, - 0.007995178995770402, - 0.005997051004669629, - 0.006997745003900491, - 0.004993588001525495, - 0.008005704003153369, - 0.006990105001023039, - 0.006995146999543067, - 0.005073426000308245, - 0.004920890998619143, - 0.0059980399964842945, - 0.007995448999281507, - 0.007998480999958701, - 0.005996535997837782, - 0.007998177003173623, - 0.008020055000088178, - 0.0029996860030223615, - 0.0051511749989003874, - 0.007813076001184527, - 0.006992137998167891, - 0.004994652001187205, - 0.009000073994684499, - 0.005999971006531268, - 0.0069903709954814985, - 0.011029333996702917, - 0.006963497995457146, - 0.00800347999756923, - 0.007992267004738096, - 0.007995338994078338, - 0.006995563999225851, - 0.01299818199913716, - 0.006993303999479394, - 0.008004436000192072, - 0.008990827998786699, - 0.006003394999424927, - 0.01599517700378783, - 0.007982932002050802, - 0.005852788999618497, - 0.0021479599963640794, - 0.0029833920052624308, - 0.005007559004297946, - 0.006995424002525397, - 0.0037986339957569726, - 0.005230298003880307, - 0.00796315900515765, - 0.006997393000347074, - 0.0032062879981822334, - 0.003691432997584343, - 0.005789907001599204, - 0.006349739000143018, - 0.0009090469975490123, - 0.000746277000871487, - 0.0006884399990667589, - 0.0006869100034236908, - 0.0006851080033811741, - 0.0059694340015994385, - 0.0008657170037622564, - 0.0007557309945696034, - 0.0006722140024066903, - 0.0006962150000617839, - 0.0032273239994538017, - 0.0008274489955510944, - 0.0014326700038509443, - 0.0006895219994476065, - 0.0006960410028113984, - 0.0006961570034036413, - 0.0006820950002293102, - 0.0006781010015401989, - 0.003402275004191324, - 0.0008694520001881756, - 0.00221835700358497, - 0.0008050780015764758, - 0.0007178400046541356, - 0.0006849049968877807, - 0.003605756995966658, - 0.0008457680014544167, - 0.0007746379997115582, - 0.0007048979969113134, - 0.0007008329994278029, - 0.0006844059971626848, - 0.004356330995506141, - 0.0008868889999575913, - 0.0007678030015085824, - 0.0006801830022595823, - 0.002252910999231972, - 0.0007556990021839738, - 0.0007373370026471093, - 0.000681685000017751, - 0.002459171002556104, - 0.0007965889963088557, - 0.0006795590015826747, - 0.0006857449989183806, - 0.0006838350018369965, - 0.0006798940012231469, - 0.005128924000018742, - 0.0008792980006546713, - 0.0013881859995308332, - 0.0006769930041627958, - 0.0007179349995567463, - 0.0006908819996169768, - 0.004187295999145135, - 0.000846171002194751, - 0.0007155860002967529, - 0.0006807919999118894, - 0.0007119149959180504, - 0.002507345998310484, - 0.0026793809956870973, - 0.0008651879979879595, - 0.0008058780003921129, - 0.0006841579961474054, - 0.0007056450049276464, - 0.006231170002138242, - 0.001000873999146279, - 0.0007408430028590374, - 0.0006897839994053356, - 0.0006977509983698837, - 0.0006829920021118596, - 0.006534415006171912, - 0.0008568129997001961, - 0.0006933609984116629, - 0.0006993349961703643, - 0.0006861590009066276, - 0.004283005000615958, - 0.0008340540007338859, - 0.0008601770023233257, - 0.0007257500037667342, - 0.0006986429943935946, - 0.005902028002310544, - 0.0008340530039276928, - 0.0007802109976182692, - 0.0006797250025556423, - 0.0006838879999122582, - 0.003155979997245595, - 0.0008725090010557324, - 0.0007890009947004728, - 0.002178786999138538, - 0.0040939079990494065, - 0.0012693489989032969, - 0.0007007160020293668, - 0.0006861789952381514, - 0.002230211997812148, - 0.0007200789987109601, - 0.0007376720022875816, - 0.0006964499989408068, - 0.0006927349968464114, - 0.002174620000005234, - 0.002794586995150894, - 0.0008092079951893538, - 0.0006766240039723925 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil", - "fullname": "TestCalculateNabla2ForW", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.0003208999987691641, - "max": 0.06402748999971664, - "mean": 0.003404689042697311, - "stddev": 0.0037307441009177666, - "rounds": 1687, - "median": 0.0015808429961907677, - "iqr": 0.006198535498697311, - "q1": 0.0003456052509136498, - "q3": 0.0065441407496109605, - "iqr_outliers": 5, - "stddev_outliers": 242, - "outliers": "242;5", - "ld15iqr": 0.0003208999987691641, - "hd15iqr": 0.01901425200048834, - "ops": 293.71257916927584, - "total": 5.743710415030364, - "data": [ - 0.0006079320010030642, - 0.000561126995307859, - 0.0005768699993495829, - 0.0005684620045940392, - 0.0006446689949370921, - 0.0005989019991829991, - 0.0005796589975943789, - 0.0005980950008961372, - 0.000573858997086063, - 0.0010199649987043813, - 0.005994811006530654, - 0.0060005850027664565, - 0.006994929994107224, - 0.008032661993638612, - 0.006999516001087613, - 0.00829950999468565, - 0.006643805005296599, - 0.007998031003808137, - 0.007995243999175727, - 0.00799275500321528, - 0.007395872999040876, - 0.006340387000818737, - 0.00424814700090792, - 0.008171932000550441, - 0.007830046997696627, - 0.005990630997985136, - 0.007988632998603862, - 0.008076662001258228, - 0.007928008999442682, - 0.007988360004674178, - 0.007988852004928049, - 0.005996225001581479, - 0.005994807004753966, - 0.0059947920017293654, - 0.0060023030018783174, - 0.007112163002602756, - 0.006883369998831768, - 0.004981685997336172, - 0.0069955090002622455, - 0.007995728999958374, - 0.005999653003527783, - 0.00467156100057764, - 0.006312996003543958, - 0.007028164000075776, - 0.006962283005123027, - 0.006997178999881726, - 0.004207951002172194, - 0.00978377600404201, - 0.007995945001312066, - 0.005513951000466477, - 0.006562123999174219, - 0.00392160900082672, - 0.006979531994147692, - 0.007997863998753019, - 0.005234782001934946, - 0.005820648999360856, - 0.003929541999241337, - 0.005985764000797644, - 0.00611862799996743, - 0.003975550003815442, - 0.004128284999751486, - 0.007763616005831864, - 0.006991772002947982, - 0.007995790998393204, - 0.006998056000156794, - 0.006993318995228037, - 0.007995558997208718, - 0.009998810994147789, - 0.006202839998877607, - 0.006807275000028312, - 0.00233237099746475, - 0.00362934400618542, - 0.0039776530029485, - 0.010020032001193613, - 0.008014694998564664, - 0.009983962998376228, - 0.005985484000120778, - 0.006244430995138828, - 0.008747551000851672, - 0.005994027000269853, - 0.006996905998676084, - 0.00699369400535943, - 0.007004885999776889, - 0.007315121001738589, - 0.007670396000321489, - 0.005992170998069923, - 0.005998152999382, - 0.005992249003611505, - 0.0059954109965474345, - 0.0060000599987688474, - 0.004216436995193362, - 0.007369670995103661, - 0.00639607799530495, - 0.0035196289973100647, - 0.004727105995698366, - 0.0017655740011832677, - 0.006029654003214091, - 0.007915475995105226, - 0.00800113299919758, - 0.0043595189999905415, - 0.004614613004378043, - 0.006987483000557404, - 0.008003244998690207, - 0.005995964995236136, - 0.008012776001123711, - 0.007966185999976005, - 0.005996143001539167, - 0.007014295995759312, - 0.006979084006161429, - 0.006995146002736874, - 0.005712140002287924, - 0.007961376002640463, - 0.007316738003282808, - 0.0039724980015307665, - 0.006132966998848133, - 0.0018912579980678856, - 0.00599178699485492, - 0.004993180999008473, - 0.007003011000051629, - 0.008031125995330513, - 0.007956720000947826, - 0.007992468003067188, - 0.006978138997510541, - 0.006941366002138238, - 0.007996963999175932, - 0.006984994004596956, - 0.007019240001682192, - 0.0069729370006825775, - 0.005983846996969078, - 0.007021023004199378, - 0.007980141999723855, - 0.007898898998973891, - 0.010140145001059864, - 0.0049724850032362156, - 0.005949776001216378, - 0.006038737999915611, - 0.007960400005686097, - 0.007721456000581384, - 0.007193782999820542, - 0.008026884999708273, - 0.005993124002998229, - 0.0020022020034957677, - 0.003806733999226708, - 0.0014629189972765744, - 0.005225802000495605, - 0.0015022349980426952, - 0.006173503999889363, - 0.0037803009981871583, - 0.001982439003768377, - 0.005999212997267023, - 0.004991471003449988, - 0.006000371002301108, - 0.006996470998274162, - 0.006992465001530945, - 0.008003249997273088, - 0.007988504999957513, - 0.005996510000841226, - 0.003004739999596495, - 0.000386570995033253, - 0.0003432890007388778, - 0.0003495959972497076, - 0.00033679199987091124, - 0.00033332500606775284, - 0.00034080399927916005, - 0.00033736800105543807, - 0.00033249899570364505, - 0.0003379640038474463, - 0.00033406099828425795, - 0.0052557210001396015, - 0.0003959200039389543, - 0.0003405269962968305, - 0.00033512099616928026, - 0.00034421400050632656, - 0.00033699499908834696, - 0.0003326599980937317, - 0.0003422540030442178, - 0.0003390260026208125, - 0.00033091400109697133, - 0.006201976997544989, - 0.00037862199678784236, - 0.0003351660052430816, - 0.0003421340006752871, - 0.0003321999975014478, - 0.00033518100099172443, - 0.00033756199991330504, - 0.0003305829959572293, - 0.0003351819977979176, - 0.0003347309975652024, - 0.0074696550000226125, - 0.0008036910003283992, - 0.00034833700192393735, - 0.0003420830034883693, - 0.00033528100175317377, - 0.0003314019995741546, - 0.0003380900016054511, - 0.00033214800350833684, - 0.00035786999796982855, - 0.006271018995903432, - 0.0003992989950347692, - 0.0013763549941359088, - 0.0003787579989875667, - 0.00033927400363609195, - 0.0003329779938212596, - 0.0019533829981810413, - 0.00035313300031702965, - 0.00033387200528522953, - 0.00033364800037816167, - 0.00034895499993581325, - 0.00033541299490025267, - 0.0003283039986854419, - 0.00033832000190159306, - 0.0003328599996166304, - 0.0003361309936735779, - 0.004133356997044757, - 0.0003776030061999336, - 0.00033205700310645625, - 0.000347517998307012, - 0.00033796900243032724, - 0.00033426199661334977, - 0.00038335700082825497, - 0.0003376559980097227, - 0.0003391779973753728, - 0.004925535999063868, - 0.0020490099996095523, - 0.000430410000262782, - 0.00040711299516260624, - 0.0032331009933841415, - 0.0007193459969130345, - 0.00035014199966099113, - 0.0003340600014780648, - 0.0003314649948151782, - 0.0004264650051482022, - 0.00033992499811574817, - 0.00033218700264114887, - 0.0003370609992998652, - 0.0003363379946677014, - 0.0003314780042273924, - 0.003985691997513641, - 0.00043293400085531175, - 0.0003462129971012473, - 0.00033112899836851284, - 0.0003414879975025542, - 0.0003325540019432083, - 0.00033056000393116847, - 0.0003347389938426204, - 0.0003365259981364943, - 0.00033096699917223305, - 0.004507024001213722, - 0.000561530003324151, - 0.00035528400621842593, - 0.00033354799961671233, - 0.0005896849979762919, - 0.0003700930028571747, - 0.0003450559961493127, - 0.00033315100154140964, - 0.00033106400223914534, - 0.005695518004358746, - 0.000399835997086484, - 0.0004098049976164475, - 0.0003398469998501241, - 0.0018629320038598962, - 0.0003946939978050068, - 0.00034306099405512214, - 0.00045792799937771633, - 0.00032674800604581833, - 0.00033087599877035245, - 0.0003288360021542758, - 0.00033019800321199, - 0.0028870559981442057, - 0.00037475599674507976, - 0.0003268270011176355, - 0.00033031900238711387, - 0.0003304880010546185, - 0.00032686600025044754, - 0.0003208999987691641, - 0.0003330830004415475, - 0.0003218339988961816, - 0.00032325799838872626, - 0.003912018997652922, - 0.001295890993787907, - 0.0003293449990451336, - 0.00032220600405707955, - 0.0004515530017670244, - 0.000331805000314489, - 0.00033029200130840763, - 0.0003345850054756738, - 0.00032213299709837884, - 0.00032236400147667155, - 0.004158295996603556, - 0.00039495599776273593, - 0.0003390769998077303, - 0.00035663999733515084, - 0.000339015998179093, - 0.00032774599822005257, - 0.00032987300073727965, - 0.0003298819938208908, - 0.0003274260016041808, - 0.005160971995792352, - 0.0003817879987764172, - 0.00032928700238699093, - 0.000328219000948593, - 0.0011285670043434948, - 0.00039908500184537843, - 0.00033106800401583314, - 0.00033144099870696664, - 0.00032647500484017655, - 0.0003216460027033463, - 0.003235809002944734, - 0.00034389000211376697, - 0.00032526399445487186, - 0.0021649679983966053, - 0.00035353600105736405, - 0.00033157099824165925, - 0.00032282999745802954, - 0.00033598799927858636, - 0.00032328000088455155, - 0.001944390001881402, - 0.00033614000130910426, - 0.0003443609966780059, - 0.0003293529953225516, - 0.0003225780019420199, - 0.003139894994092174, - 0.0003376609965926036, - 0.0003244540057494305, - 0.0018621039998834021, - 0.0003476530037005432, - 0.0003357419991516508, - 0.0003241559970774688, - 0.0003255849951528944, - 0.00032918999932007864, - 0.0003276369970990345, - 0.0003273780021118, - 0.004453412999282591, - 0.00033398800587747246, - 0.00033032499777618796, - 0.00032732999534346163, - 0.00032792399724712595, - 0.00032404999365098774, - 0.00032830899726832286, - 0.00032158400426851586, - 0.00032490299781784415, - 0.00032823999936226755, - 0.0003273539987276308, - 0.00032508900039829314, - 0.003996256004029419, - 0.00034262700501130894, - 0.00033003499993355945, - 0.00033727000118233263, - 0.0003378199980943464, - 0.0003303830017102882, - 0.0003252660026191734, - 0.004682671002228744, - 0.001151872995251324, - 0.00038002599467290565, - 0.0003548620006768033, - 0.0003346399971633218, - 0.0003359609981998801, - 0.0003262309983256273, - 0.00033032899955287576, - 0.0003468389986664988, - 0.0003366210003150627, - 0.00032826800452312455, - 0.005471345997648314, - 0.0003549699977156706, - 0.00033853299828479066, - 0.00032708499929867685, - 0.00032327399821951985, - 0.0003334780049044639, - 0.0003309879975859076, - 0.00032474700128659606, - 0.00035785799991572276, - 0.0004708499982370995, - 0.0003474199984339066, - 0.01308392800274305, - 0.00699561600049492, - 0.008000709000043571, - 0.00799032599752536, - 0.008329307005624287, - 0.008682552994287107, - 0.014000860996020492, - 0.00695665900275344, - 0.006995037998422049, - 0.00798692399985157, - 0.006989351997617632, - 0.007995137995749246, - 0.0069956639999873005, - 0.00999543399666436, - 0.00799971700325841, - 0.007984791998751462, - 0.0089963060017908, - 0.005091300001367927, - 0.005900258998735808, - 0.00999971399869537, - 0.006991481997829396, - 0.0059960470025544055, - 0.006998526994721033, - 0.008017240004846826, - 0.007973492996825371, - 0.006014150996634271, - 0.007977847999427468, - 0.007996795000508428, - 0.008002082999155391, - 0.006988193003053311, - 0.005996125997626223, - 0.007995109001058154, - 0.007322777004446834, - 0.007670508995943237, - 0.006987171997025143, - 0.007001630001468584, - 0.006983853003475815, - 0.005995104998874012, - 0.007995350999408402, - 0.00610569299897179, - 0.0088882409982034, - 0.005407092998211738, - 0.0055816219974076375, - 0.006026750001183245, - 0.007967092999024317, - 0.00699530600104481, - 0.007018377000349574, - 0.005417435000708792, - 0.005547528002352919, - 0.007992769998963922, - 0.0059957060002489015, - 0.012683961002039723, - 0.010828283004229888, - 0.0006085019995225593, - 0.00039033500070217997, - 0.00035183200088795274, - 0.00035363399365451187, - 0.0003333770000608638, - 0.0003335249930387363, - 0.00034430900268489495, - 0.0003368400066392496, - 0.0003339550021337345, - 0.0030058629999984987, - 0.00035058000503340736, - 0.0015120230018510483, - 0.00037589699786622077, - 0.00034309100010432303, - 0.00033290799910901114, - 0.00033738800266291946, - 0.00034115600283257663, - 0.00033042199356714264, - 0.003850280001643114, - 0.00035318799928063527, - 0.00033572699612705037, - 0.00034618900099303573, - 0.00034026500361505896, - 0.0003391479986021295, - 0.00033370200253557414, - 0.00035729700175579637, - 0.00033810500463005155, - 0.0003333209970151074, - 0.005378326000936795, - 0.00035703100002137944, - 0.00033893799991346896, - 0.0003458789942669682, - 0.0003316810034448281, - 0.0003328259990666993, - 0.00033679999614832923, - 0.0003354179934831336, - 0.00033267099934164435, - 0.004661209000914823, - 0.0003377269968041219, - 0.00036383100086823106, - 0.00033514900133013725, - 0.00034325999877182767, - 0.00036431899934541434, - 0.00033401400287402794, - 0.00033925400202861056, - 0.00033149999944726005, - 0.00032996200025081635, - 0.0064745839990791865, - 0.0003821279969997704, - 0.0003353670035721734, - 0.000343929001246579, - 0.0003371390048414469, - 0.000331880000885576, - 0.00034399700234644115, - 0.0003332960041007027, - 0.0003352920030010864, - 0.00034011900424957275, - 0.0003306159997009672, - 0.00033243800135096535, - 0.0047095929985516705, - 0.0003546010048012249, - 0.00033702899963827804, - 0.0003344550059409812, - 0.00033486699976492673, - 0.00033739199716364965, - 0.0003304880010546185, - 0.00033646600059000775, - 0.000341566999850329, - 0.011602451995713636, - 0.0069878840004093945, - 0.006996018004429061, - 0.00799723000091035, - 0.006997385004069656, - 0.006993556002271362, - 0.005033866000303533, - 0.0029606370007968508, - 0.0059956719996989705, - 0.006988449000346009, - 0.0069964610011084005, - 0.00700010600121459, - 0.0059988779976265505, - 0.01118648900592234, - 0.008858707005856559, - 0.005927088001044467, - 0.009992792998673394, - 0.005989933000819292, - 0.00699290099873906, - 0.007992403996468056, - 0.0070063499952084385, - 0.0053065070023876615, - 0.008688170004461426, - 0.007980556998518296, - 0.006996128999162465, - 0.01099701500061201, - 0.0069948230011505075, - 0.008000237998203374, - 0.0069894109983579256, - 0.00799405300494982, - 0.007001058998866938, - 0.0059919670020462945, - 0.006995306997851003, - 0.008017104999453295, - 0.005969876998278778, - 0.004995688999770209, - 0.005995569001242984, - 0.006996830001298804, - 0.00799770699813962, - 0.006996028998401016, - 0.006998387005296536, - 0.007143690003431402, - 0.007843858998967335, - 0.006996014999458566, - 0.00799622900376562, - 0.00699672300106613, - 0.0070002360007492825, - 0.007002963000559248, - 0.008993824005301576, - 0.007001860001764726, - 0.007970226004545111, - 0.00799713499873178, - 0.007996309002919588, - 0.004995144001441076, - 0.007994266001333017, - 0.007665469995117746, - 0.009333302004961297, - 0.007988555997144431, - 0.0045471120029105805, - 0.004453358000318985, - 0.007983224000781775, - 0.005994876999466214, - 0.00800107100076275, - 0.00799778399959905, - 0.006989836998400278, - 0.007999696004844736, - 0.006995518997428007, - 0.006992433001869358, - 0.0037029840023024008, - 0.003325127996504307, - 0.0004945619948557578, - 0.0003414170059841126, - 0.00034870600211434066, - 0.0010123679967364296, - 0.0003356300003360957, - 0.00033359900407958776, - 0.0004003310023108497, - 0.0003478360013104975, - 0.00033916099346242845, - 0.0003395730018382892, - 0.00033609499951126054, - 0.00047478799388045445, - 0.0003431839941185899, - 0.00033354799961671233, - 0.00034544600202934816, - 0.0031429390000994317, - 0.0003861090008285828, - 0.00035232800291851163, - 0.00033426199661334977, - 0.00033926399919437245, - 0.0009990110047510825, - 0.000341867002134677, - 0.0003382099966984242, - 0.0007072119988151826, - 0.004206192999845371, - 0.0004540719965007156, - 0.00034978999610757455, - 0.0003368719990248792, - 0.0004208679965813644, - 0.00036217599699739367, - 0.0003364410003996454, - 0.00035173899959772825, - 0.0003390419951756485, - 0.0003403549999347888, - 0.005493372002092656, - 0.00039081700379028916, - 0.0003583270008675754, - 0.00034500999754527584, - 0.00033808500302257016, - 0.0003737360020750202, - 0.00034824400063371286, - 0.0007989740042830817, - 0.0009072350003407337, - 0.005382843999541365, - 0.0004993910042685457, - 0.0003649440041044727, - 0.0006359889957820997, - 0.0003573640060494654, - 0.0003420069988351315, - 0.0004061090003233403, - 0.0003478010039543733, - 0.0003480039958958514, - 0.0045221109976409934, - 0.0004067799964104779, - 0.0003667759956442751, - 0.0003463179964455776, - 0.0006425770043279044, - 0.00038042500091250986, - 0.00033904299925779924, - 0.00036021000414621085, - 0.00033956800325540826, - 0.0032071250025182962, - 0.00042620900057954714, - 0.0004852560014114715, - 0.0031186659980448894, - 0.00041664400487206876, - 0.00037335600063670427, - 0.00035141099942848086, - 0.00034774900268530473, - 0.000340304002747871, - 0.0027548250000108965, - 0.00039913799992064014, - 0.0016017489979276434, - 0.00041476600017631426, - 0.0003616339963627979, - 0.00038865899841766804, - 0.003844601997116115, - 0.0003819429985014722, - 0.0003475599951343611, - 0.00033919700217666104, - 0.00034231100289616734, - 0.0005155010003363714, - 0.0003397479958948679, - 0.00038018199848011136, - 0.0003429930002312176, - 0.0004860440021730028, - 0.004985670995665714, - 0.0004928000053041615, - 0.00034367700573056936, - 0.0006540960021084175, - 0.0003430029973969795, - 0.00035077699430985376, - 0.0003470650044619106, - 0.0006060300001990981, - 0.0003565769948181696, - 0.00034225000126753, - 0.0037389970020740293, - 0.0012319480010773987, - 0.0014338700057123788, - 0.00035226599720772356, - 0.00046837099944241345, - 0.00040978300239657983, - 0.00035040800139540806, - 0.0006729430024279281, - 0.004383897998195607, - 0.0005079420006950386, - 0.00034340900310780853, - 0.00039595099951839074, - 0.00034986100217793137, - 0.00053587400179822, - 0.0003634449967648834, - 0.0003475500052445568, - 0.00034903200139524415, - 0.0003404809976927936, - 0.0003472600001259707, - 0.0059577260035439394, - 0.00041181199776474386, - 0.0003492989999358542, - 0.00035864800156559795, - 0.0004825979995075613, - 0.0012233529996592551, - 0.0019150070002069697, - 0.0024374049971811473, - 0.0012584659998537973, - 0.0003473150063655339, - 0.00034996599424630404, - 0.00034338900150032714, - 0.0003389249977772124, - 0.00034835700353141874, - 0.00035072000173386186, - 0.0003382280046935193, - 0.003117801999906078, - 0.003328998005599715, - 0.0003630559949669987, - 0.0003904000041075051, - 0.00038360399776138365, - 0.0006178010007715784, - 0.00176658700365806, - 0.003524382002069615, - 0.0003748010058188811, - 0.002202830000896938, - 0.0009884259998216294, - 0.0004189350001979619, - 0.0003402980000828393, - 0.00034873000549850985, - 0.000342480001563672, - 0.0003390840065549128, - 0.0023688270011916757, - 0.0007818160011083819, - 0.0003547099986462854, - 0.0003410639983485453, - 0.0003425719987717457, - 0.00040643400279805064, - 0.0003699079970829189, - 0.0007935440007713623, - 0.0017874260011012666, - 0.005652380998071749, - 0.002977707001264207, - 0.006001362002280075, - 0.006991996000579093, - 0.0070022560030338354, - 0.007005139996181242, - 0.006979753001360223, - 0.008003351002116688, - 0.006584994000149891, - 0.007391286002530251, - 0.0059858050008188, - 0.005990473000565544, - 0.006988585999351926, - 0.006998516997555271, - 0.0059922360014752485, - 0.006996220996370539, - 0.007995473002665676, - 0.006598899999517016, - 0.005397824999818113, - 0.0079924360034056, - 0.0059956980039714836, - 0.005995620995236095, - 0.00601442699553445, - 0.009981451003113762, - 0.006993214999965858, - 0.004995506998966448, - 0.008000255998922512, - 0.007996118998562451, - 0.007277871998667251, - 0.006714181006827857, - 0.005991288002405781, - 0.005995727995468769, - 0.004734663001727313, - 0.006256655004108325, - 0.009572833005222492, - 0.005435596001916565, - 0.007983796000189614, - 0.005995312996674329, - 0.008004328003153205, - 0.007990998004970606, - 0.007987798999238294, - 0.0029944729976705275, - 0.006997393997153267, - 0.007996430002094712, - 0.006999801000347361, - 0.006990829999267589, - 0.0059933820011792704, - 0.006000594003126025, - 0.0069915810017846525, - 0.005996414001856465, - 0.006000459994538687, - 0.007999387999007013, - 0.007988564000697806, - 0.005300400000123773, - 0.0066444929980207235, - 0.006049975003406871, - 0.005990054996800609, - 0.007996451000508387, - 0.003996157000074163, - 0.006996558004175313, - 0.007997762993909419, - 0.006997017997491639, - 0.004013658995972946, - 0.005977084001642652, - 0.004990185996575747, - 0.005004254999221303, - 0.007989577999978792, - 0.0069957349987817, - 0.00600198900065152, - 0.007992862003447954, - 0.005083067997475155, - 0.0069014700056868605, - 0.0059958609999739565, - 0.007998322005732916, - 0.005995178995362949, - 0.0059961329970974475, - 0.0069962670022505336, - 0.007997624001291115, - 0.006000286994094495, - 0.00415893299941672, - 0.00382895499933511, - 0.00039198999729705974, - 0.00034943100035889074, - 0.00036292699951445684, - 0.00034656599746085703, - 0.000341302999004256, - 0.004256861000612844, - 0.0047009749978315085, - 0.0003855030008708127, - 0.00034775100357364863, - 0.0009558849997119978, - 0.005396043001383077, - 0.0004073090021847747, - 0.00035006100370083004, - 0.0003434970058151521, - 0.006401257996913046, - 0.0003873149980790913, - 0.000350488000549376, - 0.00034598100319271907, - 0.00034414300171192735, - 0.0027784310004790314, - 0.007055848000163678, - 0.0023285770002985373, - 0.0003886159975081682, - 0.0036082099977647886, - 0.0011318559991195798, - 0.0005931769992457703, - 0.0003475359990261495, - 0.0003405419993214309, - 0.00033856899972306564, - 0.0008270649996120483, - 0.002070075999654364, - 0.005172204000700731, - 0.0012619090048247017, - 0.0003626160032581538, - 0.0003448140050750226, - 0.00034994599991478026, - 0.00034594199678394943, - 0.0003404669987503439, - 0.0013501919966074638, - 0.0006155540031613782, - 0.0003560960030881688, - 0.0003404379967832938, - 0.0008138530029100366, - 0.0003687409989652224, - 0.00034528799733379856, - 0.0003424820024520159, - 0.0003416620020288974, - 0.0024211150011979043, - 0.0003807199973380193, - 0.00034867799695348367, - 0.00039912600186653435, - 0.0003440309956204146, - 0.003354663000209257, - 0.00037990199780324474, - 0.00034791199868777767, - 0.00036505499883787706, - 0.0003481599997030571, - 0.00034558799961814657, - 0.0035540239987312816, - 0.00037581799551844597, - 0.0003578570031095296, - 0.0003446400005486794, - 0.0003408720003790222, - 0.00038537100044777617, - 0.0003473719989415258, - 0.0003555750008672476, - 0.0003488669972284697, - 0.0003396180036361329, - 0.00034445899655111134, - 0.0029436379991238937, - 0.00038655000389553607, - 0.00034122900251531973, - 0.0003469479997875169, - 0.0003451979937381111, - 0.0003431130026001483, - 0.0003491079987725243, - 0.0003459310028119944, - 0.0003415970058995299, - 0.000665783001750242, - 0.0038963120023254305, - 0.000396932999137789, - 0.00035540899989428, - 0.0003448369971010834, - 0.00034691499604377896, - 0.0007295550021808594, - 0.0003468970026005991, - 0.004232778999721631, - 0.00037365599564509466, - 0.00034948199754580855, - 0.0003499649974401109, - 0.0003453009994700551, - 0.0003402360016480088, - 0.0004066769979544915, - 0.00034662699908949435, - 0.0003403230002732016, - 0.004085421998752281, - 0.005970776001049671, - 0.003994259997853078, - 0.005998203996568918, - 0.006994022995058913, - 0.005997513995680492, - 0.005997478998324368, - 0.007997818000148982, - 0.005996901003527455, - 0.006997103002504446, - 0.00899800500337733, - 0.007996560001629405, - 0.0060003190010320395, - 0.006993844996031839, - 0.0057115950039587915, - 0.00528409999969881, - 0.008006527998077217, - 0.00799275899771601, - 0.007000473000516649, - 0.00799037900287658, - 0.006989966997934971, - 0.007999831999768503, - 0.00799555499543203, - 0.006007224001223221, - 0.008011503006855492, - 0.01901425200048834, - 0.007745226997940335, - 0.006814860003942158, - 0.014391746997716837, - 0.040002701003686525, - 0.06402748999971664, - 0.008943223998358008, - 0.024204378998547327, - 0.007741574998362921, - 0.007184078000136651, - 0.004786650002643, - 0.004096472002856899, - 0.004901552005321719, - 0.009057937997567933, - 0.004836994005017914, - 0.006035561003955081, - 0.0018834860020433553, - 0.004099577003216837, - 0.00395853299414739, - 0.00695017799444031, - 0.0069628280034521595, - 0.003968842996982858, - 0.009907469000609126, - 0.0040763709985185415, - 0.003990640994743444, - 0.006998917000601068, - 0.006994586001383141, - 0.007997974993486423, - 0.005989559998852201, - 0.006068900001992006, - 0.004960814003425185, - 0.0029032279999228194, - 0.0020055770000908524, - 0.0019949529960285872, - 0.0031015029962873086, - 0.007898049996583723, - 0.006030497999745421, - 0.007963652002217714, - 0.0059932550066150725, - 0.003003220001119189, - 0.0029834360029781237, - 0.0029845200042473152, - 0.005005873004847672, - 0.0037488620000658557, - 0.004231840001011733, - 0.0050031189966830425, - 0.004989109002053738, - 0.0029865360047551803, - 0.007998398003110196, - 0.005997761996695772, - 0.002982243000587914, - 0.006997592994594015, - 0.0050008649996016175, - 0.003987349002272822, - 0.007008797001617495, - 0.006994805997237563, - 0.003990329001680948, - 0.004993396993086208, - 0.0060049710009479895, - 0.003976135994889773, - 0.006000102002872154, - 0.003994906001025811, - 0.004131052002776414, - 0.003861065997625701, - 0.0019984079990535975, - 0.007248100999277085, - 0.0037144779998925515, - 0.007005748004303314, - 0.005970552003418561, - 0.006982782993873116, - 0.005000488999939989, - 0.00598558100318769, - 0.004651408002246171, - 0.0015199219997157343, - 0.007890052998845931, - 0.006929865005076863, - 0.0029566489974968135, - 0.005018317999201827, - 0.0059767870043287985, - 0.006995269002800342, - 0.00598833299591206, - 0.004273892998753581, - 0.0037085610019857995, - 0.00699457099835854, - 0.008003130002180114, - 0.003986056995927356, - 0.004996277995815035, - 0.0049982620039372705, - 0.005003090998798143, - 0.00398404600127833, - 0.004999769997084513, - 0.006990196998231113, - 0.006992144997639116, - 0.0069991260024835356, - 0.02220677800505655, - 0.007768221003061626, - 0.003967384996940382, - 0.008005844996660016, - 0.007974235006258823, - 0.007990893995156512, - 0.00601404099870706, - 0.007968099002027884, - 0.0030201320041669533, - 0.0004088220011908561, - 0.00036636999720940366, - 0.00037117199826752767, - 0.0003634189997683279, - 0.0003634669992607087, - 0.000370935995306354, - 0.0003661839946289547, - 0.0003667880009743385, - 0.0003621120049501769, - 0.0003620260031311773, - 0.007315823997487314, - 0.0069251239983714186, - 0.006369351001922041, - 0.007620943004440051, - 0.006997975993726868, - 0.006994278999627568, - 0.003805975000432227, - 0.004184800993243698, - 0.003993451995484065, - 0.008002817994565703, - 0.00799490200006403, - 0.008994117000838742, - 0.007999512999958824, - 0.010993601004884113, - 0.0059996749987476505, - 0.010996719996910542, - 0.006990532005147543, - 0.006059437997464556, - 0.0019408379957894795, - 0.003995334998762701, - 0.005995772997266613, - 0.007992177001142409, - 0.006000369001412764, - 0.006991116002609488, - 0.006992873997660354, - 0.004996354997274466, - 0.006583873000636231, - 0.005408908000390511, - 0.004991009998775553, - 0.004991360001440626, - 0.007996903004823253, - 0.007992503997229505, - 0.007997396001883317, - 0.0069923080009175465, - 0.007000530000368599, - 0.006994200994085986, - 0.006992889997491147, - 0.007000242003414314, - 0.0049960629985434934, - 0.006995497999014333, - 0.006004242000926752, - 0.006992423004703596, - 0.005001697994885035, - 0.007994504005182534, - 0.006987407999986317, - 0.007006623003690038, - 0.008023807997233234, - 0.00700074100313941, - 0.005914276996918488, - 0.006982658000197262, - 0.0069879669972578995, - 0.005769253002654295, - 0.0022195179990376346, - 0.006100703998527024, - 0.0068956740069552325, - 0.0022707190000801347, - 0.004713260001153685, - 0.006008926997310482, - 0.007994865998625755, - 0.006989701003476512, - 0.006999610006459989, - 0.00038019999919924885, - 0.0003607379985623993, - 0.000359678000677377, - 0.000342024999554269, - 0.0003340600014780648, - 0.00034275500365765765, - 0.00033444000291638076, - 0.0003347550009493716, - 0.0003397879991098307, - 0.00032829499832587317, - 0.00033385199640179053, - 0.0032400960044469684, - 0.00267910700495122, - 0.0003622719959821552, - 0.0004232760038576089, - 0.00035042800300288945, - 0.0003514380005071871, - 0.0003385780000826344, - 0.0003336180016049184, - 0.0003391999998711981, - 0.00033151800016639754, - 0.00033411799813620746, - 0.004674144998716656, - 0.00037822000012965873, - 0.00036399300006451085, - 0.00033979499858105555, - 0.0003456570048001595, - 0.00034225299896206707, - 0.0003358670001034625, - 0.00034326599416090176, - 0.0003315639987704344, - 0.00033470900234533474, - 0.004648852998798247, - 0.0003947529985453002, - 0.00034836499980883673, - 0.00033140899904537946, - 0.00033136000274680555, - 0.0003468650029390119, - 0.00033352999889757484, - 0.00033622099726926535, - 0.00033960000291699544, - 0.0003304639976704493, - 0.00033253299625357613, - 0.0033019360053003766, - 0.0028909070024383254, - 0.00036094000097364187, - 0.00034945300285471603, - 0.0003576300005079247, - 0.000340604005032219, - 0.00033118599822046235, - 0.0003422679947107099, - 0.00033715899917297065, - 0.0003371359998709522, - 0.0003392059952602722, - 0.009871415000816341, - 0.0004598699961206876, - 0.00036238999746274203, - 0.0044453559967223555, - 0.000391028996091336, - 0.007933651002531406, - 0.00043638899660436437, - 0.0003691480014822446, - 0.00036464699951466173, - 0.0003646529949037358, - 0.00033874799555633217, - 0.00033858299866551533, - 0.00035280900192447007, - 0.0003860529977828264, - 0.0003385149975656532, - 0.005235490003542509, - 0.0005137510015629232, - 0.0003747999944607727, - 0.0003435970065766014, - 0.00033700900530675426, - 0.0003351990017108619, - 0.0003770569965126924, - 0.00037473500560736284, - 0.000366891996236518, - 0.000335090997396037, - 0.00033330100268358365, - 0.0038808640019851737, - 0.0003600959971663542, - 0.00034705899452092126, - 0.0003400389978196472, - 0.00037820199941052124, - 0.00034555100137367845, - 0.0003353710053488612, - 0.00033880999399116263, - 0.00033762300154194236, - 0.0003342970012454316, - 0.004795862005266827, - 0.00041822600178420544, - 0.0003875639959005639, - 0.0003597350005293265, - 0.0003843809972750023, - 0.0003396219981368631, - 0.0003342199997860007, - 0.0003431549994274974, - 0.0003396170068299398, - 0.0003413459999137558, - 0.0003403649971005507, - 0.0003317910013720393, - 0.000331916002323851, - 0.0047830339972279035, - 0.0004215219960315153, - 0.0006291540048550814, - 0.0006908770010340959, - 0.0003311749969725497, - 0.0003379789995960891, - 0.00033209400135092437, - 0.0003322529955767095, - 0.0003369330006535165, - 0.00033322799572488293, - 0.00033398000232409686, - 0.005139573993801605, - 0.007993505001650192, - 0.006997499003773555, - 0.005997887994453777, - 0.007997788998181932, - 0.005543473998841364, - 0.006455506001657341, - 0.0059923549997620285, - 0.009000566002214327, - 0.006992834001721349, - 0.006998245997237973, - 0.006999686003837269, - 0.006994905001192819, - 0.005996961997880135, - 0.007004384002357256, - 0.005055669003922958, - 0.003930754006432835, - 0.005997523003316019, - 0.005999368004268035, - 0.00799819399981061, - 0.005989244004013017, - 0.005998091997753363, - 0.004995991999749094, - 0.007003121994785033, - 0.001014364002912771, - 0.00035623699659481645, - 0.00033610100217629224, - 0.00034059100289596245, - 0.0003591780041460879, - 0.0003583719953894615, - 0.00034291699557797983, - 0.0003401989961275831, - 0.0003293350018793717, - 0.0021546900024986826, - 0.0012101069951313548, - 0.0003408539996598847, - 0.00033855299989227206, - 0.00033564700424904004, - 0.0003339480026625097, - 0.000329869995766785, - 0.00034163700183853507, - 0.000335248994815629, - 0.0004392349947011098, - 0.00035336000291863456, - 0.000402956000471022, - 0.0020952170016244054, - 0.0031527499959338456, - 0.00040201299998443574, - 0.0003464960027486086, - 0.00035235499672126025, - 0.0003339810064062476, - 0.00033491800422780216, - 0.0018938560024253093, - 0.0003596980022848584, - 0.0003440849977778271, - 0.00033852000342449173, - 0.0003516039942041971, - 0.0003495989949442446, - 0.000336951001372654, - 0.000342751998687163, - 0.00034346199390711263, - 0.00033041000278899446, - 0.005269553999823984, - 0.0003772729978663847, - 0.0003476799975032918, - 0.00034643400431377813, - 0.0003411349971429445, - 0.0003346419980516657, - 0.00034132500150008127, - 0.00033746700501069427, - 0.00039903500146465376, - 0.00038175599911483005, - 0.0004014200021629222, - 0.006077894999179989, - 0.00039378300425596535, - 0.0003647460034699179, - 0.0007571269961772487, - 0.0003702429967233911, - 0.0003381489950697869, - 0.00034007099748123437, - 0.0003304999991087243, - 0.005017094998038374, - 0.00044784799683839083, - 0.0003641460061771795, - 0.00044414900185074657, - 0.0003833470036624931, - 0.00034342599974479526, - 0.003236902004573494, - 0.00039766699774190784, - 0.0003715429993462749, - 0.0015808429961907677, - 0.0003577600000426173, - 0.00033825499849626794, - 0.00033480000274721533, - 0.0003303320045233704, - 0.00034094799775630236, - 0.0003482749962131493, - 0.00033188700035680085, - 0.0003332490014145151, - 0.0003362460047355853, - 0.00033337600325467065, - 0.005469229996378999, - 0.00044032500591129065, - 0.0003924830016330816, - 0.00033893399813678116, - 0.0009361509946756996, - 0.00036659000033978373, - 0.0006687290006084368, - 0.00036625799839384854, - 0.0034799009954440407, - 0.0003762699998333119, - 0.0003551049958332442, - 0.00033677800092846155, - 0.00033271100255660713, - 0.000333559000864625, - 0.0003391619975445792, - 0.0003354069995111786, - 0.0003414370003156364, - 0.000340379003318958, - 0.00033191499824170023, - 0.0003343360003782436, - 0.005299935000948608, - 0.0005588900021393783, - 0.0005593329988187179, - 0.00034158400376327336, - 0.0003470129959168844, - 0.00033811300090746954, - 0.0003354170039528981, - 0.00033582199830561876, - 0.00035420100175542757, - 0.00034639400109881535, - 0.004854209000768606, - 0.000408816005801782, - 0.0003626260004239157, - 0.00035066699638264254, - 0.0003407279946259223, - 0.000333186995703727, - 0.000329132002661936, - 0.00033963700116146356, - 0.00033384200651198626, - 0.0003325529978610575, - 0.0003363049981999211, - 0.000331564006046392, - 0.000333447998855263, - 0.003812912000284996, - 0.001285672995436471, - 0.0004290040014893748, - 0.00037456300196936354, - 0.0006105889988248236, - 0.00035072000173386186, - 0.0008358830018551089, - 0.00034709899773588404, - 0.00033726400579325855, - 0.0003391520003788173, - 0.00033822099794633687, - 0.0003386390017112717, - 0.00033286799589404836, - 0.0003294669950264506, - 0.0029699339938815683, - 0.00037547899410128593, - 0.0003631110012065619, - 0.0003601459975470789, - 0.0003648019992397167, - 0.0003356909946887754, - 0.0022922220014152117, - 0.0003728119991137646, - 0.00035180600389139727, - 0.0003969649987993762, - 0.0003314279965707101, - 0.00032994200591929257, - 0.00037567999970633537, - 0.0003479879960650578, - 0.0003328360035084188, - 0.004702327001723461, - 0.0004018280014861375, - 0.0003665550029836595, - 0.0003409609998925589, - 0.0003377940010977909, - 0.0003789360052905977, - 0.00037018700095359236, - 0.0007959619979374111, - 0.0003891440064762719, - 0.002614908000396099, - 0.00036073500086786225, - 0.0003588279942050576, - 0.00033485300082247704, - 0.00033470900234533474, - 0.00033754600008251145, - 0.00033177300065290183, - 0.00036730599822476506, - 0.0006849120036349632, - 0.00259893399925204, - 0.0003611189968069084, - 0.00035471100272843614, - 0.00034490899997763336, - 0.0003339670001878403, - 0.00037026599602540955, - 0.0003331420011818409, - 0.00033969400101341307, - 0.00041550499736331403, - 0.00034713599598035216, - 0.0003303679986856878, - 0.005075136999948882, - 0.0009563219937263057, - 0.00036515599640551955, - 0.00033766999695217237, - 0.00033126900234492496, - 0.00034037000295938924, - 0.0003348910031490959, - 0.0003285869970568456, - 0.00033874100336106494, - 0.0033243580037378706, - 0.0013972380038467236, - 0.00034245300048496574, - 0.0003298470037407242, - 0.00035028000274905935, - 0.0003343270000186749, - 0.00032949799788184464, - 0.001198190002469346, - 0.0007047780018183403, - 0.0010975190016324632, - 0.0006642439984716475, - 0.0005068199970992282, - 0.0003804599982686341, - 0.001363026000035461, - 0.001266707004106138, - 0.00040148400148609653, - 0.0009699019938125275, - 0.0014226780040189624, - 0.00035355400177650154, - 0.00034656200296012685, - 0.0003322729971841909, - 0.00040867699863156304, - 0.0011120729977847077, - 0.0010177989970543422, - 0.0003477339996607043, - 0.0003312900007585995, - 0.0003360230039106682, - 0.0003313410052214749, - 0.0003311179971206002, - 0.0003546880034264177, - 0.00033046200405806303, - 0.000550528995518107, - 0.00034687399602262303, - 0.0003316630027256906, - 0.0030811129981884733, - 0.0003972119957325049, - 0.00036533999809762463, - 0.0018188899994129315, - 0.0007284880048246123, - 0.0005999049972160719, - 0.0003538269957061857, - 0.00033275199530180544, - 0.0010128290014108643, - 0.0004099259967915714, - 0.00048682299529900774, - 0.0003428719937801361, - 0.000345947002642788, - 0.00033115900441771373, - 0.00033829399762907997, - 0.002136458999302704, - 0.0003580739939934574, - 0.00035494000621838495, - 0.00035566599399317056, - 0.0003374419975443743, - 0.00033380100649083033, - 0.0003340300027048215, - 0.00046059900341788307, - 0.0003366089949849993, - 0.0003396460015210323, - 0.0004900620042462833, - 0.0004729959982796572, - 0.0004402699996717274, - 0.00038046999543439597, - 0.0007025980012258515, - 0.00041856700408970937, - 0.000543087997357361, - 0.0009719109948491678, - 0.0021323830005712807, - 0.0003671590020530857, - 0.00034345199674135074, - 0.00043884000479010865, - 0.0003524049971019849, - 0.0003333000058773905, - 0.0006396320022759028, - 0.00035688700154423714, - 0.0003381989954505116, - 0.005003923004551325, - 0.005997938002110459, - 0.005999710003379732, - 0.00534947599953739, - 0.006650882001849823, - 0.007989257996086963, - 0.006994721996306907, - 0.007997549000720028, - 0.006000475004839245, - 0.007992471997567918, - 0.007000270001299214, - 0.0069940649991622195, - 0.006123029001173563, - 0.007876436997321434, - 0.005992509999487083, - 0.005996497995511163, - 0.005996592000883538, - 0.008998211000289302, - 0.006997957003477495, - 0.007997235996299423, - 0.005996894993586466, - 0.007997346001502592, - 0.006999543998972513, - 0.00799531300435774, - 0.006996903000981547, - 0.007997668006282765, - 0.00699722900026245, - 0.006997065000177827, - 0.0021210720005910844, - 0.005873653994058259, - 0.005996575993776787, - 0.007998480003152508, - 0.00799729199934518, - 0.005998668995744083, - 0.0069971159973647445, - 0.006995693998760544, - 0.005997504005790688, - 0.0059976850025122985, - 0.006997693999437615, - 0.0059973619936499745, - 0.006997626995143946, - 0.0060047179940738715, - 0.006989596004132181, - 0.007996979999006726, - 0.005997313004627358, - 0.006007017000229098, - 0.007990345999132842, - 0.006000303998007439, - 0.00799194900173461, - 0.005996451996907126, - 0.00800452999828849, - 0.005991685997287277, - 0.005995174004056025, - 0.0069953999991412275, - 0.0036488920013653114, - 0.0063485140053671785, - 0.00799561199528398, - 0.0029933170008007437, - 0.004827908000152092, - 0.0061674959943047725, - 0.008998534001875669, - 0.005993191996822134, - 0.006999769000685774, - 0.006989927998802159, - 0.006998434997512959, - 0.007997590997547377, - 0.008221493000746705, - 0.006822995994298253, - 0.007967829995322973, - 0.00818151599378325, - 0.007781528001942206, - 0.007996264001121745, - 0.004000038999947719, - 0.007992355996975675, - 0.00803933700080961, - 0.003953842999180779, - 0.007995790001587011, - 0.006997349002631381, - 0.007995181003934704, - 0.006994249997660518, - 0.004003722999186721, - 0.0019910820046789013, - 0.006014858001435641, - 0.00797651699394919, - 0.006995037001615856, - 0.0069948669988662004, - 0.005936735004070215, - 0.004056620004121214, - 0.007998044995474629, - 0.007998270004463848, - 0.005988349003018811, - 0.009012355003505945, - 0.00698074400133919, - 0.010996320001140703, - 0.005303003003064077, - 0.009051724002347328, - 0.0068764719981118105, - 0.00774690000253031, - 0.00400462000106927, - 0.0021951539965812117, - 0.0004182010015938431, - 0.00039402599941240624, - 0.0003839780038106255, - 0.00037358299596235156, - 0.00036949400237062946, - 0.0005243510022410192, - 0.0004104570034542121, - 0.0003876589980791323, - 0.0003887369966832921, - 0.00039736100006848574, - 0.0003906290003214963, - 0.0003792569987126626, - 0.0003695279956446029, - 0.0003657159977592528, - 0.00037451000389410183, - 0.0003658909990917891, - 0.000364948995411396, - 0.011098492999735754, - 0.006995621995883994, - 0.007995075000508223, - 0.007996273998287506, - 0.006995712996285874, - 0.006998084005317651, - 0.007996428998012561, - 0.0069956919978722, - 0.005997360000037588, - 0.005994390994601417, - 0.00800274599896511, - 0.00799267399997916, - 0.006995401003223378, - 0.0080002750037238, - 0.007995681997272186, - 0.007992900005774572, - 0.007995371997822076, - 0.007051690001389943, - 0.005965917996945791, - 0.005963916002656333, - 0.008504183002514765, - 0.008149733002937865, - 0.004430597000464331, - 0.0029058610016363673, - 0.0011942669952986762, - 0.007604354002978653, - 0.005183182001928799, - 0.00500052300048992, - 0.006009607001033146, - 0.011053734997403808, - 0.008925728994654492, - 0.004025303001981229, - 0.009668286998930853, - 0.00691161699796794, - 0.004373660995042883, - 0.004575676997774281, - 0.004413971000758465, - 0.0036455899971770123, - 0.006490191000921186, - 0.004859070999373216, - 0.0029956060025142506, - 0.007995987005415373, - 0.00799724899843568, - 0.008040572996833362, - 0.00594420400011586, - 0.009001247999549378, - 0.008029924996662885, - 0.007952166000904981, - 0.007001766003668308, - 0.006979294994380325, - 0.006114135998359416, - 0.004955919001076836, - 0.009919262003677431, - 0.004778567003086209, - 0.005205548004596494, - 0.00599804800003767, - 0.00599569999758387, - 0.007992536993697286, - 0.007995772997674067, - 0.00599810600397177, - 0.0071082560025388375, - 0.003726499999174848, - 0.004033803998026997, - 0.006968291003431659, - 0.007355195004492998, - 0.004782460993737914, - 0.006984003004617989, - 0.007093458996678237, - 0.007893705995229539, - 0.007994055005838163, - 0.004994954004359897, - 0.001986459996260237, - 0.007000956000410952, - 0.0059957080011372454 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil", - "fullname": "TestCalculateNabla2ForZ", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.00035895800101570785, - "max": 0.016018908994738013, - "mean": 0.0027472810595105318, - "stddev": 0.0031088569312507455, - "rounds": 2338, - "median": 0.0005580219985859003, - "iqr": 0.005586072002188303, - "q1": 0.00038300600135698915, - "q3": 0.005969078003545292, - "iqr_outliers": 3, - "stddev_outliers": 591, - "outliers": "591;3", - "ld15iqr": 0.00035895800101570785, - "hd15iqr": 0.014607814999180846, - "ops": 363.9962487777514, - "total": 6.4231431171356235, - "data": [ - 0.0004140509990975261, - 0.00044366600195644423, - 0.0004188559978501871, - 0.00041580700053600594, - 0.005287389998557046, - 0.0005029899984947406, - 0.0005002940015401691, - 0.0004611670010490343, - 0.00041869200504152104, - 0.00042370099981781095, - 0.0004143650003243238, - 0.0004359239974291995, - 0.00042533400119282305, - 0.0038606399975833483, - 0.001633487998333294, - 0.0004825510040973313, - 0.0005229309972492047, - 0.0004471479987842031, - 0.00047233199438778684, - 0.0004306800037738867, - 0.00041627299651736394, - 0.0004221319977659732, - 0.0004166260041529313, - 0.0034631289963726886, - 0.0005176039994694293, - 0.0005098589972476475, - 0.0004349829978309572, - 0.00042922399734379724, - 0.0004168619998381473, - 0.0004268080010660924, - 0.00042930500057991594, - 0.00042749400017783046, - 0.006008299998939037, - 0.0005888410014449619, - 0.0005254650022834539, - 0.0011446429998613894, - 0.00043248000292805955, - 0.00042646699876058847, - 0.0004225630036671646, - 0.0010899629996856675, - 0.0023013120007817633, - 0.00045683000644203275, - 0.00043338200339348987, - 0.00043205700058024377, - 0.0004286640032660216, - 0.0004370280003058724, - 0.0004323449975345284, - 0.00042588100041029975, - 0.002596524005639367, - 0.0018057289998978376, - 0.0004780590024893172, - 0.000468391997856088, - 0.0004647800014936365, - 0.00042600299639161676, - 0.0004159840027568862, - 0.001228666995302774, - 0.0006767410013708286, - 0.0004356989957159385, - 0.0004229130063322373, - 0.000421370001276955, - 0.003198434002115391, - 0.00046781599667156115, - 0.0004300419968785718, - 0.0004553840044536628, - 0.002133403002517298, - 0.0007502570006181486, - 0.0004495730026974343, - 0.0028173550017527305, - 0.00045386199781205505, - 0.0004539360015769489, - 0.0004373980045784265, - 0.0006019520005793311, - 0.0004231409984640777, - 0.0004594280035234988, - 0.00042466699960641563, - 0.002762434996839147, - 0.0024598960008006543, - 0.00043407100019976497, - 0.00041407099342904985, - 0.00041877300100168213, - 0.0004063140004291199, - 0.00045782300003338605, - 0.0012933019970660098, - 0.002371221999055706, - 0.000522616996022407, - 0.0004471749998629093, - 0.003189793002093211, - 0.0004897629987681285, - 0.00047349300439236686, - 0.0004688070039264858, - 0.00041774199780775234, - 0.00041000400233315304, - 0.00043254900083411485, - 0.001120640998124145, - 0.007742198999039829, - 0.0005203279943089001, - 0.0004594529964379035, - 0.00044730299850925803, - 0.00041459599742665887, - 0.00040717799856793135, - 0.0005344809978851117, - 0.00042000700341304764, - 0.00743685600173194, - 0.0005363379968912341, - 0.00045368399878498167, - 0.00041070699808187783, - 0.00042312999721616507, - 0.0004169890016783029, - 0.00041188299655914307, - 0.00041509799484629184, - 0.00041877500189002603, - 0.007403035000606906, - 0.0005669579986715689, - 0.0005736629973398522, - 0.00045636600407306105, - 0.0014895029962644912, - 0.0004575099956127815, - 0.00920232100179419, - 0.0008042279951041564, - 0.0018714010002440773, - 0.008289607998449355, - 0.0006571490011992864, - 0.0004916659963782877, - 0.00042833499901462346, - 0.004470268002478406, - 0.003996019993792288, - 0.007993464001629036, - 0.007006854997598566, - 0.00798574999498669, - 0.007993809995241463, - 0.007992188002390321, - 0.008012352998775896, - 0.006981681006436702, - 0.010151741997106, - 0.0038314729972626083, - 0.01099706300010439, - 0.007055682995996904, - 0.007931519001431298, - 0.006980485995882191, - 0.008237932001065928, - 0.00575435500650201, - 0.005990351994114462, - 0.008006046002265066, - 0.005989697005134076, - 0.006985066000197548, - 0.007000675999734085, - 0.006986558000789955, - 0.007997806002094876, - 0.00799000099505065, - 0.005004209000617266, - 0.0060467719958978705, - 0.007936084002722055, - 0.009087471997190733, - 0.0059015250008087605, - 0.011090057996625546, - 0.0068985129983047955, - 0.006997726995905396, - 0.006995210998866241, - 0.007996459004061762, - 0.005102041002828628, - 0.006896781997056678, - 0.004989832996216137, - 0.003995190003479365, - 0.0070019519989728, - 0.008055153994064312, - 0.006939474995306227, - 0.006998568998824339, - 0.005405789001088124, - 0.008592287005740218, - 0.007063969002047088, - 0.006919307001226116, - 0.006991959999140818, - 0.007004003004112747, - 0.006991059999563731, - 0.005992942999000661, - 0.006993287999648601, - 0.004992825000954326, - 0.005008669999369886, - 0.006987597997067496, - 0.005039597999711987, - 0.003943995005101897, - 0.010000472000683658, - 0.005943233998550568, - 0.00405624600534793, - 0.003983121001510881, - 0.009005366002384108, - 0.007165765004174318, - 0.00637686499976553, - 0.004438549003680237, - 0.005253150004136842, - 0.0037401330046122894, - 0.006982700004300568, - 0.007999766996363178, - 0.005997486994601786, - 0.007993342995177954, - 0.006560968002304435, - 0.0054232699985732324, - 0.005995608997181989, - 0.007001988000411075, - 0.004034286001115106, - 0.0029534819987020455, - 0.006000722001772374, - 0.00909722299547866, - 0.003919066999515053, - 0.00047703299787826836, - 0.00045021699770586565, - 0.00042362399835838005, - 0.0003818460027105175, - 0.0020734770005219616, - 0.0004013919970020652, - 0.00038493899774039164, - 0.003833574999589473, - 0.0005231989998719655, - 0.00045532100193668157, - 0.0003863829988404177, - 0.005379233996791299, - 0.00975620200188132, - 0.004988663000403903, - 0.00508182100020349, - 0.007916356000350788, - 0.00600334900082089, - 0.00798862199735595, - 0.007999109002412297, - 0.0029962150001665577, - 0.006995124000241049, - 0.005019011005060747, - 0.0029705820052186027, - 0.005988978999084793, - 0.007000502999289893, - 0.006998679004027508, - 0.005995452003844548, - 0.004922141000861302, - 0.011071179993450642, - 0.010001913004089147, - 0.0059880420012632385, - 0.00799023600120563, - 0.006985281994275283, - 0.005342881995602511, - 0.004654676005884539, - 0.006708577006065752, - 0.0033227650055778213, - 0.00405949899868574, - 0.00522220300626941, - 0.003677854998386465, - 0.006977454999287147, - 0.004993960996216629, - 0.0070042020015534945, - 0.007991542996023782, - 0.00799761700181989, - 0.011998729001788888, - 0.006986483997025061, - 0.007998117005627137, - 0.006991756999923382, - 0.008996274002129212, - 0.006994299001235049, - 0.006997849995968863, - 0.007996533997356892, - 0.006995314004598185, - 0.007000033001531847, - 0.005992521000734996, - 0.007997998000064399, - 0.005995808998704888, - 0.010005408999859355, - 0.007989370998984668, - 0.008372561998839956, - 0.006631907002883963, - 0.004820007998205256, - 0.0031683370034443215, - 0.003985973999078851, - 0.0050001690033241175, - 0.007291185000212863, - 0.007695932996284682, - 0.009992970997700468, - 0.007000969002547208, - 0.006680530997982714, - 0.005310260006808676, - 0.005998469001497142, - 0.006992440998146776, - 0.007997619999514427, - 0.005997407002723776, - 0.0069947509982739575, - 0.008001032001629937, - 0.009015193005325273, - 0.006964349995541852, - 0.002990836997923907, - 0.004999479999241885, - 0.006002023998007644, - 0.0070967869978630915, - 0.008894918995792978, - 0.006716226998833008, - 0.006260681999265216, - 0.004000174994871486, - 0.0005162379966350272, - 0.000446805999672506, - 0.00042101899452973157, - 0.00038525900163222104, - 0.0009576969969202764, - 0.0004785639976034872, - 0.00039591499808011577, - 0.007426702999509871, - 0.0004925419998471625, - 0.0004554859988274984, - 0.0004198989990982227, - 0.0003831879948847927, - 0.0003871300068567507, - 0.0006031629964127205, - 0.00047457100299652666, - 0.0003988349999417551, - 0.008095345998299308, - 0.0004911629948765039, - 0.000431597996794153, - 0.0005431540048448369, - 0.00037205099943093956, - 0.00037054499989608303, - 0.000371279995306395, - 0.0003675010011647828, - 0.006601565000892151, - 0.0005023050034651533, - 0.00045903900172561407, - 0.00037458999577211216, - 0.0003755930010811426, - 0.004293347999919206, - 0.00048046900337794796, - 0.0004466960017452948, - 0.0004096850025234744, - 0.002286572998855263, - 0.00042334199679316953, - 0.0006345540023176, - 0.00044504999823402613, - 0.00037472300027729943, - 0.00038213100197026506, - 0.0003714639969985001, - 0.000372456997865811, - 0.00037282499397406355, - 0.00037400000292109326, - 0.0003761970001505688, - 0.00040711300243856385, - 0.000409926004067529, - 0.0004354639968369156, - 0.0004063399974256754, - 0.0004020970009150915, - 0.0003857749979943037, - 0.00039673099672654644, - 0.00041930199950002134, - 0.00040859400178305805, - 0.00039602400647709146, - 0.00037990800046827644, - 0.00037522999627981335, - 0.0003735190039151348, - 0.0003698680011439137, - 0.0013711850042454898, - 0.00043276999349473044, - 0.00041099100053543225, - 0.0005583059974014759, - 0.0003968129967688583, - 0.0003834180024568923, - 0.0014161520011839457, - 0.00038436999602708966, - 0.00037100099871167913, - 0.0003752140037249774, - 0.0003847600019071251, - 0.0003707860014401376, - 0.0031067880045156926, - 0.0004649599941330962, - 0.0004370840033516288, - 0.00044596799853025004, - 0.0004209020044072531, - 0.00037809700006619096, - 0.00037441300082718953, - 0.0003733589983312413, - 0.005021436998504214, - 0.00047339600132545456, - 0.000403789003030397, - 0.0004259729976183735, - 0.0003830420027952641, - 0.00038196099922060966, - 0.00038751200190745294, - 0.0003748720046132803, - 0.0003712269972311333, - 0.004436781004187651, - 0.00045262800267664716, - 0.00044063299719709903, - 0.00041149599564960226, - 0.0004173500055912882, - 0.00038459899951703846, - 0.0003734459969564341, - 0.00037860700103919953, - 0.0004138079966651276, - 0.003187724003510084, - 0.00044432499998947605, - 0.0004359540034784004, - 0.003351890998601448, - 0.0004867919997195713, - 0.00044638399413088337, - 0.00044518199865706265, - 0.00039282899524550885, - 0.003055229004530702, - 0.00045554599637398496, - 0.00042501799907768145, - 0.0009560520047671162, - 0.00039523200393887237, - 0.0003944220006815158, - 0.00041568199958419427, - 0.00041326000064145774, - 0.0003843099984806031, - 0.00333593699906487, - 0.0005085590019007213, - 0.0004513300009421073, - 0.00043278600060148165, - 0.00037455900019267574, - 0.0003749859970412217, - 0.0003717029976542108, - 0.0003765419969568029, - 0.00041334699926665053, - 0.00038061100349295884, - 0.0030726309996680357, - 0.0004449190018931404, - 0.00043227399874012917, - 0.00043478500447236, - 0.00038915300683584064, - 0.0003676210035337135, - 0.000371841000742279, - 0.0003703280017361976, - 0.005234926000412088, - 0.0004880820051766932, - 0.0004709369968622923, - 0.0004779480004799552, - 0.0004016839957330376, - 0.0003816539974650368, - 0.0003701519963215105, - 0.0003687549979076721, - 0.0006257210043258965, - 0.003848644999379758, - 0.0005417880020104349, - 0.0004994330010958947, - 0.00037106799572939053, - 0.0003791510025621392, - 0.00037546100065810606, - 0.00037531700218096375, - 0.0003843399972538464, - 0.0003697080028359778, - 0.00041938100184779614, - 0.0005174370016902685, - 0.0004850650002481416, - 0.00044155100476928055, - 0.0005484049979713745, - 0.0006291190002229996, - 0.0029671660013264045, - 0.00044863000221084803, - 0.00040304900176124647, - 0.0004024989975732751, - 0.0003937499932362698, - 0.00037947300006635487, - 0.0026863800012506545, - 0.00048358899948652834, - 0.00042644399945857003, - 0.00039236299926415086, - 0.00037748699833173305, - 0.0019357109995326027, - 0.007078795002598781, - 0.0016596900022705086, - 0.00566951999644516, - 0.0018750140006886795, - 0.0004440789998625405, - 0.00039388800360029563, - 0.0003890369989676401, - 0.004531346996373031, - 0.0011955500012845732, - 0.000605918001383543, - 0.0071289260013145395, - 0.0006917010032339022, - 0.00046417499834205955, - 0.0008108300025924109, - 0.003696810999826994, - 0.0016490959969814867, - 0.00047164699935819954, - 0.0005271510017337278, - 0.00039817200013203546, - 0.006119832003605552, - 0.0005535480013350025, - 0.001832829999329988, - 0.0004921150029986165, - 0.005663279000145849, - 0.0023323849964071997, - 0.005370434999349527, - 0.0005251860056887381, - 0.0029006459953961894, - 0.0036518920023809187, - 0.0032011270013754256, - 0.0005239019956206903, - 0.0006195989990374073, - 0.004058310994878411, - 0.0010147129942197353, - 0.00047914800234138966, - 0.00040908200026024133, - 0.002113497997925151, - 0.005144672002643347, - 0.0011181020017829724, - 0.000633118994301185, - 0.002138050003850367, - 0.002395135998085607, - 0.000770108999859076, - 0.002512959996238351, - 0.0008428569999523461, - 0.0006617850012844428, - 0.0004497659974731505, - 0.004221559996949509, - 0.0014367510011652485, - 0.0008608889984316193, - 0.00046497499715769663, - 0.004852925005252473, - 0.0017218450011569075, - 0.001360777001536917, - 0.0004069110000273213, - 0.005697462001990061, - 0.0019709530024556443, - 0.003067439000005834, - 0.0029213170055299997, - 0.0012776119983755052, - 0.0005419920053100213, - 0.0005085560042061843, - 0.004000357999757398, - 0.007411423997837119, - 0.0007262400031322613, - 0.0004448079998837784, - 0.00701927300542593, - 0.0015292329990188591, - 0.0005912029955652542, - 0.0023391270005959086, - 0.004260970999894198, - 0.0013137080022715963, - 0.0007042680008453317, - 0.0004685160020017065, - 0.002293539000675082, - 0.0007311989975278266, - 0.0012841510033467785, - 0.0005899280004086904, - 0.002732073000515811, - 0.006643168002483435, - 0.0006460590011556633, - 0.0035388070027693175, - 0.0032595989978290163, - 0.0032976099973893724, - 0.004710697001428343, - 0.007538799996837042, - 0.0036577430000761524, - 0.0011083299978054129, - 0.006336974001897033, - 0.0006778679962735623, - 0.0004269880009815097, - 0.00040404299943475053, - 0.00041804399370448664, - 0.00039801600360078737, - 0.005278955002722796, - 0.006996829004492611, - 0.0072004190005827695, - 0.008788207000179682, - 0.010236914000415709, - 0.0008257970039267093, - 0.0013575209959526546, - 0.007510394003475085, - 0.0006660069993813522, - 0.00046391999785555527, - 0.0004260120040271431, - 0.0014344890005304478, - 0.006867966003483161, - 0.002306941998540424, - 0.0004818330053240061, - 0.0077698790046270005, - 0.0011752060017897747, - 0.0033538440038682893, - 0.006264814997848589, - 0.0006612220022361726, - 0.0005438320004031993, - 0.0005004809936508536, - 0.00041933000466087833, - 0.00040901800093706697, - 0.0004993190013919957, - 0.0004193109998595901, - 0.008265176002169028, - 0.0016342390008503571, - 0.0007132280006771907, - 0.005612319000647403, - 0.0026217100021312945, - 0.0005591770022874698, - 0.0004446340026333928, - 0.006918474995472934, - 0.0008524849981768057, - 0.0004452949942788109, - 0.0004235079977661371, - 0.0004092109957127832, - 0.0051381769953877665, - 0.0006321320033748634, - 0.0005467570008477196, - 0.00046061100147198886, - 0.0004221990020596422, - 0.0004259710040059872, - 0.00040692000038689, - 0.004883347006398253, - 0.0037136789978831075, - 0.0024061520016402937, - 0.0010941309956251644, - 0.0004674359952332452, - 0.0004327339993324131, - 0.00042928699986077845, - 0.0004223180003464222, - 0.0005450440003187396, - 0.00047538199578411877, - 0.0004660600025090389, - 0.0004607880036928691, - 0.00047118299698922783, - 0.0004140960008953698, - 0.0004090729999006726, - 0.0030885240048519336, - 0.0005257860029814765, - 0.0004529699945123866, - 0.0004607459995895624, - 0.0005112689977977425, - 0.00041189700277755037, - 0.00040510699909646064, - 0.0006782370037399232, - 0.0004626079971785657, - 0.0037757160025648773, - 0.002997572002641391, - 0.0004980010053259321, - 0.00043811099749291316, - 0.0004135009949095547, - 0.000407152998377569, - 0.00043478600127855316, - 0.00044380599865689874, - 0.006973716997890733, - 0.0006104500062065199, - 0.00045188099466031417, - 0.0006069959999877028, - 0.0005332700020517223, - 0.0004251040008966811, - 0.00040936000004876405, - 0.00040821100265020505, - 0.007251574999827426, - 0.0005337359980330803, - 0.00043636400368995965, - 0.0004115800038562156, - 0.00041839899495244026, - 0.0004104810068383813, - 0.00040689200250199065, - 0.0004119930017623119, - 0.0004058400008943863, - 0.00040150699351215735, - 0.00816755200503394, - 0.0005207549984334037, - 0.00047659099800512195, - 0.0004472540022106841, - 0.00039542200102005154, - 0.0003901529998984188, - 0.00038995000068098307, - 0.0007088830025168136, - 0.00045397099893307313, - 0.005952709005214274, - 0.000522521004313603, - 0.0005918890019529499, - 0.0006295420025708154, - 0.000444516001152806, - 0.0004103290048078634, - 0.0003985980001743883, - 0.005972048995317891, - 0.0007233290016301908, - 0.00041706399497343227, - 0.0006839470006525517, - 0.00044549599988386035, - 0.00039563899917993695, - 0.0004026020033052191, - 0.0066707980004139245, - 0.0004651589988498017, - 0.0004059530037920922, - 0.00039631599793210626, - 0.00040371900104219094, - 0.0003996820014435798, - 0.0004017570026917383, - 0.0003884250036207959, - 0.00039489300252171233, - 0.008062321001489181, - 0.005989365999994334, - 0.006995048999669962, - 0.00549262399727013, - 0.009500183005002327, - 0.007034439004200976, - 0.007976654000231065, - 0.006966836997889914, - 0.007998660003067926, - 0.006998270000622142, - 0.005994515995553229, - 0.007788286995491944, - 0.00814127099874895, - 0.008383763997699134, - 0.009061459997610655, - 0.0072888430004240945, - 0.0076302029992803, - 0.004655575001379475, - 0.0050043709998135455, - 0.007055861999106128, - 0.007431532001646701, - 0.006494246001238935, - 0.008014195002033375, - 0.007959402006235905, - 0.005904103003558703, - 0.006946082001377363, - 0.006112585993832909, - 0.0073237659962615, - 0.009799597995879594, - 0.007870788002037443, - 0.006952035997528583, - 0.00599970299663255, - 0.00899260299775051, - 0.008031704004679341, - 0.007469238000339828, - 0.00557438100076979, - 0.005762018001405522, - 0.0011323130020173267, - 0.0017735719957272522, - 0.005514348995347973, - 0.0020801889986614697, - 0.002931732997240033, - 0.005659451999235898, - 0.0019531579964677803, - 0.0005477060039993376, - 0.002013669000007212, - 0.007304304002900608, - 0.00696612899628235, - 0.004509796002821531, - 0.002382320999458898, - 0.002334796001377981, - 0.005328131002897862, - 0.005749876996560488, - 0.003648542995506432, - 0.004370138005469926, - 0.0009830800045165233, - 0.0005173140016268007, - 0.0012401150015648454, - 0.007537277000665199, - 0.0039038280010572635, - 0.0005995229948894121, - 0.0004459850024431944, - 0.0025335269965580665, - 0.0063028099975781515, - 0.0014205199986463413, - 0.005969078003545292, - 0.0029240090007078834, - 0.00048398699436802417, - 0.00043285900028422475, - 0.006426681997254491, - 0.0018499639991205186, - 0.007073485998262186, - 0.0007830870017642155, - 0.006390301998180803, - 0.001863947996753268, - 0.0004560679954010993, - 0.0005385429976740852, - 0.0069591079954989254, - 0.0007190760006778874, - 0.0005039409952587448, - 0.00044664100278168917, - 0.006279884997638874, - 0.0006280640009208582, - 0.0004427640014910139, - 0.00043123800423927605, - 0.0004198809983790852, - 0.005387548997532576, - 0.0008363640008610673, - 0.0004454970039660111, - 0.004930656003125478, - 0.008288304001325741, - 0.011370198997610714, - 0.003674066007079091, - 0.0009259719954570755, - 0.0005051629996160045, - 0.0004419430042617023, - 0.006557111002621241, - 0.010046265000710264, - 0.0023348600007011555, - 0.00898833400424337, - 0.005144144997757394, - 0.0006461419980041683, - 0.00618706300156191, - 0.0027538770009414293, - 0.006450879001931753, - 0.00959150600101566, - 0.0067002759969909675, - 0.010779655996884685, - 0.00690123100503115, - 0.008162703998095822, - 0.003496016994176898, - 0.005688471996109001, - 0.008859174005920067, - 0.014607814999180846, - 0.005277606003801338, - 0.0024553809998906218, - 0.007817917001375463, - 0.007527359004598111, - 0.01041611500113504, - 0.007827055997040588, - 0.00702271300542634, - 0.006624101995839737, - 0.010486819002835546, - 0.009321360004832968, - 0.012841766998462845, - 0.01473491499928059, - 0.01000237599510001, - 0.010249975995975547, - 0.009740733003127389, - 0.00698519100114936, - 0.011409195001760963, - 0.005558388002100401, - 0.010003848001360893, - 0.00696829999651527, - 0.008986134998849593, - 0.00841199400019832, - 0.011303933002636768, - 0.006248430996492971, - 0.008989720001409296, - 0.007985351003299002, - 0.010375309000664856, - 0.005603840996627696, - 0.007997354005055968, - 0.009008136003103573, - 0.004969843997969292, - 0.008013299004232977, - 0.00887489099841332, - 0.006097329001931939, - 0.005988717995933257, - 0.00883997899654787, - 0.004148940999584738, - 0.007996148997335695, - 0.007994526000402402, - 0.005993952996504959, - 0.0050044070012518205, - 0.006989510002313182, - 0.00900041899876669, - 0.006989186003920622, - 0.0034071879999828525, - 0.011005698004737496, - 0.002569539996329695, - 0.006366059998981655, - 0.007679649002966471, - 0.007944228003907483, - 0.006993106995651033, - 0.006996018004429061, - 0.006176098999276292, - 0.007964346004882827, - 0.005842778002261184, - 0.0029894340041209944, - 0.006470146996434778, - 0.0075279939992469735, - 0.007993496998096816, - 0.00799939099670155, - 0.006995066003582906, - 0.007994835999852512, - 0.011913241003639996, - 0.004077363002579659, - 0.006995370997174177, - 0.007997713000804652, - 0.005990571997244842, - 0.005000704004487488, - 0.0079947539998102, - 0.007995161002327222, - 0.006994935996772256, - 0.00599286599754123, - 0.0035776980002992786, - 0.0054189910006243736, - 0.0069947509982739575, - 0.0049954450005316176, - 0.006996461997914594, - 0.008992773000500165, - 0.008725902000151109, - 0.007270561000041198, - 0.007023370002571028, - 0.006963202002225444, - 0.009001382000860758, - 0.0056517819975852035, - 0.009340860997326672, - 0.007992571998329367, - 0.008004889001313131, - 0.0070893119991524145, - 0.006881989000248723, - 0.006997436998062767, - 0.006996012998570222, - 0.00799529400683241, - 0.006994034993113019, - 0.006992627997533418, - 0.007997740998689551, - 0.00999571599822957, - 0.006767796003259718, - 0.0062208279996411875, - 0.005994321996695362, - 0.002992472000187263, - 0.007001494996075053, - 0.007002696998824831, - 0.00665695500356378, - 0.004327804999775253, - 0.0029971209951327182, - 0.004994697002985049, - 0.006998012002441101, - 0.00745626100251684, - 0.007532178999099415, - 0.013996752000821289, - 0.007997300002898555, - 0.0069953999991412275, - 0.005992387996229809, - 0.01047622799524106, - 0.005517865996807814, - 0.0046886179989087395, - 0.008016909996513277, - 0.005344783996406477, - 0.006935541998245753, - 0.010995270997227635, - 0.005134498002007604, - 0.006875686995044816, - 0.005974987005174626, - 0.005994104998535477, - 0.005996063002385199, - 0.005997107997245621, - 0.006996482006798033, - 0.004997452997486107, - 0.006997823002166115, - 0.0069967300005373545, - 0.0068677290037157945, - 0.004130178000195883, - 0.004993798000214156, - 0.007998056993528735, - 0.006997111995588057, - 0.007000444995355792, - 0.006993735005380586, - 0.007997902001079638, - 0.006997313997999299, - 0.005998165004712064, - 0.006629903000430204, - 0.005364264994568657, - 0.007997745000466239, - 0.007996460000867955, - 0.006999862998782191, - 0.007864503000746481, - 0.0061467380000976846, - 0.007978166999237146, - 0.007001380996371154, - 0.005002827994758263, - 0.004985327999747824, - 0.00541704199713422, - 0.00557714299793588, - 0.007995622996531893, - 0.009003184000903275, - 0.008008247998077422, - 0.00797773899830645, - 0.007996735002961941, - 0.00799719199858373, - 0.008998534001875669, - 0.006030694996297825, - 0.003967287004343234, - 0.0069941429974278435, - 0.009993180996389128, - 0.005998447006277274, - 0.007995533000212163, - 0.005998717999318615, - 0.005004856000596192, - 0.007987110999238212, - 0.006997810000029858, - 0.00599985099688638, - 0.006993399998464156, - 0.007997865999641363, - 0.007999525005288888, - 0.006996068004809786, - 0.006995349998760503, - 0.006998127995757386, - 0.0020052110048709437, - 0.008000182999239769, - 0.006989338995481376, - 0.007990912003151607, - 0.007000974997936282, - 0.003964764000556897, - 0.006022254005074501, - 0.007995833999302704, - 0.006998090000706725, - 0.0070028720001573674, - 0.006001690999255516, - 0.009992915001930669, - 0.00399707599717658, - 0.006562319002114236, - 0.0034257969964528456, - 0.006999000004725531, - 0.007997633998456877, - 0.0059921950014540926, - 0.005992139995214529, - 0.008998379998956807, - 0.005992656006128527, - 0.003994323000370059, - 0.005997034000756685, - 0.00799977699352894, - 0.006991725997067988, - 0.0069170350034255534, - 0.005725265000364743, - 0.009348410996608436, - 0.005998212000122294, - 0.008995969998068176, - 0.007188987001427449, - 0.007806802001141477, - 0.00486449299933156, - 0.005130035999172833, - 0.003996728999482002, - 0.006996216005063616, - 0.006000123001285829, - 0.00899270400259411, - 0.00729529200179968, - 0.004699049997725524, - 0.006997424003202468, - 0.006997382006375119, - 0.007978147004905622, - 0.0070180080001591705, - 0.006992296999669634, - 0.014003683005284984, - 0.006990604997554328, - 0.00699647500005085, - 0.006997975004196633, - 0.006996720003371593, - 0.006997084004979115, - 0.007998585002496839, - 0.006996557000093162, - 0.0079996070053312, - 0.005993729995680042, - 0.008002107999345753, - 0.016018908994738013, - 0.007956580004247371, - 0.007991065001988318, - 0.003995364000729751, - 0.005998191001708619, - 0.008003605995327234, - 0.0069917729997541755, - 0.007998335997399408, - 0.004637555997760501, - 0.009358353003335651, - 0.007078882001223974, - 0.0039032880013110116, - 0.007999789995665196, - 0.009994915999413934, - 0.005998887994792312, - 0.005997133994242176, - 0.008022601003176533, - 0.008964499000285286, - 0.006688927998766303, - 0.007303152997337747, - 0.006993086994043551, - 0.00800195799820358, - 0.007989841993548907, - 0.007997066000825725, - 0.005358565998903941, - 0.002855907005141489, - 0.008778176001214888, - 0.006987063999986276, - 0.0059985620027873665, - 0.004624547000275925, - 0.00571625999873504, - 0.0056465169982402585, - 0.010287511002388783, - 0.007631568005308509, - 0.006503120996057987, - 0.001539037999464199, - 0.0011194329999852926, - 0.0010202689954894595, - 0.0010065560054499656, - 0.0009719079971546307, - 0.0009888390050036833, - 0.0032464730029460043, - 0.0011274319986114278, - 0.0010194529968430288, - 0.0009601269994163886, - 0.0010391370014986023, - 0.0010504489982849918, - 0.0010544649994699284, - 0.0010741779988165945, - 0.0009973650012398139, - 0.0009776389997568913, - 0.0010499589989194646, - 0.0011051399997086264, - 0.0010293230006936938, - 0.0009976910005207174, - 0.000963730999501422, - 0.0009805700028664432, - 0.0010448430039105006, - 0.0011197819985682145, - 0.0029723469997406937, - 0.006998197000939399, - 0.0069974520010873675, - 0.006010705001244787, - 0.007983839997905307, - 0.007007628999417648, - 0.007986927004822064, - 0.006996832002187148, - 0.00799792700127, - 0.006996332005655859, - 0.006999247001658659, - 0.00699710899789352, - 0.006064532994059846, - 0.0071882560005178675, - 0.008067789996857755, - 0.0076751290034735575, - 0.0046990629998617806, - 0.006108765002863947, - 0.004161983000813052, - 0.00900230900151655, - 0.006987210996157955, - 0.0070042489969637245, - 0.007980497997778002, - 0.006990553003561217, - 0.0069951729965396225, - 0.005987491997075267, - 0.0059873259961022995, - 0.006000003995723091, - 0.004437174000486266, - 0.006629466995946132, - 0.004944615000567865, - 0.004987460000847932, - 0.0071674849968985654, - 0.004801838003913872, - 0.005992183003399987, - 0.004341022999142297, - 0.006646655005170032, - 0.007987685996340588, - 0.006995536001340952, - 0.005990466997900512, - 0.006991727001150139, - 0.008002793001651298, - 0.00798414199380204, - 0.008010363002540544, - 0.0059735969989560544, - 0.005038477000198327, - 0.007966582001245115, - 0.007978023997566197, - 0.007001748999755364, - 0.006999564997386187, - 0.007988513003510889, - 0.00998583500040695, - 0.006997862001298927, - 0.006984675994317513, - 0.007003232996794395, - 0.00698351099708816, - 0.004483938995690551, - 0.006503951997729018, - 0.005993316997773945, - 0.00499188200046774, - 0.0070048259949544445, - 0.010992589996021707, - 0.006994458999542985, - 0.006998730998020619, - 0.006996983000135515, - 0.007118180001270957, - 0.00685453400365077, - 0.006000301000312902, - 0.007990139005414676, - 0.007997962995432317, - 0.006991435002419166, - 0.006998580000072252, - 0.0069975270016584545, - 0.0038444230012828484, - 0.006150745000923052, - 0.007014331000391394, - 0.007980241003679112, - 0.0070277729973895475, - 0.006961405000765808, - 0.007000356999924406, - 0.007994601997779682, - 0.007998184002644848, - 0.007994333995156921, - 0.008000680005352478, - 0.007991922997462098, - 0.007997346001502592, - 0.002998274998390116, - 0.004007684998214245, - 0.004986879997886717, - 0.00699568800337147, - 0.008010163001017645, - 0.005989019002299756, - 0.005994058003125247, - 0.007990846002940089, - 0.004998556003556587, - 0.0029608370023197494, - 0.007030155997199472, - 0.007003514998359606, - 0.005989031997160055, - 0.0059980009973514825, - 0.006006595998769626, - 0.007988241995917633, - 0.005997776002914179, - 0.007059878000291064, - 0.007930842999485321, - 0.00799679200281389, - 0.003564033002476208, - 0.006432709000364412, - 0.0059941010040347464, - 0.008999154000775889, - 0.0051271410047775134, - 0.004868085001362488, - 0.007994580999366008, - 0.007999264998943545, - 0.007995744002982974, - 0.007996330001333263, - 0.005996018000587355, - 0.005998203996568918, - 0.004997166004613973, - 0.0038777750014560297, - 0.006116178003139794, - 0.007999042005394585, - 0.007703183997364249, - 0.001600689000042621, - 0.0006335260040941648, - 0.0004048870032420382, - 0.0003695210034493357, - 0.00037437499850057065, - 0.000368420995073393, - 0.000606647998210974, - 0.0032171540005947463, - 0.0004377650038804859, - 0.0004393780036480166, - 0.00042078100523212925, - 0.0003915310007869266, - 0.0004600999964168295, - 0.00037130699638510123, - 0.00037943000643281266, - 0.0003712899997481145, - 0.00037198799691395834, - 0.003222731000278145, - 0.0004586709983414039, - 0.0004656849996536039, - 0.00042689199472079054, - 0.0003949000019929372, - 0.000384079001378268, - 0.0003802389983320609, - 0.00037139200139790773, - 0.0003702469985000789, - 0.0003720919994520955, - 0.000373823000700213, - 0.00037000000156695023, - 0.0003677479980979115, - 0.00037311099731596187, - 0.0003777329984586686, - 0.004401366000820417, - 0.0004561730020213872, - 0.000448337996203918, - 0.00043945600191364065, - 0.0003964770003221929, - 0.0003762619962799363, - 0.00037120799970580265, - 0.00037028999940957874, - 0.0003768110036617145, - 0.00037370400241343305, - 0.005293395995977335, - 0.0005094230000395328, - 0.00044086600246373564, - 0.0004367619985714555, - 0.00038017999759176746, - 0.00037802000588271767, - 0.00036696300230687484, - 0.00036983600148232654, - 0.0003792139978031628, - 0.0003677790009533055, - 0.00040965199877973646, - 0.003933099003916141, - 0.0005755879974458367, - 0.0006007750052958727, - 0.0005046410005888902, - 0.00040627399721415713, - 0.0003826219981419854, - 0.000375659998098854, - 0.00045753100130241364, - 0.0004857559979427606, - 0.0005104369993205182, - 0.0036487909965217113, - 0.0004647409950848669, - 0.00043814500531880185, - 0.00042429099994478747, - 0.0003694519982673228, - 0.0007313299938687123, - 0.00036828599695581943, - 0.00043959999311482534, - 0.0004888839976047166, - 0.0005479179963003844, - 0.00043467200157465413, - 0.0004678490004152991, - 0.0008695120050106198, - 0.0005476970036397688, - 0.0005063589997007512, - 0.000500185000419151, - 0.0007951640000101179, - 0.0007217500024125911, - 0.0003711300014401786, - 0.00038306299393298104, - 0.002090355999825988, - 0.0014058500019018538, - 0.0003926289937226102, - 0.00037849799991818145, - 0.0003698110012919642, - 0.0003671000013127923, - 0.00037461199826793745, - 0.00036892899515805766, - 0.000406348000979051, - 0.008908864998375066, - 0.0006992049966356717, - 0.0004943929961882532, - 0.0004166020007687621, - 0.0010372550023021176, - 0.0005182179957046174, - 0.0004124180049984716, - 0.0028490510012488812, - 0.0004489499988267198, - 0.0005522249994101003, - 0.0005527199973585084, - 0.0005113099978188984, - 0.00040452399844070897, - 0.00039076599932741374, - 0.00038160000258358195, - 0.00039837000076659024, - 0.0003897040005540475, - 0.0003802409992204048, - 0.0006240500006242655, - 0.0004929970018565655, - 0.006986663996940479, - 0.006995612995524425, - 0.007994899999175686, - 0.006999596000241581, - 0.008006661999388598, - 0.007995512998604681, - 0.007985559001099318, - 0.005993644001137, - 0.007007587002590299, - 0.006989276997046545, - 0.007452541001839563, - 0.0075358489993959665, - 0.0069956669976818375, - 0.0069964719950803556, - 0.0058726689967443235, - 0.005132131998834666, - 0.003984253999078646, - 0.004996541996661108, - 0.006996055999479722, - 0.004996586001652759, - 0.006995878000452649, - 0.007996756998181809, - 0.006995780000579543, - 0.007996897998964414, - 0.006032063000020571, - 0.007960445997014176, - 0.004001803004939575, - 0.003991113000665791, - 0.006995368996285833, - 0.007997232998604886, - 0.006995840005401988, - 0.003997024999989662, - 0.0029971699987072498, - 0.004996110001229681, - 0.006995115996687673, - 0.005997502004902344, - 0.005613226006971672, - 0.00537868600076763, - 0.005997516003844794, - 0.008009897996089421, - 0.006011566998495255, - 0.006993884002440609, - 0.006970517002628185, - 0.005995929001073819, - 0.008995210002467502, - 0.006996331998379901, - 0.006999656994594261, - 0.006002850997901987, - 0.006994789000600576, - 0.008041141998546664, - 0.007067257996823173, - 0.003534023002430331, - 0.004290420001780149, - 0.00695964100304991, - 0.005996669999149162, - 0.007003573999099899, - 0.003345555000123568, - 0.00043788199400296435, - 0.0003941269969800487, - 0.0003810799971688539, - 0.0003777759993681684, - 0.00038190999475773424, - 0.0003728950032382272, - 0.0033458000034443103, - 0.00040585299575468525, - 0.00037987399991834536, - 0.0036005779984407127, - 0.0004214209984638728, - 0.0003798260004259646, - 0.0003772810014197603, - 0.0027418960016802885, - 0.0004193150016362779, - 0.000408358006097842, - 0.00037299899850040674, - 0.0003646340046543628, - 0.00036960899888072163, - 0.0003692219979711808, - 0.0003647679986897856, - 0.0003636230030679144, - 0.00036400100361788645, - 0.005443250003736466, - 0.0004109559959033504, - 0.00038788900565123186, - 0.00036343800456961617, - 0.0003634020031313412, - 0.00036722000368172303, - 0.0003594349982449785, - 0.0003617100010160357, - 0.00036628200177801773, - 0.0003650090002338402, - 0.0003721349930856377, - 0.003115703002549708, - 0.00044712500675814226, - 0.0003752310003619641, - 0.0034322479987167753, - 0.00037896600406384096, - 0.0003692720056278631, - 0.0003733409976121038, - 0.0003645449978648685, - 0.00036656800511991605, - 0.000363152998033911, - 0.00036432999331736937, - 0.0040755359950708225, - 0.0004056050020153634, - 0.00037533500290010124, - 0.0004115880001336336, - 0.00036586499481927603, - 0.00036329199792817235, - 0.0003760290055652149, - 0.00036017000093124807, - 0.00036568900395650417, - 0.00036102399462834, - 0.00036031300260219723, - 0.006232289000763558, - 0.0004096830016351305, - 0.0003714400008902885, - 0.00036977600393584, - 0.00036927300243405625, - 0.00036235399602446705, - 0.0003672519960673526, - 0.00037069700192660093, - 0.00036635099968407303, - 0.0003650069993454963, - 0.00036081000143894926, - 0.0036529339995468035, - 0.0017211790036526509, - 0.00038301199674606323, - 0.000368066001101397, - 0.0004232279970892705, - 0.00036457600072026253, - 0.00039790399750927463, - 0.00037336399691412225, - 0.00036190600076224655, - 0.0010250699997413903, - 0.003206482004316058, - 0.00040145399543689564, - 0.00036540700239129364, - 0.00037824500032002106, - 0.00039083300362108275, - 0.0004279929999029264, - 0.0003765380024560727, - 0.0003660240035969764, - 0.00037540400080615655, - 0.0003625030003604479, - 0.00036111800000071526, - 0.005445697002869565, - 0.0005187219940125942, - 0.00037142300425330177, - 0.0003666420016088523, - 0.0003970729958382435, - 0.00037042400072095916, - 0.0003688400029204786, - 0.0003772999989450909, - 0.005212340998696163, - 0.00040933999844128266, - 0.0003727509974851273, - 0.00036093100061407313, - 0.0003632240041042678, - 0.00036143499892205, - 0.00036700799682876095, - 0.00039114899846026674, - 0.0003848850028589368, - 0.000365411993698217, - 0.00510140199912712, - 0.00041671900544315577, - 0.0003647720004664734, - 0.0003662189992610365, - 0.00036139100120635703, - 0.0003646969998953864, - 0.00036777200148208067, - 0.00036017000093124807, - 0.00036573900433722883, - 0.0034111839995603077, - 0.0004376010037958622, - 0.00038286999915726483, - 0.0004559780063573271, - 0.001340415001322981, - 0.0003949730016756803, - 0.0003724089983734302, - 0.0003691120000439696, - 0.00036683399957837537, - 0.00036188800004310906, - 0.001451856005587615, - 0.000764926997362636, - 0.0003865289982059039, - 0.00037138599873287603, - 0.00036849900061497465, - 0.0003858099953504279, - 0.0003654079991974868, - 0.00036702999932458624, - 0.00036360800004331395, - 0.00036455399822443724, - 0.0003595180023694411, - 0.00035936700442107394, - 0.0015772109982208349, - 0.002601000996946823, - 0.0004702730002463795, - 0.0022448740055551752, - 0.00040729900501901284, - 0.00037150400021346286, - 0.0003639760034275241, - 0.0003655399996205233, - 0.00036493899824563414, - 0.00036656600423157215, - 0.0003843200029223226, - 0.00037002500175731257, - 0.0004278470005374402, - 0.0012268150021554902, - 0.0011823259992524981, - 0.002719372001593001, - 0.0004051610012538731, - 0.0003766230001929216, - 0.00036795000050915405, - 0.0003656589979073033, - 0.0003684510011225939, - 0.00035979300446342677, - 0.0003618600021582097, - 0.00036824799462920055, - 0.00035985999420518056, - 0.0018368360033491626, - 0.0009445399991818704, - 0.002929930000391323, - 0.0007241880011861213, - 0.00038164200668688864, - 0.0003703379989019595, - 0.0003751240001292899, - 0.00036777400237042457, - 0.000365199004590977, - 0.0003766369991353713, - 0.00036814699706155807, - 0.0013691130006918684, - 0.0013593969997600652, - 0.0010212279958068393, - 0.0006232150044525042, - 0.00042591399687808007, - 0.000372260001313407, - 0.0003687860007630661, - 0.00035948699951404706, - 0.00036105899926042184, - 0.0003655260006780736, - 0.00036445300065679476, - 0.00036301799991633743, - 0.0003609709965530783, - 0.0005978679982945323, - 0.0005777300029876642, - 0.0014775390009162948, - 0.0005321880016708747, - 0.0011671409956761636, - 0.0017253920013899915, - 0.00040429700311506167, - 0.00037061899638501927, - 0.00036737499613082036, - 0.00039917499816510826, - 0.0003620450006565079, - 0.0003656629996839911, - 0.00036793600156670436, - 0.0020148680050624534, - 0.0007285480023710988, - 0.0006314089987426996, - 0.00041896200127666816, - 0.0005608400024357252, - 0.00039259400364244357, - 0.0003805129963438958, - 0.00035970599856227636, - 0.0003716079954756424, - 0.00036710700078401715, - 0.0003616680041886866, - 0.0003637790068751201, - 0.0003619859999162145, - 0.00036856399674434215, - 0.001370888996461872, - 0.001327385994954966, - 0.0028646430000662804, - 0.000494122999953106, - 0.0019554789978428744, - 0.00038325100467773154, - 0.0003851019937428646, - 0.0005250260001048446, - 0.0009731649988680147, - 0.0006577039966941811, - 0.0005948129983153194, - 0.0015450889986823313, - 0.0004072520023328252, - 0.0003720899985637516, - 0.00041172299825120717, - 0.0003654300016933121, - 0.00036144199839327484, - 0.0003682969982037321, - 0.0003694460028782487, - 0.00223918299889192, - 0.0012733559997286648, - 0.000957137999648694, - 0.0004770500017912127, - 0.000374399998690933, - 0.00036731399450218305, - 0.0003598019975470379, - 0.0003652780069387518, - 0.0003641170042101294, - 0.00036325599648989737, - 0.0003645389951998368, - 0.00036422399716684595, - 0.00036309900315245613, - 0.0070947940039332025, - 0.006999965000431985, - 0.007432170998072252, - 0.007555486001365352, - 0.006367672998749185, - 0.007276303003891371, - 0.007344992001890205, - 0.0079963910029619, - 0.006995326002652291, - 0.007996555003046524, - 0.005996104999212548, - 0.006535824999446049, - 0.008463412996206898, - 0.007990891004737932, - 0.007993942002940457, - 0.007996466993063223, - 0.0035585570003604516, - 0.007433637001668103, - 0.005996380998112727, - 0.007996312997420318, - 0.005996357002004515, - 0.004998254000383895, - 0.00999882599717239, - 0.005992797996441368, - 0.004997759002435487, - 0.009996795000915881, - 0.011995179003861267, - 0.006994572002440691, - 0.0053104010003153235, - 0.009682629999588244, - 0.0010310689976904541, - 0.0004061270010424778, - 0.0003670650039566681, - 0.0003753689961740747, - 0.0003607749968068674, - 0.0010691600036807358, - 0.0003898549985024147, - 0.00036209400423103943, - 0.0003622450021794066, - 0.00036709299456560984, - 0.00036464699951466173, - 0.0012542820040835068, - 0.004260190995410085, - 0.0020556640010909177, - 0.0003998539978056215, - 0.00040022200118983164, - 0.00044168900058139116, - 0.00039117300184443593, - 0.00036818700027652085, - 0.00036614500277210027, - 0.00036807199649047107, - 0.002074005999020301, - 0.002233412000350654, - 0.00038300600135698915, - 0.00041949300066335127, - 0.004008162002719473, - 0.0030723849995411, - 0.0027539840011741035, - 0.0025276169981225394, - 0.002484085001924541, - 0.0004066749970661476, - 0.0003658620044006966, - 0.0003631190047599375, - 0.0003683760005515069, - 0.0003613049993873574, - 0.0014687889997730963, - 0.0003960820031352341, - 0.00036609199742088094, - 0.0003678380016935989, - 0.0003607239996199496, - 0.0003630330029409379, - 0.0003626550023909658, - 0.00037075000000186265, - 0.003347447003761772, - 0.0018770809983834624, - 0.0003901269956259057, - 0.0003712580000865273, - 0.002255613995657768, - 0.00040072200499707833, - 0.0003741850014193915, - 0.0003732940022018738, - 0.0003649540012702346, - 0.000359409001248423, - 0.0003717380022862926, - 0.00036335500044515356, - 0.004371881004772149, - 0.00039906299934955314, - 0.0003642509982455522, - 0.0003691989986691624, - 0.0003639010028564371, - 0.0003735579957719892, - 0.00037416999839479104, - 0.00036220999754732475, - 0.0003715249986271374, - 0.0003653129970189184, - 0.0003619959970819764, - 0.0007902900033514015, - 0.0006890189979458228, - 0.0003720640015671961, - 0.00036048400215804577, - 0.0019521279973560013, - 0.0004131359964958392, - 0.00036775100306840613, - 0.00037057799636386335, - 0.00036262300272937864, - 0.0009828779948293231, - 0.0022786129993619397, - 0.00040567200630903244, - 0.000375540999812074, - 0.0003680669979075901, - 0.00037674700433854014, - 0.00036274299782235175, - 0.00038859000051161274, - 0.000375494004401844, - 0.0003612989967223257, - 0.00036368500150274485, - 0.0003700909946928732, - 0.00035982899862574413, - 0.0016153820033650845, - 0.001892235995910596, - 0.0003938040026696399, - 0.0012264389952179044, - 0.00038842600042698905, - 0.0003705409981193952, - 0.00037134800368221477, - 0.00036559000000124797, - 0.0003650979997473769, - 0.00037797500408487394, - 0.0003706259958562441, - 0.0005591559965978377, - 0.002155182999558747, - 0.00111412099795416, - 0.0003733100020326674, - 0.0011496560036903247, - 0.0005635790002997965, - 0.0003730509997694753, - 0.0013078080010018311, - 0.00036657199962064624, - 0.0003744120040209964, - 0.0003622879958129488, - 0.00036831099714618176, - 0.00036781800008611754, - 0.0003643320014816709, - 0.0003643650052254088, - 0.0018416169987176545, - 0.0013942190053057857, - 0.0004908099945168942, - 0.0013150140002835542, - 0.00040371100476477295, - 0.0003663440002128482, - 0.0007001619960647076, - 0.0003671750018838793, - 0.0012795040020137094, - 0.0018052889936370775, - 0.0007430420009768568, - 0.0010243430006084964, - 0.0003961849943152629, - 0.00036929100315319374, - 0.0003620100033003837, - 0.002273605998198036, - 0.0004129349981667474, - 0.00036254600126994774, - 0.0003751149997697212, - 0.0003650860016932711, - 0.00036092599475523457, - 0.0003656920016510412, - 0.000365282001439482, - 0.0007036930037429556, - 0.004074411997862626, - 0.00039759599894750863, - 0.0018991179967997596, - 0.0013765379990218207, - 0.0003757500016945414, - 0.0003721999964909628, - 0.0003691989986691624, - 0.00036455199733609334, - 0.0003632500011008233, - 0.00036251400160836056, - 0.0031864190023043193, - 0.0006442500016419217, - 0.0003751839976757765, - 0.00036781599919777364, - 0.00037386400072136894, - 0.0003612449945649132, - 0.0003627149999374524, - 0.00036584700137609616, - 0.00036477299727266654, - 0.001992136996705085, - 0.0009754819984664209, - 0.00037242400139803067, - 0.0003613660010159947, - 0.0003637350018834695, - 0.0011725780059350654, - 0.0014394520039786585, - 0.00037589400017168373, - 0.0003860180004267022, - 0.0003623059965320863, - 0.0003671279991976917, - 0.00036422000266611576, - 0.0003692699974635616, - 0.00036932600050931796, - 0.00036168900260236114, - 0.0003673899991554208, - 0.001571098000567872, - 0.0013810829987050965, - 0.000381181002012454, - 0.00036360599915497005, - 0.00036279799678595737, - 0.0011700909963110462, - 0.001967837000847794, - 0.0003892029999406077, - 0.00036679099866887555, - 0.0003705660055857152, - 0.00036230300611350685, - 0.00037097700260346755, - 0.00037118600448593497, - 0.0003653149979072623, - 0.0006831399950897321, - 0.0022579259966732934, - 0.0007091880033840425, - 0.000373926006432157, - 0.0013439250033115968, - 0.0011661850003292784, - 0.0003874000030918978, - 0.00043122399802086875, - 0.001336676999926567, - 0.0003718959997058846, - 0.00037580200296361, - 0.00035904099786421284, - 0.0003645389951998368, - 0.00036659200122812763, - 0.00036540900327963755, - 0.0004826579970540479, - 0.0028712099956464954, - 0.0015067009953781962, - 0.0004767989958054386, - 0.00037182400410529226, - 0.0003606000027502887, - 0.0017811239958973601, - 0.00040004600305110216, - 0.0006048079958418384, - 0.00039337199996225536, - 0.005964534997474402, - 0.0004674559968407266, - 0.0004144480044487864, - 0.001310762992943637, - 0.0019100950012216344, - 0.0015566280053462833, - 0.0008717739983694628, - 0.00036753099993802607, - 0.0005156010010978207, - 0.00037018899456597865, - 0.0022796789999119937, - 0.0009493469988228753, - 0.0003770670009544119, - 0.0003648470010375604, - 0.00037020499439677224, - 0.000370671994460281, - 0.00036012700002174824, - 0.0012980150058865547, - 0.0015409170009661466, - 0.0003828150001936592, - 0.000367038999684155, - 0.00035895800101570785, - 0.00036617700243368745, - 0.000366161999409087, - 0.0003604339945013635, - 0.0009848699992289767, - 0.0003855199975077994, - 0.0003688889992190525, - 0.00036521499714581296, - 0.0003638560010585934, - 0.00036592400283552706, - 0.00036741499934578314, - 0.0003628720005508512, - 0.0003654040046967566, - 0.0017635160038480535, - 0.0019487140016281046, - 0.00039230800030054525, - 0.0003684330004034564, - 0.0003659089998109266, - 0.001350140999420546, - 0.0005309340049279854, - 0.0006627120019402355, - 0.0009950919993571006, - 0.00036488499608822167, - 0.00037161499494686723, - 0.0003694560000440106, - 0.0003632459993241355, - 0.0003651220031315461, - 0.0003669059951789677, - 0.00036252200516173616, - 0.001060669994330965, - 0.0005525189990294166, - 0.0006341490006889217, - 0.0006738879965269007, - 0.0013270019990159199, - 0.0005095050000818446, - 0.00046680100058438256, - 0.00039445199945475906, - 0.00036648099921876565, - 0.0020418429994606413, - 0.0004069540009368211, - 0.000395139999454841, - 0.00037649000296369195, - 0.0003641660005087033, - 0.00036459299735724926, - 0.00036987299972679466, - 0.0003667800046969205, - 0.0003974029968958348, - 0.000477920999401249, - 0.0005297890020301566, - 0.0005010359964217059, - 0.0011155199972563423, - 0.0003684920011437498, - 0.0003657220004242845, - 0.0003646059994935058, - 0.0006821959977969527, - 0.0005705430012312718, - 0.0007310070068342611, - 0.00038678900455124676, - 0.0003746959991985932, - 0.0003694650004035793, - 0.0006682530001853593, - 0.0008362239968846552, - 0.0005033769994042814, - 0.00039050299528753385, - 0.0003681520029203966, - 0.00036462300340645015, - 0.0003610240019042976, - 0.000363370003469754, - 0.00037160099600441754, - 0.0003596919996198267, - 0.00036757699854206294, - 0.00045699600013904274, - 0.00047878499754006043, - 0.0011182170055690221, - 0.0010721569997258484, - 0.0014958890023990534, - 0.0004134500049985945, - 0.0003802130013355054, - 0.00036843000270891935, - 0.00037233200418995693, - 0.00037425900518428534, - 0.0003735170030267909, - 0.000453905995527748, - 0.0017509789977339096, - 0.0017361279969918542, - 0.00040217099740402773, - 0.0003784549990086816, - 0.0003770219991565682, - 0.00038217399560380727, - 0.00037438899744302034, - 0.00042582800233503804, - 0.0009014129973365925, - 0.0005011399989598431, - 0.0005337389957276173, - 0.00039439100510207936, - 0.0003758890015888028, - 0.0014500469987979159, - 0.0003950909958803095, - 0.0004047109978273511, - 0.00040055200224742293, - 0.00038095399941084906, - 0.0005975250023766421, - 0.0006656800032942556, - 0.0005345949975890107, - 0.0005066340017947368, - 0.00046584800293203443, - 0.0003943539995816536, - 0.0003737360020750202, - 0.00037098299799254164, - 0.00038143999699968845, - 0.00037227800203254446, - 0.0003754639983526431, - 0.0005116080064908601, - 0.0009914740003296174, - 0.0003803490035352297, - 0.0003797210010816343, - 0.0003742379994946532, - 0.0003704549963003956, - 0.0003766840018215589, - 0.000371848997019697, - 0.0009938899966073222, - 0.0005026150029152632, - 0.0005107359975227155, - 0.0004437320021679625, - 0.00037743400025647134, - 0.00037661399983335286, - 0.00048789300490170717, - 0.0005061549964011647, - 0.0004860090048168786, - 0.00040779599657980725, - 0.00038290899829007685, - 0.00037743800203315914, - 0.001621804003661964, - 0.0004363709958852269, - 0.0015707499987911433, - 0.0004154419948463328, - 0.00040340099803870544, - 0.0003823489969363436, - 0.0003670690057333559, - 0.0003780900005949661, - 0.00038348000089172274, - 0.00037760600389447063, - 0.00037772399809909984, - 0.00037146500108065084, - 0.005977310000162106, - 0.0004243599978508428, - 0.00038277499697869644, - 0.00037186200643191114, - 0.00037807899934705347, - 0.00037541599886026233, - 0.0003771459960262291, - 0.0003765150031540543, - 0.000370609006495215, - 0.003084600997681264, - 0.002301788001204841, - 0.0005803960011689924, - 0.00037981700006639585, - 0.00037733699718955904, - 0.0003701899986481294, - 0.00037272199551807716, - 0.0003751949989236891, - 0.0003744260029634461, - 0.00037508400419028476, - 0.00036943899613106623, - 0.0021946210035821423, - 0.0031664760026615113, - 0.0004098720019101165, - 0.0003833390001091175, - 0.000380418001441285, - 0.0003746049987967126, - 0.0003824340019491501, - 0.0003699160006362945, - 0.0003704619957716204, - 0.003511104005156085, - 0.000666888001433108, - 0.0004135790004511364, - 0.0003785399967455305, - 0.0014757579992874525, - 0.0004081349979969673, - 0.0003951450053136796, - 0.00037421699380502105, - 0.00037461600004462525, - 0.00037585399695672095, - 0.0003726490031112917, - 0.00037192500167293474, - 0.0003777859965339303, - 0.0003711300014401786, - 0.003895068002748303, - 0.0005003659971407615, - 0.001984049995371606, - 0.0004056429970660247, - 0.00037715499638579786, - 0.00038094100455055013, - 0.00037079100002301857, - 0.0004568049989757128, - 0.0022070710037951358, - 0.0004983489998267032, - 0.00040772299689706415, - 0.0003750869946088642, - 0.00037037699803477153, - 0.0020814070012420416, - 0.0004067190020577982, - 0.00037364300078479573, - 0.00038002399378456175, - 0.00037627599522238597, - 0.00036961500154575333, - 0.00037448800139827654, - 0.000373869996110443, - 0.0003793279975070618, - 0.0003850330031127669, - 0.00037842700112378225, - 0.005014601003495045, - 0.0016021679984987713, - 0.0004215439985273406, - 0.000391934001527261, - 0.0003734799975063652, - 0.000413590001699049, - 0.0015045089967316017, - 0.00040691200410947204, - 0.0003819389967247844, - 0.00037405900366138667, - 0.0003801970015047118, - 0.00036919400008628145, - 0.000375737996364478, - 0.000518875996931456, - 0.0005154919999768026, - 0.0010643139976309612, - 0.0010014479994424619, - 0.00043667799764079973, - 0.0003747769951587543, - 0.0003880459989886731, - 0.00037220699596218765, - 0.0003707879950525239, - 0.0003806910026469268, - 0.0003702100002556108, - 0.00043989199912175536, - 0.0005169760042917915, - 0.0004710239954874851, - 0.001475363998906687, - 0.0006531610051752068, - 0.0003755060024559498, - 0.0007521099978475831, - 0.0005577379997703247, - 0.00038330600364133716, - 0.0003701449968502857, - 0.0016143319953698665, - 0.0006999030010774732, - 0.0003881579978042282, - 0.00037509999674512073, - 0.0003785640001296997, - 0.0003715290004038252, - 0.00037515500298468396, - 0.0003758449965971522, - 0.00037149399577174336, - 0.0033180630052811466, - 0.0004538569992291741, - 0.0012509750013123266, - 0.000400630000513047, - 0.0003746309957932681, - 0.0003741970067494549, - 0.0017424290053895675, - 0.0003962240007240325, - 0.000371848997019697, - 0.0003756420046556741, - 0.0003796620003413409, - 0.00037770999915665016, - 0.0003741109976544976, - 0.0003773540011025034, - 0.00037608699494739994, - 0.001151086005847901, - 0.0023179160052677616, - 0.00038217499968595803, - 0.00036879099934594706, - 0.0003864980026264675, - 0.0003886050035362132, - 0.001051132996508386, - 0.00043656699563143775, - 0.0003819209960056469, - 0.000524319002579432, - 0.0016547010018257424, - 0.0003931699975510128, - 0.00037088400131324306, - 0.00037441099993884563, - 0.0003795669981627725, - 0.0003703159964061342, - 0.0003738070008694194, - 0.0003718699954333715, - 0.00037821599835297093, - 0.0013888739995309152, - 0.004214919994410593, - 0.0003995340011897497, - 0.0003822270009550266, - 0.0003690750017995015, - 0.00037303799763321877, - 0.0006709350054734387, - 0.0017347600005450659, - 0.0004150040040258318, - 0.00040538900066167116, - 0.0003884410034515895, - 0.00036968800122849643, - 0.0003805899978033267, - 0.00037483100459212437, - 0.0010623830021359026, - 0.0013794270053040236, - 0.0003850919965771027, - 0.0003735610007424839, - 0.0003743620036402717, - 0.0003747420050785877, - 0.0003766999943763949, - 0.00037023999902885407, - 0.00037307800084818155, - 0.001635459004319273, - 0.00039320399810094386, - 0.00038369700632756576, - 0.0003828589979093522, - 0.001405964998411946, - 0.0010328330026823096, - 0.00039123000169638544, - 0.0003720540044014342, - 0.001393113998346962, - 0.00039111699879867956, - 0.0003822979997494258, - 0.00037410900404211134, - 0.0003740379979717545, - 0.0003802270002779551, - 0.00037191300361882895, - 0.0003708560034283437, - 0.004016879000118934, - 0.00038832499558338895, - 0.000374324998119846, - 0.0003759340033866465, - 0.0003710099990712479, - 0.0003703900001710281, - 0.0013920510027674027, - 0.0018650310012162663, - 0.0003947299992432818, - 0.0003811349961324595, - 0.0003820709971478209, - 0.00037638100184267387, - 0.0003889700019499287, - 0.0003851020010188222, - 0.00037388200144050643, - 0.0013146880010026507, - 0.0005469220050144941, - 0.0023067420042934828, - 0.0004117009957553819, - 0.00037545899976976216, - 0.00037921700277365744, - 0.000377212003513705, - 0.000367683001968544, - 0.003143187001114711, - 0.0003995520019088872, - 0.0003778730024350807, - 0.00038077500357758254, - 0.00037138199695618823, - 0.0003994729995611124, - 0.0003838920019916259, - 0.0003735210048034787, - 0.0037342220020946115, - 0.00040378399717155844, - 0.00038693899841746315, - 0.0003757719969144091, - 0.0003762540000025183, - 0.0003752369957510382, - 0.00037457900180015713, - 0.003417459003685508, - 0.0004059490020154044, - 0.0003845329993055202, - 0.0003804620064329356, - 0.0003682819951791316, - 0.00037455900019267574, - 0.0003680279987747781, - 0.00037002700264565647, - 0.00037283299752743915, - 0.0003697280044434592, - 0.0003718699954333715, - 0.004990822999388911, - 0.0004068260022904724, - 0.0003810199996223673, - 0.0013842560001648962, - 0.00037642499955836684, - 0.000388917003874667, - 0.0004309940050006844, - 0.0003777879974222742, - 0.0003790709961322136, - 0.00036924299638485536, - 0.0003710039964062162, - 0.004317398001148831, - 0.00045600999874295667, - 0.00045712799328612164, - 0.00044775499554816633, - 0.00038920400402275845, - 0.000413297995692119, - 0.00039670099795330316, - 0.0004134920018259436, - 0.0003785900044022128, - 0.0004003040012321435, - 0.00037847799831070006, - 0.0004153100016992539, - 0.0003816379976342432, - 0.00037774199881823733, - 0.0003864640020765364, - 0.0004007980023743585, - 0.00038593800127273425, - 0.0003755290017579682, - 0.00038249399949563667, - 0.0003788230023928918, - 0.00040149699634639546, - 0.0004019090047222562, - 0.0003803409999818541, - 0.0003723029949469492, - 0.0003876610062434338, - 0.0004102729944861494, - 0.00041203499858966097, - 0.000402575999032706, - 0.0003869240026688203, - 0.00038180000410648063, - 0.00038969200249994174, - 0.0003828129993053153, - 0.00037781900027766824, - 0.0004295089966035448, - 0.0004026300011901185, - 0.00041524400148773566, - 0.0003981529953307472, - 0.0003771269985008985, - 0.0003859770004055463, - 0.0004034279991174117, - 0.00040859800355974585, - 0.0003816210009972565, - 0.00039868600288173184, - 0.00038156300433911383, - 0.00039616399590158835, - 0.0003894809997291304, - 0.00037822200101800263, - 0.00037947200326016173, - 0.0003922890027752146, - 0.0003822959988610819, - 0.00041611299820942804, - 0.0003985920047853142, - 0.0004058939957758412, - 0.00038145799771882594, - 0.00037401999725261703, - 0.00042539300193311647, - 0.00037587000406347215, - 0.000375165996956639, - 0.00041127699660137296, - 0.00039568400097778067, - 0.00043003899918403476, - 0.000378586002625525, - 0.0003735840000445023, - 0.0003966069998568855, - 0.0004047999973408878 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil", - "fullname": "TestCalculateNabla2OfTheta", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.00032102699333336204, - "max": 0.01520376700500492, - "mean": 0.004275450127302015, - "stddev": 0.003297357114006507, - "rounds": 999, - "median": 0.004997969001124147, - "iqr": 0.006618793997404282, - "q1": 0.0003801100010605296, - "q3": 0.006998903998464812, - "iqr_outliers": 0, - "stddev_outliers": 541, - "outliers": "541;0", - "ld15iqr": 0.00032102699333336204, - "hd15iqr": 0.01520376700500492, - "ops": 233.89350132147166, - "total": 4.271174677174713, - "data": [ - 0.006984973995713517, - 0.006992690003244206, - 0.006997310003498569, - 0.007996078005817253, - 0.00700949600286549, - 0.005983716997434385, - 0.0069981410051696, - 0.006994752999162301, - 0.007997875000000931, - 0.006997529999352992, - 0.006000068999128416, - 0.010994078998919576, - 0.006998311997449491, - 0.0070006470050429925, - 0.0069955450017005205, - 0.005997569998726249, - 0.006994153998675756, - 0.006994381001277361, - 0.004996586998458952, - 0.006997406999289524, - 0.007997730004717596, - 0.0069984199944883585, - 0.007996056003321428, - 0.006996420001087245, - 0.0020023620018037036, - 0.007994080995558761, - 0.006998621996899601, - 0.006998946002568118, - 0.007993274994078092, - 0.006997928998316638, - 0.00737710500106914, - 0.006619380998017732, - 0.00799547399947187, - 0.006005750001349952, - 0.007997777996934019, - 0.009986240002035629, - 0.006995626004936639, - 0.006997351003519725, - 0.0079969859943958, - 0.007994388004590292, - 0.00436803499906091, - 0.003628400998422876, - 0.007997287997568492, - 0.006994522002059966, - 0.005998131004162133, - 0.006995939998887479, - 0.006997491997026373, - 0.007997339002031367, - 0.006997274998866487, - 0.007007005006016698, - 0.010997275996487588, - 0.006989007000811398, - 0.007991954000317492, - 0.008995993004646152, - 0.007995328000106383, - 0.00705143999948632, - 0.007943524993606843, - 0.00699511499988148, - 0.006998199001827743, - 0.0029975979996379465, - 0.007013187001575716, - 0.007986879005329683, - 0.007989890997123439, - 0.007998652996320743, - 0.007994134997716174, - 0.004997682000976056, - 0.00399577799544204, - 0.006996984004217666, - 0.007999215005838778, - 0.00699679899844341, - 0.009002370999951381, - 0.007991680002305657, - 0.006998955002927687, - 0.007993758001248352, - 0.007000559999141842, - 0.006994221999775618, - 0.00727083799574757, - 0.005721720997826196, - 0.007997401000466198, - 0.005997225001920015, - 0.007998459004738834, - 0.007995143998414278, - 0.007996895001269877, - 0.005996945001243148, - 0.005998986998747569, - 0.006000043998938054, - 0.0009992349951062351, - 0.007993629995326046, - 0.006000631001370493, - 0.005989610006508883, - 0.0060017700016032904, - 0.006991946000198368, - 0.00701313200261211, - 0.007985910000570584, - 0.006996627002081368, - 0.006991652997385245, - 0.004300234002585057, - 0.009699831003672443, - 0.005992489001073409, - 0.006157001000246964, - 0.00889921100315405, - 0.007983283001522068, - 0.006035528997017536, - 0.009906015999149531, - 0.008013913997274358, - 0.00797904199862387, - 0.006994355004280806, - 0.007994355000846554, - 0.00699586900009308, - 0.005996217005304061, - 0.007997107000846881, - 0.005060413001046982, - 0.006941678999282885, - 0.0029939379965071566, - 0.005997730004310142, - 0.006993203001911752, - 0.006995746000029612, - 0.005995924999297131, - 0.007998164001037367, - 0.006998535005550366, - 0.007995258005394135, - 0.007998916000360623, - 0.005996933003189042, - 0.007995954001671635, - 0.008000782996532507, - 0.007993911000085063, - 0.005997617001412436, - 0.003995906001364347, - 0.007000036996032577, - 0.00750819099630462, - 0.009043978003319353, - 0.004448690000572242, - 0.006978816003538668, - 0.007996804000867996, - 0.007994648003659677, - 0.007495939004002139, - 0.006498335002106614, - 0.00799965899932431, - 0.007994935003807768, - 0.0070095420014695264, - 0.008983485997305252, - 0.004992276000848506, - 0.004995490999135654, - 0.0069972739947843365, - 0.007999446002941113, - 0.007998583001608495, - 0.007143599999835715, - 0.009847595996689051, - 0.007996608997927979, - 0.007996115004061721, - 0.00899668299825862, - 0.006996894000621978, - 0.006998589000431821, - 0.007997364999027923, - 0.007996872998774052, - 0.00799685299716657, - 0.007141705995309167, - 0.006860362002043985, - 0.00799654299771646, - 0.007164488000853453, - 0.00781908399949316, - 0.006997321004746482, - 0.006996441996307112, - 0.007999550995009486, - 0.007001357000262942, - 0.0069915400017634965, - 0.00801001800573431, - 0.006988752997131087, - 0.006994379000389017, - 0.007003781000094023, - 0.009017794000101276, - 0.007969828002387658, - 0.006997386000875849, - 0.006996762997005135, - 0.008001395006431267, - 0.005031955995946191, - 0.004960555997968186, - 0.007992225997440983, - 0.00699774399981834, - 0.006996303993219044, - 0.006997028001933359, - 0.011003243002051022, - 0.0059994420007569715, - 0.009072877997823525, - 0.0061119779929867946, - 0.008064431996899657, - 0.005731060002290178, - 0.009996307002438698, - 0.0060002440004609525, - 0.005993490005494095, - 0.00799779299995862, - 0.0059992140013491735, - 0.005062194999482017, - 0.003928201003873255, - 0.007007134998275433, - 0.006988651999563444, - 0.0069951369951013476, - 0.006000138004310429, - 0.00799426800222136, - 0.007996979002200533, - 0.004001512999820989, - 0.007991644997673575, - 0.008001713002158795, - 0.006991638001636602, - 0.007098807000147644, - 0.00589173399930587, - 0.004996985000616405, - 0.008003962997463532, - 0.006990112000494264, - 0.006999953999184072, - 0.006993978000537027, - 0.009996578999562189, - 0.007996202002686914, - 0.004014504003862385, - 0.004986134998034686, - 0.007990580001205672, - 0.006433636001020204, - 0.007561573998827953, - 0.006999944998824503, - 0.006994946998020168, - 0.0110012570003164, - 0.006993249000515789, - 0.005996589999995194, - 0.008017694002774078, - 0.0059750720029114746, - 0.006999523997365031, - 0.005994585997541435, - 0.006997551994572859, - 0.0069976740051060915, - 0.006995832998654805, - 0.0069982929999241605, - 0.006997109005169477, - 0.007996995002031326, - 0.005996954001602717, - 0.006997267002589069, - 0.007002095000643749, - 0.005010242995922454, - 0.007981183000083547, - 0.0059952090014121495, - 0.006000854999001604, - 0.007992412996827625, - 0.005997938998916652, - 0.006998265002039261, - 0.007997253000212368, - 0.005996977000904735, - 0.004560000001220033, - 0.0024287759952130727, - 0.007002170001214836, - 0.004994868002540898, - 0.00799673000437906, - 0.005005580002034549, - 0.006994899005803745, - 0.007991498998308089, - 0.005016288996557705, - 0.0019765850011026487, - 0.00800045499636326, - 0.005997906999255065, - 0.0080940970001393, - 0.004899724001006689, - 0.0015261240041581914, - 0.010470002998772543, - 0.007997354994586203, - 0.005042529999627732, - 0.0059499880007933825, - 0.003995086000941228, - 0.005996343999868259, - 0.006447238003602251, - 0.007548109002527781, - 0.006996978998358827, - 0.006995614996412769, - 0.00799677599570714, - 0.00799864800501382, - 0.006997980999585707, - 0.007994789993972518, - 0.006997131000389345, - 0.006999655997788068, - 0.005993085003865417, - 0.00499778900120873, - 0.005994610000925604, - 0.006997074997343589, - 0.00799996400019154, - 0.007994651998160407, - 0.0095213020031224, - 0.007479905005311593, - 0.007991297003172804, - 0.007003617000009399, - 0.010985806002281606, - 0.0079964060059865, - 0.00599595299718203, - 0.003991832003521267, - 0.003600643998652231, - 0.005395983003836591, - 0.003996906001702882, - 0.00699585500115063, - 0.007007544998486992, - 0.005983262999507133, - 0.0035851760039804503, - 0.006409059998986777, - 0.004995536000933498, - 0.007998100998520385, - 0.005995566003548447, - 0.007997132001037244, - 0.004997969001124147, - 0.007988192002812866, - 0.008629534997453447, - 0.0063666700007161126, - 0.008061868000368122, - 0.006928741997398902, - 0.00700378500187071, - 0.006989779001742136, - 0.009001348000310827, - 0.003991306002717465, - 0.004996572999516502, - 0.007997462002094835, - 0.006997313997999299, - 0.007998305998626165, - 0.00799674400332151, - 0.006998251999903005, - 0.006996869997237809, - 0.007998863999091554, - 0.006998390002991073, - 0.006994835996010806, - 0.0065555590044823475, - 0.006444970997108612, - 0.005986995005514473, - 0.006996379997872282, - 0.006996894997428171, - 0.00699731500208145, - 0.005996874999254942, - 0.006997557997237891, - 0.008001848000276368, - 0.0069948089949321, - 0.010997270001098514, - 0.010997033001331147, - 0.007996424996235874, - 0.006990556001255754, - 0.007014104994595982, - 0.007980907997989561, - 0.00800008400256047, - 0.00863332399603678, - 0.0053523440001299605, - 0.006991460999415722, - 0.006995628995355219, - 0.008001070003956556, - 0.006992939997871872, - 0.006995149997237604, - 0.006994864001171663, - 0.00699690000328701, - 0.00599609700293513, - 0.007006995998381171, - 0.00599614399834536, - 0.006988907000049949, - 0.009011039997858461, - 0.008993674004159402, - 0.007982639996043872, - 0.007992679995368235, - 0.008000273002835456, - 0.006986266998865176, - 0.005994237995764706, - 0.00700410900026327, - 0.006991027003095951, - 0.005010911998397205, - 0.00697817699983716, - 0.005995539002469741, - 0.009997004002798349, - 0.006388171997969039, - 0.007610621003550477, - 0.007991521000803914, - 0.010997260993462987, - 0.007996054002433084, - 0.003997037005319726, - 0.004996643998310901, - 0.007996225001988932, - 0.009014486000523902, - 0.004978967001079582, - 0.0059974269970553, - 0.007998761997441761, - 0.005997489002766088, - 0.00702854800329078, - 0.007966092001879588, - 0.006995124997047242, - 0.007997109998541418, - 0.005007817002478987, - 0.0069872990061412565, - 0.007997166001587175, - 0.008995699005026836, - 0.005997788000968285, - 0.007002974998613354, - 0.0060012839967384934, - 0.004986289000953548, - 0.007998549001058564, - 0.007999689994903747, - 0.007993926003109664, - 0.007994346000486985, - 0.007012278001639061, - 0.010988633999659214, - 0.004855423998378683, - 0.00717923000047449, - 0.007950148996314965, - 0.005997567001031712, - 0.00699833600083366, - 0.009999597001296934, - 0.005993667000439018, - 0.005917317001149058, - 0.007078119000652805, - 0.0049949999956879765, - 0.007997544002137147, - 0.007997332002560142, - 0.007997218999662437, - 0.00633523100259481, - 0.009255267003027257, - 0.007402013005048502, - 0.006997216994932387, - 0.008001776004675776, - 0.008007846998225432, - 0.006986673994106241, - 0.007992044993443415, - 0.006997775999479927, - 0.007997232998604886, - 0.006997634001891129, - 0.00699747700127773, - 0.0069975940059521236, - 0.0050088620046153665, - 0.0079862319980748, - 0.005996829000650905, - 0.007997667999006808, - 0.00800661499670241, - 0.005154282000148669, - 0.003885659003572073, - 0.008020697998290416, - 0.003917261004971806, - 0.003994628001237288, - 0.011001033999491483, - 0.006994262002990581, - 0.0034946810046676546, - 0.0004527990022324957, - 0.00043163500231457874, - 0.0004298269996070303, - 0.0003974389983341098, - 0.0003523900013533421, - 0.0003519420060911216, - 0.0003916970017598942, - 0.0003672239981824532, - 0.0003677380009321496, - 0.00037817199336132035, - 0.00037580500065814704, - 0.0003639409987954423, - 0.00036331800220068544, - 0.0003621830037445761, - 0.00036697100586025044, - 0.00034526700619608164, - 0.0003698470027302392, - 0.00034986899845534936, - 0.00035766700602835044, - 0.0003563779973774217, - 0.00035997099621454254, - 0.00036033400101587176, - 0.00038186399615369737, - 0.0003582139979698695, - 0.0003507090004859492, - 0.0003692760001285933, - 0.00035535799543140456, - 0.00036249400000087917, - 0.00036737200571224093, - 0.0026363150027464144, - 0.005998158005240839, - 0.006903517998580355, - 0.00910072099941317, - 0.007996668995474465, - 0.006990565998421516, - 0.0036763090029126033, - 0.0033135400008177385, - 0.005999979002808686, - 0.008004757000890095, - 0.005998123000608757, - 0.00799141400057124, - 0.006991515998379327, - 0.005997537002258468, - 0.006085228000301868, - 0.00790819300164003, - 0.006998870994721074, - 0.007002361002378166, - 0.008004479997907765, - 0.007983091003552545, - 0.0069978320025256835, - 0.005114170002343599, - 0.005880386001081206, - 0.0059985230036545545, - 0.006997960997978225, - 0.006996653995884117, - 0.00799751700105844, - 0.0056184549976023845, - 0.005383323994465172, - 0.006014660000801086, - 0.007965069999045227, - 0.006997013995714951, - 0.006997486998443492, - 0.008003274000657257, - 0.007991194004716817, - 0.006998914999712724, - 0.004222789000777993, - 0.005770982999820262, - 0.00799618499877397, - 0.006062578999262769, - 0.008186512997781392, - 0.006972081006097142, - 0.007765162998111919, - 0.009997977998864371, - 0.00799778399959905, - 0.006996959004027303, - 0.008005524003237952, - 0.007998436005436815, - 0.004215786997519899, - 0.003767068003071472, - 0.005998326996632386, - 0.006997775999479927, - 0.007003165999776684, - 0.003990388002421241, - 0.008997739998449106, - 0.004201996001938824, - 0.0073382999980822206, - 0.006450009997934103, - 0.007995913001650479, - 0.007996518994332291, - 0.006000936002237722, - 0.007996950000233483, - 0.00799791300232755, - 0.007062894001137465, - 0.006930089999514166, - 0.006994170995312743, - 0.006999815996096004, - 0.0069929269957356155, - 0.008003308001207188, - 0.0035697380008059554, - 0.007416977001412306, - 0.007997213004273362, - 0.007998068998858798, - 0.004996790004952345, - 0.006997861994022969, - 0.005997413994919043, - 0.005997175998345483, - 0.006999025004915893, - 0.005995204002829269, - 0.007996757994988002, - 0.006997291995503474, - 0.005996813997626305, - 0.005999391003570054, - 0.005997704000037629, - 0.005994649996864609, - 0.006999368000833783, - 0.008134334995702375, - 0.006857931999547873, - 0.006996639000135474, - 0.007997575004992541, - 0.006997185999352951, - 0.0060008649961673655, - 0.007994351995876059, - 0.0059353579999879, - 0.004058509999595117, - 0.005998549997457303, - 0.006995648000156507, - 0.00900121599988779, - 0.006992977003392298, - 0.00599863899697084, - 0.006995611998718232, - 0.008003529001143761, - 0.007990815000084694, - 0.006995383002504241, - 0.007997829001396894, - 0.007997339002031367, - 0.006998051001573913, - 0.00799703899974702, - 0.005997230997309089, - 0.007997969994903542, - 0.002997398994921241, - 0.005997535998176318, - 0.006010589000652544, - 0.0008661829997436143, - 0.0003530999965732917, - 0.0003322789998492226, - 0.00033334000181639567, - 0.0007260739948833361, - 0.00032644900056766346, - 0.00032974100031424314, - 0.0003259049990447238, - 0.00032102699333336204, - 0.0011691049949149601, - 0.00413738999486668, - 0.0017375999959767796, - 0.0034710460022324696, - 0.01520376700500492, - 0.0047395259971381165, - 0.0011269399983575568, - 0.001818118995288387, - 0.00033739000355126336, - 0.004491113999392837, - 0.0012041230002068914, - 0.0003456170015851967, - 0.0022464819994638674, - 0.006000230001518503, - 0.0015569419992971234, - 0.00036522899608826265, - 0.0004555949999485165, - 0.0018946039999718778, - 0.006195572001161054, - 0.0012625320014194585, - 0.0021339170052669942, - 0.0021223619987722486, - 0.002946824999526143, - 0.0047316600030171685, - 0.003625165998528246, - 0.0027611109981080517, - 0.00042310500430176035, - 0.0004185660000075586, - 0.0004274580060155131, - 0.0003483240070636384, - 0.004307762996177189, - 0.0038111059984657913, - 0.0027202019991818815, - 0.0024305550032295287, - 0.0003975789950345643, - 0.0003932050021830946, - 0.002763243996014353, - 0.009257906000129879, - 0.0006581400011782534, - 0.0004849680044571869, - 0.0020241759993950836, - 0.004847732998314314, - 0.0004962129969499074, - 0.00045683400094276294, - 0.0010477190007804893, - 0.0007689110061619431, - 0.005152023004484363, - 0.0005109410049044527, - 0.0005039410025347024, - 0.0031840930023463443, - 0.0034398179996060207, - 0.0025307660034741275, - 0.0026709020021371543, - 0.0034371229994576424, - 0.0014445790002355352, - 0.004465276004339103, - 0.001990336000744719, - 0.00042533600208116695, - 0.00043008299689972773, - 0.00042331599979661405, - 0.00034913299896288663, - 0.0003479820006759837, - 0.003233905001252424, - 0.0006590659977518953, - 0.0010565600023255683, - 0.001259722004760988, - 0.003471005999017507, - 0.003374254003574606, - 0.0004449219995876774, - 0.00040633699973113835, - 0.0011635550035862252, - 0.0003654929969343357, - 0.0003600389973144047, - 0.00454403900221223, - 0.0005896549992030486, - 0.0030390199972316623, - 0.00041127100121229887, - 0.00040589099808130413, - 0.002301972999703139, - 0.00045099700218997896, - 0.006417743999918457, - 0.004343672997492831, - 0.0004620770050678402, - 0.0003801220009336248, - 0.00036718399496749043, - 0.002468377999321092, - 0.003875064998283051, - 0.0010827139994944446, - 0.00043372400250518695, - 0.0035635969979921356, - 0.0023208560014609247, - 0.0004123170001548715, - 0.0022204749984666705, - 0.0009259030048269778, - 0.003927190999093, - 0.0004570099990814924, - 0.0004425510051078163, - 0.00037357200199039653, - 0.0003537479933584109, - 0.00035489599395077676, - 0.00036464900040300563, - 0.0003520619939081371, - 0.00034689299354795367, - 0.001930587000970263, - 0.0007647169986739755, - 0.002492441999493167, - 0.0005839560035383329, - 0.0003972510021412745, - 0.0003618199989432469, - 0.001416247003362514, - 0.0003649700011010282, - 0.0004189949977444485, - 0.0003784500004258007, - 0.0003609449995565228, - 0.00035427200054982677, - 0.003056800000194926, - 0.0004564519986161031, - 0.00041762299952097237, - 0.0003780190018005669, - 0.0003505729982862249, - 0.00035253499663667753, - 0.0003643870004452765, - 0.0003507189976517111, - 0.00035011600266443565, - 0.002727311999478843, - 0.0020096239968552254, - 0.0009702740062493831, - 0.0003600510026444681, - 0.0003668470017146319, - 0.00034989499545190483, - 0.00035194300289731473, - 0.00035339199530426413, - 0.00038584099820582196, - 0.0008637219943921082, - 0.000884501998370979, - 0.002734488996793516, - 0.00039407900476362556, - 0.00036482799623627216, - 0.0003517639997880906, - 0.00035326099896337837, - 0.0003531789989210665, - 0.0003551660047378391, - 0.0003475519988569431, - 0.004730702996312175, - 0.0004120939993299544, - 0.0003747609953279607, - 0.00035058600042248145, - 0.0003565039951354265, - 0.0003529759997036308, - 0.00034792600490618497, - 0.00035280700103612617, - 0.0003533929993864149, - 0.0003464580004219897, - 0.003503068001009524, - 0.001816238000174053, - 0.00037608099955832586, - 0.0003530280009726994, - 0.00035545099672162905, - 0.0003529979949234985, - 0.00034844600304495543, - 0.0003618650007410906, - 0.00034896300348918885, - 0.00034738900285447016, - 0.0022737760009476915, - 0.0004112800015718676, - 0.00037227400025585666, - 0.00034924699866678566, - 0.00034901400067610666, - 0.00038977300573606044, - 0.0003567809981177561, - 0.00034674800554057583, - 0.0003586200036806986, - 0.0003513610063237138, - 0.0003574820002540946, - 0.0011878509976668283, - 0.004361118997621816, - 0.00038928700087126344, - 0.0003906719939550385, - 0.00038560599932679906, - 0.0003822370053967461, - 0.0003624679957283661, - 0.0003501499959384091, - 0.0003546719963196665, - 0.0003532940027071163, - 0.0003531180045683868, - 0.0029839519993402064, - 0.0015895980031928048, - 0.0003626630059443414, - 0.00038997599767753854, - 0.000393501999496948, - 0.0004219250040478073, - 0.0003754110002773814, - 0.00035290999949211255, - 0.0003578910036594607, - 0.0003534499992383644, - 0.00034692100598476827, - 0.001747299000271596, - 0.004023920999316033, - 0.00043043200275860727, - 0.0005720129993278533, - 0.00044277499546296895, - 0.0007121189992176369, - 0.0004164909987594001, - 0.0003535169962560758, - 0.0003567949970602058, - 0.003023934994416777, - 0.0004761259988299571, - 0.0003556119991117157, - 0.0003628900012699887, - 0.0003534130009938963, - 0.0003490009985398501, - 0.00035562099947128445, - 0.00035536199720809236, - 0.00034765000600600615, - 0.00035335800203029066, - 0.00034834499820135534, - 0.0003472860043984838, - 0.005248487999779172, - 0.0012642079964280128, - 0.00035950299934484065, - 0.00215434399433434, - 0.00038352100091287866, - 0.00035168600152246654, - 0.00037793899537064135, - 0.0003528710003593005, - 0.0003535880023264326, - 0.002543146001698915, - 0.0003775079967454076, - 0.0003655520049505867, - 0.0010267900070175529, - 0.00036328800342744216, - 0.0003538719975040294, - 0.00035993299388792366, - 0.0003888989958795719, - 0.0047141679970081896, - 0.0009351390035590157, - 0.0003712070028996095, - 0.0005298830001265742, - 0.0003801340062636882, - 0.0003569210020941682, - 0.00036620799801312387, - 0.0004509959981078282, - 0.00036050199560122564, - 0.0003502389954519458, - 0.0003488750007818453, - 0.003595347996451892, - 0.0012843879958381876, - 0.0003516020005918108, - 0.0003413009981159121, - 0.0003412369987927377, - 0.0003459450017544441, - 0.0003421200017328374, - 0.0003391049976926297, - 0.0003817190008703619, - 0.00034119900374207646, - 0.00033787600114010274, - 0.005184669003938325, - 0.0004814870044356212, - 0.00044363200140651315, - 0.0003449630021350458, - 0.0003443989990046248, - 0.00034702599805314094, - 0.00034597899502841756, - 0.00034583499655127525, - 0.0003532190021360293, - 0.0003465469999355264, - 0.0003381830028956756, - 0.004252855993399862, - 0.0003719039959833026, - 0.00034612000308698043, - 0.0003500720049487427, - 0.00034161800431320444, - 0.00033888900361489505, - 0.0003479879960650578, - 0.0003475859994068742, - 0.00033856299705803394, - 0.003944708005292341, - 0.0003777960009756498, - 0.00034446499921614304, - 0.00034251400211360306, - 0.00035402199864620343, - 0.00033962899760808796, - 0.000342930004990194, - 0.0003473739998298697, - 0.00033912199432961643, - 0.0003437889972701669, - 0.001682207002886571, - 0.0024013669972191565, - 0.0006232640007510781, - 0.00038544800190720707, - 0.0003485149936750531, - 0.00033859799441415817, - 0.0003497030047583394, - 0.00034574799792608246, - 0.0003375930027686991, - 0.00035222100268583745, - 0.00034340099955443293, - 0.00033771400194382295, - 0.0021279079956002533, - 0.002630884999234695, - 0.00035462200321489945, - 0.00039554799877805635, - 0.0003819419944193214, - 0.0015745139971841127, - 0.00037266899744281545, - 0.000346180000633467, - 0.00034654000046430156, - 0.0003375080050318502, - 0.00034973500441992655, - 0.0003388920013094321, - 0.0009435870015295222, - 0.003124517999822274, - 0.00036971100053051487, - 0.0003576200033421628, - 0.00034505700023146346, - 0.00034270600008312613, - 0.00036047599860467017, - 0.0003401990034035407, - 0.0003449020005064085, - 0.0003464839974185452, - 0.00034109599801013246, - 0.0004170239990344271, - 0.0027174480055691674, - 0.001506355001765769, - 0.001335121996817179, - 0.000367023007129319, - 0.0003440729997237213, - 0.0004510930011747405, - 0.0003442840024945326, - 0.0003382970025995746, - 0.0003408860065974295, - 0.0029606810057885014, - 0.000971839006524533, - 0.00038410400156863034, - 0.0003450049989623949, - 0.00034451999817974865, - 0.0003618039991124533, - 0.0003503689949866384, - 0.00033865799923660234, - 0.0003482039974187501, - 0.0003391009959159419, - 0.00034070200490532443, - 0.0036719550043926574, - 0.00042298399785067886, - 0.0003703109978232533, - 0.0003415989995119162, - 0.00035257599665783346, - 0.00033920299756573513, - 0.0003371920029167086, - 0.00034641799720702693, - 0.00034966199746122584, - 0.00033905999589478597, - 0.0029907779971836135, - 0.0011875810014316812, - 0.0003747040027519688, - 0.0003538980017765425, - 0.0003451950033195317, - 0.00034227500145789236, - 0.0003419619970372878, - 0.00034339100238867104, - 0.00034250199678353965, - 0.00034131499705836177, - 0.00035098299849778414, - 0.00034236400097142905, - 0.0003495090059004724, - 0.004321589003666304, - 0.0004049639974255115, - 0.0003470559968263842, - 0.0003654709944385104, - 0.00033889499900396913, - 0.00034641000092960894, - 0.0003513769988785498, - 0.00034269000025233254, - 0.0003443630048423074, - 0.0003443320019869134, - 0.0015906819971860386, - 0.0004619510000338778, - 0.0004587030052789487, - 0.001660018999245949, - 0.0007079730057739653, - 0.0004549670047708787, - 0.00037070100370328873, - 0.000343779000104405, - 0.0003425350005272776, - 0.0003532459959387779, - 0.00033976600388996303, - 0.00034017599682556465, - 0.0010180190001847222, - 0.002102543003275059, - 0.0023904840054456145, - 0.0003801060011028312, - 0.0003495089986245148, - 0.0003507920046104118, - 0.000339202000759542, - 0.00036809599987464026, - 0.0003376649983692914, - 0.0003433940000832081, - 0.0003611720021581277, - 0.00034242699621245265, - 0.0003445940019446425, - 0.005047108999860939, - 0.0004053719967487268, - 0.0003835640018223785, - 0.00035122400004183874, - 0.00034347400651313365, - 0.0003377210014150478, - 0.000347546003467869, - 0.0003386470052646473, - 0.0003424850001465529, - 0.0031757479955558665, - 0.00038355300057446584, - 0.0003575270020519383, - 0.0003905909979948774 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil[none]", - "fullname": "TestCalculateNabla4[none]", - "params": { - "static_variant": [ - "none", - null - ] - }, - "param": "none", - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.00047410900151589885, - "max": 0.020068984995305073, - "mean": 0.004361228765102155, - "stddev": 0.003397980529931836, - "rounds": 1868, - "median": 0.004725374499685131, - "iqr": 0.00642587999755051, - "q1": 0.0005770005009253509, - "q3": 0.007002880498475861, - "iqr_outliers": 2, - "stddev_outliers": 1019, - "outliers": "1019;2", - "ld15iqr": 0.00047410900151589885, - "hd15iqr": 0.01715502899605781, - "ops": 229.29317718938242, - "total": 8.146775333210826, - "data": [ - 0.0007394010026473552, - 0.0015758150038891472, - 0.0008321380009874701, - 0.0005799630016554147, - 0.0005317830000421964, - 0.000610720002441667, - 0.0005400189984356984, - 0.000521087997185532, - 0.000520650006365031, - 0.0005336539979907684, - 0.0028133050000178628, - 0.0009912879977491684, - 0.001233732997206971, - 0.0005386670018197037, - 0.0005188860013731755, - 0.0005368119964259677, - 0.0005163239984540269, - 0.0005330700005288236, - 0.0005153439997229725, - 0.001604570003109984, - 0.0009312179972766899, - 0.0012574610009323806, - 0.0005350260034902021, - 0.0019517420005286112, - 0.0005285850056679919, - 0.0005284509970806539, - 0.0005236290016910061, - 0.0022395409978344105, - 0.0006843059964012355, - 0.0005359589995350689, - 0.000519512002938427, - 0.0005288009997457266, - 0.0005195050034672022, - 0.0005256699951132759, - 0.0005205730049056001, - 0.002160669995646458, - 0.001965328003279865, - 0.0005421160021796823, - 0.0005302299978211522, - 0.0005150509969098493, - 0.0005286950035952032, - 0.0005161020017112605, - 0.0005241930048214272, - 0.0005145040049683303, - 0.0024454379963572137, - 0.0005321610005921684, - 0.0013459019974106923, - 0.0007870170011301525, - 0.0007725800023763441, - 0.0005569310014834628, - 0.0005201360036153346, - 0.0006209660059539601, - 0.0011220890010008588, - 0.0007597830044687726, - 0.000540070999704767, - 0.0005662549956468865, - 0.0018649199992069043, - 0.0005397950008045882, - 0.0005292779969749972, - 0.000514482002472505, - 0.0012727540015475824, - 0.0006016389961587265, - 0.0005313540023053065, - 0.0005153789970790967, - 0.0007122550014173612, - 0.000901696999790147, - 0.0008243659976869822, - 0.0005817140045110136, - 0.0005363160016713664, - 0.0005149520002305508, - 0.0005371320003177971, - 0.0005237039949861355, - 0.0012805660007870756, - 0.00308199899882311, - 0.0011627689964370802, - 0.0011035990028176457, - 0.0011801840009866282, - 0.0025037439991137944, - 0.0026382560026831925, - 0.0005869849992450327, - 0.0005326440004864708, - 0.0005281970006763004, - 0.0005248759989626706, - 0.0009825550005189143, - 0.004952340997988358, - 0.0005555809984798543, - 0.0005230499955359846, - 0.0005211379975662567, - 0.0006192779983393848, - 0.0005356219990062527, - 0.004568520002067089, - 0.0005422289978014305, - 0.0005203820037422702, - 0.0005266520020086318, - 0.0005319529955158941, - 0.0005205310008022934, - 0.0005287570020300336, - 0.0005125410025357269, - 0.0030275259996415116, - 0.005999230001179967, - 0.006998839999141637, - 0.005998171000101138, - 0.006991921996814199, - 0.008000019995961338, - 0.005991736004943959, - 0.007996804000867996, - 0.00799907800683286, - 0.005997635998937767, - 0.007998537999810651, - 0.006992436996370088, - 0.0069938660017214715, - 0.008003507995454129, - 0.006998034994467162, - 0.0079962319941842, - 0.006997337004577275, - 0.007993222003278788, - 0.005000899996957742, - 0.0069946369985700585, - 0.007543596999312285, - 0.009444023002288304, - 0.008022879999771249, - 0.00597681499493774, - 0.006987303000641987, - 0.005995384999550879, - 0.008001119997061323, - 0.007993680002982728, - 0.008001571994100232, - 0.006997517004492693, - 0.009995334999985062, - 0.007995929001481272, - 0.005993648002913687, - 0.007238513993797824, - 0.003751688993361313, - 0.005996831998345442, - 0.005997209998895414, - 0.006998507000389509, - 0.016001390999008436, - 0.005997054002364166, - 0.007992537000973243, - 0.008000048997928388, - 0.006007675001455937, - 0.006103829997300636, - 0.008932659002311993, - 0.008054345002165064, - 0.004845805997319985, - 0.006985775995417498, - 0.008012418998987414, - 0.005967571996734478, - 0.007003199003520422, - 0.007985030002600979, - 0.00699576400074875, - 0.008037208004679997, - 0.007585111998196226, - 0.006323320994852111, - 0.007980141999723855, - 0.005994867999106646, - 0.005989452998619527, - 0.006988893997913692, - 0.0070235009989119135, - 0.006969524998567067, - 0.006996787000389304, - 0.006975360003707465, - 0.00698752400057856, - 0.006997413998760749, - 0.0074142609955742955, - 0.005576016003033146, - 0.006974996002099942, - 0.00600399800168816, - 0.0069897470020805486, - 0.006991515001573134, - 0.008000258996617049, - 0.006971882001380436, - 0.008003827002539765, - 0.007983365998370573, - 0.005638588001602329, - 0.003763446002267301, - 0.0035612620049505495, - 0.008813262000330724, - 0.00470346200017957, - 0.007499544997699559, - 0.00629791399842361, - 0.003628696002124343, - 0.006018798005243298, - 0.005669760001183022, - 0.0042990820002160035, - 0.005991208003251813, - 0.00599773800058756, - 0.007001161000516731, - 0.007992459999513812, - 0.006989556997723412, - 0.007996127002115827, - 0.006993675000558142, - 0.0069984090005164035, - 0.0069994310033507645, - 0.006995541996730026, - 0.007004037004662678, - 0.00798375599697465, - 0.00800085200171452, - 0.005993594997562468, - 0.008002349000889808, - 0.007996139996976126, - 0.005001612000341993, - 0.004006869996374007, - 0.003979383000114467, - 0.0080072310011019, - 0.006014840997522697, - 0.007966730001498945, - 0.006997908996709157, - 0.00799202499911189, - 0.008008156000869349, - 0.007979248999617994, - 0.0070704350000596605, - 0.008498552000673953, - 0.007862418999138754, - 0.004516435001278296, - 0.005007182000554167, - 0.0021029639974585734, - 0.003993264996097423, - 0.005874155998753849, - 0.008006543001101818, - 0.006980313999520149, - 0.007002368001849391, - 0.007974412001203746, - 0.007001960002526175, - 0.007974590000230819, - 0.006997664997470565, - 0.006999953002377879, - 0.006983687999309041, - 0.007992939004907385, - 0.00798309800302377, - 0.0059990360023221, - 0.006990342000790406, - 0.006991656999161933, - 0.006838437002443243, - 0.006142352998722345, - 0.0058164749934803694, - 0.007189365001977421, - 0.004063937005412299, - 0.008676698998897336, - 0.0062235839941422455, - 0.00926838000304997, - 0.006716698997479398, - 0.007991365004272666, - 0.00798825600213604, - 0.008001387002877891, - 0.010065725997264963, - 0.006940082996152341, - 0.006928339003934525, - 0.006982150000112597, - 0.007000021003477741, - 0.0069788100008736365, - 0.008710979003808461, - 0.004834398998355027, - 0.006935326004168019, - 0.007669817001442425, - 0.006792485000914894, - 0.003109230994596146, - 0.0038825960000394844, - 0.007995507003215607, - 0.0068332630035001785, - 0.007146903000830207, - 0.005992420003167354, - 0.00802031400235137, - 0.005963767995126545, - 0.007005652005318552, - 0.006982154001889285, - 0.006998655000643339, - 0.005076544999610633, - 0.00790771200263407, - 0.007007410000369418, - 0.00697839700296754, - 0.007998747001693118, - 0.006996491996687837, - 0.010519561998080462, - 0.006456151997554116, - 0.005988826000248082, - 0.007009521003055852, - 0.0069800679993932135, - 0.007001605001278222, - 0.007009549997746944, - 0.005966764998447616, - 0.005040048999944702, - 0.003947356999560725, - 0.007993849001650233, - 0.006999949000601191, - 0.00698650699632708, - 0.007013092996203341, - 0.007981440998264588, - 0.00506303200381808, - 0.007926219994260464, - 0.004984769999282435, - 0.007005814004514832, - 0.006981194994295947, - 0.008008106000488624, - 0.0050638589964364655, - 0.0045076480018906295, - 0.005776196005172096, - 0.00635845800570678, - 0.0063308069948107, - 0.006930650997674093, - 0.006986891996348277, - 0.006990251000388525, - 0.0059828880039276555, - 0.009994077001465484, - 0.008004103001439944, - 0.006981110003835056, - 0.007023090001894161, - 0.006961810002394486, - 0.006999874000030104, - 0.009381782998389099, - 0.013631627996801399, - 0.006953860000066925, - 0.008010110999748576, - 0.006964041996980086, - 0.0070003500004531816, - 0.005985859999782406, - 0.007000923003943171, - 0.007993115999852307, - 0.006970941001782194, - 0.007012545000179671, - 0.00697467899590265, - 0.007005378996836953, - 0.007977681001648307, - 0.006010995995893609, - 0.007989413999894168, - 0.006983691004279535, - 0.008022889000130817, - 0.011570770002435893, - 0.0033909829944605008, - 0.009974494998459704, - 0.008017422005650587, - 0.005996189000143204, - 0.008002898997801822, - 0.007960344999446534, - 0.008020567998755723, - 0.007971530998474918, - 0.008008885000890587, - 0.007979475005413406, - 0.0079940430005081, - 0.0079800730018178, - 0.008038589003263041, - 0.009948269995220471, - 0.0059943009982816875, - 0.00921421499515418, - 0.004747286999190692, - 0.01002882000466343, - 0.007850508998672012, - 0.00619264499982819, - 0.003890799001965206, - 0.005978920999041293, - 0.008036346000153571, - 0.0069487279979512095, - 0.008025212002394255, - 0.006963223000639118, - 0.007007995998719707, - 0.00797129800048424, - 0.00699872199766105, - 0.00475746799929766, - 0.004317598999477923, - 0.007908425999630708, - 0.006467265004175715, - 0.006528167999931611, - 0.009941983997123316, - 0.005030653002904728, - 0.006013794001773931, - 0.006232940002519172, - 0.007109372003469616, - 0.0016172589967027307, - 0.020068984995305073, - 0.01495579699985683, - 0.010956115998851601, - 0.013024546999076847, - 0.007959997994475998, - 0.003984212002251297, - 0.00847266599885188, - 0.010555030996329151, - 0.010955822996038478, - 0.014991067997470964, - 0.015023434003524017, - 0.014083371999731753, - 0.01087433999782661, - 0.007944409997435287, - 0.007999622997886036, - 0.005967748998955358, - 0.007004086997767445, - 0.015993039996828884, - 0.007979075002367608, - 0.010254036998958327, - 0.005706294999981765, - 0.007982702998560853, - 0.00633415599440923, - 0.003747164002561476, - 0.002901416999520734, - 0.007011663001321722, - 0.006975504002184607, - 0.006000936002237722, - 0.007993595994776115, - 0.0080047969968291, - 0.011133744999824557, - 0.0038561250039492734, - 0.007006316998740658, - 0.006982197999604978, - 0.0069960059990989976, - 0.007000255995080806, - 0.0069908209989080206, - 0.007001013997069094, - 0.004204311000648886, - 0.0077869290034868754, - 0.005997127002046909, - 0.005998171996907331, - 0.006997136995778419, - 0.006993508999585174, - 0.0070014150041970424, - 0.004019807995064184, - 0.005243682004220318, - 0.0038924019972910173, - 0.007827761000953615, - 0.0029889060024288483, - 0.0030801399989286438, - 0.002912312003900297, - 0.004322585002228152, - 0.007670503997360356, - 0.008530535000318196, - 0.007462116001988761, - 0.006962541003304068, - 0.007345788995735347, - 0.007683853997150436, - 0.006012974001350813, - 0.008986825996544212, - 0.007989270998223219, - 0.006313840996881481, - 0.010677440004656091, - 0.00699343100131955, - 0.006009092001477256, - 0.00599286599754123, - 0.006987411994487047, - 0.006651761999819428, - 0.006798276997869834, - 0.006539449997944757, - 0.007995857995410915, - 0.006987173001107294, - 0.008008226002857555, - 0.007992537000973243, - 0.006988238004851155, - 0.009083791002922226, - 0.0032121170006575994, - 0.01031359799526399, - 0.008377373000257649, - 0.004079580998222809, - 0.0029058259970042855, - 0.0036051880015293136, - 0.00738855099916691, - 0.005994762999762315, - 0.006119735997344833, - 0.007359267998253927, - 0.006631023999943864, - 0.0031263950004358776, - 0.003634535001765471, - 0.008634316000097897, - 0.0006319390013231896, - 0.0005526129971258342, - 0.013224150003225077, - 0.006399597004929092, - 0.0006918229992152192, - 0.0005204679982853122, - 0.00808299399795942, - 0.000765503995353356, - 0.0005363560048863292, - 0.013496302002749871, - 0.0071198939986061305, - 0.0028120030037825927, - 0.0005508730027941056, - 0.0020471139941946603, - 0.010363253000832628, - 0.0007369049999397248, - 0.0005361310031730682, - 0.010021766000136267, - 0.0006948510053916834, - 0.0005135270039318129, - 0.0030572089963243343, - 0.007441387999278959, - 0.00655068299965933, - 0.010029773999121971, - 0.005974184001388494, - 0.008652053998957854, - 0.0012315109997871332, - 0.004267575000994839, - 0.008292330996482633, - 0.0007263699953909963, - 0.0005568319975282066, - 0.010053079000499565, - 0.008857414999511093, - 0.005079226000816561, - 0.0050685050009633414, - 0.0007818619997124188, - 0.0032963979974738322, - 0.0055241830050363205, - 0.0014400069994735532, - 0.004497620997426566, - 0.004270277997420635, - 0.0008020390014280565, - 0.0005778159975307062, - 0.0031904200004646555, - 0.00764632999926107, - 0.0007341220043599606, - 0.0005659599992213771, - 0.0005828619978274219, - 0.006295387000136543, - 0.004800255999725778, - 0.006993160997808445, - 0.0006780949988751672, - 0.001259887001651805, - 0.0010430530019220896, - 0.0012911000012536533, - 0.0005622810058412142, - 0.0034172470041085035, - 0.0006206409962032922, - 0.000539273998583667, - 0.0005278500029817224, - 0.0005322610013536178, - 0.0022555239993380383, - 0.00309545000345679, - 0.0031236220020218752, - 0.0019189580052625388, - 0.001540174002002459, - 0.0022525970052811317, - 0.001349915997707285, - 0.0029210020002210513, - 0.0005582200028584339, - 0.001615563000086695, - 0.0007027370011201128, - 0.0005899989992030896, - 0.002755761001026258, - 0.00775825600430835, - 0.0006130749970907345, - 0.0010027210009866394, - 0.0005515779994311742, - 0.0010576490021776408, - 0.003761883002880495, - 0.0005761850043199956, - 0.0015480610018130392, - 0.002860959000827279, - 0.0005892099943594076, - 0.0006676630000583827, - 0.0026068369988934137, - 0.004124061000766233, - 0.0006287930009420961, - 0.0005326489990693517, - 0.0008319210028275847, - 0.0006633480006712489, - 0.0005543899969779886, - 0.0005170169970369898, - 0.0018736800047918223, - 0.003804977000982035, - 0.0036141729942755774, - 0.0010415729993837886, - 0.0005657189976773225, - 0.0005316040042089298, - 0.0005221889950917102, - 0.0005268789973342791, - 0.0007119539950508624, - 0.004032645003462676, - 0.0006183619989315048, - 0.004379064994282089, - 0.0008348129995283671, - 0.0023583010042784736, - 0.0005503649954334833, - 0.0029464980034390464, - 0.0009717659995658323, - 0.0006319989988696761, - 0.0006697320059174672, - 0.0006822330033173785, - 0.0013186470023356378, - 0.0005329679988790303, - 0.0034664760023588315, - 0.0005849579974892549, - 0.0010998799989465624, - 0.0008302990027004853, - 0.0009784210051293485, - 0.0005533880030270666, - 0.000528094002220314, - 0.0005318580006132834, - 0.0009541289982735179, - 0.004067782996571623, - 0.0012356879960861988, - 0.0017335299999103881, - 0.0006818149995524436, - 0.0006986119988141581, - 0.0006666200060863048, - 0.0008083760039880872, - 0.0006231110019143671, - 0.0006058380022295751, - 0.0005315169983077794, - 0.0007271639988175593, - 0.000685689999954775, - 0.0012037679989589378, - 0.0005584069949691184, - 0.0005973219958832487, - 0.0005709820034098811, - 0.0005616509952233173, - 0.0008254679996753111, - 0.0005718350003007799, - 0.0005207380017964169, - 0.0005141170040587895, - 0.0005038599992985837, - 0.004565392002405133, - 0.0007244579974212684, - 0.0010427160013932735, - 0.0005707579985028133, - 0.0049377299947082065, - 0.0035482520033838227, - 0.0006457170020439662, - 0.0005368170022848062, - 0.0008682289990247227, - 0.0005657349975081161, - 0.0006086749999667518, - 0.0005065539953648113, - 0.0070189679972827435, - 0.0007099889990058728, - 0.000547543000720907, - 0.0005153360034455545, - 0.0005132470032549463, - 0.0005103830044390634, - 0.004897726001217961, - 0.0006717129945172928, - 0.0005983500013826415, - 0.0005355780012905598, - 0.0005123509981785901, - 0.0005123700029798783, - 0.0005151080040377565, - 0.004880778993538115, - 0.0007126029959181324, - 0.0005593059977400117, - 0.0005513709984370507, - 0.0005175679980311543, - 0.0005117440014146268, - 0.000513809995027259, - 0.0005053229979239404, - 0.003949432997615077, - 0.0008446270003332756, - 0.000548490002984181, - 0.0005179859945201315, - 0.000516728003276512, - 0.0005072330022812821, - 0.0006839779962319881, - 0.0045930160049465485, - 0.0007261680002557114, - 0.0005145879986230284, - 0.0023417330012307502, - 0.0005807790003018454, - 0.0006109050009399652, - 0.0005399199944804423, - 0.0005128959965077229, - 0.0032796310042613186, - 0.0006383410000125878, - 0.0005450269964057952, - 0.0005093910003779456, - 0.000517225002113264, - 0.0005103720031911507, - 0.0037088409953867085, - 0.0006087229994591326, - 0.0005376560002332553, - 0.000525085000845138, - 0.0005066689991508611, - 0.0005357900008675642, - 0.0005148879936314188, - 0.005255652002233546, - 0.0016208559973165393, - 0.0005383120005717501, - 0.0005191349991946481, - 0.0005128479970153421, - 0.000511136997374706, - 0.000518192995514255, - 0.0005097609973745421, - 0.004434854999999516, - 0.0005852600006619468, - 0.0005214510019868612, - 0.00051921699923696, - 0.0005134590028319508, - 0.0005174649995751679, - 0.0005121599970152602, - 0.0005045389989390969, - 0.007201606000307947, - 0.0006212950029294007, - 0.0005197439968469553, - 0.0005075340013718233, - 0.003562635996786412, - 0.0006148839966044761, - 0.0005986260002828203, - 0.004181133997917641, - 0.0006000290013616905, - 0.0006968600064283237, - 0.000557343999389559, - 0.000516878004418686, - 0.0005033280031057075, - 0.004152428999077529, - 0.0022827999928267673, - 0.0020054480046383105, - 0.0006314979982562363, - 0.00208354299684288, - 0.002383307000854984, - 0.0005381829978432506, - 0.0005550360001507215, - 0.0005204059998504817, - 0.002971568006614689, - 0.0006868860000395216, - 0.0005564629973378032, - 0.0005208689981373027, - 0.0005237790028331801, - 0.0005126690011820756, - 0.000798786997620482, - 0.0036861479966319166, - 0.0006205890022101812, - 0.0005196249985601753, - 0.0005058050010120496, - 0.0005105220043333247, - 0.0004889979973086156, - 0.0016162980027729645, - 0.0005010910026612692, - 0.0004977540011168458, - 0.0005024230049457401, - 0.000495462998514995, - 0.0005231410032138228, - 0.0005708089956897311, - 0.003226884000468999, - 0.0006054639961803332, - 0.0005672859988408163, - 0.0004929759961669333, - 0.0009966860015993007, - 0.0005017900039092638, - 0.0005036939983256161, - 0.000500381996971555, - 0.0004994830014766194, - 0.0006237159977899864, - 0.0005070909974165261, - 0.0005012309993617237, - 0.001675055995292496, - 0.0005745050002587959, - 0.0005577720003202558, - 0.002243944996735081, - 0.0006148279935587198, - 0.0005269859975669533, - 0.000702931996784173, - 0.0005306510065565817, - 0.0004925339962937869, - 0.0004987509964848869, - 0.0004933610034640878, - 0.002072317998681683, - 0.0005508350004674867, - 0.0005106550015625544, - 0.0004939290010952391, - 0.0005065229997853749, - 0.0005008779990021139, - 0.0044686620021821, - 0.0006342510023387149, - 0.0005231230024946854, - 0.0004917620026390068, - 0.0005008779990021139, - 0.0004941939987475052, - 0.0005027130027883686, - 0.0004933500022161752, - 0.0006386419991031289, - 0.0008447450018138625, - 0.0005226209977990948, - 0.0018002729993895628, - 0.0005357420013751835, - 0.0005563840022659861, - 0.0005053080021752976, - 0.000490355996589642, - 0.0044113849944551475, - 0.0006304880007519387, - 0.0005346959951566532, - 0.0005039790048613213, - 0.0005290739936754107, - 0.0004979160003131256, - 0.0005496710000443272, - 0.0004966839987901039, - 0.0005033039997215383, - 0.0004988319997210056, - 0.0004985300038242713, - 0.0004967590066371486, - 0.004242101997078862, - 0.0006386929962900467, - 0.0005166889968677424, - 0.0004955129988957196, - 0.0007376599969575182, - 0.0005496449957718141, - 0.0007589430024381727, - 0.0005330880012479611, - 0.0004954609976266511, - 0.0005751310018240474, - 0.0004960459991707467, - 0.0005054049979662523, - 0.0004984290062566288, - 0.0011796789985965006, - 0.000532445999851916, - 0.001146750000771135, - 0.0005021580000175163, - 0.0005015300048398785, - 0.0004991020032321103, - 0.00048824900295585394, - 0.0005156550032552332, - 0.0005007740037399344, - 0.0019376820055185817, - 0.0005709569959435612, - 0.0005517879981198348, - 0.0005131389989401214, - 0.0004968659995938651, - 0.0032197749969782308, - 0.0009554569987813011, - 0.0005980189962428994, - 0.0005274330032989383, - 0.0005088620018796064, - 0.003423825000936631, - 0.0005907809972995892, - 0.0005106620010337792, - 0.0004939439968438819, - 0.0005039480020059273, - 0.0004990460001863539, - 0.0006733929985784926, - 0.0005545579988393001, - 0.0005058000024291687, - 0.0004909060007776134, - 0.0005022960031055845, - 0.0004932250012643635, - 0.003742332999536302, - 0.0008724369981791824, - 0.0005158549975021742, - 0.0004964110048604198, - 0.0011276680015726015, - 0.0012013439991278574, - 0.0005522510036826134, - 0.0004922279986203648, - 0.0005579299977398477, - 0.000498125002195593, - 0.000529708995600231, - 0.0005009820015402511, - 0.0004919749990222044, - 0.000499302004755009, - 0.000495116000820417, - 0.0005158919957466424, - 0.0014470019959844649, - 0.0005537480028579012, - 0.0004999150041840039, - 0.0005001449972041883, - 0.0004948569985572249, - 0.0010359649968449958, - 0.0019460120020085014, - 0.0008464310012641363, - 0.0005499420003616251, - 0.0006354040015139617, - 0.0005647579964715987, - 0.0031342860020231456, - 0.0006944460037630051, - 0.0006789340040995739, - 0.0010383659973740578, - 0.000777467001171317, - 0.0004971619928255677, - 0.0004919109996990301, - 0.0004982050013495609, - 0.0005534369993256405, - 0.0005538840050576255, - 0.0005043330020271242, - 0.0020176260004518554, - 0.0005516829987755045, - 0.0005849029985256493, - 0.0005200899977353401, - 0.0005088140023872256, - 0.0005071089981356636, - 0.0005986699979985133, - 0.002448512001137715, - 0.0014399930005311035, - 0.000624947999313008, - 0.0005150970027898438, - 0.0005096220047562383, - 0.0016716280006221496, - 0.000521302004926838, - 0.0005076399975223467, - 0.0004972530005034059, - 0.0013601899991044775, - 0.0007734169994364493, - 0.0005036760048824362, - 0.0004922709995298646, - 0.0004992429967387579, - 0.0004906050016870722, - 0.0004964679974364117, - 0.0005169440046302043, - 0.000499652000144124, - 0.0005014459966332652, - 0.0006344800058286637, - 0.0005033859997638501, - 0.000497902998176869, - 0.0004953580064466223, - 0.000614461001532618, - 0.0005921539996052161, - 0.0005069000035291538, - 0.0005018150040996261, - 0.0004956249977112748, - 0.004628852002497297, - 0.0006249159996514209, - 0.0017076070071198046, - 0.0005790710056317039, - 0.0005207729991525412, - 0.0004989300068700686, - 0.0004903129956801422, - 0.0005012570036342368, - 0.0004937109988532029, - 0.0014041780013940297, - 0.0005353379965526983, - 0.0014553299988619983, - 0.0004982259997632354, - 0.0005704770010197535, - 0.0005159610009286553, - 0.0005472230041050352, - 0.0005128230041009374, - 0.0005161029985174537, - 0.0005086800010758452, - 0.0004944910033373162, - 0.0005026789949624799, - 0.000493977997393813, - 0.0004907210022793151, - 0.0005045989964855835, - 0.0005013479967601597, - 0.0005194190016482025, - 0.0005258469973341562, - 0.0011191079975105822, - 0.0005785020039184019, - 0.000513687999045942, - 0.000726341997506097, - 0.0005119889974594116, - 0.0004919459970551543, - 0.0004905210007564165, - 0.0005017179937567562, - 0.0004978310025762767, - 0.0006332470002234913, - 0.0005479610044858418, - 0.0007682560026296414, - 0.0005265709987725131, - 0.0005177809944143519, - 0.0004901000065729022, - 0.000502314003824722, - 0.0004979559962521307, - 0.0005168460047570989, - 0.0005523340005311184, - 0.0005662130060954951, - 0.0005175600017537363, - 0.000496477005071938, - 0.0005139810018590651, - 0.0005061400006525218, - 0.0005051960033597425, - 0.0005031170003348961, - 0.0005179810032132082, - 0.0005211859970586374, - 0.0005161000008229166, - 0.0008437910000793636, - 0.0004965789994457737, - 0.0016457040037494153, - 0.0008901360051822849, - 0.0005027340012020431, - 0.0004953729949193075, - 0.0004952150047756732, - 0.0005276509982650168, - 0.0005100670023239218, - 0.0007287209955393337, - 0.0008740760022192262, - 0.0004963500032317825, - 0.0006995750009082258, - 0.0010580590023892, - 0.0004942279992974363, - 0.0005566330000874586, - 0.0004987630018149503, - 0.0008639169973321259, - 0.0005476829974213615, - 0.0006178089970489964, - 0.0005158679996384308, - 0.00050605800061021, - 0.0004992480025975965, - 0.0004984000042895786, - 0.0004969990040990524, - 0.000540821005415637, - 0.000525770999956876, - 0.0005130839999765158, - 0.001187476998893544, - 0.0005320700001902878, - 0.0005028640007367358, - 0.0004913899974781089, - 0.0005033420020481572, - 0.0016586299971095286, - 0.0005008240041206591, - 0.0005042859993409365, - 0.000490430997160729, - 0.0005450809985632077, - 0.0009657300033723004, - 0.0005334079978638329, - 0.0005547429973375984, - 0.000499802001286298, - 0.0013712709987885319, - 0.010081985994474962, - 0.007927642996946815, - 0.0013628539963974617, - 0.005661258001055103, - 0.0032291309980791993, - 0.0037014319968875498, - 0.005668764999427367, - 0.009357442999316845, - 0.006957977995625697, - 0.006352076001348905, - 0.002593732002424076, - 0.00655027199536562, - 0.005377114001021255, - 0.00834187600412406, - 0.006820123002398759, - 0.007870754998293705, - 0.007982650997291785, - 0.007172319004894234, - 0.005679552996298298, - 0.006119910998677369, - 0.005981959999189712, - 0.0069857350026723, - 0.005999373002850916, - 0.005982301998301409, - 0.011014181000064127, - 0.00396653100324329, - 0.0070135030036908574, - 0.007967146993905772, - 0.006982630002312362, - 0.005906190002860967, - 0.004074220996699296, - 0.008072970995272044, - 0.005968752004264388, - 0.005932615000347141, - 0.004986324995115865, - 0.006994994000706356, - 0.009000170000945218, - 0.006940400002349634, - 0.003018930998223368, - 0.005998194006679114, - 0.004000229004304856, - 0.007994589999725576, - 0.007990026999323163, - 0.007983555995451752, - 0.007995915999345016, - 0.005986448006296996, - 0.0029764439968857914, - 0.007005829000263475, - 0.007992497994564474, - 0.005982940005196724, - 0.007994669998879544, - 0.005979402994853444, - 0.007017926996923052, - 0.007977173001563642, - 0.006985626998357475, - 0.007998059001693036, - 0.00801621100254124, - 0.007962420997500885, - 0.006022688998200465, - 0.0006855609972262755, - 0.0005729389959014952, - 0.0006028049974702299, - 0.0019549299977370538, - 0.0010334489998058416, - 0.0020144149966654368, - 0.0006032330056768842, - 0.0005519999976968393, - 0.000542135996511206, - 0.0005444959970191121, - 0.0035940939997090027, - 0.0006457350027631037, - 0.0013301790022524074, - 0.0005625630001304671, - 0.0005439480009954423, - 0.0005452069963212125, - 0.0005357019981602207, - 0.0005426589996204711, - 0.0005823899991810322, - 0.004436088995134924, - 0.0006396559983841144, - 0.0005532270006369799, - 0.000551744997210335, - 0.0005430259989225306, - 0.0005465259964694269, - 0.0006441410005209036, - 0.005220683000516146, - 0.0007224729997687973, - 0.0006001580040901899, - 0.0005946269957348704, - 0.0005473009950947016, - 0.0005559600031119771, - 0.0005466740039992146, - 0.004547736003587488, - 0.0006345060028252192, - 0.0005829639994772151, - 0.0005602359960903414, - 0.0005408829965745099, - 0.0005499949984368868, - 0.00053252399811754, - 0.005510820003109984, - 0.0006714069968438707, - 0.000609780996455811, - 0.0007430640034726821, - 0.0005781209983979352, - 0.0005520199993043207, - 0.005832536000525579, - 0.0006586399977095425, - 0.0005472470002132468, - 0.0005288120009936392, - 0.0005172780001885258, - 0.0026106160003109835, - 0.0006140779951238073, - 0.0013874910000595264, - 0.0005504709988599643, - 0.0005283100035740063, - 0.0005179400031920522, - 0.0005242059996817261, - 0.0005271049958537333, - 0.0005248660017969087, - 0.000517576998390723, - 0.0031594619940733537, - 0.000626329994702246, - 0.0005297019961290061, - 0.0005236979995970614, - 0.0005263150014798157, - 0.0005275990042719059, - 0.0005303349971654825, - 0.0005700709953089245, - 0.005895873997360468, - 0.0006953760021133348, - 0.0005819910002173856, - 0.0005236150027485564, - 0.0005275570001685992, - 0.0005199049992370419, - 0.005033016997913364, - 0.00067352999758441, - 0.0005585050021181814, - 0.0005321520002325997, - 0.0005344180026440881, - 0.0005253099952824414, - 0.0034527489988249727, - 0.007996607993845828, - 0.007996651002031285, - 0.005996921994665172, - 0.005762587999925017, - 0.006230942999536637, - 0.005995340005028993, - 0.005995615996653214, - 0.007017299001745414, - 0.008004402996448334, - 0.005977321998216212, - 0.008006315998500213, - 0.007990689999132883, - 0.007995415006007534, - 0.005988144999719225, - 0.00799887599714566, - 0.006989321998844389, - 0.010546183002588805, - 0.010429057001601905, - 0.010127574001671746, - 0.008160028999554925, - 0.004648727001040243, - 0.007004259998211637, - 0.00797134100139374, - 0.00801604799926281, - 0.007996742999239359, - 0.0069855550027568825, - 0.006970105001528282, - 0.007996366999577731, - 0.006998705997830257, - 0.007977025998116005, - 0.005812533003336284, - 0.006172341003548354, - 0.006003205002343748, - 0.012989143004233483, - 0.006986221000261139, - 0.006988451001234353, - 0.005969132005702704, - 0.006989786001213361, - 0.0077943729993421584, - 0.008185516999219544, - 0.00585776699881535, - 0.004081742998096161, - 0.007994816005520988, - 0.0070736069974373095, - 0.0078979590034578, - 0.004191761996480636, - 0.012244944999110885, - 0.010156027004995849, - 0.00636435500200605, - 0.005006986997614149, - 0.007979212998179719, - 0.0060148129996377975, - 0.010021133995905984, - 0.0059533800013014115, - 0.007989964993612375, - 0.0010016859960160218, - 0.005989823002892081, - 0.006004605005728081, - 0.007987361997948028, - 0.008044160000281408, - 0.0029535780049627647, - 0.004992266003682744, - 0.006994831994234119, - 0.005995769999572076, - 0.005992637001327239, - 0.0069950870019965805, - 0.004120655998121947, - 0.007873500995629001, - 0.005993753002258018, - 0.007999121000466403, - 0.007997082000656519, - 0.0029928329968242906, - 0.004995572002371773, - 0.00799886099412106, - 0.006002817994158249, - 0.007993017003173009, - 0.0069953150014043786, - 0.00799505000031786, - 0.00799699700291967, - 0.007997246997547336, - 0.007994448998942971, - 0.008040721993893385, - 0.0069546650047414005, - 0.005995308005367406, - 0.0069992919961805455, - 0.007994390005478635, - 0.006994467003096361, - 0.007997758002602495, - 0.005994340004690457, - 0.007999044995813165, - 0.006995827003265731, - 0.005995462997816503, - 0.006996894000621978, - 0.007998429995495826, - 0.005999464003252797, - 0.005992489001073409, - 0.00800378600251861, - 0.005990061006741598, - 0.007995391002623364, - 0.0018314729968551546, - 0.0005138319975230843, - 0.0005005269995308481, - 0.0004835989966522902, - 0.0004773960026795976, - 0.00047840000479482114, - 0.00047410900151589885, - 0.000480472001072485, - 0.0004793479965883307, - 0.0012727950015687384, - 0.0026000090001616627, - 0.00057496899535181, - 0.0005666219949489459, - 0.0004924730019411072, - 0.0005307659957907163, - 0.0009394280059495941, - 0.0019289609990664758, - 0.0005951610000920482, - 0.0015346799991675653, - 0.0004883939982391894, - 0.0005697250016964972, - 0.0005142119989614002, - 0.0005053600034443662, - 0.0038869650015840307, - 0.0006703499966533855, - 0.0015207010001176968, - 0.0008763239966356196, - 0.0005032709959778003, - 0.0004911959986202419, - 0.0004944910033373162, - 0.00047605799773009494, - 0.0004866179951932281, - 0.00047928199637681246, - 0.0030969290019129403, - 0.0005918680035392754, - 0.0005297860043356195, - 0.0004923029991914518, - 0.00048651900578988716, - 0.00048044299910543486, - 0.0004890389973297715, - 0.00048726400564191863, - 0.00048057000094559044, - 0.003068243997404352, - 0.0006350270050461404, - 0.0006524410055135377, - 0.0026321999976062216, - 0.0006189060004544444, - 0.0005502839994733222, - 0.00047715999971842393, - 0.0007548600042355247, - 0.00244519799889531, - 0.0005725150040234439, - 0.0005187230053707026, - 0.00047987099969759583, - 0.0004791040046256967, - 0.0004867060051765293, - 0.0004848159951507114, - 0.004689861001679674, - 0.0006202030053827912, - 0.0005725360024371184, - 0.0004955120020895265, - 0.00048148600035347044, - 0.0004840789988520555, - 0.0004753519970108755, - 0.0034294010038138367, - 0.001585136000358034, - 0.0005427790019894019, - 0.0005130490026203915, - 0.0004971950038452633, - 0.0035531030007405207, - 0.0006017439955030568, - 0.0010947820046567358, - 0.0009624859958421439, - 0.0004977659991709515, - 0.0004847409945796244, - 0.00048435700591653585, - 0.0004969610017724335, - 0.0005020539974793792, - 0.0004887149989372119, - 0.0004842349953833036, - 0.0009547150038997643, - 0.004106669999600854, - 0.00704060000134632, - 0.00695593799900962, - 0.005995116996928118, - 0.005998895001539495, - 0.006579913999303244, - 0.004407380001794081, - 0.006068251001124736, - 0.007917313996586017, - 0.008396101999096572, - 0.007991109996510204, - 0.0060898090014234185, - 0.01453382900217548, - 0.007988813005795237, - 0.007989395002368838, - 0.007998057000804693, - 0.00636661200405797, - 0.0074818100038100965, - 0.0070855320009286515, - 0.006997848002356477, - 0.0052690069933305494, - 0.00871587800065754, - 0.007999089997611009, - 0.006982736995269079, - 0.004987913001968991, - 0.007001644997217227, - 0.006284319999394938, - 0.006682127001113258, - 0.007004807994235307, - 0.009983050003938843, - 0.006224540004041046, - 0.0037272170011419803, - 0.00630442699912237, - 0.005809383001178503, - 0.002834872000676114, - 0.007028154999716207, - 0.01715502899605781, - 0.005816879005578812, - 0.009589287998096552, - 0.009155782005109359, - 0.0031799280041013844, - 0.0052002159936819226, - 0.005526769004063681, - 0.008885057999577839, - 0.002915213000960648, - 0.003955786000005901, - 0.003890660998877138, - 0.011014404997695237, - 0.006974937001359649, - 0.005991011996229645, - 0.0054878099981579, - 0.003541696998581756, - 0.006862946996989194, - 0.002574641002865974, - 0.005465225003717933, - 0.007945586999994703, - 0.007000270001299214, - 0.0053780829985043965, - 0.003604845995141659, - 0.006993672999669798, - 0.007996232001460157, - 0.007996952001121826, - 0.00800117600010708, - 0.007994184998096898, - 0.008994194002298173, - 0.004996987001504749, - 0.009004853003716562, - 0.006989694004005287, - 0.008002104004845023, - 0.0039913700020406395, - 0.005996337000397034, - 0.005995455998345278, - 0.008012801001314074, - 0.006277558000874706, - 0.003710229000716936, - 0.00500086099782493, - 0.007998943998245522, - 0.007157651998568326, - 0.006817950998083688, - 0.005966009004623629, - 0.004031914002553094, - 0.005492534997756593, - 0.009075683999981266, - 0.005401962996984366, - 0.010013305996835697, - 0.006994050003413577, - 0.0019434909991105087, - 0.004998345997591969, - 0.009012515001813881, - 0.0058396700042067096, - 0.005183774999750312, - 0.0029638309933943674, - 0.0033433190037612803, - 0.0010972529998980463, - 0.002505853997718077, - 0.010284798001521267, - 0.010737712000263855, - 0.009992626997700427, - 0.00971870400098851, - 0.011130496000987478, - 0.007102784002199769, - 0.014085866998357233, - 0.005298078998748679, - 0.0022641289979219437, - 0.0032537579972995445, - 0.003702248002809938, - 0.006272013000852894, - 0.007034112000837922, - 0.006999315999564715, - 0.004916607998893596, - 0.004054587996506598, - 0.004992084999685176, - 0.00404078999417834, - 0.003947420002077706, - 0.0028779540007235482, - 0.0013368060026550665, - 0.003771422001591418, - 0.0027704699969035573, - 0.004219585003738757, - 0.0033101310036727227, - 0.0009340629985672422, - 0.002086962995235808, - 0.006861129004391842, - 0.0016219099998124875, - 0.00580018699838547, - 0.008365756999410223, - 0.0022286349994828925, - 0.00976655999693321, - 0.012005224998574704, - 0.012758668002788909, - 0.011216245999094099, - 0.01276543400308583, - 0.005176562997803558, - 0.010999460995662957, - 0.006987286004005, - 0.008230022998759523, - 0.006738842996128369, - 0.006984241997997742, - 0.00644435100548435, - 0.005537445002119057, - 0.008491768996464089, - 0.004487902006076183, - 0.01109063400508603, - 0.010919578999164514, - 0.006112568997195922, - 0.006766243001038674, - 0.005829902002005838, - 0.007128367004042957, - 0.005009533997508697, - 0.00630234200070845, - 0.0046741289988858625, - 0.008129968999128323, - 0.004858182001044042, - 0.0060007539941580035, - 0.008379675004107412, - 0.008626593997178134, - 0.007208920003904495, - 0.007744941998680588, - 0.0020401780056999996, - 0.00494050399720436, - 0.004336007004894782, - 0.0036537650012178347, - 0.009984847005398478, - 0.003003031997650396, - 0.004215897002723068, - 0.0029123650019755587, - 0.005866660998435691, - 0.007004409999353811, - 0.008027420000871643, - 0.009577871998772025, - 0.004355413999292068, - 0.00798952500190353, - 0.01046536799549358, - 0.003522957005770877, - 0.003917444999387953, - 0.012068736999935936, - 0.00896650899812812, - 0.008024224000109825, - 0.007838944999093655, - 0.005094163003377616, - 0.00698383100097999, - 0.005988302000332624, - 0.006992648006416857, - 0.006005212999298237, - 0.011924813996301964, - 0.005049182000220753, - 0.00799643900245428, - 0.00699255800282117, - 0.011002156003087293, - 0.006986892003624234, - 0.007012747002590913, - 0.00597944000037387, - 0.010000884001783561, - 0.005987644006381743, - 0.008002616996236611, - 0.008024841001315508, - 0.005960000999039039, - 0.012000869995972607, - 0.00799370899767382, - 0.006994062001467682, - 0.010000965994549915, - 0.006991989001107868, - 0.0019959970013587736, - 0.00599872899329057, - 0.0020021730015287176, - 0.005988584998704027, - 0.005001939003705047, - 0.006992167000134941, - 0.0039619499948457815, - 0.0040372849980485626, - 0.009991041995817795, - 0.009996120003052056, - 0.008000939997145906, - 0.00799420499970438, - 0.005993254002532922, - 0.008996151002065744, - 0.006996112002525479, - 0.0069983729990781285, - 0.007364974997472018, - 0.007627771003171802, - 0.004997497002477758, - 0.010999828999047168, - 0.006992861002800055, - 0.0070008020047680475, - 0.006069944000046235, - 0.00892199900408741, - 0.007994833002157975, - 0.00610433299880242, - 0.007888781998190098, - 0.00502070999937132, - 0.0016877599991858006, - 0.0022807470013503917, - 0.004657313002098817, - 0.009339504002127796, - 0.007999590001418255, - 0.011001544000464492, - 0.008072231998085044, - 0.005956490997050423, - 0.007933034998131916, - 0.010005280993937049, - 0.007996312997420318, - 0.006973970004764851, - 0.007992749000550248, - 0.0074526509997667745, - 0.006499630006146617, - 0.005151516001205891, - 0.00787606299854815, - 0.004007725998235401, - 0.010428636000142433, - 0.006538374997035135, - 0.008006732998182997, - 0.006979813995712902, - 0.005993313003273215, - 0.007988268000190146, - 0.00799568399816053, - 0.0059964300016872585, - 0.00801206399773946, - 0.006976657998166047, - 0.0049896050040842965, - 0.008641170999908354, - 0.009712799997942057, - 0.01143495400174288, - 0.00919833700027084, - 0.010832031002792064, - 0.006152105997898616, - 0.0070025619934313, - 0.006987190994550474, - 0.00799828399613034, - 0.005993735998345073, - 0.00599569700716529, - 0.004991952999262139, - 0.0070023009975557216, - 0.00799066499894252, - 0.007999413996003568, - 0.006937451005796902, - 0.0035341890034032986, - 0.004521375005424488, - 0.005993476996081881, - 0.00699436499417061, - 0.005996235995553434, - 0.007998631001100875, - 0.003997205996711273, - 0.004905781999696046, - 0.004087528999662027, - 0.007779794999805745, - 0.007215837998955976, - 0.005996396001137327, - 0.007997966000402812, - 0.007995275002031121, - 0.008000625995919108, - 0.0069934519997332245, - 0.007997389002412092, - 0.00899731800018344, - 0.00799592299881624, - 0.006997832999331877, - 0.005997661006404087, - 0.006996466996497475, - 0.007997358996362891, - 0.005004190999898128, - 0.008992916002171114, - 0.010998178993759211, - 0.00799195299623534, - 0.0059974520045216195, - 0.004997044998162892, - 0.005030856002122164, - 0.003741566004464403, - 0.006223771997611038, - 0.00798541700351052, - 0.007997018001333345, - 0.006001341003866401, - 0.009993081002903637, - 0.00799661500059301, - 0.0040215660046669655, - 0.007971260994963814, - 0.006000865003443323, - 0.008002889997442253, - 0.006981922997510992, - 0.007018454001809005, - 0.007209501003671903, - 0.0017464850025135092, - 0.003018930998223368, - 0.0064329469969379716, - 0.007537308993050829, - 0.005996402003802359, - 0.006998418997682165, - 0.0069918420049361885, - 0.007994527004484553, - 0.009009538000100292, - 0.005370662998757325, - 0.007929485000204295, - 0.005900617004954256, - 0.007764419999148231, - 0.007992186001501977, - 0.005414474006101955, - 0.001569842999742832, - 0.0060031859975424595, - 0.008002187001693528, - 0.009412167994014453, - 0.004562098998576403, - 0.005998543005262036, - 0.006990915004280396, - 0.006998331002250779, - 0.007993138999154326, - 0.005705491996195633, - 0.00429517800512258, - 0.007018474003416486, - 0.007181521003076341, - 0.0077720629997202195, - 0.007002313999691978, - 0.0059775580011773854, - 0.006928491995495278, - 0.005060566996689886, - 0.00599687900103163, - 0.0070141709948075, - 0.009591582995199133, - 0.004379929006972816, - 0.007011830995907076, - 0.0069796099996892735, - 0.007848856002965476, - 0.005177806997380685, - 0.006962099003430922, - 0.007990769001480658, - 0.006994139002927113, - 0.005040536001615692, - 0.010946690999844577, - 0.007996674998139497, - 0.006996498996159062, - 0.006986647997109685, - 0.007995403997483663, - 0.008001146001333836, - 0.007998035005584825, - 0.00699046299996553, - 0.007999822999408934, - 0.006996055999479722, - 0.007992904000275303, - 0.0059940290011581965, - 0.010996430006343871, - 0.008002456001122482, - 0.010004759002185892, - 0.007983534000231884, - 0.007991015001607593, - 0.007991553000465501, - 0.00999761600542115, - 0.006991836002271157, - 0.007165472001361195, - 0.0073406590017839335, - 0.006478931005403865, - 0.0040280190005432814, - 0.005946752993622795, - 0.0049357789976056665, - 0.007081062998622656, - 0.0069717880032840185, - 0.007996071995876264, - 0.003988649004895706, - 0.010006186996179167, - 0.007985192998603452, - 0.0079995490013971, - 0.005992911996145267, - 0.00799916800315259, - 0.005932520994974766, - 0.004061883999384008, - 0.007990846002940089, - 0.004995374001737218, - 0.00499885300087044, - 0.00799400100368075, - 0.004995684001187328, - 0.0059973330062348396, - 0.006995910000114236, - 0.0068002219995833, - 0.0061930739975650795, - 0.007999928006029222, - 0.005987730000924785, - 0.008002112997928634, - 0.005993847000354435, - 0.00594705199910095, - 0.006042098997568246, - 0.007996514003025368, - 0.00799954000103753, - 0.007994419996975921, - 0.006997898999543395, - 0.007996331005415414, - 0.009000285004731268, - 0.007987536999280564, - 0.006000305998895783, - 0.005993796999973711, - 0.006988258995988872, - 0.008000140995136462, - 0.0069926889991620556, - 0.007996585001819767, - 0.00596805100212805, - 0.006024416004947852, - 0.006994434996158816, - 0.00800204899860546, - 0.007991136000782717, - 0.007006877000094391, - 0.007983338997291867, - 0.004995469003915787, - 0.007997314001841005, - 0.007997351000085473, - 0.007996185006049927, - 0.007998891996976454, - 0.004994146998797078, - 0.009000716003356501, - 0.007997296001121867, - 0.007991798003786243, - 0.004994357994291931, - 0.004996064999431837, - 0.006998496006417554, - 0.006996348005486652, - 0.007005848005064763, - 0.007992742001079023, - 0.007993457002157811, - 0.007993018996785395, - 0.00799348500004271, - 0.007994683997821994, - 0.0070467860059579834, - 0.007955574998049997, - 0.005995102001179475, - 0.00898672600305872, - 0.004976998003257904, - 0.007990491001692135, - 0.006990309004322626, - 0.0060161940054968, - 0.009986344004573766, - 0.007984252995811403, - 0.007992603997990955, - 0.007991598999069538, - 0.007192427001427859, - 0.0041777160004130565, - 0.002551345001847949, - 0.005021859004045837, - 0.008411338996666018, - 0.006603191002795938, - 0.008097873003862333, - 0.0027088429997093044, - 0.0042375980046927, - 0.006938415004697163, - 0.004314541001804173, - 0.0020167830007267185, - 0.0046545329969376326, - 0.0018365440046181902, - 0.00914630700572161, - 0.006997784999839496, - 0.00598726899625035, - 0.008005449999473058, - 0.007991657003003638, - 0.005320367999956943, - 0.00466614100150764, - 0.0060342180004226975, - 0.007967779005412012, - 0.0019962809965363704, - 0.006396839999069925, - 0.002413619004073553, - 0.004173381996224634, - 0.005997095002385322, - 0.004338079001172446, - 0.005670648999512196, - 0.00798859799397178, - 0.0069919669986120425, - 0.0050759419973474, - 0.009914877999108285, - 0.006005524999636691, - 0.006332716999168042, - 0.007651239000551868, - 0.006996506999712437, - 0.005998999004077632, - 0.007005813000432681, - 0.004323608998674899, - 0.004660295002395287, - 0.005992965001496486, - 0.004402030004712287, - 0.002449307001370471, - 0.0041371059996890835, - 0.0032451079969177954, - 0.006757465002010576, - 0.007990853999217506, - 0.008001274996786378, - 0.007001860001764726, - 0.005984833995171357, - 0.005141576999449171, - 0.0068532999939634465, - 0.003991375997429714, - 0.005998152002575807, - 0.002169856998079922, - 0.0005292900023050606, - 0.000507662000018172, - 0.0004958450008416548, - 0.0004967380009475164, - 0.0005288890024530701, - 0.0033688079929561354, - 0.0006284429982770234, - 0.0004957030032528564, - 0.00047619500401197, - 0.0006343670029309578, - 0.0004924609966110438, - 0.0004959060024702922, - 0.00048599800356896594, - 0.0017200299989781342, - 0.001611024999874644, - 0.0004933469972456805, - 0.000490771999466233, - 0.0004839160028495826, - 0.0004870510019827634, - 0.00047933799942256883, - 0.0030964869947638363, - 0.0006509719969471917, - 0.0005806929984828457, - 0.0004883159999735653, - 0.0005116990068927407, - 0.0004865160008193925 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil[compile_time_domain]", - "fullname": "TestCalculateNabla4[compile_time_domain]", - "params": { - "static_variant": [ - "compile_time_domain", - [ - "horizontal_start", - "horizontal_end", - "vertical_start", - "vertical_end" - ] - ] - }, - "param": "compile_time_domain", - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.00046226000267779455, - "max": 0.010999489997630008, - "mean": 0.00587169544827639, - "stddev": 0.002629615661344385, - "rounds": 201, - "median": 0.006990451001911424, - "iqr": 0.002989786247781012, - "q1": 0.004998075502953725, - "q3": 0.007987861750734737, - "iqr_outliers": 19, - "stddev_outliers": 45, - "outliers": "45;19", - "ld15iqr": 0.0005246520013315603, - "hd15iqr": 0.010999489997630008, - "ops": 170.30856058679706, - "total": 1.1802107851035544, - "data": [ - 0.005879707001440693, - 0.0029924499976914376, - 0.00599567099561682, - 0.0069937080043018796, - 0.007001646998105571, - 0.007995829000719823, - 0.007997226995939855, - 0.00799333699978888, - 0.007996729997103103, - 0.005997448002744932, - 0.007996221000212245, - 0.00799657199968351, - 0.00600060499709798, - 0.007999983004992828, - 0.0060135450039524585, - 0.005975626001600176, - 0.0060025749990018085, - 0.007992749000550248, - 0.00799916699907044, - 0.00799494500097353, - 0.005993551996652968, - 0.0059974629984935746, - 0.005996026004140731, - 0.006998340999416541, - 0.0069970710028428584, - 0.008003531002032105, - 0.007996400003321469, - 0.007997838998562656, - 0.003992122001363896, - 0.004998385004000738, - 0.008001025002158713, - 0.009992678002163302, - 0.0069963470014045015, - 0.006301901004917454, - 0.007694865002122242, - 0.006995060997724067, - 0.006999372002610471, - 0.005995259998599067, - 0.006996748001256492, - 0.007002188998740166, - 0.006993192997470032, - 0.008000257999810856, - 0.006992627000727225, - 0.007003309001447633, - 0.0069922779948683456, - 0.00599809100094717, - 0.006992724993324373, - 0.0069971129996702075, - 0.006998631994065363, - 0.006997204000072088, - 0.007997406995855272, - 0.005997021995426621, - 0.007037445000605658, - 0.010959307001030538, - 0.008992159004264977, - 0.004992802001652308, - 0.007443670998327434, - 0.007552561000920832, - 0.00623557699873345, - 0.004823118993954267, - 0.007934261993796099, - 0.007994428000529297, - 0.008995389005576726, - 0.0029961340042063966, - 0.007999401997949462, - 0.00799698300397722, - 0.005995986000925768, - 0.007996690001164097, - 0.008051806995354127, - 0.0077058849929017015, - 0.006233462001546286, - 0.008000407004146837, - 0.00699450899992371, - 0.006997554002737161, - 0.005997936001222115, - 0.006001771005685441, - 0.0062146780037437566, - 0.005772820004494861, - 0.007004440005403012, - 0.00798716300050728, - 0.008003205999557395, - 0.010993026997311972, - 0.0066997650064877234, - 0.005287652995320968, - 0.005996983003569767, - 0.005996944004436955, - 0.00904589000128908, - 0.005971897000563331, - 0.0049795790037023835, - 0.004363877000287175, - 0.0066294610005570576, - 0.003988098003901541, - 0.004988390006474219, - 0.006995556999754626, - 0.007994685001904145, - 0.006009087999700569, - 0.0029883989991503768, - 0.00399132700113114, - 0.0059987249987898394, - 0.008002952999959234, - 0.008000746995094232, - 0.006987572000070941, - 0.006995160001679324, - 0.008996663003927097, - 0.007996238004125189, - 0.007996811000339221, - 0.006997085998591501, - 0.007003491002251394, - 0.00799124199693324, - 0.007001277001108974, - 0.00799218699830817, - 0.00699805399926845, - 0.010999489997630008, - 0.006990909998421557, - 0.007996145999641158, - 0.006999235003604554, - 0.006010536999383476, - 0.002981050005473662, - 0.003996644998551346, - 0.0039980790024856105, - 0.0059969789945171215, - 0.006996971002081409, - 0.007323489000555128, - 0.00767282499873545, - 0.0029951070027891546, - 0.0070003780056140386, - 0.007001926998782437, - 0.006990451001911424, - 0.0070041940052760765, - 0.007993673003511503, - 0.007989958001417108, - 0.007998467997822445, - 0.006008105003274977, - 0.005993944003421348, - 0.0059900600026594475, - 0.008004317001905292, - 0.006988209999690298, - 0.00799684900266584, - 0.007995733001735061, - 0.009302718004619237, - 0.006692304996249732, - 0.006003520997182932, - 0.0059926940011791885, - 0.007000110002991278, - 0.006991146998188924, - 0.005998713000735734, - 0.006996267999056727, - 0.006996083997364622, - 0.0038977930016699247, - 0.006094568001572043, - 0.006996232004894409, - 0.006016744002408814, - 0.006979163001233246, - 0.005997991000185721, - 0.00599528600287158, - 0.005427557000075467, - 0.007569247994979378, - 0.00799190499674296, - 0.00799738299974706, - 0.004996807001589332, - 0.004997146999812685, - 0.006997058000706602, - 0.00699678499950096, - 0.010998465004377067, - 0.006998607997957151, - 0.007995640000444837, - 0.00371095599984983, - 0.0006368269969243556, - 0.0006570960031240247, - 0.000490534002892673, - 0.0004639079998014495, - 0.00047927199921105057, - 0.00046651899901917204, - 0.004659404999983963, - 0.0006018599960952997, - 0.0005882080004084855, - 0.0005246520013315603, - 0.00047045199607964605, - 0.0004705519968410954, - 0.00047603800339857116, - 0.005322265002178028, - 0.0006261819944484159, - 0.000532285004737787, - 0.00046465500054182485, - 0.0004750200023408979, - 0.00046341899724211544, - 0.0004703439990407787, - 0.0004676610042224638, - 0.005487378999532666, - 0.0006017209962010384, - 0.0005281549965729937, - 0.0004712630034191534, - 0.0004707930056611076, - 0.0004683530059992336, - 0.00046947700320743024, - 0.0004725960025098175, - 0.0053124500045669265, - 0.0005711749981855974, - 0.00056279799900949, - 0.0004704289967776276, - 0.00046226000267779455 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil[compile_time_vertical]", - "fullname": "TestCalculateNabla4[compile_time_vertical]", - "params": { - "static_variant": [ - "compile_time_vertical", - [ - "vertical_start", - "vertical_end" - ] - ] - }, - "param": "compile_time_vertical", - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.0004770460000145249, - "max": 0.05026232600357616, - "mean": 0.004794945990073818, - "stddev": 0.004631223442377899, - "rounds": 295, - "median": 0.004902200002106838, - "iqr": 0.0061894879981991835, - "q1": 0.0008117057514027692, - "q3": 0.007001193749601953, - "iqr_outliers": 7, - "stddev_outliers": 18, - "outliers": "18;7", - "ld15iqr": 0.0004770460000145249, - "hd15iqr": 0.0163589250005316, - "ops": 208.55292261271225, - "total": 1.4145090670717764, - "data": [ - 0.007137049004086293, - 0.00684582599933492, - 0.0069967580056982115, - 0.0060692240003845654, - 0.0069221090016071685, - 0.00530778500251472, - 0.008063152999966405, - 0.007256682998558972, - 0.002352803996473085, - 0.004993515998648945, - 0.007994608000444714, - 0.004072004994668532, - 0.007919269999547396, - 0.00800405699555995, - 0.007988495999597944, - 0.00799554099648958, - 0.016001743999368045, - 0.020018267998239025, - 0.0080606940027792, - 0.005755172001954634, - 0.006465951002610382, - 0.0004966080014128238, - 0.0022576810006285086, - 0.0005393889950937591, - 0.0005659749949700199, - 0.0005341940050129779, - 0.00048635499842930585, - 0.005001200996048283, - 0.0005059659961261787, - 0.0020630740036722273, - 0.0005009599990444258, - 0.00048472400521859527, - 0.0004830469988519326, - 0.0004861839988734573, - 0.0004893000004813075, - 0.0004803460033144802, - 0.0054404069960583, - 0.0005641469979309477, - 0.0004880250053247437, - 0.00048005599819589406, - 0.00048595700354781, - 0.0004770460000145249, - 0.0019983739985036664, - 0.005394217005232349, - 0.0024059479983407073, - 0.0005504310029209591, - 0.0020809890047530644, - 0.0011438220026320778, - 0.0004838090026169084, - 0.0004897879989584908, - 0.0004808119992958382, - 0.0163589250005316, - 0.003898666000168305, - 0.0020181139989290386, - 0.004329633004090283, - 0.0023032130047795363, - 0.003520837999531068, - 0.002509514000848867, - 0.0009956470021279529, - 0.0006967180015635677, - 0.0005274309951346368, - 0.0005430579985841177, - 0.0005190399970160797, - 0.000494989006256219, - 0.004902200002106838, - 0.0010331579978810623, - 0.002316696998605039, - 0.0019601920066634193, - 0.0009617469986551441, - 0.0022438310043071397, - 0.0005540679994737729, - 0.0005129130004206672, - 0.002872598997782916, - 0.0016857480004546233, - 0.0009172999998554587, - 0.0014660669985460117, - 0.0014938009990146384, - 0.0005511709969141521, - 0.006539677000546362, - 0.020539933997497428, - 0.002902250998886302, - 0.009907497005769983, - 0.004075721000845078, - 0.006495897003333084, - 0.010411124996608123, - 0.01038513399544172, - 0.013831720003508963, - 0.05026232600357616, - 0.0116395660006674, - 0.010186127998167649, - 0.010040342996944673, - 0.01725114400323946, - 0.006244434000109322, - 0.0013356659983401187, - 0.001719653999316506, - 0.0014041030008229427, - 0.0008047890005400404, - 0.0006100199971115217, - 0.0005548530025407672, - 0.0008391700030188076, - 0.0005985190000501461, - 0.0008009180019143969, - 0.0008335430029546842, - 0.0012335570063441992, - 0.0009021379955811426, - 0.0008709499961696565, - 0.0008324560039909557, - 0.0017583509979886003, - 0.003801519997068681, - 0.004018353996798396, - 0.003974027997173835, - 0.004012217003037222, - 0.0026324229984311387, - 0.005376260996854398, - 0.003998118998424616, - 0.004075926000950858, - 0.001821718004066497, - 0.0024635410009068437, - 0.004694048002420459, - 0.0058699239962152205, - 0.002158035000320524, - 0.00383406100445427, - 0.005167246999917552, - 0.002831375000823755, - 0.005994599006953649, - 0.0020010539956274442, - 0.005033477005781606, - 0.002947214998130221, - 0.007691500999499112, - 0.0009218659979524091, - 0.007294686998648103, - 0.0007242170031531714, - 0.0006327240043901838, - 0.0005993330050841905, - 0.0007046190003165975, - 0.0006460419972427189, - 0.0005945160010014661, - 0.0006139949982753024, - 0.0006072989999665879, - 0.0006330660035018809, - 0.0006153610011097044, - 0.000747146004869137, - 0.0005840699959662743, - 0.0005736189996241592, - 0.0005496730009326711, - 0.0006354430006467737, - 0.0006136570009402931, - 0.0005812739982502535, - 0.0005773489974671975, - 0.0005623360048048198, - 0.0005792760057374835, - 0.0005694939973182045, - 0.0005670140017173253, - 0.0007226849993458018, - 0.0006400700003723614, - 0.0006861529982415959, - 0.000617511999735143, - 0.0005996260006213561, - 0.0005836349955643527, - 0.0005691480037057772, - 0.000607560999924317, - 0.0005842839964316227, - 0.0005848680011695251, - 0.0005749139963882044, - 0.0005716619998565875, - 0.000575615995330736, - 0.0005847740030731075, - 0.0005774330056738108, - 0.0005743069996242411, - 0.0006348989991238341, - 0.0006836549946456216, - 0.006104185995354783, - 0.023009905002254527, - 0.01653653199900873, - 0.008784257996012457, - 0.006903753994265571, - 0.0030825770008959807, - 0.002975693001644686, - 0.007054506997519638, - 0.006940392006072216, - 0.007992837003257591, - 0.007994809995579999, - 0.006005950999679044, - 0.006185893995279912, - 0.0021820769979967736, - 0.006715337003697641, - 0.007890558001236059, - 0.0010787280043587089, - 0.0039058179972926155, - 0.009050659005879425, - 0.007979904003150295, - 0.006949811999220401, - 0.004996126001060475, - 0.008001784000953194, - 0.007998278000741266, - 0.007987598997715395, - 0.007994719002454076, - 0.007006672996794805, - 0.0069990530028007925, - 0.003252406997489743, - 0.005732953002734575, - 0.00498962499841582, - 0.005997446998662781, - 0.007005744002526626, - 0.010989810005412437, - 0.0059878040046896785, - 0.00100208700314397, - 0.0020723319976241328, - 0.007415889005642384, - 0.0054905970027903095, - 0.006050415999197867, - 0.008261948001745623, - 0.0017828150012064725, - 0.006295002000115346, - 0.0037843370009795763, - 0.003279700002167374, - 0.00451088699628599, - 0.008003101997019257, - 0.007993809995241463, - 0.004993319002096541, - 0.007998581997526344, - 0.006222267002158333, - 0.007792795993736945, - 0.010980032995576039, - 0.007703174000198487, - 0.0072787680037436076, - 0.006983986000705045, - 0.005994124003336765, - 0.00700521600083448, - 0.006988240995269734, - 0.005996778003463987, - 0.007996555999852717, - 0.005992961996525992, - 0.005991799996991176, - 0.007997428001544904, - 0.002991362001921516, - 0.00600293400202645, - 0.007997344997420441, - 0.007994543004315346, - 0.0046153800067259, - 0.0043745799994212575, - 0.001728390998323448, - 0.0032625649982946925, - 0.006999420002102852, - 0.007993956001882907, - 0.007996349006134551, - 0.0069938379965606146, - 0.006997468000918161, - 0.007994330000656191, - 0.007994036001036875, - 0.0070028670015744865, - 0.006991825001023244, - 0.007990569996763952, - 0.008003356000699569, - 0.004986152998753823, - 0.004003321002528537, - 0.010993066003720742, - 0.006002747002639808, - 0.008011675003217533, - 0.0079753059981158, - 0.00713815900235204, - 0.005858003998582717, - 0.0069820629942114465, - 0.007000388999585994, - 0.006999678997090086, - 0.007992819002538454, - 0.008008197997696698, - 0.007000017998507246, - 0.006976855998800602, - 0.006996937998337671, - 0.008236128000135068, - 0.004755375994136557, - 0.0069912089966237545, - 0.005996743006107863, - 0.006001637004374061, - 0.005982467999274377, - 0.007002068996371236, - 0.007995617001142818, - 0.006993629998760298, - 0.00699410900415387, - 0.006995506999373902, - 0.006992925002123229, - 0.007995951003977098, - 0.005812378003611229, - 0.004181711999990512, - 0.0070014619996072724, - 0.007989810001163278, - 0.0061094159973436035, - 0.006880783999804407, - 0.007993921004526783, - 0.00599906500428915, - 0.005990644000121392, - 0.005993978993501514, - 0.00600642099743709, - 0.00498650899680797 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil", - "fullname": "TestEnhanceDiffusionCoefficientForGridPointColdPools", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.00032675699912942946, - "max": 0.02308472900040215, - "mean": 0.005575253752455827, - "stddev": 0.004989503482430912, - "rounds": 319, - "median": 0.005988128999888431, - "iqr": 0.007595933497213991, - "q1": 0.00039615625246369746, - "q3": 0.007992089749677689, - "iqr_outliers": 4, - "stddev_outliers": 138, - "outliers": "138;4", - "ld15iqr": 0.00032675699912942946, - "hd15iqr": 0.019948140004999004, - "ops": 179.36403335175783, - "total": 1.778505947033409, - "data": [ - 0.006990530004259199, - 0.008011363999685273, - 0.007984112999110948, - 0.005995724000968039, - 0.005992070997308474, - 0.00999461900210008, - 0.00599915500060888, - 0.005988128999888431, - 0.0039968269993551075, - 0.00398686999687925, - 0.008006832002138253, - 0.005193342003622092, - 0.009827343004872091, - 0.005099105001136195, - 0.009053759000380524, - 0.006787723999877926, - 0.00799335099873133, - 0.006995679999818094, - 0.006008814998494927, - 0.007988617006049026, - 0.006985851003264543, - 0.005993574995954987, - 0.006960134000109974, - 0.010004557996580843, - 0.007983142000739463, - 0.00798075000056997, - 0.008729814995604102, - 0.007241919003718067, - 0.006979917001444846, - 0.00702725000155624, - 0.00697073400078807, - 0.006950850001885556, - 0.008002223999937996, - 0.005912470005569048, - 0.008072659999015741, - 0.006996232004894409, - 0.005129640005179681, - 0.007854882001993246, - 0.006990925001446158, - 0.008004032002645545, - 0.006996452997555025, - 0.004982154001481831, - 0.006999956000072416, - 0.007994580002559815, - 0.006997727003181353, - 0.006993479997618124, - 0.004995670999051072, - 0.015187088996754028, - 0.00882696799817495, - 0.008987290995719377, - 0.017006351001327857, - 0.004955503005476203, - 0.006972403003601357, - 0.008998045996122528, - 0.010990208000293933, - 0.00598137699853396, - 0.007988531004230026, - 0.012992788004339673, - 0.010112870993907563, - 0.008869654004229233, - 0.007981614995514974, - 0.01901512900076341, - 0.014980136998929083, - 0.020981080997444224, - 0.01396004200069001, - 0.013992708001751453, - 0.009445616997254547, - 0.017545544003951363, - 0.02308472900040215, - 0.01585110700398218, - 0.013978128001326695, - 0.014220035998732783, - 0.010755169001640752, - 0.011131314000522252, - 0.015856104000704363, - 0.014992936005000956, - 0.014985130001150537, - 0.01700440899730893, - 0.0060284249993856065, - 0.009946557998773642, - 0.015986610000254586, - 0.016170169998076744, - 0.012791940003808122, - 0.01296895900304662, - 0.01598095900408225, - 0.013980620002257638, - 0.013543672001105733, - 0.016452370000479277, - 0.016968137002550066, - 0.01196940999943763, - 0.005963436000456568, - 0.01898782900389051, - 0.017989705003856216, - 0.021972690003167372, - 0.007371469997451641, - 0.006598294996365439, - 0.00598206900031073, - 0.007999414003279526, - 0.006986209002207033, - 0.007992936996743083, - 0.007992011000169441, - 0.007992115999513771, - 0.0069858709975960664, - 0.007007523003267124, - 0.006986118998611346, - 0.005988993994833436, - 0.00699617899954319, - 0.007997000000614207, - 0.0056007830062299035, - 0.007378855996648781, - 0.010010566002165433, - 0.007999878995178733, - 0.006988319000811316, - 0.007986532997165341, - 0.007983491006598342, - 0.01119328800268704, - 0.007278004995896481, - 0.006897195999044925, - 0.007602302001032513, - 0.006991982998442836, - 0.00799241600179812, - 0.007992253995325882, - 0.006998880999162793, - 0.0060043550038244575, - 0.00399054899753537, - 0.0029820030031260103, - 0.013000737002585083, - 0.006995513002038933, - 0.010000784001022112, - 0.0059952249939669855, - 0.006055912002921104, - 0.006922979002411012, - 0.00699458499730099, - 0.007998747998499312, - 0.00698975599516416, - 0.00700706199859269, - 0.006972426999709569, - 0.0069962489942554384, - 0.008003569004358724, - 0.006988336994254496, - 0.006983455998124555, - 0.007998788998520467, - 0.008004565999726765, - 0.007985476004250813, - 0.01099274899752345, - 0.003992608995758928, - 0.005021883996960241, - 0.007973218002007343, - 0.005008276006265078, - 0.006998028002271894, - 0.005981820002489258, - 0.0059912139986408874, - 0.007994486004463397, - 0.005986449999909382, - 0.002999976997671183, - 0.00699211799656041, - 0.006665054999757558, - 0.00732279299700167, - 0.008017346997803543, - 0.006973376002861187, - 0.006967276000068523, - 0.0069851829975959845, - 0.006992969996645115, - 0.007018674004939385, - 0.006961560997297056, - 0.00799658100004308, - 0.007984359996044077, - 0.007035455004370306, - 0.019948140004999004, - 0.0150564569994458, - 0.008906843002478126, - 0.009734052000567317, - 0.010623919995850883, - 0.001754100994730834, - 0.000358658995537553, - 0.0003879689975292422, - 0.004888009003479965, - 0.00812591199792223, - 0.006490635001682676, - 0.002095473995723296, - 0.0009534450000501238, - 0.00040359400009037927, - 0.00047569999878760427, - 0.016232754998782184, - 0.005279530996631365, - 0.006174891001137439, - 0.003896269998222124, - 0.007949286002258305, - 0.0057760010022320785, - 0.0007660199989913963, - 0.00041854400478769094, - 0.000482161995023489, - 0.00039510200440417975, - 0.0003798409961746074, - 0.0003417189946048893, - 0.0008952060015872121, - 0.000366694999684114, - 0.0049784770017140545, - 0.0005400269947131164, - 0.0003584440055419691, - 0.0003572429995983839, - 0.00035052900057053193, - 0.0018337299989070743, - 0.00038951800524955615, - 0.00035957599902758375, - 0.00034600499930093065, - 0.0003435120015637949, - 0.002735018999374006, - 0.00037224799598334357, - 0.010286965996783692, - 0.0006361740015563555, - 0.004280385001038667, - 0.0005077250025351532, - 0.0015107170038390905, - 0.0006287740034167655, - 0.0012837699978263117, - 0.0003794169970205985, - 0.00034147000405937433, - 0.0003485209963400848, - 0.00034138500632252544, - 0.00033619000168982893, - 0.0003453169993008487, - 0.0003393360020709224, - 0.000338596997607965, - 0.0027553870022529736, - 0.00039931899664225057, - 0.00035090000164927915, - 0.0003426269977353513, - 0.000393196998629719, - 0.00040060500032268465, - 0.00035201099672121927, - 0.00035752799885813147, - 0.00033633800194365904, - 0.0003446910050115548, - 0.005115212996315677, - 0.0003832749935099855, - 0.0003441340013523586, - 0.00040847000491339713, - 0.001033707994793076, - 0.002365585998632014, - 0.0005363700038287789, - 0.0003583590005291626, - 0.0003447900016908534, - 0.0013381859971559606, - 0.00035584800207288936, - 0.0003743819979717955, - 0.0003533019989845343, - 0.00033931100188056007, - 0.00034875600249506533, - 0.0004523050010902807, - 0.00035462199593894184, - 0.0003545400031725876, - 0.0003447400013101287, - 0.0036605630011763424, - 0.0004805910066352226, - 0.0009017880001920275, - 0.0011612730013439432, - 0.0029923359979875386, - 0.0051763499941444024, - 0.00036451800406211987, - 0.0003312939952593297, - 0.0003328319944557734, - 0.0003329089959152043, - 0.0004481899959500879, - 0.00034066200169036165, - 0.00032995399669744074, - 0.00032675699912942946, - 0.0046758079988649115, - 0.000880372004758101, - 0.0003379260015208274, - 0.0003396869942662306, - 0.0003350870028953068, - 0.0004139010052313097, - 0.0003768110036617145, - 0.0003363389987498522, - 0.00035370799741940573, - 0.005641416006255895, - 0.0004364119959063828, - 0.0003430939977988601, - 0.00032941300014499575, - 0.00032760600151959807, - 0.000335512995661702, - 0.00032967400329653174, - 0.00033939399872906506, - 0.0003383049988769926, - 0.0003274510017945431, - 0.0003303240009699948, - 0.004657896002754569, - 0.0009201900029438548, - 0.00037760000122943893, - 0.0003339729955769144, - 0.00044322699977783486, - 0.00036473299405770376, - 0.0003359500042279251, - 0.0003300809985375963, - 0.00033434500073781237, - 0.00032932800240814686, - 0.00032947800355032086, - 0.005056998998043127, - 0.0008358019986189902, - 0.0003508160007186234, - 0.0003400799978408031, - 0.0003296919967397116, - 0.00032891299633774906, - 0.0022865089995320886, - 0.00036510900099528953, - 0.00033676799648674205, - 0.00033469699701527134, - 0.00033059400448109955, - 0.0003290029999334365, - 0.002880165004171431, - 0.014814965004916303, - 0.0068359880024218, - 0.004813850006030407, - 0.004957701996318065, - 0.003302805998828262, - 0.0017566320020705462, - 0.0009305509956902824, - 0.0005061899937572889 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil", - "fullname": "TestTemporaryFieldForGridPointColdPoolsEnhancement", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.0009878880009637214, - "max": 0.010999470003298484, - "mean": 0.0066737883276331445, - "stddev": 0.0017856254882766165, - "rounds": 372, - "median": 0.006996029998845188, - "iqr": 0.0019980180004495196, - "q1": 0.005995185998472152, - "q3": 0.007993203998921672, - "iqr_outliers": 24, - "stddev_outliers": 76, - "outliers": "76;24", - "ld15iqr": 0.0030675300004077144, - "hd15iqr": 0.0109974009974394, - "ops": 149.83993361902887, - "total": 2.4826492578795296, - "data": [ - 0.005993339000269771, - 0.006996619995334186, - 0.008281041002192069, - 0.006704343999444973, - 0.007995282001502346, - 0.006995339004788548, - 0.002685065999685321, - 0.004304442001739517, - 0.006001161003950983, - 0.006992818998696748, - 0.0069995169978938065, - 0.006991820999246556, - 0.005997134001518134, - 0.007610255001054611, - 0.009389061997353565, - 0.006991111004026607, - 0.0109974009974394, - 0.005993252001644578, - 0.006997030999627896, - 0.007996421001735143, - 0.006213474000105634, - 0.0017723139972076751, - 0.007001316000241786, - 0.007058518000121694, - 0.007118503999663517, - 0.006805659999372438, - 0.006993797003815416, - 0.005991634003294166, - 0.007872592999774497, - 0.0064812229975359514, - 0.003631610998127144, - 0.00542544500058284, - 0.007569599001726601, - 0.005989469005726278, - 0.0070000830019125715, - 0.007995226995262783, - 0.006002697999065276, - 0.005993822000164073, - 0.0019868800009135157, - 0.005523749001440592, - 0.006503607997728977, - 0.005967853001493495, - 0.009994117994210683, - 0.006993046001298353, - 0.006991472000663634, - 0.006002586000249721, - 0.0019819019944407046, - 0.006006055999023374, - 0.007076586000039242, - 0.006921123000211082, - 0.006995011994149536, - 0.009991367005568463, - 0.0058256569973309524, - 0.007163946000218857, - 0.007996589003596455, - 0.007994529994903132, - 0.004992220994608942, - 0.0044128279987489805, - 0.004575088998535648, - 0.007999803994607646, - 0.006997671000135597, - 0.006996033996983897, - 0.007997909000550862, - 0.00699297699611634, - 0.006996509000600781, - 0.006996903000981547, - 0.006994014998781495, - 0.009861658996669576, - 0.005128580996824894, - 0.010999470003298484, - 0.007997184999112505, - 0.007994658000825439, - 0.007995505999133456, - 0.004328023002017289, - 0.001667498996539507, - 0.005002542995498516, - 0.001986488998227287, - 0.006498646995169111, - 0.007597811003506649, - 0.007890058994235005, - 0.00985801599745173, - 0.006134955998277292, - 0.006297029001871124, - 0.009696658002212644, - 0.00799659600306768, - 0.006992267997702584, - 0.0079946959958761, - 0.007994851999683306, - 0.005861095996806398, - 0.00413301499793306, - 0.007995476000360213, - 0.005045807003625669, - 0.007951276005769614, - 0.001994915997784119, - 0.003652733998023905, - 0.008337295999808703, - 0.007997280001291074, - 0.007311840003239922, - 0.00768156999401981, - 0.003679761000967119, - 0.010362057997554075, - 0.006945073000679258, - 0.007997196000360418, - 0.006995533003646415, - 0.005724536000343505, - 0.005266643995128106, - 0.0059965549953631125, - 0.003997548999905121, - 0.0070036280012573116, - 0.006988164001086261, - 0.0051855349956895225, - 0.0028076129965484142, - 0.004996784002287313, - 0.007733876998827327, - 0.0052619819980463944, - 0.005996605003019795, - 0.006996675998379942, - 0.010998908001056407, - 0.005995471998176072, - 0.006995679999818094, - 0.007998958993994165, - 0.00799624300270807, - 0.007998155997483991, - 0.006998164994001854, - 0.005995784005790483, - 0.00699775799876079, - 0.00799736799672246, - 0.005997161999403033, - 0.006739383999956772, - 0.006254006999370176, - 0.00599906500428915, - 0.009999740999774076, - 0.007989548998011742, - 0.006995012998231687, - 0.004997611002181657, - 0.009000243997434154, - 0.006996026000706479, - 0.007997255997906905, - 0.005993145998218097, - 0.0059970770016661845, - 0.007997830005479045, - 0.008013720995222684, - 0.0039779939979780465, - 0.005996014995616861, - 0.007015581999439746, - 0.007974599997396581, - 0.0069955760045559146, - 0.007998960994882509, - 0.007995760002813768, - 0.008996632997877896, - 0.008006774995010346, - 0.007987728997250088, - 0.004996331997972447, - 0.006997783995757345, - 0.006996511998295318, - 0.007000302000960801, - 0.007010183995589614, - 0.007982066003023647, - 0.006011215999023989, - 0.007983452000189573, - 0.007999746005225461, - 0.007994685998710338, - 0.007999360001122113, - 0.00747067500196863, - 0.009632132001570426, - 0.00688524099678034, - 0.006997370997851249, - 0.006995233001362067, - 0.006997729004069697, - 0.006997569995291997, - 0.007997225002327468, - 0.006996072996116709, - 0.004997357995307539, - 0.007995455001946539, - 0.007997556000191253, - 0.00699607799469959, - 0.006998208998993505, - 0.006996377996983938, - 0.006998039003519807, - 0.003996797000581864, - 0.004997099997126497, - 0.00799729300342733, - 0.008002303002285771, - 0.006998776996624656, - 0.008991732000140473, - 0.007995011997991242, - 0.005998323002131656, - 0.0069975320002413355, - 0.007995678999577649, - 0.005996920997858979, - 0.007782640001096297, - 0.0072130889966501854, - 0.0050014839944196865, - 0.00999788699846249, - 0.006992697002715431, - 0.006997270000283606, - 0.008003470000403468, - 0.009000806996482424, - 0.007041308002953883, - 0.007939508999697864, - 0.005995808998704888, - 0.00799314599862555, - 0.005996978004986886, - 0.007997834996785969, - 0.007996663996891584, - 0.007997183995030355, - 0.005996229003358167, - 0.005998528999043629, - 0.005997047999699134, - 0.008001316993613727, - 0.007993261999217793, - 0.0076784900011261925, - 0.00833239899657201, - 0.0009878880009637214, - 0.006052620999980718, - 0.006932129996130243, - 0.006001629000820685, - 0.007993326005816925, - 0.006558918998052832, - 0.002433943998767063, - 0.005999674001941457, - 0.007232138996187132, - 0.0030944099999032915, - 0.0076568759977817535, - 0.003999565997219179, - 0.003992794001533184, - 0.006248577999940608, - 0.0077373640015139244, - 0.006999873003223911, - 0.00992064899764955, - 0.0030675300004077144, - 0.0050000240007648245, - 0.009999025001889095, - 0.0069930719982949086, - 0.00799718700000085, - 0.0059933049997198395, - 0.00599637400591746, - 0.005011684996134136, - 0.004980872996384278, - 0.008000802998139989, - 0.0069937130028847605, - 0.008055436999711674, - 0.00693396099813981, - 0.007996781998372171, - 0.006997741998929996, - 0.006996041003731079, - 0.006996932002948597, - 0.006996466996497475, - 0.003999047999968752, - 0.008994430994789582, - 0.002949540998088196, - 0.004042694999952801, - 0.006874605001939926, - 0.002116584997565951, - 0.006001836998621002, - 0.007996982996701263, - 0.005995880004775245, - 0.006561019996297546, - 0.008432829003140796, - 0.00406036800268339, - 0.0029317489970708266, - 0.006996792995778378, - 0.006994762996328063, - 0.0069986779999453574, - 0.006996768002863973, - 0.0069943260023137555, - 0.0070007690010243095, - 0.005115406995173544, - 0.007876370997109916, - 0.003991957004473079, - 0.005996838001010474, - 0.010008625002228655, - 0.007988231998751871, - 0.008477714996843133, - 0.007514860000810586, - 0.00799465399904875, - 0.007002512997132726, - 0.006997342003160156, - 0.007991950005816761, - 0.005999310997140128, - 0.005990137004118878, - 0.006240373993932735, - 0.0027498990020831116, - 0.0055172180000226945, - 0.006478862997028045, - 0.005993841004965361, - 0.00599476999923354, - 0.006140505000075791, - 0.007854837000195403, - 0.0035907860001316294, - 0.003402213995286729, - 0.007997392996912822, - 0.007997351996891666, - 0.005998949003696907, - 0.0013490150013240054, - 0.005646443001751322, - 0.007995398998900782, - 0.009996740998758469, - 0.00699727200117195, - 0.00699756400717888, - 0.005555924006330315, - 0.0025710839981911704, - 0.007862487000238616, - 0.009997509994718712, - 0.005997041997034103, - 0.006308424999588169, - 0.0066873209943878464, - 0.009001396996609401, - 0.0052305540011730045, - 0.008756814000662416, - 0.006095189004554413, - 0.007909141000709496, - 0.00698696100153029, - 0.003994527003669646, - 0.005003822996513918, - 0.0069913979968987405, - 0.005995691004500259, - 0.006995757998083718, - 0.0039972459999262355, - 0.006998315999226179, - 0.006998304001172073, - 0.006997440003033262, - 0.0069956610022927634, - 0.0069980650005163625, - 0.006995998999627773, - 0.0069981010019546375, - 0.006996519005042501, - 0.00599653999961447, - 0.009998289002396632, - 0.006999350000114646, - 0.006997563999902923, - 0.008014341001398861, - 0.005976365995593369, - 0.00699650299793575, - 0.006994492003286723, - 0.002671956994163338, - 0.004331809999712277, - 0.007982579001691192, - 0.0050024719967041165, - 0.009996715001761913, - 0.005992678001348395, - 0.007995678999577649, - 0.007996933993126731, - 0.006994470000790898, - 0.0072802610011422075, - 0.002715952003200073, - 0.0067521200035116635, - 0.007235794997541234, - 0.006001703004585579, - 0.005994899998768233, - 0.008055714999500196, - 0.010942159999103751, - 0.006101341998146381, - 0.006161362995044328, - 0.008062776003498584, - 0.0026459470027475618, - 0.005324484998709522, - 0.008674017000885215, - 0.005994054998154752, - 0.008000349000212736, - 0.004992526999558322, - 0.009997563000069931, - 0.007837821001885459, - 0.007149466997361742, - 0.004999924000003375, - 0.005381415998272132, - 0.001128861003962811, - 0.007477123006538022, - 0.005871414003195241, - 0.0061207019971334375, - 0.001990708005905617, - 0.007002815000305418 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil", - "fullname": "TestTemporaryFieldsForTurbulenceDiagnostics", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.00043558600009419024, - "max": 0.016960643995844293, - "mean": 0.0011528848425870221, - "stddev": 0.0016986498828167875, - "rounds": 1913, - "median": 0.0004994359987904318, - "iqr": 0.00029274399821588304, - "q1": 0.00046294100138766225, - "q3": 0.0007556849996035453, - "iqr_outliers": 347, - "stddev_outliers": 181, - "outliers": "181;347", - "ld15iqr": 0.00043558600009419024, - "hd15iqr": 0.0012006299948552623, - "ops": 867.389320303704, - "total": 2.2054687038689735, - "data": [ - 0.002500215996406041, - 0.0005546089960262179, - 0.0005493860007845797, - 0.0005240680038696155, - 0.0009673000022303313, - 0.0023072110052453354, - 0.0005586560000665486, - 0.0005295049995766021, - 0.0005271539994282648, - 0.0005280129989841953, - 0.0005271929985610768, - 0.001106764997530263, - 0.0010908250042120926, - 0.0005344899982446805, - 0.0005233769988990389, - 0.0005224329943303019, - 0.0005233049960224889, - 0.002658787001564633, - 0.0008739629993215203, - 0.0005311649947543629, - 0.0005268149980111048, - 0.00053804500203114, - 0.0005216499994276091, - 0.001169491995824501, - 0.001207651999720838, - 0.0005210170056670904, - 0.0007723200033069588, - 0.0017566279930179007, - 0.0012209849955979735, - 0.000511959005962126, - 0.001345530996331945, - 0.0007736919942544773, - 0.0005096699969726615, - 0.0005091169950901531, - 0.0005199180013732985, - 0.0005445160059025511, - 0.0038229109995882027, - 0.0005453490011859685, - 0.0005121559952385724, - 0.0005098100009490736, - 0.0005122179936734028, - 0.0027845990043715574, - 0.0009355090005556121, - 0.0005201149979257025, - 0.0005058940005255863, - 0.0005134480015840381, - 0.0005192679964238778, - 0.003234133000660222, - 0.001413938996847719, - 0.0005230640017543919, - 0.0005237980003585108, - 0.0005073150023235939, - 0.0024662939977133647, - 0.0005257969969534315, - 0.0005194339973968454, - 0.0005094820007798262, - 0.0005197190039325505, - 0.0006441730001824908, - 0.0026548050009296276, - 0.0011918979944312014, - 0.0005167729977983981, - 0.0005161919980309904, - 0.0005108199984533712, - 0.0013693439977942035, - 0.0022909620020072907, - 0.0005286500017973594, - 0.0005247950030025095, - 0.0005062619966338389, - 0.0006612040015170351, - 0.0015561160034849308, - 0.0007556290001957677, - 0.0005350630017346703, - 0.000508270000864286, - 0.0005169529977138154, - 0.0005138900014571846, - 0.0005151669975020923, - 0.0005051199987065047, - 0.0006356119993142784, - 0.0006422669976018369, - 0.0011603780003497377, - 0.0007270420028362423, - 0.0005323969962773845, - 0.0013935070019215345, - 0.0005907329978072084, - 0.0007605659993714653, - 0.0006240910006454214, - 0.0005201000021770597, - 0.0005161349981790408, - 0.0005124000017531216, - 0.0005201269959798083, - 0.0005031860055169091, - 0.0005463870038511232, - 0.0006682370003545657, - 0.0006572220008820295, - 0.0006450689979828894, - 0.0020939860041835345, - 0.000535341001523193, - 0.0005135029932716861, - 0.000504837000335101, - 0.0005389500001911074, - 0.0005181679953238927, - 0.0005659270027535968, - 0.0005103200019220822, - 0.0005378689966164529, - 0.0005114560044603422, - 0.000506937998579815, - 0.0005127580006956123, - 0.000654414005111903, - 0.002115827999659814, - 0.003614158005802892, - 0.0005688019955414347, - 0.0005809989961562678, - 0.0005423249967861921, - 0.0005037450027884915, - 0.0024104030017042533, - 0.0008976839962997474, - 0.0005091480052215047, - 0.0004894230005447753, - 0.0012908669959870167, - 0.0025790110012167133, - 0.0005913630011491477, - 0.0005156150000402704, - 0.0005023449994041584, - 0.0005010520035284571, - 0.003880220996506978, - 0.0005530319976969622, - 0.0005316289971233346, - 0.0004975389965693466, - 0.0006224280004971661, - 0.0016220149991568178, - 0.0005162430024938658, - 0.0004922610023641028, - 0.0023644770044484176, - 0.000563677996979095, - 0.0005257329976302572, - 0.0004937859994242899, - 0.0005128690027049743, - 0.0004974890034645796, - 0.0004942399973515421, - 0.0004944690008414909, - 0.0004997079959139228, - 0.000488634999783244, - 0.0004958780045853928, - 0.0004893170043942519, - 0.0004985659979865886, - 0.0005039770039729774, - 0.000494697000249289, - 0.0004963689934811555, - 0.004418094002176076, - 0.0006521729956148192, - 0.0005367939957068302, - 0.0005113989973324351, - 0.000507768003444653, - 0.0004995820054318756, - 0.0005017119983676821, - 0.000499012996442616, - 0.0005042340053478256, - 0.0004964749969076365, - 0.000496572996780742, - 0.000487571996927727, - 0.0005025460050092079, - 0.0004906579997623339, - 0.004279706001398154, - 0.0005519480037037283, - 0.0005184570036362857, - 0.0004880479973508045, - 0.0004949599970132113, - 0.000494241998239886, - 0.0004994330010958947, - 0.0025898789972416125, - 0.0005299030017340556, - 0.0005178569990675896, - 0.0004933759992127307, - 0.0004935310062137432, - 0.0035572640044847503, - 0.0005690889956895262, - 0.000589032999414485, - 0.0005256889999145642, - 0.000759775000915397, - 0.0005016479990445077, - 0.0004906750036752783, - 0.000499809997563716, - 0.0004899119958281517, - 0.0004930029972456396, - 0.0005917689995840192, - 0.004215330998704303, - 0.0006386460008798167, - 0.0005117300024721771, - 0.0005016110008000396, - 0.00049790799675975, - 0.000494815998536069, - 0.0004992530011804774, - 0.0004918260019621812, - 0.0012950450036441907, - 0.0005201229942031205, - 0.0004962929961038753, - 0.0008216540009016171, - 0.0004944009997416288, - 0.0004954250034643337, - 0.0004986540006939322, - 0.00048593599785817787, - 0.0004979420045856386, - 0.0004953309980919585, - 0.0006558029999723658, - 0.001479046004533302, - 0.0005946650053374469, - 0.0005972430008114316, - 0.0020166389949736185, - 0.0005148769996594638, - 0.0006245569966267794, - 0.0005902789998799562, - 0.0005271069967420772, - 0.0005007559957448393, - 0.0022566480038221925, - 0.0005376769986469299, - 0.0005034470013924874, - 0.0004970520021743141, - 0.002326339999854099, - 0.00052143499488011, - 0.0009600870034773834, - 0.0005003859987482429, - 0.0004934050011797808, - 0.0005063000062364154, - 0.00048701299965614453, - 0.0005273239949019626, - 0.005464560003019869, - 0.0005709030010621063, - 0.0005305070008034818, - 0.00048160000005736947, - 0.0004892229990218766, - 0.0004892290016869083, - 0.00047993199405027553, - 0.0036285339956521057, - 0.000930781003262382, - 0.0004984570041415282, - 0.0004910619973088615, - 0.0004899940031464212, - 0.0004889070041826926, - 0.0004833569983020425, - 0.0005478410021169111, - 0.0004752490058308467, - 0.004668460998800583, - 0.0010048650001408532, - 0.0005144919996382669, - 0.00048747500113677233, - 0.0004753719986183569, - 0.00048820700612850487, - 0.0004812359984498471, - 0.004248584002198186, - 0.0013303020023158751, - 0.0004905140012851916, - 0.0004840339970542118, - 0.00048028300079749897, - 0.0004832400009036064, - 0.0004840850015170872, - 0.0037776950048282743, - 0.0004889709962299094, - 0.0004800870010512881, - 0.0004898039987892844, - 0.00048187100037466735, - 0.000482490002468694, - 0.0004785980054293759, - 0.004095159994903952, - 0.0022518269979627803, - 0.0005039560055593029, - 0.0004883399960817769, - 0.000482461997307837, - 0.00047875099699012935, - 0.0005035440044593997, - 0.00047976199857657775, - 0.005350382998585701, - 0.00048727299872552976, - 0.0004818660017917864, - 0.00047771700337762013, - 0.00048579199938103557, - 0.0004851459962083027, - 0.013012787996558473, - 0.00728095299564302, - 0.007306749997951556, - 0.00868252199870767, - 0.005994970000756439, - 0.007001367994234897, - 0.006993047994910739, - 0.008002595001016743, - 0.003774708995479159, - 0.0005326769969542511, - 0.00048111700016306713, - 0.0004709390050265938, - 0.0004762500029755756, - 0.00047140999959083274, - 0.0004657419995055534, - 0.0005264529972919263, - 0.0005296899980749004, - 0.0005119929992360994, - 0.00047305099724326283, - 0.00048041199625004083, - 0.00046542700147256255, - 0.000469496997538954, - 0.00048345999675802886, - 0.003924541000742465, - 0.0005320260024745949, - 0.0004802350013051182, - 0.0005072109997854568, - 0.0004696489995694719, - 0.0004726070037577301, - 0.000465149998490233, - 0.003642707997641992, - 0.000549869000678882, - 0.0004924629974993877, - 0.0004625450019375421, - 0.0004708380001829937, - 0.0004653899959521368, - 0.0004856710002059117, - 0.0004669339978136122, - 0.0010669059993233532, - 0.003821632999461144, - 0.0005963319999864325, - 0.00048349199641961604, - 0.00046768800530117005, - 0.00046260900126071647, - 0.0004681330028688535, - 0.00047862700012046844, - 0.00046163200022419915, - 0.0028026049985783175, - 0.000686266997945495, - 0.0018114129998139106, - 0.0004974929979653098, - 0.0004723449965240434, - 0.0004609140014508739, - 0.0004756170019390993, - 0.0004666899985750206, - 0.002746491998550482, - 0.0005113779989187606, - 0.0004774310000357218, - 0.0004630269977496937, - 0.0005808600035379641, - 0.00047632500354666263, - 0.0004777529975399375, - 0.0051534850063035265, - 0.0005301970013533719, - 0.00048433799383929, - 0.0007312330053537153, - 0.00046202300291042775, - 0.0004763959950651042, - 0.0004673240036936477, - 0.003424980997806415, - 0.0006488279977929778, - 0.0006247549972613342, - 0.000616295998042915, - 0.0005560439967666753, - 0.00048103399603860453, - 0.0004615970028680749, - 0.00046134800504660234, - 0.0004749290019390173, - 0.00046105000365059823, - 0.0013044549996266142, - 0.002343343003303744, - 0.0005197099962970242, - 0.0004646259985747747, - 0.0004677679971791804, - 0.0004911439973511733, - 0.0004841230038437061, - 0.000463245996797923, - 0.0004599509993568063, - 0.0026399139969726093, - 0.0005082909992779605, - 0.0022733599980711006, - 0.0018722110035014339, - 0.0005234999989625067, - 0.0004821940019610338, - 0.00047252499643946066, - 0.0004656099990825169, - 0.000468956000986509, - 0.002322252999874763, - 0.002222355004050769, - 0.0005056430018157698, - 0.0004758100039907731, - 0.0004635880031855777, - 0.00046636699698865414, - 0.0004643079955712892, - 0.00045166399650042877, - 0.00044813800195697695, - 0.00044326899660518393, - 0.0038046519985073246, - 0.000492366001708433, - 0.00045340199721977115, - 0.00045643900375580415, - 0.00044586700096260756, - 0.000449972998467274, - 0.00045539200073108077, - 0.0006221129951882176, - 0.003464292996795848, - 0.0010907140022027306, - 0.0004987569991499186, - 0.00045991600200068206, - 0.00046140699851093814, - 0.0009676519985077903, - 0.0005223929983912967, - 0.001976794002985116, - 0.003084547999606002, - 0.0005287860039970838, - 0.0004672939976444468, - 0.00046424999891314656, - 0.0004598900050041266, - 0.0004578180014505051, - 0.0004506519981077872, - 0.0032458589994348586, - 0.0007860530022298917, - 0.0006419450000976212, - 0.0006052269964129664, - 0.0005546370011870749, - 0.00046509299863828346, - 0.00045445100113283843, - 0.00045898599637439474, - 0.00045501899876398966, - 0.00045192900142865255, - 0.0042948989939759485, - 0.0007209479954326525, - 0.0005396729975473136, - 0.00046027699863770977, - 0.0004599479943863116, - 0.000444366000010632, - 0.00046103000204311684, - 0.0004515230029937811, - 0.00402059900079621, - 0.0005085090015199967, - 0.0004711439978564158, - 0.0004627660018741153, - 0.00045240300096338615, - 0.0004539299989119172, - 0.00045735100138699636, - 0.0004559330045594834, - 0.00420877799479058, - 0.0007347439968725666, - 0.0004916160032735206, - 0.0004653549985960126, - 0.00045588200009660795, - 0.0004523070019786246, - 0.00045878399396315217, - 0.0004572429970721714, - 0.0007676759996684268, - 0.0037465670029632747, - 0.0005089350015623495, - 0.0011168909986736253, - 0.0005307729952619411, - 0.00048070599586935714, - 0.00046188099804567173, - 0.0004504390017245896, - 0.0004590640019159764, - 0.000463313001091592, - 0.0006032210003468208, - 0.0030921629950171337, - 0.0004959370053256862, - 0.0004664860025513917, - 0.0004496949986787513, - 0.00045162700553191826, - 0.00044548000005306676, - 0.00045533700176747516, - 0.00045295399468159303, - 0.009440199995879084, - 0.008995810996566433, - 0.005995127001369838, - 0.007997086002433207, - 0.00799639499746263, - 0.007006609994277824, - 0.010986007997416891, - 0.007002136997471098, - 0.0005082420029793866, - 0.00047118899965425953, - 0.0004552789978333749, - 0.00045960400166222826, - 0.0004597610022756271, - 0.00045326999679673463, - 0.000464115000795573, - 0.0004651509952964261, - 0.005922935997659806, - 0.0009477419953327626, - 0.0005463300039991736, - 0.0004745050027850084, - 0.0004665119995479472, - 0.00044817400339525193, - 0.008081576001131907, - 0.0005021229953854345, - 0.00044674999662674963, - 0.00044633499783230945, - 0.000450888997875154, - 0.0004531239974312484, - 0.00045223299821373075, - 0.005395601001509931, - 0.0005767749971710145, - 0.0005216360004851595, - 0.0004515839973464608, - 0.0004470269996090792, - 0.0004560000015771948, - 0.00044276200060267, - 0.004364204003650229, - 0.00045591600064653903, - 0.000490345002617687, - 0.0011551379939191975, - 0.0005350630017346703, - 0.00046427400229731575, - 0.00044062099914299324, - 0.0005040579999331385, - 0.0004598419982357882, - 0.0004564489936456084, - 0.0004416349984239787, - 0.00043711999751394615, - 0.002841401001205668, - 0.0006397249962901697, - 0.00045744200178887695, - 0.0015790839970577508, - 0.00045609300286741927, - 0.00044837300083599985, - 0.0004500270006246865, - 0.000747653997677844, - 0.0005393859973992221, - 0.0004471360007300973, - 0.0004662339997594245, - 0.00045107999903848395, - 0.0004454750014701858, - 0.0004420490004122257, - 0.00043824200110975653, - 0.001428718001989182, - 0.0016085710012703203, - 0.0018761440005619079, - 0.0004635909936041571, - 0.00045227900409372523, - 0.00044358099694363773, - 0.0004378040030132979, - 0.0006460770018748008, - 0.0014852669992251322, - 0.0010332239980925806, - 0.0007989329969859682, - 0.00047990400344133377, - 0.0004542839960777201, - 0.00044012699800077826, - 0.0004464880039449781, - 0.000450474995886907, - 0.002372319002461154, - 0.001351812003122177, - 0.00045085899910191074, - 0.0004416639931150712, - 0.0004482179938349873, - 0.0004546749987639487, - 0.0007605610007885844, - 0.0008454120034002699, - 0.0008326209936058149, - 0.0004498630005400628, - 0.0004513540043262765, - 0.0004409949979162775, - 0.00044303599861450493, - 0.00044586500007426366, - 0.0004386059954413213, - 0.001426277005521115, - 0.0005404760013334453, - 0.0005442429974209517, - 0.0004593070043483749, - 0.0004476929971133359, - 0.00044849899859400466, - 0.00047492599696852267, - 0.00044503000390250236, - 0.0012864079981227405, - 0.0016559220021008514, - 0.00048145199980353937, - 0.0004403470011311583, - 0.00046484799531754106, - 0.00044535000051837415, - 0.0004377860022941604, - 0.00044255400280235335, - 0.0004366869980003685, - 0.002160411997465417, - 0.00046601700159953907, - 0.0004478450064198114, - 0.001621879004233051, - 0.001499496996984817, - 0.0005339370036381297, - 0.00046715899952687323, - 0.0004596250000759028, - 0.00043831700168084353, - 0.00046386699978029355, - 0.00044216100650373846, - 0.00044508600694825873, - 0.00044216700189281255, - 0.00044264699681662023, - 0.00044240600254852325, - 0.0007434819999616593, - 0.0020464449989958666, - 0.003162611996231135, - 0.0008043900015763938, - 0.0004610959949786775, - 0.0008291740014101379, - 0.0010812479958985932, - 0.0007499350031139329, - 0.00044930900185136124, - 0.00044502800301415846, - 0.0019157099959556945, - 0.0006636010002694093, - 0.0009213900048052892, - 0.00045943199802422896, - 0.0004521529990597628, - 0.00045450600009644404, - 0.00045064000005368143, - 0.00045959399722050875, - 0.0004596329963533208, - 0.0009389340048073791, - 0.0004808879966731183, - 0.00046064500202191994, - 0.0004689449997385964, - 0.0004516890039667487, - 0.0006479630028479733, - 0.0008434529954683967, - 0.0016159160004463047, - 0.001677633001236245, - 0.0013642760022776201, - 0.0004824679999728687, - 0.0004593150006257929, - 0.0008370860014110804, - 0.0009457790001761168, - 0.000687565996486228, - 0.00047238499973900616, - 0.0004808119992958382, - 0.0004609399984474294, - 0.0004715000031865202, - 0.0004556260028039105, - 0.0004519380017882213, - 0.0043368339975131676, - 0.00047119500231929123, - 0.00045835199853172526, - 0.0004553459948510863, - 0.001004586993076373, - 0.0018210960042779334, - 0.0004900209969491698, - 0.0004612280026776716, - 0.00045018000673735514, - 0.00045769099961034954, - 0.0004554900006041862, - 0.003316497000923846, - 0.0007062320000841282, - 0.00048165800399146974, - 0.0004543510003713891, - 0.00045854000200051814, - 0.00045999900612514466, - 0.0012351569966995157, - 0.0018465379980625585, - 0.0007081540024955757, - 0.0004636470039258711, - 0.0004498389971558936, - 0.00044731799425790086, - 0.00045231399417389184, - 0.0004576499995891936, - 0.0017877729987958446, - 0.00047467800322920084, - 0.0004544299954432063, - 0.00046173999726306647, - 0.0004543050017673522, - 0.000452756998129189, - 0.0017734479988575913, - 0.0009885929976007901, - 0.00046968200331320986, - 0.00045463399874279276, - 0.0004569610027829185, - 0.00045332299487199634, - 0.0007021909987088293, - 0.0016900199989322573, - 0.0004935309989377856, - 0.0004624190041795373, - 0.0004578769949148409, - 0.0004545760020846501, - 0.0017447339996579103, - 0.0005749289994128048, - 0.00046516599832102656, - 0.000462314004835207, - 0.00044702400191454217, - 0.00045563499588752165, - 0.00045443300041370094, - 0.00046581999777117744, - 0.0015195250016404316, - 0.0013146929995855317, - 0.0004802900002687238, - 0.0004599269959726371, - 0.0004569200027617626, - 0.0004499220012803562, - 0.0017734189968905412, - 0.0020787460016435944, - 0.0005595059992629103, - 0.0004571409954223782, - 0.0004700499994214624, - 0.000460948002000805, - 0.0004523899988271296, - 0.003800415994192008, - 0.0004909260023850948, - 0.00046385700261453167, - 0.0004567829964798875, - 0.000453934000688605, - 0.0009167949974653311, - 0.0008757579998928122, - 0.0019347240013303235, - 0.0004822349947062321, - 0.00045874500210629776, - 0.0004496530018514022, - 0.0005500370025401935, - 0.0004611029944499023, - 0.0012411480056471191, - 0.0006245859985938296, - 0.0004617590020643547, - 0.0004503540039877407, - 0.000463964999653399, - 0.00045330099965212867, - 0.00044813600106863305, - 0.0024093320025713183, - 0.0004906389949610457, - 0.00045465599396266043, - 0.00044971800525672734, - 0.002022979002504144, - 0.0006575299994437955, - 0.00047777799773029983, - 0.00047256299876607955, - 0.0004542880051303655, - 0.00045617800060426816, - 0.0004460120035219006, - 0.0004751720043714158, - 0.00045840899838367477, - 0.0010351720047765411, - 0.0010775769987958483, - 0.004507588004344143, - 0.0005474810022860765, - 0.001636310997128021, - 0.0006929939991096035, - 0.0025333059966214933, - 0.0006067659996915609, - 0.00048429600428789854, - 0.0015911119990050793, - 0.001382935995934531, - 0.00048209400119958445, - 0.00046326700248755515, - 0.0004773230029968545, - 0.00045047399908071384, - 0.0005293799986247905, - 0.0007715059982729144, - 0.0028124359960202128, - 0.000520435998623725, - 0.0004651869967347011, - 0.00047644699952797964, - 0.00047545599954901263, - 0.0007576899952255189, - 0.0006036129998392425, - 0.0006016309998813085, - 0.0005920969997532666, - 0.0004731919980258681, - 0.0018115940038114786, - 0.0009381789932376705, - 0.0006944679989828728, - 0.00046459399891318753, - 0.00045908300671726465, - 0.00045506500464398414, - 0.0004557779975584708, - 0.0004465849997359328, - 0.0008922500055632554, - 0.0017154809975181706, - 0.0004578849984682165, - 0.0004639550024876371, - 0.0008468070009257644, - 0.001612021995242685, - 0.0007449359982274473, - 0.0004599029998644255, - 0.0018686770054046065, - 0.0004768190046888776, - 0.00045418400259222835, - 0.0006866019975859672, - 0.0004596389990183525, - 0.00045954099914524704, - 0.0004544990006252192, - 0.0006398750047083013, - 0.0006360120023600757, - 0.0016760400030761957, - 0.0007517320045735687, - 0.00047586199798388407, - 0.0004626520021702163, - 0.00045432800106937066, - 0.0019297150065540336, - 0.0004978740034857765, - 0.0005050370018579997, - 0.00045351300650509074, - 0.0004492590014706366, - 0.00045795299956807867, - 0.00045158200373407453, - 0.0007713489976595156, - 0.0017269460004172288, - 0.001654789004533086, - 0.0008234289998654276, - 0.0004630730036296882, - 0.0004493769956752658, - 0.00045440299436450005, - 0.0004578619991661981, - 0.0004590719981933944, - 0.0026022980018751696, - 0.004608885996276513, - 0.0005818050049128942, - 0.000488748999487143, - 0.00046126500092213973, - 0.0004624279972631484, - 0.00044784800411434844, - 0.0004512659943429753, - 0.0004538869980024174, - 0.0004578700027195737, - 0.003875088004861027, - 0.0012565479992190376, - 0.0004785270066349767, - 0.004098010998859536, - 0.0006097020013839938, - 0.0004696039977716282, - 0.004248498997185379, - 0.0007939259958220646, - 0.0004800300011993386, - 0.0004567700016195886, - 0.0004714329988928512, - 0.0004635090008378029, - 0.004298405998270027, - 0.0004953819952788763, - 0.00045909700565971434, - 0.00048505399900022894, - 0.0004525840049609542, - 0.004098473000340164, - 0.001065848999132868, - 0.0006027150011505, - 0.0009301329992013052, - 0.0007746250048512593, - 0.000461338997411076, - 0.0010042169960797764, - 0.00045869400491937995, - 0.00046203500096453354, - 0.0012497409989009611, - 0.0016333960011252202, - 0.00047972099855542183, - 0.0004635130026144907, - 0.002252015001431573, - 0.0004964519976056181, - 0.000461331001133658, - 0.0006840179994469509, - 0.0013136029956513084, - 0.0005172929959371686, - 0.00046535600267816335, - 0.0004568639997160062, - 0.00045948199840495363, - 0.000456868001492694, - 0.0004472150030778721, - 0.0010162639955524355, - 0.0028136990003986284, - 0.0010849649988813326, - 0.0004676870012190193, - 0.00045902000420028344, - 0.0004544809999060817, - 0.00045628500083694234, - 0.0011708820020430721, - 0.001237775999470614, - 0.0009239460050594062, - 0.0009699080037535168, - 0.0007232110001496039, - 0.000461563999124337, - 0.0004594389974954538, - 0.0008516180023434572, - 0.0020971460035070777, - 0.00046340299741132185, - 0.0004527630007942207, - 0.0004584260022966191, - 0.0004946159970131703, - 0.0006407570035662502, - 0.0014023770054336637, - 0.0006997549935476854, - 0.001381478999974206, - 0.0012772909976774827, - 0.00046254799963207915, - 0.001096506995963864, - 0.0007457180035999045, - 0.0011061510012950748, - 0.0005991830039420165, - 0.0004518189962254837, - 0.0004643180000130087, - 0.0004516460030572489, - 0.0004971039961674251, - 0.0013907609973102808, - 0.0007670409977436066, - 0.0015016580000519753, - 0.0004693119990406558, - 0.0004511130027822219, - 0.00046005599870113656, - 0.0004535249972832389, - 0.0010578259971225634, - 0.0017833800011430867, - 0.002007388000492938, - 0.00045968100312165916, - 0.00046417699923040345, - 0.00045379099901765585, - 0.0013491960053215735, - 0.000848046998726204, - 0.0009851659997366369, - 0.0004628410024452023, - 0.00046387899783439934, - 0.0004654909935197793, - 0.00045299500197870657, - 0.0011623230020632036, - 0.00177189199894201, - 0.0007367360012722202, - 0.00047731200174894184, - 0.00047760999586898834, - 0.00046728100278414786, - 0.00046055599523242563, - 0.0004475800014915876, - 0.0037456179998116568, - 0.0004877519968431443, - 0.00045427499571815133, - 0.00045443700219038874, - 0.00045372400199994445, - 0.0004543369941529818, - 0.0006959039965295233, - 0.0011946919985348359, - 0.0012283520045457408, - 0.0005757870021625422, - 0.00047166100557660684, - 0.0004517999987001531, - 0.0004658230027416721, - 0.0005285869992803782, - 0.0016871879997779615, - 0.001833210997574497, - 0.001054783999279607, - 0.000486001001263503, - 0.00048168699868256226, - 0.0004523759998846799, - 0.001055932996678166, - 0.002087809996737633, - 0.0004707379994215444, - 0.0004503329982981086, - 0.0004565720009850338, - 0.0004593190024024807, - 0.000981190001766663, - 0.002861000997654628, - 0.00050381499750074, - 0.00045381999370874837, - 0.00045300300553208217, - 0.00069213799724821, - 0.004876448001596145, - 0.000515522995556239, - 0.0004595220016199164, - 0.00045633199624717236, - 0.00046171100257197395, - 0.00045093000517226756, - 0.0004577349973260425, - 0.00045141200098441914, - 0.0032522980036446825, - 0.007936868998513091, - 0.00699567799892975, - 0.006995073999860324, - 0.0069955110011505894, - 0.006995408999500796, - 0.006996374999289401, - 0.005999491004331503, - 0.006994656003371347, - 0.007994719999260269, - 0.007996340005774982, - 0.00699608900322346, - 0.006996823001827579, - 0.00799674900190439, - 0.006996381998760626, - 0.006996572999923956, - 0.007996426997124217, - 0.007997840002644807, - 0.004996335999749135, - 0.004993663002096582, - 0.006996829004492611, - 0.007995694999408443, - 0.006993324001086876, - 0.006994664996454958, - 0.00800841399905039, - 0.00713530600478407, - 0.006838492001406848, - 0.008000134999747388, - 0.007070294996083248, - 0.006875767998280935, - 0.00895590699656168, - 0.007941555995785166, - 0.006947462999960408, - 0.007992585000465624, - 0.005999380999128334, - 0.008080681996943895, - 0.009004013998492155, - 0.007988008001120761, - 0.003982483998697717, - 0.006891662000271026, - 0.0070838160027051345, - 0.007941274001495913, - 0.003542523001669906, - 0.010457783995661885, - 0.007809755006746855, - 0.007715156003541779, - 0.010311013997124974, - 0.004928082002152223, - 0.008031402998312842, - 0.006048629998986144, - 0.00586709899653215, - 0.005970295002043713, - 0.016960643995844293, - 0.007664700999157503, - 0.008317312000144739, - 0.0057630419978522696, - 0.004221736002364196, - 0.007995873005711474, - 0.007995309999387246, - 0.0059971910013700835, - 0.007996371998160612, - 0.007996382002602331, - 0.007838252000510693, - 0.003189328999724239, - 0.0005027320003136992, - 0.0004761890013469383, - 0.00044869099656352773, - 0.00044442799844546244, - 0.0006317170045804232, - 0.0004693869996117428, - 0.00044154100032756105, - 0.0004397809971123934, - 0.002801043003273662, - 0.00044863000221084803, - 0.00044198500108905137, - 0.00044224999874131754, - 0.00043749299948103726, - 0.00044607799645746127, - 0.0004468869956326671, - 0.0004359199956525117, - 0.0037526169980992563, - 0.0004818599991267547, - 0.0004439410040504299, - 0.00044185099977767095, - 0.00044915499893249944, - 0.0004458390030777082, - 0.003670144003990572, - 0.0004913649972877465, - 0.00044449100096244365, - 0.0004967939967173152, - 0.00043897799332626164, - 0.0004741649972856976, - 0.0004451020067790523, - 0.00043990400445181876, - 0.005299377000483219, - 0.0004845969961024821, - 0.0004614490026142448, - 0.000440787996922154, - 0.00044149299355922267, - 0.0004808970043086447, - 0.00044215899833943695, - 0.0004576540013658814, - 0.000440855001215823, - 0.006010131997754797, - 0.0005191420059418306, - 0.0004550199955701828, - 0.0004454080044524744, - 0.000446236997959204, - 0.0004463360019144602, - 0.0004467659964575432, - 0.00043955799628747627, - 0.004799529000592884, - 0.0004957809942425229, - 0.00046028300130274147, - 0.0012720730010187253, - 0.0005680340036633424, - 0.00062081800570013, - 0.0005044400022597983, - 0.0004509409991442226, - 0.0004505930046434514, - 0.0004446140010259114, - 0.0004408589957165532, - 0.00043662100506480783, - 0.0005858709992025979, - 0.0005949769983999431, - 0.0006170859996927902, - 0.0009835290038608946, - 0.0004532570019364357, - 0.000439835996075999, - 0.00044793300185119733, - 0.0004399919998832047, - 0.0004408930035424419, - 0.0004453840010683052, - 0.00043558600009419024, - 0.0011671690008370206, - 0.000672187001327984, - 0.001299160998314619, - 0.0006985460058785975, - 0.0011649599982774816, - 0.00048260299809044227, - 0.0004477880065678619, - 0.00045138200221117586, - 0.0004370359965832904, - 0.0006541369948536158, - 0.0006370929986587726, - 0.0006141800040495582, - 0.0006032070014043711, - 0.000488290999783203, - 0.0024642760035931133, - 0.0005280510013108142, - 0.0009146980009973049, - 0.0007840399994165637, - 0.0005434879931272008, - 0.0004602590051945299, - 0.00044849699770566076, - 0.00044940000225324184, - 0.0004459209958440624, - 0.000442386997747235, - 0.00044001900096191093, - 0.0004805529970326461, - 0.0005819060024805367, - 0.0006038450010237284, - 0.000593951997871045, - 0.0006407999971997924, - 0.0013633080016006716, - 0.0004500249997363426, - 0.0004506130062509328, - 0.000566182003240101, - 0.00047509999421890825, - 0.0004609399984474294, - 0.00045168799988459796, - 0.0005636109999613836, - 0.0009029110005940311, - 0.0007137529973988421, - 0.0005452569967019372, - 0.00047751099918968976, - 0.0005824850013596006, - 0.0005486599984578788, - 0.0018798529999912716, - 0.00046054999984335154, - 0.0004530399965005927, - 0.006594334998226259, - 0.0012520370000856929, - 0.0008944080036599189, - 0.0004950759976054542, - 0.000458736001746729, - 0.001966736999747809, - 0.0005986910036881454, - 0.0008183640020433813, - 0.0008116289973258972, - 0.00047701299627078697, - 0.0004518450004979968, - 0.00045733700244454667, - 0.0004476989997783676, - 0.0006645039975410327, - 0.001153732999227941, - 0.0007519490027334541, - 0.0012976799989701249, - 0.0005849219960509799, - 0.00045650800166185945, - 0.000468226004159078, - 0.00045578699791803956, - 0.00045302099897526205, - 0.0004572420002659783, - 0.00046075899444986135, - 0.0010280749993398786, - 0.0006065199995646253, - 0.0006360679981298745, - 0.0006229849968804047, - 0.0012006299948552623, - 0.0005002550024073571, - 0.00047032200382091105, - 0.00045369999861577526, - 0.0004551799938781187, - 0.0005052720007370226, - 0.0005071050036349334, - 0.0005995290048304014, - 0.0011685719946399331, - 0.0004916459947708063, - 0.0004604720015777275, - 0.000508498000272084, - 0.00047525799891445786, - 0.0006156029994599521, - 0.000755852997826878, - 0.0008996239994303323, - 0.0004817359949811362, - 0.00045936699461890385, - 0.00046790899796178564, - 0.0004528659992502071, - 0.0005944310032646172, - 0.0006125299987616017, - 0.0004966300039086491, - 0.0005011229950468987, - 0.0005933669963269494, - 0.0005562060032389127, - 0.0005982839938951656, - 0.0004850650002481416, - 0.00046059300075285137, - 0.00046116800513118505, - 0.0004526619959506206, - 0.0012388960021780804, - 0.000720693999028299, - 0.0008003179973457009, - 0.0005146469993633218, - 0.0013740610011154786, - 0.0004795830027433112, - 0.0004601239998009987, - 0.00045142499584471807, - 0.0013201789988670498, - 0.0005980089990771376, - 0.0010146240019821562, - 0.0004935750039294362, - 0.00045657299779122695, - 0.00045747299736831337, - 0.00044641599379247054, - 0.0005357499976526015, - 0.0006545209980686195, - 0.0006090050010243431, - 0.000647281005512923, - 0.001055590997566469, - 0.00045608700020238757, - 0.00046341100096469745, - 0.0007175979990279302, - 0.001272019995667506, - 0.0005828319990541786, - 0.0011135039967484772, - 0.0005107640026835725, - 0.0004546489944914356, - 0.00045842299732612446, - 0.00045098899863660336, - 0.0004628100068657659, - 0.0004524930045590736, - 0.000448673999926541, - 0.0005966640019323677, - 0.0006104319982114248, - 0.0006599120024475269, - 0.0006137250020401552, - 0.0015644420054741204, - 0.0004906030007987283, - 0.00046466100320685655, - 0.00045555999531643465, - 0.0004532570019364357, - 0.0005253740018815733, - 0.0004682690050685778, - 0.0005839290024596266, - 0.0005633159962599166, - 0.0004700700010289438, - 0.0004555240011541173, - 0.00045166799827711657, - 0.000453672997537069, - 0.00045688899990636855, - 0.000582236003538128, - 0.0006326250004349276, - 0.001840765995439142, - 0.0004902220025542192, - 0.00047284699394367635, - 0.00047287900088122115, - 0.0007506149995606393, - 0.00048150400107260793, - 0.00047808400267967954, - 0.0005055190049461089, - 0.0006235870023374446, - 0.0006290809978963807, - 0.000635169999441132, - 0.00048320600035367534, - 0.0004720439974335022, - 0.00046979100443422794, - 0.00046394600212806836, - 0.00047045199607964605, - 0.0006214069944689982, - 0.0012972830008948222, - 0.000486422999529168, - 0.0004725699982373044, - 0.00046508899686159566, - 0.0005668640005751513, - 0.0006156580056995153, - 0.0006220450013643131, - 0.0005502500061993487, - 0.0004690019995905459, - 0.0005083819996798411, - 0.0015630869966116734, - 0.0005583319943980314, - 0.0009834200027398765, - 0.00046486999781336635, - 0.00047918899508658797, - 0.00046243600081652403, - 0.0005237900040810928, - 0.0006129769972176291, - 0.00126590600120835, - 0.0006551879996550269, - 0.00047279200225602835, - 0.0004659890037146397, - 0.0004895559977740049, - 0.0004669230038416572, - 0.0004997919968445785, - 0.0006306829964159988, - 0.0009138180030276999, - 0.0006361500054481439, - 0.0006368949980242178, - 0.0005388459976529703, - 0.0004681020000134595, - 0.0004690869973273948, - 0.0004626949958037585, - 0.00047341999743366614, - 0.000589079994824715, - 0.0009057560018845834, - 0.0016662599955452606, - 0.0009373240027343854, - 0.00048792699817568064, - 0.000476135995995719, - 0.0004647060050047003, - 0.0004679449994000606, - 0.00046185099927242845, - 0.001136767998104915, - 0.0006134670038591139, - 0.0005512200004886836, - 0.0004771390013047494, - 0.00046959100291132927, - 0.00046447299973806366, - 0.0004651359995477833, - 0.0004650190021493472, - 0.000457444999483414, - 0.001568929001223296, - 0.0014634149993071333, - 0.0004729770007543266, - 0.00046180199569789693, - 0.0016788499997346662, - 0.000574049998249393, - 0.0005017880030209199, - 0.0008842430033837445, - 0.0006492920001619495, - 0.000592088996199891, - 0.0005001769968657754, - 0.00047569599701091647, - 0.00046790099440841004, - 0.0004625310029950924, - 0.0005006809951737523, - 0.0004782550022355281, - 0.0005402069946285337, - 0.0020321309930295683, - 0.0007184369969763793, - 0.001078648005204741, - 0.0010502479999559, - 0.0005035440044593997, - 0.0004762050011777319, - 0.00046771000052103773, - 0.0005159979991731234, - 0.0019049249967793003, - 0.0006416779942810535, - 0.00048815600166562945, - 0.00048023199633462355, - 0.00048181800229940563, - 0.00046279699745355174, - 0.0004930510040139779, - 0.0006702399987261742, - 0.0012607229946297593, - 0.0007697119945078157, - 0.0006830679994891398, - 0.0004956390039296821, - 0.00047183599963318557, - 0.00046011099766474217, - 0.00048464699648320675, - 0.0004611280019162223, - 0.0011411580053390935, - 0.0006360129991662689, - 0.0005935799999861047, - 0.0012131949988543056, - 0.0004913450029562227, - 0.00047714399988763034, - 0.0004602449989761226, - 0.0005107030010549352, - 0.0004690179994213395, - 0.0006289719967753626, - 0.0006212190055521205, - 0.0008015819985303096, - 0.0005654030028381385, - 0.0006332490011118352, - 0.0005196240017539822, - 0.0004846619995078072, - 0.0004606220027199015, - 0.0005374969987315126, - 0.0016637409935356118, - 0.0004948209971189499, - 0.00046703599946340546, - 0.001051127001119312, - 0.00046971999836387113, - 0.00046019799628993496, - 0.0005699429966625758, - 0.000621778002823703, - 0.0005864290069439448, - 0.0004829419995076023, - 0.0005048460006946698, - 0.0004678069963119924, - 0.0004686440006480552, - 0.00046363400178961456, - 0.0004630429975804873, - 0.00046008000208530575, - 0.0006065999987185933, - 0.0006175689995870925, - 0.003885705998982303, - 0.0005725220034946688, - 0.0005135389947099611, - 0.0004890319978585467, - 0.000516353000421077, - 0.005891453001822811, - 0.0008573590021114796, - 0.0010659519975888543, - 0.000511504003952723, - 0.0004693410010077059, - 0.00048344099923269823, - 0.00046325500443344936, - 0.004565624003589619, - 0.0004866880044573918, - 0.0004950200018356554, - 0.00047235999954864383, - 0.0006303080008365214, - 0.0006015179969836026, - 0.0005492330019478686, - 0.0004904480010736734, - 0.00046262300020316616, - 0.0004705440005636774, - 0.00046715499775018543, - 0.00047231200005626306, - 0.0004604099958669394, - 0.005500633000337984, - 0.0007056729955365881, - 0.0006461609955294989, - 0.0006349230025080033, - 0.0005067640013294294, - 0.002516916996682994, - 0.0006675810000160709, - 0.0044176329975016415, - 0.0004988229993614368, - 0.0007506859983550385, - 0.0006512320032925345, - 0.0006270360026974231, - 0.002632404997712001, - 0.0005090620034025051, - 0.0011688469967339188, - 0.0006355970035656355, - 0.002547529998992104, - 0.0005258300006971695, - 0.0006109800015110523, - 0.0005682720002369024, - 0.0004987029969925061, - 0.004481914002099074, - 0.0006675810000160709, - 0.0016904520016396418, - 0.0004990090019418858, - 0.004648479000024963, - 0.0005099269983475097, - 0.000634174000879284, - 0.0008088930044323206, - 0.004645165005058516, - 0.0005526990062207915, - 0.000546125003893394, - 0.0006510370003525168, - 0.0006709269946441054, - 0.005549524001253303, - 0.0006862990048830397, - 0.0006893789977766573, - 0.0005220099992584437, - 0.0004798459995072335, - 0.0035872260050382465, - 0.0007604649945278652, - 0.0005512970019481145, - 0.00047639600234106183, - 0.0004754699984914623, - 0.0004620240069925785, - 0.00046799799747532234, - 0.0004772749962285161, - 0.003777366000576876, - 0.005516616998647805, - 0.0005585459948633797, - 0.0005009619999327697, - 0.00047780299792066216, - 0.00047248600458260626, - 0.00047365100181195885, - 0.0004693549999501556, - 0.0022570049986825325, - 0.0004894480007351376, - 0.0017360450001433492, - 0.001492070994572714, - 0.0005043229975854047, - 0.00047032199654495344, - 0.003057289999560453, - 0.004996011994080618, - 0.006004079004924279, - 0.00807432399597019, - 0.007910516993433703, - 0.007986378994246479, - 0.006994411000050604, - 0.007995280000614002, - 0.00599553400388686, - 0.007996601998456754, - 0.004002104004030116, - 0.0005571330038947053, - 0.0005325719976099208, - 0.0004763149991049431, - 0.0023455740010831505, - 0.0005355100001906976, - 0.0004977179996785708, - 0.0004643220017896965, - 0.00046233300236053765, - 0.00047549699957016855, - 0.0004622999986167997, - 0.006715913004882168, - 0.0005879349992028438, - 0.0005519410042325035, - 0.00047632600035285577, - 0.000496526001370512, - 0.0004667000030167401, - 0.0004622529959306121, - 0.007183898000221234, - 0.0005725030059693381, - 0.000539620996278245, - 0.0011271820039837621, - 0.00047512199671473354, - 0.00047153599734883755, - 0.0004681330028688535, - 0.00046372500219149515, - 0.00046883500181138515, - 0.0004574609993142076, - 0.0010540099974605255, - 0.0035883439995814115, - 0.0005941000054008327, - 0.0005581410005106591, - 0.0025823110045166686, - 0.0005407889984780923, - 0.0005379770009312779, - 0.0005132890000822954, - 0.0004885609960183501, - 0.0004633369971998036, - 0.00046817600377835333, - 0.0004757860006066039, - 0.005966798002191354, - 0.0005888410014449619, - 0.0005426049974630587, - 0.0004732469969894737, - 0.0004595680002239533, - 0.000489286998345051, - 0.0004642929998226464, - 0.003482008003629744, - 0.0005878800002392381, - 0.000593561002460774, - 0.0005038980016252026, - 0.00046722200204385445, - 0.00046598599874414504, - 0.0004782110045198351, - 0.0004719720018329099, - 0.005884987003810238, - 0.0005845569976372644, - 0.0005619720031972975, - 0.00046984200162114576, - 0.00046660199586767703, - 0.00046424200263572857, - 0.0004584669950418174, - 0.004655320000892971, - 0.0005807019988424145, - 0.0005772049989900552, - 0.0004766729980474338, - 0.00047907100088195875, - 0.00046637299965368584, - 0.00046107899834169075, - 0.00048514099762542173, - 0.00047137799992924556, - 0.0004621760017471388, - 0.005177481005375739, - 0.0005658309964928776, - 0.0004931250005029142, - 0.000464086995634716, - 0.0005388179997680709, - 0.00047498699859716, - 0.0004685989988502115, - 0.0004579480009851977, - 0.006200010000611655, - 0.0005441750035970472, - 0.0005064170036348514, - 0.00046854300308041275, - 0.00045479200343834236, - 0.00045194899576017633, - 0.00044494900066638365, - 0.005323228004272096, - 0.0005844220067956485, - 0.0004942500017932616, - 0.000454234002972953, - 0.0004557850043056533, - 0.00045878699893364683, - 0.0007123690011212602, - 0.002632263996929396, - 0.0008469349995721132, - 0.00045487099851015955, - 0.0004506550030782819, - 0.00046024400216992944, - 0.00063539799884893, - 0.0005996320032863878, - 0.0004776700006914325, - 0.0004685399981099181, - 0.0004488179984036833, - 0.0005307730025378987, - 0.0029234570029075257, - 0.0005612639943137765, - 0.0004840769979637116, - 0.0004632190029951744, - 0.0004994359987904318, - 0.0006500570016214624, - 0.0012717360004899092, - 0.0006119819954619743, - 0.0005157339983270504, - 0.00045906900049885735, - 0.0004613439959939569, - 0.0004488529957598075, - 0.00044837900350103155, - 0.0006105299980845302, - 0.0006367820024024695, - 0.0006571970006916672, - 0.0008518409958924167, - 0.00048280299961334094, - 0.0004508259953581728, - 0.0004630999974324368, - 0.0004571309982566163, - 0.0006398699988494627, - 0.0004581179964588955, - 0.0004645269946195185, - 0.0004598090017680079, - 0.0004515429973253049, - 0.00045233799755806103, - 0.00045407099969452247, - 0.00044852500286651775, - 0.00044468399573815987, - 0.000570497999433428, - 0.000608973998168949, - 0.0006806059973314404, - 0.0006356899975799024, - 0.0006623429944738746, - 0.0021488729980774224, - 0.00048638899897923693, - 0.0004932550000376068, - 0.0004755670015583746, - 0.0010095760007970966, - 0.0006286650022957474, - 0.0005990909994579852, - 0.0005801450024591759, - 0.0004807069999515079, - 0.00045587999920826405, - 0.00044703000457957387, - 0.00045682799827773124, - 0.0004500499999267049, - 0.0005511389972525649, - 0.0035541050019674003, - 0.0005987350014038384, - 0.00048350999713875353, - 0.00045938100083731115, - 0.0004552239988697693, - 0.0005815670010633767, - 0.000898990998393856, - 0.0006172319990582764, - 0.0005829690053360537, - 0.00047675300447735935, - 0.0004857599997194484, - 0.00047000400081742555, - 0.00046470599772874266, - 0.0004986730054952204, - 0.0009641380020184442, - 0.0009273040050175041, - 0.00046970199764473364, - 0.000465421995613724, - 0.0031967370014172047, - 0.000789368998084683, - 0.0007967700003064238, - 0.000483703006466385, - 0.00046457799908239394, - 0.0004668059991672635, - 0.00046412999654421583, - 0.0004659219994209707, - 0.00047003200597828254, - 0.004178594004770275, - 0.0015497150016017258, - 0.000505452997458633, - 0.0005599669966613874, - 0.0005196599959162995, - 0.00046867499622749165, - 0.00046138700417941436, - 0.0004747740022139624, - 0.0004724399987026118, - 0.0010164799969061278, - 0.004130403001909144, - 0.0005793180025648326, - 0.0004905829991912469, - 0.0004661249986384064, - 0.00046280300011858344, - 0.0004873020006925799, - 0.0004613130004145205, - 0.000469954997242894, - 0.0004606500006048009, - 0.0030915190000087023, - 0.0005576019975706004, - 0.0005242059996817261, - 0.0004694379967986606, - 0.0004586869981721975, - 0.00047882399667287245, - 0.00046909199591027573, - 0.005187303999264259, - 0.0005670099999406375, - 0.0005408129945863038, - 0.00044376999721862376, - 0.0004477760012377985, - 0.00044814199645770714, - 0.00044571900070877746, - 0.0004940180006087758, - 0.00044475699542090297, - 0.00569863899727352, - 0.000567408002098091, - 0.0005948920006630942, - 0.0004614380013663322, - 0.0004561870009638369, - 0.0004522769959294237, - 0.000449020997621119, - 0.002488326994352974, - 0.0026775799997267313, - 0.0005234920026850887, - 0.0005136039981152862, - 0.00045226699876366183, - 0.00045123599556973204, - 0.0004561339956126176, - 0.0004477869952097535, - 0.0035613470026873983, - 0.0005620299998554401, - 0.0005401270027505234, - 0.0005215080018388107, - 0.00046119200123939663, - 0.0004559270018944517, - 0.00045674500142922625, - 0.0005119749985169619, - 0.006031659999280237, - 0.0005491960037034005, - 0.0004960469959769398, - 0.00045324000529944897, - 0.0004510889993980527, - 0.0004560209999908693, - 0.0004552219979814254, - 0.0037363110022852197, - 0.001411837998603005, - 0.0004586020004353486, - 0.0004512779996730387, - 0.0004527589990175329, - 0.0004560629968182184, - 0.00045189200318418443, - 0.0004492110019782558, - 0.0004480839997995645, - 0.004959329002304003, - 0.0005632690008496866, - 0.00047443299990845844, - 0.0004971239977749065, - 0.0004527380006038584, - 0.00045376699563348666, - 0.0004492660009418614, - 0.001201238002977334, - 0.0016507019972777925, - 0.00046691799798281863, - 0.0004573679980239831, - 0.0014283440032158978, - 0.0004904040033579804, - 0.0004674989977502264, - 0.0004523899988271296, - 0.0004567279975162819, - 0.0005385289987316355, - 0.0005048540042480454, - 0.0004583120025927201, - 0.00045599300210596994, - 0.00045579300058307126, - 0.00044949300354346633, - 0.0010134350013686344, - 0.00047070300206542015, - 0.0004582350011332892, - 0.00047212500066962093, - 0.0005324559970176779, - 0.0004507289995672181, - 0.0004497580011957325, - 0.0004587069997796789, - 0.0004516039989539422, - 0.0004797299989149906, - 0.0004466619939194061, - 0.0004511090010055341, - 0.00045650600077351555, - 0.0006176079987199046, - 0.000470567996671889, - 0.0004629219984053634, - 0.0004704120001406409, - 0.00046943699999246746, - 0.000456911999208387, - 0.00044999200326856226, - 0.000451830004749354, - 0.0004450270062079653, - 0.000452606996987015, - 0.00045141600276110694, - 0.000475129003461916, - 0.0004561039968393743, - 0.00045787199633195996, - 0.00044905200047651306, - 0.0004476429967326112, - 0.0005435619968920946, - 0.0027471659996081144, - 0.0005204909975873306, - 0.000506595999468118, - 0.0005050650070188567, - 0.0012245699981576763, - 0.0010182469995925203, - 0.0004603080014931038, - 0.00044628999603446573, - 0.00045697600580751896, - 0.00045727600081590936, - 0.0004578849984682165, - 0.0004484449964365922, - 0.0005117250038892962, - 0.00047425500088138506, - 0.0029384430017671548, - 0.0007325469996430911, - 0.0005235410062596202, - 0.00044995999633101746, - 0.0004574489939841442, - 0.0011644690021057613, - 0.00047806000657146797, - 0.0005183700050110929, - 0.0007328819992835633, - 0.00047331699897767976, - 0.00044626599992625415, - 0.0004537470013019629, - 0.00045216400030767545, - 0.0004509510035859421, - 0.0004713140006060712, - 0.00045338100608205423, - 0.0020472030009841546, - 0.0005528520050575025, - 0.0005948290054220706, - 0.0005076789966551587, - 0.0004766269994433969, - 0.000463005002529826, - 0.00046749800094403327, - 0.0004591710021486506, - 0.0004992619942640886, - 0.0009475789993302897, - 0.0010109270006068982, - 0.001057032000971958, - 0.00046300200483528897, - 0.0004713710004580207, - 0.0004629460017895326, - 0.00046056000428507105, - 0.0004924289969494566, - 0.0004783480035257526, - 0.00046712699986528605, - 0.0004630250041373074, - 0.00046539200411643833, - 0.00046328999451361597, - 0.00048157299897866324, - 0.00048361100198235363, - 0.000519100998644717, - 0.0004900229978375137, - 0.00047472900041611865, - 0.0004683980005211197, - 0.00046612200094386935, - 0.0004629260001820512, - 0.00047083499521249905, - 0.00046422100422205403, - 0.0005040550022386014, - 0.00047786500363145024, - 0.0004824759962502867, - 0.000481041002785787, - 0.00047223099682014436, - 0.00048249700193991885, - 0.0004566749994410202, - 0.0005262070044409484, - 0.0010732279988587834, - 0.00046543700591428205, - 0.0004668930050684139, - 0.0004620789986802265, - 0.0004645249937311746, - 0.0004869800031883642, - 0.0010862670023925602, - 0.0004867489988100715, - 0.00048693999997340143, - 0.0014267639999161474, - 0.0004770969972014427, - 0.0004868120013270527, - 0.00046522600314347073, - 0.0004652169955079444, - 0.0004667250032071024, - 0.0015479319990845397, - 0.002575611993961502, - 0.0005191219970583916, - 0.00047790000098757446, - 0.00047786300274310634, - 0.00046357700193766505, - 0.0004664469961426221, - 0.0004687609980464913, - 0.0007434480066876858, - 0.000975096998445224, - 0.0011309530018479563, - 0.0011541400017449632, - 0.00047519500367343426, - 0.0004693449955084361, - 0.00047209699550876394, - 0.0004627680027624592, - 0.00046663299872307107, - 0.00046011200174689293, - 0.00046667500282637775, - 0.0029499299998860806, - 0.0005215229975874536, - 0.0004750360021716915, - 0.0004631470001186244, - 0.0005222619947744533, - 0.00046121099876472726, - 0.0004642749991035089, - 0.0005023060002713464, - 0.0004965870029991493, - 0.0008486220031045377, - 0.0007503529996029101, - 0.0005551349968300201, - 0.0014408610004466027, - 0.001329742997768335, - 0.0018722469976637512, - 0.0007306479965336621, - 0.0004718290001619607, - 0.0004908409973722883, - 0.00047290799557231367 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil", - "fullname": "TestTrulyHorizontalDiffusionNablaOfThetaOverSteepPoints", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.00043865200132131577, - "max": 0.01299809200281743, - "mean": 0.005433601697726129, - "stddev": 0.002916803648316478, - "rounds": 334, - "median": 0.006328357496386161, - "iqr": 0.004274203005479649, - "q1": 0.0035999009996885434, - "q3": 0.007874104005168192, - "iqr_outliers": 0, - "stddev_outliers": 94, - "outliers": "94;0", - "ld15iqr": 0.00043865200132131577, - "hd15iqr": 0.01299809200281743, - "ops": 184.0399896110315, - "total": 1.8148229670405271, - "data": [ - 0.001974742997845169, - 0.005991189005726483, - 0.009000550999189727, - 0.006991414004005492, - 0.007001578000199515, - 0.006994010000198614, - 0.006997163000050932, - 0.007995958003448322, - 0.006995802999881562, - 0.005996199994115159, - 0.006018956999469083, - 0.006973362003918737, - 0.0071339560017804615, - 0.007860489997256082, - 0.006996161995630246, - 0.005015518996515311, - 0.003978914006438572, - 0.007992282000486739, - 0.006995340998400934, - 0.006994711002334952, - 0.005994097999064252, - 0.00544539799739141, - 0.008554105996154249, - 0.007000958998105489, - 0.006986596999922767, - 0.005006983999919612, - 0.006985355998040177, - 0.010004787000070792, - 0.006988712004385889, - 0.004989676002878696, - 0.0035999009996885434, - 0.004390064001199789, - 0.005999424000037834, - 0.008001123002031818, - 0.007989593003003392, - 0.005003239995858166, - 0.004988310996850487, - 0.007001017998845782, - 0.006993556002271362, - 0.006996714000706561, - 0.006998297001700848, - 0.007992247999936808, - 0.008000565001566429, - 0.007993757993972395, - 0.007998884000699036, - 0.00498649100336479, - 0.005005370003345888, - 0.005989564000628889, - 0.006994236005994026, - 0.005601311000646092, - 0.0043902879988309, - 0.005650762999721337, - 0.010345394999603741, - 0.00800501800404163, - 0.0069852609958616085, - 0.010001710004871711, - 0.007991350001248065, - 0.007007741995039396, - 0.006987105000007432, - 0.006994623996433802, - 0.006999433004239108, - 0.007995441003004089, - 0.006994821997068357, - 0.006998480996116996, - 0.007997773995157331, - 0.004383501996926498, - 0.004607748000125866, - 0.006997113996476401, - 0.005997098000079859, - 0.006998528006079141, - 0.0069963889982318506, - 0.004002338006102946, - 0.0055955049974727444, - 0.007530669005063828, - 0.007885408005677164, - 0.007964916003402323, - 0.00799650899716653, - 0.007995707004738506, - 0.010001860006013885, - 0.005989666999084875, - 0.007998255001439247, - 0.006993564005824737, - 0.007995579995622393, - 0.01101143000414595, - 0.007987722994585056, - 0.002987391002534423, - 0.007005019004282076, - 0.005986613999994006, - 0.006008823998854496, - 0.010106360001373105, - 0.007874104005168192, - 0.005003666003176477, - 0.003263197999331169, - 0.0057198280046577565, - 0.005000937002478167, - 0.006996062002144754, - 0.006994974995905068, - 0.006996826996328309, - 0.007995678999577649, - 0.005997674998070579, - 0.00799658599862596, - 0.00599930300086271, - 0.007000842997513246, - 0.00599088599847164, - 0.006995800998993218, - 0.008003256996744312, - 0.005990116005705204, - 0.007996366999577731, - 0.004998793003323954, - 0.006996833995799534, - 0.006995094001467805, - 0.01299809200281743, - 0.006999208002525847, - 0.005994849998387508, - 0.006996675001573749, - 0.00800123299995903, - 0.004645459994208068, - 0.0063461239988100715, - 0.010003677998611238, - 0.00699224800337106, - 0.007001341000432149, - 0.007982658004038967, - 0.00856472400482744, - 0.006437834999815095, - 0.004790803002833854, - 0.009191071992972866, - 0.006994742005190346, - 0.004003381996881217, - 0.007991778002178762, - 0.007997699998668395, - 0.005990095000015572, - 0.005998540997097734, - 0.00799470700439997, - 0.008999022997159045, - 0.006711800997436512, - 0.008280941998236813, - 0.007005444997048471, - 0.007990449004864786, - 0.007995879001100548, - 0.005994004997774027, - 0.0069981410051696, - 0.00408851799875265, - 0.007906076003564522, - 0.007999010995263234, - 0.007994106999831274, - 0.0030264739980339073, - 0.005968646000837907, - 0.0059971549999318086, - 0.008009050994587597, - 0.007985228003235534, - 0.003996015999291558, - 0.003717268002219498, - 0.007280263002030551, - 0.005995504005113617, - 0.007997016000445, - 0.006997060998401139, - 0.006018002997734584, - 0.00797720999980811, - 0.00613597099436447, - 0.006861289999505971, - 0.005993583996314555, - 0.006001155001285952, - 0.0076487129990709946, - 0.0063404139946214855, - 0.004996001996914856, - 0.008015557999897283, - 0.007089963000908028, - 0.00788709899643436, - 0.007997032000275794, - 0.0049968439998338, - 0.007000988996878732, - 0.005991933998302557, - 0.006998119002673775, - 0.007995038999069948, - 0.00699873200210277, - 0.0059983889950672165, - 0.008290698002383579, - 0.005698961998859886, - 0.008002511000086088, - 0.007990065001649782, - 0.008000920999620575, - 0.007994004001375288, - 0.0060702460032189265, - 0.007926733000203967, - 0.007991951999429148, - 0.0020022589960717596, - 0.004993943999579642, - 0.007997232001798693, - 0.00556829300330719, - 0.007057952003378887, - 0.007364971999777481, - 0.00699617500504246, - 0.006997907999902964, - 0.006010372002492659, - 0.010984045002260245, - 0.008000554000318516, - 0.0069925749994581565, - 0.006997967000643257, - 0.006996511998295318, - 0.006996566000452731, - 0.006999626006290782, - 0.007996460997674149, - 0.005998517997795716, - 0.0059949010028503835, - 0.0069965720031177625, - 0.006995342002483085, - 0.008010443001694512, - 0.007983816001797095, - 0.00807205299497582, - 0.005924641001911368, - 0.007992898994416464, - 0.0049989170001936145, - 0.001994942002056632, - 0.006001251000270713, - 0.009996341999794822, - 0.006996201002039015, - 0.007009569999354426, - 0.005992614998831414, - 0.0029895109983044676, - 0.0049970100008067675, - 0.007996861000719946, - 0.005995997002173681, - 0.009997639994253404, - 0.00600160300382413, - 0.00799373300105799, - 0.007998348002729472, - 0.006991235997702461, - 0.005997046995616984, - 0.007999383997230325, - 0.0069934519997332245, - 0.007001720005064271, - 0.007999565998034086, - 0.007990493999386672, - 0.005494175995409023, - 0.007498842001950834, - 0.00599625900213141, - 0.005995519000862259, - 0.0069984400033717975, - 0.0069979149993741885, - 0.00799809299496701, - 0.0029960610045236535, - 0.005997669999487698, - 0.0059970380025333725, - 0.003995432001829613, - 0.005996744999720249, - 0.006019512002239935, - 0.006975092997890897, - 0.00800279800023418, - 0.007992077997187153, - 0.007003685997915454, - 0.006575943996722344, - 0.009417635999852791, - 0.006990684996708296, - 0.004999432996555697, - 0.0069948259988450445, - 0.007995234002009965, - 0.004678716999478638, - 0.006316300998150837, - 0.00404667900147615, - 0.0006232809973880649, - 0.0005330519998096861, - 0.0004933329983032309, - 0.0004446890015969984, - 0.0004407110027386807, - 0.0004930610011797398, - 0.0004519720023381524, - 0.0004728269996121526, - 0.0017756599991116673, - 0.0004546629934338853, - 0.0004396450021886267, - 0.0004416840019985102, - 0.0004401259939186275, - 0.00044349800009513274, - 0.0004404499995871447, - 0.00043865200132131577, - 0.0027661889980663545, - 0.0005029629974160343, - 0.0004405919971759431, - 0.0007621610056958161, - 0.002285353002662305, - 0.0008558879999327473, - 0.0006801280032959767, - 0.0012495810005930252, - 0.0004579379965434782, - 0.0004506350014708005, - 0.00044887799595016986, - 0.0010036720050266013, - 0.0013806799979647622, - 0.0005710850018658675, - 0.0004599349995260127, - 0.0004619920000550337, - 0.00046049900265643373, - 0.0004525660042418167, - 0.0005878629963262938, - 0.0008274319989141077, - 0.0011609160064836033, - 0.0008610100048827007, - 0.0005931089981459081, - 0.00046643800305901095, - 0.0004572990001179278, - 0.00045643899647984654, - 0.0004507630001171492, - 0.0005164570029592142, - 0.001232244998391252, - 0.0006568030003109016, - 0.0005342459990060888, - 0.0013320750003913417, - 0.0008081330015556887, - 0.0005203359978622757, - 0.0004650750051951036, - 0.0005771090000052936, - 0.0013719560010940768, - 0.0005489649993251078, - 0.0004547159987851046, - 0.0004655149969039485, - 0.00044797000009566545, - 0.00045815500197932124, - 0.0004518229980021715, - 0.0004522119998000562, - 0.0008975939999800175, - 0.0008318710024468601, - 0.0008007409996935166, - 0.00047246599569916725, - 0.00045867999870097265, - 0.0004511660008574836, - 0.00047887899563647807, - 0.0006941660030861385, - 0.0007910339991212822, - 0.0008062839988269843, - 0.0004884090012637898, - 0.0004576439969241619, - 0.000460267998278141, - 0.0004626869995263405, - 0.0004585720016621053 - ], - "iterations": 1 - } - }, - { - "group": null, - "name": "test_stencil", - "fullname": "TestUpdateThetaAndExner", - "params": null, - "param": null, - "extra_info": {}, - "options": { - "disable_gc": false, - "timer": "perf_counter", - "min_rounds": 5, - "max_time": 1.0, - "min_time": 5e-06, - "warmup": 30 - }, - "stats": { - "min": 0.0003912679967470467, - "max": 0.015237425999657717, - "mean": 0.0013409496156103144, - "stddev": 0.0021647318011531686, - "rounds": 419, - "median": 0.0004286800030968152, - "iqr": 0.00018946225281979423, - "q1": 0.0004108402463316452, - "q3": 0.0006003024991514394, - "iqr_outliers": 84, - "stddev_outliers": 53, - "outliers": "53;84", - "ld15iqr": 0.0003912679967470467, - "hd15iqr": 0.0008869759985827841, - "ops": 745.7401742457445, - "total": 0.5618578889407218, - "data": [ - 0.006992770999204367, - 0.006002845999319106, - 0.007988664998265449, - 0.007998503002454527, - 0.0069941710025887005, - 0.004002454996225424, - 0.00799194900173461, - 0.007994188003067393, - 0.006059844003175385, - 0.006943410000531003, - 0.0005575140021392144, - 0.00043153799924766645, - 0.00041680599679239094, - 0.00041543800034560263, - 0.00042569299694150686, - 0.0005926720041316003, - 0.00041415000305278227, - 0.0004103050014236942, - 0.0017139010014943779, - 0.001903217998915352, - 0.0004176619986537844, - 0.00042315600148867816, - 0.0004184680001344532, - 0.0006799550028517842, - 0.0006761240001651458, - 0.0009831689967541024, - 0.000668533997668419, - 0.00041593299829401076, - 0.00040881700260797516, - 0.0004080930011696182, - 0.00041492799937259406, - 0.00040742500277701765, - 0.00041404600051464513, - 0.00040775500383460894, - 0.0004060770006617531, - 0.0008869759985827841, - 0.0023436600022250786, - 0.000502120004966855, - 0.0004662139981519431, - 0.0004103460014448501, - 0.00042253600258845836, - 0.00042197899892926216, - 0.00041369199607288465, - 0.0004148919979343191, - 0.00040894099947763607, - 0.0008331249991897494, - 0.0009188339972752146, - 0.0011976000023423694, - 0.0005343790035112761, - 0.0005851380046806298, - 0.0021103570034028962, - 0.0008271670012618415, - 0.0004581070024869405, - 0.0004140880046179518, - 0.0004116240015719086, - 0.0004176039947196841, - 0.00042191799730062485, - 0.0004241760034346953, - 0.0004107369968551211, - 0.0008361369982594624, - 0.0007823870037100278, - 0.00041465400136075914, - 0.0004115510018891655, - 0.0004210299957776442, - 0.0004142849938943982, - 0.00041533300100127235, - 0.00040430899389320984, - 0.00042454199865460396, - 0.0004089290014235303, - 0.000406731000111904, - 0.003070404003665317, - 0.0007727080010226928, - 0.00042935399687848985, - 0.001190342998597771, - 0.0005357120026019402, - 0.0004272720034350641, - 0.000412878995120991, - 0.0004288260024623014, - 0.00041081499512074515, - 0.00041749299998627976, - 0.0004127149950363673, - 0.0004079899954376742, - 0.0014516169976559468, - 0.0005152920057298616, - 0.000620179001998622, - 0.0004467609978746623, - 0.0008768599946051836, - 0.0004120699959457852, - 0.00041091599996434525, - 0.00041525800043018535, - 0.00041545099520590156, - 0.0004060279970872216, - 0.0004238260007696226, - 0.0004105970001546666, - 0.0010547290003160015, - 0.0005210900053498335, - 0.0007050790009088814, - 0.0004593269986798987, - 0.0013161509996280074, - 0.0004172219996689819, - 0.00042333199962740764, - 0.00041422699723625556, - 0.0004144199992879294, - 0.0004133120019105263, - 0.00041848199907690287, - 0.00040770499617792666, - 0.0004991290043108165, - 0.003759169005206786, - 0.015237425999657717, - 0.007782090004184283, - 0.009160306995909195, - 0.0028064559955964796, - 0.00915346800320549, - 0.0005482709966599941, - 0.0004978110009687953, - 0.0004399819954414852, - 0.0004412439957377501, - 0.0004294829996069893, - 0.0004205899967928417, - 0.0004288300042389892, - 0.00042279899935238063, - 0.007910255997558124, - 0.0006028459974913858, - 0.0005315550006343983, - 0.0004304690010030754, - 0.00042214099812554196, - 0.00042611399840097874, - 0.0004679359990404919, - 0.005422180001914967, - 0.0005011620014556684, - 0.0004977249991497956, - 0.0004438879986992106, - 0.0004302079978515394, - 0.001989188000152353, - 0.0005351639993023127, - 0.000442086995462887, - 0.0004381239996291697, - 0.0004231730054016225, - 0.0016870550025487319, - 0.000426906997745391, - 0.0004465769961825572, - 0.0004446849998203106, - 0.0004286800030968152, - 0.0004445000013220124, - 0.000426254999183584, - 0.00041831999988062307, - 0.00041862199577735737, - 0.0032311420000041835, - 0.0005486490044859238, - 0.0018845430022338405, - 0.0006083530024625361, - 0.0005672050028806552, - 0.00047016999451443553, - 0.0004272270016372204, - 0.0004228180041536689, - 0.0004223380019539036, - 0.0004181379990768619, - 0.0033738900019670837, - 0.00047595299838576466, - 0.00047661099961260334, - 0.0004590819953591563, - 0.00046570500126108527, - 0.0004320980006013997, - 0.0004133690017624758, - 0.0004072520023328252, - 0.0004059830025653355, - 0.004549698001937941, - 0.0005017610019422136, - 0.0004148820007685572, - 0.00040707200241740793, - 0.00046060900058364496, - 0.0004153459958615713, - 0.00041548700392013416, - 0.0004113510003662668, - 0.00041347099613631144, - 0.005194863995711785, - 0.00056048299302347, - 0.000490767000883352, - 0.00042168100480921566, - 0.0004323640023358166, - 0.0004092390008736402, - 0.00041155199869535863, - 0.0004220320042804815, - 0.000409783999202773, - 0.004203453994705342, - 0.00044779200106859207, - 0.00040930099930847064, - 0.00041845300438581035, - 0.0004140390010434203, - 0.0004488760023377836, - 0.00042370900337118655, - 0.00041452699952060357, - 0.004747687999042682, - 0.0004903630033368245, - 0.0004480280040297657, - 0.00041536700155120343, - 0.0004188149978290312, - 0.000415644004533533, - 0.00041204899753211066, - 0.00040947799425339326, - 0.004354295997472946, - 0.0069930769968777895, - 0.00799689099949319, - 0.00699728200561367, - 0.006997250995482318, - 0.006999044999247417, - 0.005997407999529969, - 0.006723004000377841, - 0.007832219002011698, - 0.007435739993525203, - 0.006996334996074438, - 0.005997653002850711, - 0.006998229997407179, - 0.007995398998900782, - 0.006777542002964765, - 0.0072169339982792735, - 0.0070020260027376935, - 0.007032200002868194, - 0.007956777000799775, - 0.008005229996342678, - 0.00798193299851846, - 0.004838141998334322, - 0.006166530001792125, - 0.0005497320016729645, - 0.00048432099720230326, - 0.000535990999196656, - 0.00044581400288734585, - 0.00041610899643274024, - 0.00041372799751115963, - 0.0004242160066496581, - 0.00041348899685544893, - 0.008395297001698054, - 0.000534187005541753, - 0.00047266699402825907, - 0.00041214399971067905, - 0.0004114879993721843, - 0.0037913900014245883, - 0.0005592990055447444, - 0.0004859119944740087, - 0.0004600489992299117, - 0.00041225900349672884, - 0.00041697399865370244, - 0.00042154100083280355, - 0.00041317500290460885, - 0.007000172998232301, - 0.00061266800184967, - 0.0005187950009712949, - 0.00041748499643290415, - 0.00040107100358000025, - 0.0004037179969600402, - 0.00039999699947657064, - 0.0004119789955439046, - 0.0004213430001982488, - 0.00040602299850434065, - 0.0017026019995682873, - 0.0004711169967777096, - 0.00045805399713572115, - 0.0004541180023807101, - 0.0004138450021855533, - 0.00040274299681186676, - 0.0041299000004073605, - 0.0005063479984528385, - 0.00044109100417699665, - 0.0004021089989691973, - 0.00041661599971121177, - 0.0004082930026925169, - 0.000396371993701905, - 0.00039478400140069425, - 0.0004098959980183281, - 0.0004119810037082061, - 0.0018871270003728569, - 0.0008601959998486564, - 0.00046006400225451216, - 0.00040993999573402107, - 0.00040595400059828535, - 0.0004091480004717596, - 0.0004032259967061691, - 0.0004004889997304417, - 0.00040121500205714256, - 0.00040217499918071553, - 0.0004129799999645911, - 0.0004448200052138418, - 0.0023394300005747937, - 0.00046710899914614856, - 0.00045551900257123634, - 0.0004008690011687577, - 0.00039480599662056193, - 0.001220270998601336, - 0.0018803510029101744, - 0.00048549799976171926, - 0.00044783199700759724, - 0.000429797000833787, - 0.00040246099524665624, - 0.00040947300294646993, - 0.00039920500421430916, - 0.0003967489974456839, - 0.003729116993781645, - 0.0005104090014356188, - 0.000521317997481674, - 0.0027825069992104545, - 0.00047056699986569583, - 0.0004743430035887286, - 0.00046953000128269196, - 0.0004117360003874637, - 0.00040142399666365236, - 0.004321016996982507, - 0.0005177839993848465, - 0.0005053080021752976, - 0.00040503800119040534, - 0.00043105600343551487, - 0.0004105489933863282, - 0.00039833600021665916, - 0.0007295989998965524, - 0.00040976700256578624, - 0.0004144609993090853, - 0.0004002330024377443, - 0.0004049560011480935, - 0.0004000150001957081, - 0.00040201900264946744, - 0.003993055994214956, - 0.0007822560000931844, - 0.0004687049950007349, - 0.00039691799611318856, - 0.00039831799949752167, - 0.0020261640020180494, - 0.0005143110029166564, - 0.0004976519994670525, - 0.0004184119970886968, - 0.0004058939957758412, - 0.0004086799963261001, - 0.0003960879985243082, - 0.002382086997386068, - 0.0004926879992126487, - 0.00046877199929440394, - 0.00042050400224979967, - 0.00039525600004708394, - 0.00040530000114813447, - 0.0004025189991807565, - 0.000394793001760263, - 0.002435160000459291, - 0.0004606789952958934, - 0.00044785899808630347, - 0.00043066900252597407, - 0.0004034769954159856, - 0.00040459699812345207, - 0.0003950909958803095, - 0.004040186002384871, - 0.0004892569995718077, - 0.00042297199979657307, - 0.0004027780014439486, - 0.0004029180054203607, - 0.00040767400059849024, - 0.0003985169969382696, - 0.0003951129983761348, - 0.0003979539978899993, - 0.000400479999370873, - 0.00041372000123374164, - 0.00040516800072509795, - 0.0003912679967470467, - 0.0005104939991724677, - 0.00043200599611736834, - 0.0004090770016773604, - 0.0003951390026486479, - 0.000410091997764539, - 0.0003993089994764887, - 0.0003988310054410249, - 0.00040726899896981195, - 0.00039981699956115335, - 0.0008987980036181398, - 0.0005792240044684149, - 0.00047407400415977463, - 0.0033222220008610748, - 0.0004990490051568486, - 0.0004385530046420172, - 0.0004004930015071295, - 0.00039780899533070624, - 0.0004005599985248409, - 0.00040605899994261563, - 0.00040538900066167116, - 0.0006110469985287637, - 0.00047482999798376113, - 0.0004079020000062883, - 0.0004043180015287362, - 0.0008415639968006872, - 0.0007869309993111528, - 0.000415587994211819, - 0.00040972099668579176, - 0.0003951520047849044, - 0.0004081389997736551, - 0.00040210499719250947, - 0.00040502000047126785, - 0.0004125639970880002, - 0.0003981990012107417, - 0.004230504993756767, - 0.0005000940000172704, - 0.0004890950003755279, - 0.0004110769950784743, - 0.0004119040022487752, - 0.00041098099609371275, - 0.00039607299549970776, - 0.0004067860063514672, - 0.0004223889991408214, - 0.0003944570053135976, - 0.00041221699939342216, - 0.00040148200059775263, - 0.000415548995079007, - 0.0004016379971290007, - 0.0003998000029241666, - 0.0006599919943255372, - 0.0005283650025376119, - 0.00042095599928870797, - 0.00043135399755556136, - 0.0004228710022289306, - 0.0004098369972780347, - 0.0004152910041739233, - 0.0004080099970451556, - 0.0032610690032015555, - 0.0004780099989147857, - 0.000468849997560028 - ], - "iterations": 1 - } - } - ], - "datetime": "2025-08-21T16:44:50.947945+00:00", - "version": "5.1.0" -} \ No newline at end of file From 935b11396d818a152e57fc975217442d1d57e746 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 19 Nov 2025 11:05:22 +0100 Subject: [PATCH 105/240] get grid tests to run again --- .../model/common/decomposition/definitions.py | 4 ++-- .../src/icon4py/model/common/decomposition/halo.py | 2 +- .../src/icon4py/model/common/grid/grid_manager.py | 13 +++++-------- .../grid/unit_tests/test_geometry_stencils.py | 4 ++-- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 52a2603819..061ffeee0e 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -167,7 +167,7 @@ def owner_mask(self, dim: gtx.Dimension) -> data_alloc.NDArray: def global_index( self, dim: gtx.Dimension, - entry_type: DecompositionInfo.EntryType = DecompositionInfo.EntryType.ALL, + entry_type: DecompositionInfo.EntryType = EntryType.ALL, ) -> data_alloc.NDArray: match entry_type: case DecompositionInfo.EntryType.ALL: @@ -296,7 +296,7 @@ def __call__(self, communication_handle: SingleNodeResult) -> None: communication_handle.wait() # Implementation of DaCe SDFGConvertible interface - def dace__sdfg__(self, *args: Any, dim: Dimension, wait: bool = True) -> dace.sdfg.sdfg.SDFG: + def dace__sdfg__(self, *args: Any, dim: gtx.Dimension, wait: bool = True) -> dace.sdfg.sdfg.SDFG: sdfg = DummyNestedSDFG().__sdfg__() sdfg.name = "_halo_exchange_wait_" return sdfg diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index b174a4f06c..465350d662 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -48,7 +48,7 @@ def __init__( def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: xp = data_alloc.import_array_ns(self._backend) create_arrays = functools.partial(_create_dummy_decomposition_arrays, array_ns=xp) - decomposition_info = defs.DecompositionInfo(klevels=self._num_levels) + decomposition_info = defs.DecompositionInfo() decomposition_info.set_dimension(dims.EdgeDim, *create_arrays(self._size.num_edges)) decomposition_info.set_dimension(dims.CellDim, *create_arrays(self._size.num_cells)) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 564b3cee52..10ca31249d 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -97,7 +97,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): def __call__( self, - allocator: gtx_typing.FieldBufferAlocationUtil, + allocator: gtx_typing.FieldBufferAllocationUtil, keep_skip_values: bool, decomposer: halo.Decomposer = _single_node_decomposer, run_properties=_single_process_props, @@ -105,22 +105,18 @@ def __call__( if not run_properties.single_node() and isinstance(decomposer, halo.SingleNodeDecomposer): raise InvalidConfigError("Need a Decomposer for multi node run") - self._geometry = self._read_geometry_fields(allocator) - self._grid = self._construct_grid(allocator=allocator, with_skip_values=keep_skip_values) - self._coordinates = self._read_coordinates(allocator) if not self._reader: self.open() self._construct_decomposed_grid( - backend=allocator, + allocator=allocator, with_skip_values=keep_skip_values, decomposer=decomposer, run_properties=run_properties, ) self._coordinates = self._read_coordinates(allocator) self._geometry = self._read_geometry_fields(allocator) - self._geometry = self._read_geometry_fields(allocator) self.close() @@ -453,7 +449,7 @@ def _construct_decomposed_grid( self._grid = grid def _construct_global_params( - self, backend: gtx_typing.Backend, global_size: base.HorizontalGridSize + self, allocator: gtx_typing.FieldBufferAllocationUtil, global_size: base.HorizontalGridSize ): grid_root = self._reader.attribute(gridfile.MandatoryPropertyName.ROOT) grid_level = self._reader.attribute(gridfile.MandatoryPropertyName.LEVEL) @@ -479,8 +475,8 @@ def _construct_global_params( geometry_type=geometry_type, subdivision=icon.GridSubdivision(root=grid_root, level=grid_level), ) + xp = data_alloc.import_array_ns(allocator) global_params = icon.GlobalGridParams.from_fields( - backend=backend, grid_shape=shape, radius=sphere_radius, domain_length=domain_length, @@ -494,6 +490,7 @@ def _construct_global_params( dual_edge_lengths=dual_edge_lengths, cell_areas=cell_areas, dual_cell_areas=dual_cell_areas, + array_ns=xp ) return global_params diff --git a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py index 8bd6b0af70..a8fa0bed1d 100644 --- a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py +++ b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py @@ -15,7 +15,7 @@ from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import definitions, grid_utils, serialbox -from ..fixtures import backend, experiment, grid_savepoint +from ..fixtures import backend, experiment, grid_savepoint, data_provider, download_ser_data, processor_props, ranked_data_path @pytest.mark.level("unit") @@ -28,7 +28,7 @@ def test_edge_length( keep = True grid_file = experiment.grid gm = grid_utils.get_grid_manager_from_identifier( - grid_file, keep_skip_values=keep, num_levels=1, backend=backend + grid_file, keep_skip_values=keep, num_levels=1, allocator=backend ) grid = gm.grid coordinates = gm.coordinates[dims.VertexDim] From 179d4b750673ac2a406138f29214ffc5b5655304 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 19 Nov 2025 11:11:41 +0100 Subject: [PATCH 106/240] get parallel test to run again --- .../src/icon4py/model/common/decomposition/halo.py | 2 +- .../grid/mpi_tests/test_parallel_grid_manager.py | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 465350d662..4ce9155be7 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -281,7 +281,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: cell_halo_levels[self._xp.isin(all_cells, second_halo_cells)] = ( defs.DecompositionFlag.SECOND_HALO_LINE ) - decomp_info = defs.DecompositionInfo(klevels=self._num_levels).set_dimension( + decomp_info = defs.DecompositionInfo().set_dimension( dims.CellDim, all_cells, cell_owner_mask, cell_halo_levels ) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 4729beadf8..25d4f9787c 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -40,7 +40,7 @@ from ...decomposition import utils as decomp_utils from .. import utils -from ..fixtures import backend, experiment, grid_savepoint, icon_grid, processor_props +from ..fixtures import backend, experiment, grid_savepoint, icon_grid, processor_props, data_provider, download_ser_data, ranked_data_path try: @@ -62,7 +62,7 @@ def run_gridmananger_for_multinode( ) -> gm.GridManager: manager = _grid_manager(file, vertical_config) manager( - keep_skip_values=True, backend=None, run_properties=run_properties, decomposer=decomposer + keep_skip_values=True, allocator=None, run_properties=run_properties, decomposer=decomposer ) return manager @@ -80,12 +80,11 @@ def run_grid_manager_for_singlenode( keep_skip_values=True, run_properties=defs.SingleNodeProcessProperties(), decomposer=halo.SingleNodeDecomposer(), - backend=None, + allocator=None, ) return manager -@pytest.mark.xfail("fix test, add data for APE") @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize( @@ -103,7 +102,7 @@ def test_start_end_index( experiment: definitions.Experiment, dim: gtx.Dimension, icon_grid: base.Grid, -) -> None: # fixture +) -> None: caplog.set_level(logging.INFO) grid_file = experiment.grid file = grid_utils.resolve_full_grid_file_name(grid_file) @@ -135,7 +134,7 @@ def test_grid_manager_validate_decomposer(processor_props: defs.ProcessPropertie with pytest.raises(exceptions.InvalidConfigError) as e: manager( keep_skip_values=True, - backend=None, + allocator=None, run_properties=processor_props, decomposer=halo.SingleNodeDecomposer(), ) From 45bbbf6753991cca1054d6ae29ed8414ff52dacc Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 19 Nov 2025 11:42:43 +0100 Subject: [PATCH 107/240] get parallel test to run again --- .../common/decomposition/mpi_decomposition.py | 10 ++++------ .../decomposition/mpi_tests/test_halo.py | 7 +------ .../mpi_tests/test_mpi_decomposition.py | 19 +++++-------------- .../mpi_tests/test_parallel_grid_manager.py | 3 ++- 4 files changed, 12 insertions(+), 27 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py index c86e4b9067..d21b919741 100644 --- a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py +++ b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py @@ -100,7 +100,7 @@ def get_multinode_properties( return _get_processor_properties(with_mpi=True, comm_id=comm_id) -@dataclass(frozen=False) +@dataclass(frozen=True) class MPICommProcessProperties(definitions.ProcessProperties): comm: mpi4py.MPI.Comm @@ -213,7 +213,7 @@ def _slice_field_based_on_dim(self, field: gtx.Field, dim: gtx.Dimension) -> dat trim_length = self._field_size[dim] return field.ndarray[:trim_length, :] except KeyError: - log.warn(f"Trying to trim field of invalid dimension {dim} for exchange. Not trimming.") + log.warning(f"Trying to trim field of invalid dimension {dim} for exchange. Not trimming.") def _get_applied_pattern(self, dim: gtx.Dimension, f: gtx.Field) -> str: # TODO(havogt): the cache is never cleared, consider using functools.lru_cache in a bigger refactoring. @@ -229,9 +229,7 @@ def _get_applied_pattern(self, dim: gtx.Dimension, f: gtx.Field) -> str: unstructured.make_field_descriptor( self._domain_descriptors[dim], array, - arch=util.Architecture.CPU - if isinstance(f, np.ndarray) - else util.Architecture.GPU, + arch=unstructured.Architecture.CPU if isinstance(f.ndarray, np.ndarray) else unstructured.Architecture.GPU, ) ) return self._applied_patterns_cache[key] @@ -250,7 +248,7 @@ def exchange(self, dim: gtx.Dimension, *fields: gtx.Field) -> MultiNodeResult: log.debug(f"exchange for {len(fields)} fields of dimension ='{dim.value}' initiated.") return MultiNodeResult(handle, applied_patterns) - def exchange_and_wait(self, dim: gtx.Dimension, *fields: tuple[gtx.Field, ...]) -> None: + def exchange_and_wait(self, dim: gtx.Dimension, *fields: gtx.Field) -> None: res = self.exchange(dim, *fields) res.wait() log.debug(f"exchange for {len(fields)} fields of dimension ='{dim.value}' done.") diff --git a/model/common/tests/common/decomposition/mpi_tests/test_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_halo.py index 93539b14d9..60fc057bea 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_halo.py @@ -33,14 +33,9 @@ from icon4py.model.common.decomposition import halo from icon4py.model.common.grid import base as base_grid, simple -from icon4py.model.testing import datatest_utils as dt_utils, definitions as test_defs -UGRID_FILE = ( - test_defs.grids_path() - .joinpath(dt_utils.R02B04_GLOBAL) - .joinpath("icon_grid_0013_R02B04_R_ugrid.nc") -) + backend = None diff --git a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py index ab7acf36fc..f1b1b3d230 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py @@ -15,7 +15,6 @@ from icon4py.model.common.grid import horizontal as h_grid, icon from icon4py.model.common.interpolation.interpolation_fields import compute_c_lin_e -from icon4py.model.common.utils import data_allocation as data_alloc try: @@ -31,7 +30,7 @@ from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import definitions as test_defs, parallel_helpers, serialbox from icon4py.model.testing.parallel_helpers import check_comm_size - +from ..utils import dummy_four_ranks from ...fixtures import ( backend, data_provider, @@ -47,14 +46,6 @@ ) -try: - import mpi4py # import mpi4py to check for optional mpi dependency - - from icon4py.model.common.decomposition import mpi_decomposition - - mpi_decomposition.init_mpi() -except ImportError: - pytest.skip("Skipping parallel on single node installation", allow_module_level=True) _log = logging.getLogger(__name__) @@ -262,7 +253,7 @@ def test_create_single_node_runtime_without_mpi( assert isinstance(exchange, definitions.SingleNodeExchange) -@pytest.mark.mpi +#@pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize("dimension", (dims.CellDim, dims.VertexDim, dims.EdgeDim)) def test_exchange_on_dummy_data( @@ -275,7 +266,7 @@ def test_exchange_on_dummy_data( exchange = definitions.create_exchange(processor_props, decomposition_info) grid = grid_savepoint.construct_icon_grid() - number = processor_props.rank + 10.0 + number = processor_props.rank + 10 input_field = data_alloc.constant_field( grid, number, @@ -289,8 +280,8 @@ def test_exchange_on_dummy_data( local_points = decomposition_info.local_index( dimension, definitions.DecompositionInfo.EntryType.OWNED ) - assert np.all(input_field == number) - exchange.exchange_and_wait(dimension, (input_field,)) + assert np.all(input_field.asnumpy() == number) + exchange.exchange_and_wait(dimension, input_field) result = input_field.asnumpy() _log.info(f"rank={processor_props.rank} - num of halo points ={halo_points.shape}") _log.info( diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 25d4f9787c..11700ba5e2 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -85,7 +85,7 @@ def run_grid_manager_for_singlenode( return manager -@pytest.mark.mpi +#@pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize( "experiment", @@ -103,6 +103,7 @@ def test_start_end_index( dim: gtx.Dimension, icon_grid: base.Grid, ) -> None: + processor_props = caplog.set_level(logging.INFO) grid_file = experiment.grid file = grid_utils.resolve_full_grid_file_name(grid_file) From bb190ec63ba78d556fff4e887de27468c790adea Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 20 Nov 2025 13:11:33 +0100 Subject: [PATCH 108/240] start/end index for halos (I) --- .../model/common/decomposition/definitions.py | 8 ++-- .../model/common/grid/grid_refinement.py | 17 ++++++- .../icon4py/model/common/grid/horizontal.py | 2 + .../mpi_tests/test_mpi_decomposition.py | 31 ++++++++++++- .../unit_tests/test_definitions.py | 16 ++++++- .../mpi_tests/test_parallel_grid_manager.py | 2 +- .../test_parallel_grid_refinement.py | 45 +++++++++++++++++++ .../common/grid/unit_tests/test_horizontal.py | 5 +++ .../src/icon4py/model/testing/serialbox.py | 2 +- 9 files changed, 119 insertions(+), 9 deletions(-) create mode 100644 model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 061ffeee0e..9792ff9b76 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -179,7 +179,7 @@ def global_index( case _: raise NotImplementedError() - def get_horizontal_size(self): + def get_horizontal_size(self)->base.HorizontalGridSize: return base.HorizontalGridSize( num_cells=self.global_index(dims.CellDim, self.EntryType.ALL).shape[0], num_edges=self.global_index(dims.EdgeDim, self.EntryType.ALL).shape[0], @@ -189,10 +189,10 @@ def get_horizontal_size(self): def get_halo_size(self, dim: gtx.Dimension, flag: DecompositionFlag) -> int: return np.count_nonzero(self.halo_level_mask(dim, flag)) - def halo_levels(self, dim: gtx.Dimension): + def halo_levels(self, dim: gtx.Dimension)->data_alloc.NDArray: return self._halo_levels[dim] - def halo_level_mask(self, dim: gtx.Dimension, level: DecompositionFlag): + def halo_level_mask(self, dim: gtx.Dimension, level: DecompositionFlag)->data_alloc.NDArray: return np.where(self._halo_levels[dim] == level, True, False) # TODO (@halungge): unused - delete? @@ -429,3 +429,5 @@ class DecompositionFlag(enum.IntEnum): - vertices (NOT USED) - edges that are only on the cell(SECOND_HALO_LINE) """ + + diff --git a/model/common/src/icon4py/model/common/grid/grid_refinement.py b/model/common/src/icon4py/model/common/grid/grid_refinement.py index 040da203cd..9af279aac0 100644 --- a/model/common/src/icon4py/model/common/grid/grid_refinement.py +++ b/model/common/src/icon4py/model/common/grid/grid_refinement.py @@ -14,6 +14,7 @@ import icon4py.model.common.grid.horizontal as h_grid from icon4py.model.common import dimension as dims +from icon4py.model.common.decomposition import definitions as decomposition from icon4py.model.common.utils import data_allocation as data_alloc @@ -155,7 +156,7 @@ def compute_domain_bounds( - dim: gtx.Dimension, refinement_fields: dict[gtx.Dimension, gtx.Field], array_ns: ModuleType = np + dim: gtx.Dimension, refinement_fields: dict[gtx.Dimension, gtx.Field], decomposition_info: decomposition.DecompositionInfo, rank, array_ns: ModuleType = np ) -> tuple[dict[h_grid.Domain, gtx.int32], dict[h_grid.Domain, gtx.int32]]: # type: ignore [name-defined] refinement_ctrl = refinement_fields[dim].ndarray refinement_ctrl = convert_to_non_nested_refinement_values(refinement_ctrl, dim, array_ns) @@ -168,7 +169,7 @@ def compute_domain_bounds( end_index = refinement_ctrl.shape[0] my_zone = domain.zone if ( - my_zone is h_grid.Zone.END or my_zone.is_halo() + my_zone is h_grid.Zone.END ): # TODO(halungge): implement for distributed start_index = refinement_ctrl.shape[0] end_index = refinement_ctrl.shape[0] @@ -194,6 +195,18 @@ def compute_domain_bounds( found = array_ns.where(refinement_ctrl == value)[0] start_index = array_ns.max(found).item() + 1 if found.size > 0 else 0 end_index = refinement_ctrl.shape[0] + elif my_zone.is_halo(): + flag = decomposition.DecompositionFlag(my_zone.level) + not_lateral_boundary = ((refinement_ctrl < 1) | (refinement_ctrl > h_grid.max_boundary_level(dim))) + halo_region = array_ns.where(decomposition_info.halo_level_mask(dim, flag) & not_lateral_boundary)[0] + print(f"{rank} - halo region {my_zone} {halo_region.shape} - {halo_region}") + if halo_region.size > 0: + start_index = array_ns.min(halo_region) + end_index = array_ns.max(halo_region) + 1 + else: + start_index = refinement_ctrl.shape[0] + end_index = refinement_ctrl.shape[0] + start_indices[domain] = gtx.int32(start_index) # type: ignore [attr-defined] end_indices[domain] = gtx.int32(end_index) # type: ignore [attr-defined] return start_indices, end_indices diff --git a/model/common/src/icon4py/model/common/grid/horizontal.py b/model/common/src/icon4py/model/common/grid/horizontal.py index 50d919166d..e97178eda2 100644 --- a/model/common/src/icon4py/model/common/grid/horizontal.py +++ b/model/common/src/icon4py/model/common/grid/horizontal.py @@ -294,6 +294,8 @@ def is_local(self) -> bool: EDGE_ZONES = tuple(Zone) +def max_boundary_level(dim:gtx.Dimension)->int: + return max((d.level for d in _get_zones_for_dim(dim) if d.is_lateral_boundary()), default=1) _ZONE_TO_INDEX_MAPPING = { Zone.END: lambda dim: _icon_domain_index(_ICON_END, dim), diff --git a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py index f1b1b3d230..921f1a7419 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py @@ -19,6 +19,7 @@ try: import mpi4py # import mpi4py to check for optional mpi dependency + from mpi4py import MPI except ImportError: pytest.skip("Skipping parallel on single node installation", allow_module_level=True) @@ -173,6 +174,34 @@ def test_decomposition_info_local_index( _assert_index_partitioning(all_indices, halo_indices, owned_indices) +@pytest.mark.datatest +@pytest.mark.mpi +@pytest.mark.parametrize("dim", (dims.CellDim, dims.EdgeDim, dims.VertexDim) ) +def test_decomposition_info_halo_level_mask( + dim:gtx.Dimension, + experiment:test_defs.Experiment, + decomposition_info:definitions.DecompositionInfo +)->None: + first_halo_level = decomposition_info.halo_level_mask(dim, definitions.DecompositionFlag.FIRST_HALO_LINE) + assert first_halo_level.ndim == 1 + assert np.count_nonzero(first_halo_level) == decomposition_info.get_halo_size(dim, definitions.DecompositionFlag.FIRST_HALO_LINE) + second_halo_level = decomposition_info.halo_level_mask(dim, definitions.DecompositionFlag.SECOND_HALO_LINE) + assert second_halo_level.ndim == 1 + assert np.count_nonzero(second_halo_level) == decomposition_info.get_halo_size(dim, definitions.DecompositionFlag.SECOND_HALO_LINE) + assert np.count_nonzero(first_halo_level) + np.count_nonzero(second_halo_level) == np.count_nonzero(~ decomposition_info.owner_mask(dim)) + +@pytest.mark.datatest +@pytest.mark.mpi +@pytest.mark.parametrize("dim", (dims.CellDim, dims.EdgeDim, dims.VertexDim) ) +def test_decomposition_info_third_level_is_empty( + dim:gtx.Dimension, + experiment:test_defs.Experiment, + decomposition_info:definitions.DecompositionInfo +)->None: + level = decomposition_info.halo_level_mask(dim, definitions.DecompositionFlag.THIRD_HALO_LINE) + assert np.count_nonzero(level) == 0 + + @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize("num", [1, 2, 3, 4, 5, 6, 7, 8]) @@ -253,7 +282,7 @@ def test_create_single_node_runtime_without_mpi( assert isinstance(exchange, definitions.SingleNodeExchange) -#@pytest.mark.mpi +@pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize("dimension", (dims.CellDim, dims.VertexDim, dims.EdgeDim)) def test_exchange_on_dummy_data( diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index 1184984f26..2a805a5b2d 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -9,6 +9,7 @@ import numpy as np import pytest from gt4py.next import common as gtx_common +import gt4py.next as gtx import icon4py.model.common.dimension as dims import icon4py.model.common.utils.data_allocation as data_alloc @@ -17,6 +18,7 @@ from icon4py.model.testing.fixtures import processor_props from .. import utils +from ...grid import utils as grid_utils from ..mpi_tests.test_halo import simple_neighbor_tables from ..utils import dummy_four_ranks @@ -78,7 +80,7 @@ def test_global_to_local_index(offset, rank): # TODO this duplicates and serializes a test from mpi_tests/test_halo.py -@pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) +@pytest.mark.parametrize("dim", grid_utils.main_horizontal_dims()) @pytest.mark.parametrize("rank", [0, 1, 2, 3]) def test_halo_constructor_decomposition_info_global_indices(dim, rank): simple_neighbor_tables = get_neighbor_tables_for_simple_grid() @@ -122,3 +124,15 @@ def test_horizontal_size(rank): assert ( horizontal_size.num_cells == expected_cells ), f"local size mismatch on rank={rank} for {dims.CellDim}: expected {expected_cells}, but was {horizontal_size.num_cells}" + +@pytest.mark.datatest +@pytest.mark.parametrize("dim", grid_utils.main_horizontal_dims()) +def test_decomposition_info_single_node_empty_halo(dim:gtx.Dimension, decomposition_info: definitions.DecompositionInfo, processor_props: definitions.ProcessProperties)->None: + if not processor_props.single_node(): + pytest.xfail() + for level in definitions.DecompositionFlag.__values__: + assert decomposition_info.get_halo_size(dim, level) == 0 + assert np.count_nonzero(decomposition_info.halo_level_mask(dim, level)) == 0 + + + diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 11700ba5e2..cde54eb79b 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -103,7 +103,7 @@ def test_start_end_index( dim: gtx.Dimension, icon_grid: base.Grid, ) -> None: - processor_props = + #decomp_utils.dummy_four_ranks(3) caplog.set_level(logging.INFO) grid_file = experiment.grid file = grid_utils.resolve_full_grid_file_name(grid_file) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py new file mode 100644 index 0000000000..ea83b64543 --- /dev/null +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py @@ -0,0 +1,45 @@ +import pytest +try: + import mpi4py + + import mpi4py.MPI +except ImportError: + pytest.skip("Skipping parallel on single node installation", allow_module_level=True) + +import numpy as np +import gt4py.next as gtx +from icon4py.model.testing import definitions, serialbox +from icon4py.model.common.decomposition import definitions as decomposition, mpi_decomposition +from icon4py.model.common.grid import grid_refinement, horizontal as h_grid +from icon4py.model.common import dimension as dims +from .. import utils +from ..fixtures import backend, experiment, grid_savepoint, icon_grid, data_provider, download_ser_data, ranked_data_path, processor_props +@pytest.mark.parametrize("processor_props", [True], indirect=True) +#@pytest.mark.parametrize("dim", utils.main_horizontal_dims()) +@pytest.mark.parametrize("dim", (dims.EdgeDim,)) +@pytest.mark.parametrize("halos", (h_grid.Zone.HALO,)) +@pytest.mark.parametrize("experiment", (definitions.Experiments.MCH_CH_R04B09,)) +@pytest.mark.mpi +def test_halo_start_end_index(dim: gtx.Dimension, halos: h_grid.Zone, experiment:definitions.Experiment, grid_savepoint: serialbox.IconGridSavepoint, + processor_props:decomposition.ProcessProperties)->None: + ref_grid = grid_savepoint.construct_icon_grid(None, keep_skip_values=True) + print(f"{processor_props.rank}/{processor_props.comm_size} - ref start {grid_savepoint.edge_start_index()}") + print(f"{processor_props.rank}/{processor_props.comm_size} - ref start {grid_savepoint.edge_end_index()}") + + decomposition_info = grid_savepoint.construct_decomposition_info() + refin_ctrl = {dim: grid_savepoint.refin_ctrl(dim) for dim in utils.main_horizontal_dims()} + domain = h_grid.domain(dim)(halos) + print(f"rank = {processor_props.rank}/{processor_props.comm_size} - refinement ctrl {refin_ctrl[dim]}") + processor_props.comm.Barrier() + + start_indices, end_indices = grid_refinement.compute_domain_bounds(dim, refin_ctrl, decomposition_info, processor_props.rank) + print(f"rank = {processor_props.rank}/{processor_props.comm_size} - start {start_indices}") + print(f"rank = {processor_props.rank}/{processor_props.comm_size} - end {end_indices}") + ref_start_index = ref_grid.start_index(domain) + ref_end_index = ref_grid.end_index(domain) + computed_start = start_indices[domain] + computed_end = end_indices[domain] + assert computed_start == ref_start_index, f"{processor_props.rank}/{processor_props.comm_size} - experiment = {experiment.name}: start_index for {domain} does not match: is {computed_start}, expected {ref_start_index}" + assert computed_end == ref_end_index, f"{processor_props.rank}/{processor_props.comm_size} - experiment = {experiment.name}: end_index for {domain} does not match: is {computed_end}, expected {ref_end_index}" + + diff --git a/model/common/tests/common/grid/unit_tests/test_horizontal.py b/model/common/tests/common/grid/unit_tests/test_horizontal.py index 1df2e9cec3..8d138c5dcd 100644 --- a/model/common/tests/common/grid/unit_tests/test_horizontal.py +++ b/model/common/tests/common/grid/unit_tests/test_horizontal.py @@ -59,3 +59,8 @@ def test_halo_zones(zone: h_grid.Zone) -> None: assert zone.is_halo() else: assert not zone.is_halo() + + +@pytest.mark.parametrize("dim, expected", [(dims.CellDim, 4), (dims.VertexDim, 4), (dims.EdgeDim, 8)]) +def test_max_boundary_level(dim:gtx.Dimension, expected)->None: + assert expected == h_grid.max_boundary_level(dim) diff --git a/model/testing/src/icon4py/model/testing/serialbox.py b/model/testing/src/icon4py/model/testing/serialbox.py index 9dbb16e8d0..a1a9c74de4 100644 --- a/model/testing/src/icon4py/model/testing/serialbox.py +++ b/model/testing/src/icon4py/model/testing/serialbox.py @@ -456,7 +456,7 @@ def global_index(self, dim: gtx.Dimension): def decomp_domain(self, dim): return self._read_field_for_dim("decomp_domain", self._read_int32, dim) - def construct_decomposition_info(self): + def construct_decomposition_info(self)->decomposition.DecompositionInfo: return ( decomposition.DecompositionInfo() .set_dimension(*self._get_decomposition_fields(dims.CellDim)) From e89c5e5d500aeed7335867882d7fdb82d0e64176 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 25 Nov 2025 09:22:15 +0100 Subject: [PATCH 109/240] fix parallel test for start_index/end_index --- .../test_benchmark_solve_nonhydro.py | 9 +- .../model/common/decomposition/definitions.py | 20 ++- .../model/common/decomposition/halo.py | 2 +- .../common/decomposition/mpi_decomposition.py | 19 +-- .../icon4py/model/common/grid/grid_manager.py | 8 +- .../model/common/grid/grid_refinement.py | 137 ++++++++++++------ .../icon4py/model/common/grid/horizontal.py | 70 ++++++--- .../decomposition/mpi_tests/test_halo.py | 1 - .../mpi_tests/test_mpi_decomposition.py | 46 +++--- .../unit_tests/test_definitions.py | 14 +- .../mpi_tests/test_parallel_grid_manager.py | 54 ++----- .../test_parallel_grid_refinement.py | 74 ++++++---- .../grid/unit_tests/test_geometry_stencils.py | 10 +- .../grid/unit_tests/test_grid_refinement.py | 54 ++++++- .../common/grid/unit_tests/test_horizontal.py | 6 +- .../model/testing/fixtures/benchmark.py | 5 +- .../src/icon4py/model/testing/serialbox.py | 2 +- 17 files changed, 321 insertions(+), 210 deletions(-) diff --git a/model/atmosphere/dycore/tests/dycore/integration_tests/test_benchmark_solve_nonhydro.py b/model/atmosphere/dycore/tests/dycore/integration_tests/test_benchmark_solve_nonhydro.py index 49c43deb59..95d055b22f 100644 --- a/model/atmosphere/dycore/tests/dycore/integration_tests/test_benchmark_solve_nonhydro.py +++ b/model/atmosphere/dycore/tests/dycore/integration_tests/test_benchmark_solve_nonhydro.py @@ -32,7 +32,6 @@ from icon4py.model.common.metrics import metrics_attributes, metrics_factory from icon4py.model.common.states import prognostic_state as prognostics from icon4py.model.common.utils import data_allocation as data_alloc -from icon4py.model.testing import grid_utils from icon4py.model.testing.fixtures.benchmark import ( geometry_field_source, interpolation_field_source, @@ -62,8 +61,6 @@ def solve_nonhydro( nonhydro_params = solve_nh.NonHydrostaticParams(config) - decomposition_info = grid_utils.construct_decomposition_info(mesh, allocator) - vertical_config = v_grid.VerticalGridConfig( mesh.num_levels, lowest_layer_thickness=50, @@ -204,11 +201,7 @@ def solve_nonhydro( vertical_params=vertical_grid, edge_geometry=edge_geometry, cell_geometry=cell_geometry, - owner_mask=gtx.as_field( - (dims.CellDim,), - decomposition_info.owner_mask(dims.CellDim), # type: ignore[arg-type] # mypy not take the type of owner_mask - allocator=allocator, - ), + owner_mask=grid_geometry.get("cell_owner_mask"), backend=backend_like, ) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 9792ff9b76..cfec030904 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -111,7 +111,9 @@ def set_dimension( self._owner_mask[dim] = owner_mask self._halo_levels[dim] = halo_levels - def local_index(self, dim: gtx.Dimension, entry_type: EntryType = EntryType.ALL): + def local_index( + self, dim: gtx.Dimension, entry_type: EntryType = EntryType.ALL + ) -> data_alloc.NDArray: match entry_type: case DecompositionInfo.EntryType.ALL: return self._to_local_index(dim) @@ -179,7 +181,7 @@ def global_index( case _: raise NotImplementedError() - def get_horizontal_size(self)->base.HorizontalGridSize: + def get_horizontal_size(self) -> base.HorizontalGridSize: return base.HorizontalGridSize( num_cells=self.global_index(dims.CellDim, self.EntryType.ALL).shape[0], num_edges=self.global_index(dims.EdgeDim, self.EntryType.ALL).shape[0], @@ -189,16 +191,12 @@ def get_horizontal_size(self)->base.HorizontalGridSize: def get_halo_size(self, dim: gtx.Dimension, flag: DecompositionFlag) -> int: return np.count_nonzero(self.halo_level_mask(dim, flag)) - def halo_levels(self, dim: gtx.Dimension)->data_alloc.NDArray: + def halo_levels(self, dim: gtx.Dimension) -> data_alloc.NDArray: return self._halo_levels[dim] - def halo_level_mask(self, dim: gtx.Dimension, level: DecompositionFlag)->data_alloc.NDArray: + def halo_level_mask(self, dim: gtx.Dimension, level: DecompositionFlag) -> data_alloc.NDArray: return np.where(self._halo_levels[dim] == level, True, False) - # TODO (@halungge): unused - delete? - def is_on_node(self, dim, index: int, entryType: EntryType = EntryType.ALL) -> bool: - return np.isin(index, self.global_index(dim, entry_type=entryType)).item() - class ExchangeResult(Protocol): def wait(self) -> None: ... @@ -296,7 +294,9 @@ def __call__(self, communication_handle: SingleNodeResult) -> None: communication_handle.wait() # Implementation of DaCe SDFGConvertible interface - def dace__sdfg__(self, *args: Any, dim: gtx.Dimension, wait: bool = True) -> dace.sdfg.sdfg.SDFG: + def dace__sdfg__( + self, *args: Any, dim: gtx.Dimension, wait: bool = True + ) -> dace.sdfg.sdfg.SDFG: sdfg = DummyNestedSDFG().__sdfg__() sdfg.name = "_halo_exchange_wait_" return sdfg @@ -429,5 +429,3 @@ class DecompositionFlag(enum.IntEnum): - vertices (NOT USED) - edges that are only on the cell(SECOND_HALO_LINE) """ - - diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 4ce9155be7..5f3e6ec232 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -128,7 +128,7 @@ def _validate_mapping(self, face_to_rank_mapping: data_alloc.NDArray): f"The distribution assumes more nodes than the current run is scheduled on {self._props} ", ) - def _assert_all_neighbor_tables(self): + def _assert_all_neighbor_tables(self) -> None: # make sure we have all connectivity arrays used in the halo construction relevant_dimension = [ dims.C2E2C, diff --git a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py index d21b919741..d5dca2353d 100644 --- a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py +++ b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py @@ -25,12 +25,9 @@ try: - import ghex # type: ignore [import-not-found] - import mpi4py # type: ignore [import-not-found] - from ghex import ( - unstructured, # type: ignore [import-not-found] - util, # type: ignore [import-not-found] - ) + import ghex # type: ignore [no-untyped-def] + import mpi4py + from ghex import unstructured, util from ghex.context import make_context # type: ignore [import-not-found] mpi4py.rc.initialize = False @@ -43,7 +40,7 @@ if TYPE_CHECKING: - import mpi4py.MPI # type: ignore [import-not-found] + import mpi4py.MPI CommId = Union[int, "mpi4py.MPI.Comm", None] log = logging.getLogger(__name__) @@ -213,7 +210,9 @@ def _slice_field_based_on_dim(self, field: gtx.Field, dim: gtx.Dimension) -> dat trim_length = self._field_size[dim] return field.ndarray[:trim_length, :] except KeyError: - log.warning(f"Trying to trim field of invalid dimension {dim} for exchange. Not trimming.") + log.warning( + f"Trying to trim field of invalid dimension {dim} for exchange. Not trimming." + ) def _get_applied_pattern(self, dim: gtx.Dimension, f: gtx.Field) -> str: # TODO(havogt): the cache is never cleared, consider using functools.lru_cache in a bigger refactoring. @@ -229,7 +228,9 @@ def _get_applied_pattern(self, dim: gtx.Dimension, f: gtx.Field) -> str: unstructured.make_field_descriptor( self._domain_descriptors[dim], array, - arch=unstructured.Architecture.CPU if isinstance(f.ndarray, np.ndarray) else unstructured.Architecture.GPU, + arch=unstructured.Architecture.CPU + if isinstance(f.ndarray, np.ndarray) + else unstructured.Architecture.GPU, ) ) return self._applied_patterns_cache[key] diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 10ca31249d..04ffebdc86 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -105,7 +105,6 @@ def __call__( if not run_properties.single_node() and isinstance(decomposer, halo.SingleNodeDecomposer): raise InvalidConfigError("Need a Decomposer for multi node run") - if not self._reader: self.open() @@ -425,7 +424,10 @@ def _construct_decomposed_grid( refinement_fields = self._read_grid_refinement_fields(allocator) domain_bounds_constructor = functools.partial( - refinement.compute_domain_bounds, refinement_fields=refinement_fields, array_ns=xp + refinement.compute_domain_bounds, + refinement_fields=refinement_fields, + decomposition_info=self._decomposition_info, + array_ns=xp, ) start_index, end_index = icon.get_start_and_end_index(domain_bounds_constructor) @@ -490,7 +492,7 @@ def _construct_global_params( dual_edge_lengths=dual_edge_lengths, cell_areas=cell_areas, dual_cell_areas=dual_cell_areas, - array_ns=xp + array_ns=xp, ) return global_params diff --git a/model/common/src/icon4py/model/common/grid/grid_refinement.py b/model/common/src/icon4py/model/common/grid/grid_refinement.py index 9af279aac0..e280255b5d 100644 --- a/model/common/src/icon4py/model/common/grid/grid_refinement.py +++ b/model/common/src/icon4py/model/common/grid/grid_refinement.py @@ -155,60 +155,105 @@ } +def _refinement_level_in_halo(domain: h_grid.Domain) -> int: + assert domain.zone.is_halo(), "Domain must be a halo Zone." + dim = domain.dim + match dim: + case dims.EdgeDim: + return 6 if domain.zone == h_grid.Zone.HALO else 4 + case dims.CellDim | dims.VertexDim: + return 2 if domain.zone == h_grid.Zone.HALO else 1 + case _: + raise ValueError(f"Invalid domain: {domain}, must be a HALO domain") + + def compute_domain_bounds( - dim: gtx.Dimension, refinement_fields: dict[gtx.Dimension, gtx.Field], decomposition_info: decomposition.DecompositionInfo, rank, array_ns: ModuleType = np + dim: gtx.Dimension, + refinement_fields: dict[gtx.Dimension, gtx.Field], + decomposition_info: decomposition.DecompositionInfo, + array_ns: ModuleType = np, ) -> tuple[dict[h_grid.Domain, gtx.int32], dict[h_grid.Domain, gtx.int32]]: # type: ignore [name-defined] - refinement_ctrl = refinement_fields[dim].ndarray - refinement_ctrl = convert_to_non_nested_refinement_values(refinement_ctrl, dim, array_ns) + """ + Compute the domain bounds (start_index, end_index) based on a grid Domain. + + Icon orders the field arrays according to their + + """ + refinement_ctrl = convert_to_non_nested_refinement_values( + refinement_fields[dim].ndarray, dim, array_ns + ) + owned = decomposition_info.owner_mask(dim) + halo_level_1 = decomposition_info.halo_level_mask( + dim, decomposition.DecompositionFlag.FIRST_HALO_LINE + ) + halo_level_2 = decomposition_info.halo_level_mask( + dim, decomposition.DecompositionFlag.SECOND_HALO_LINE + ) - domains = h_grid.get_domains_for_dim(dim) start_indices = {} end_indices = {} - for domain in domains: - start_index = 0 - end_index = refinement_ctrl.shape[0] - my_zone = domain.zone - if ( - my_zone is h_grid.Zone.END - ): # TODO(halungge): implement for distributed - start_index = refinement_ctrl.shape[0] - end_index = refinement_ctrl.shape[0] - elif my_zone.is_lateral_boundary(): - found = array_ns.where(refinement_ctrl == my_zone.level)[0] - start_index, end_index = ( - (array_ns.min(found).item(), array_ns.max(found).item() + 1) - if found.size > 0 - else (0, 0) - ) - elif my_zone.is_nudging(): - value = _LAST_BOUNDARY[dim].level + my_zone.level - found = array_ns.where(refinement_ctrl == value)[0] - start_index, end_index = ( - (array_ns.min(found).item(), array_ns.max(found).item() + 1) - if found.size > 0 - else (0, 0) - ) - elif my_zone is h_grid.Zone.INTERIOR: - # for the Vertex and Edges the level after the nudging zones are not ordered anymore, so - # we rely on using the end index of the nudging zone for INTERIOR - value = get_nudging_refinement_value(dim) - found = array_ns.where(refinement_ctrl == value)[0] - start_index = array_ns.max(found).item() + 1 if found.size > 0 else 0 - end_index = refinement_ctrl.shape[0] - elif my_zone.is_halo(): - flag = decomposition.DecompositionFlag(my_zone.level) - not_lateral_boundary = ((refinement_ctrl < 1) | (refinement_ctrl > h_grid.max_boundary_level(dim))) - halo_region = array_ns.where(decomposition_info.halo_level_mask(dim, flag) & not_lateral_boundary)[0] - print(f"{rank} - halo region {my_zone} {halo_region.shape} - {halo_region}") - if halo_region.size > 0: - start_index = array_ns.min(halo_region) - end_index = array_ns.max(halo_region) + 1 - else: - start_index = refinement_ctrl.shape[0] - end_index = refinement_ctrl.shape[0] + end_domain = h_grid.domain(dim)(h_grid.Zone.END) + start_indices[end_domain] = gtx.int32(refinement_ctrl.shape[0]) + end_indices[end_domain] = gtx.int32(refinement_ctrl.shape[0]) + + halo_domains = h_grid.get_halo_domains(dim) + for domain in halo_domains: + my_flag = decomposition.DecompositionFlag(domain.zone.level) + upper_boundary_level_1 = _refinement_level_in_halo(h_grid.domain(dim)(h_grid.Zone.HALO)) + not_lateral_boundary_1 = (refinement_ctrl < 1) | ( + refinement_ctrl > upper_boundary_level_1 + ) # edge 6 + halo_region_1 = array_ns.where(halo_level_1 & not_lateral_boundary_1)[0] + not_lateral_boundary_2 = (refinement_ctrl < 1) | ( + refinement_ctrl + > _refinement_level_in_halo(h_grid.domain(dim)(h_grid.Zone.HALO_LEVEL_2)) + ) + + halo_region_2 = array_ns.where(halo_level_2 & not_lateral_boundary_2)[0] + start_halo_2, end_halo_2 = ( + (array_ns.min(halo_region_2), array_ns.max(halo_region_2) + 1) + if halo_region_2.size > 0 + else (refinement_ctrl.size, refinement_ctrl.size) + ) + if my_flag == h_grid.Zone.HALO.level: + start_index = array_ns.min(halo_region_1) if halo_region_1.size > 0 else start_halo_2 + end_index = start_halo_2 + else: + start_index = start_halo_2 + end_index = end_halo_2 start_indices[domain] = gtx.int32(start_index) # type: ignore [attr-defined] end_indices[domain] = gtx.int32(end_index) # type: ignore [attr-defined] + + ordered_domains = h_grid.get_ordered_domains(dim) + for domain in ordered_domains: + value = ( + domain.zone.level + if domain.zone.is_lateral_boundary() + else _LAST_BOUNDARY[dim].level + domain.zone.level + ) + found = array_ns.where((refinement_ctrl == value) & owned)[0] + start_index, end_index = ( + (array_ns.min(found).item(), array_ns.max(found).item() + 1) + if found.size > 0 + else (0, 0) + ) + start_indices[domain] = gtx.int32(start_index) + end_indices[domain] = gtx.int32(end_index) + + interior_domain = h_grid.domain(dim)(h_grid.Zone.INTERIOR) + # for the Vertex and Edges the level after the nudging zones are not ordered anymore, so + # we rely on using the end index of the nudging zone for INTERIOR + nudging = h_grid.get_last_nudging(dim) + start_indices[interior_domain] = end_indices[nudging] + halo_1 = h_grid.domain(dim)(h_grid.Zone.HALO) + end_indices[interior_domain] = start_indices[halo_1] + + local_domain = h_grid.domain(dim)(h_grid.Zone.LOCAL) + halo_1 = h_grid.domain(dim)(h_grid.Zone.HALO) + end_indices[local_domain] = start_indices[halo_1] + start_indices[local_domain] = gtx.int32(0) + return start_indices, end_indices diff --git a/model/common/src/icon4py/model/common/grid/horizontal.py b/model/common/src/icon4py/model/common/grid/horizontal.py index e97178eda2..6fe9290df0 100644 --- a/model/common/src/icon4py/model/common/grid/horizontal.py +++ b/model/common/src/icon4py/model/common/grid/horizontal.py @@ -165,27 +165,29 @@ class Zone(enum.Enum): | from mo_impl_constants.f90 | index | | |:-------------------------------------- |:-------|:-------------------------- | | `min_rledge` (-13) | 0 |`END` | - | `min_rledge_int-2` (-10) | 1 |`HALO_LEVEL_2` | - | `min_rledge_int-1` (-9) | 2 |`HALO` | - | `min_rledge_int` (-8) | 3 |`LOCAL` | - | (-7) | 4 | | unused in icon4py (relevant for nesting) - | (-6) | 5 | | unused in icon4py (relevant for nesting) - | (-5) | 6 | | unused in icon4py (relevant for nesting) - | (-4) | 7 | | unused in icon4py (relevant for nesting) - | (-3) | 8 | | unused in icon4py (relevant for nesting) - | (-2) | 9 | | unused in icon4py (relevant for nesting) - |(-1) | 10 | | unused in icon4py (relevant for nesting) - | `0` | 11 | `INTERIOR` | - | `1` | 12 | `LATERAL_BOUNDARY` | - | `2` | 13 | `LATERAL_BOUNDARY_LEVEL_2` | - | `3` | 14 |`LATERAL_BOUNDARY_LEVEL_3` | - | `4` | 15 |`LATERAL_BOUNDARY_LEVEL_4` | - | `5` | 16 |`LATERAL_BOUNDARY_LEVEL_5` | - | `6` | 17 |`LATERAL_BOUNDARY_LEVEL_6` | - | `7` | 18 | `LATERAL_BOUNDARY_LEVEL_7` | - | `8` | 19 | `LATERAL_BOUNDARY_LEVEL_8`| - | `grf_bdywidth_e` (9) | 20 | `NUDGING` | - | `grf_bdywidth_e+1`, `max_rledge` (10) | 21 | `NUDGING_LEVEL_2` | + |(-12) | 1 | | + |(-11) | 2 | | + | `min_rledge_int-2` (-10) | 3 |`HALO_LEVEL_2` | + | `min_rledge_int-1` (-9) | 4 |`HALO` | + | `min_rledge_int` (-8) | 5 |`LOCAL` | + | (-7) | 6 | | unused in icon4py (relevant for nesting) + | (-6) | 7 | | unused in icon4py (relevant for nesting) + | (-5) | 8 | | unused in icon4py (relevant for nesting) + | (-4) | 9 | | unused in icon4py (relevant for nesting) + | (-3) | 10 | | unused in icon4py (relevant for nesting) + | (-2) | 11 | | unused in icon4py (relevant for nesting) + |(-1) | 12 | | unused in icon4py (relevant for nesting) + | `0` | 13 | `INTERIOR` | + | `1` | 14 | `LATERAL_BOUNDARY` | + | `2` | 15 | `LATERAL_BOUNDARY_LEVEL_2` | + | `3` | 16 |`LATERAL_BOUNDARY_LEVEL_3` | + | `4` | 17 |`LATERAL_BOUNDARY_LEVEL_4` | + | `5` | 18 |`LATERAL_BOUNDARY_LEVEL_5` | + | `6` | 18 |`LATERAL_BOUNDARY_LEVEL_6` | + | `7` | 20 | `LATERAL_BOUNDARY_LEVEL_7` | + | `8` | 12 | `LATERAL_BOUNDARY_LEVEL_8`| + | `grf_bdywidth_e` (9) | 22 | `NUDGING` | + | `grf_bdywidth_e+1`, `max_rledge` (10) | 23 | `NUDGING_LEVEL_2` | """ @@ -294,9 +296,12 @@ def is_local(self) -> bool: EDGE_ZONES = tuple(Zone) -def max_boundary_level(dim:gtx.Dimension)->int: + + +def max_boundary_level(dim: gtx.Dimension) -> int: return max((d.level for d in _get_zones_for_dim(dim) if d.is_lateral_boundary()), default=1) + _ZONE_TO_INDEX_MAPPING = { Zone.END: lambda dim: _icon_domain_index(_ICON_END, dim), Zone.INTERIOR: lambda dim: _icon_domain_index(_ICON_INTERIOR, dim), @@ -413,6 +418,27 @@ def get_domains_for_dim(dim: gtx.Dimension) -> Iterator[Domain]: return domains +def get_halo_domains(dim: gtx.Dimension) -> Iterator[Domain]: + get_domain = domain(dim) + domains = (get_domain(zone) for zone in _get_zones_for_dim(dim) if zone.is_halo()) + return domains + + +def get_ordered_domains(dim: gtx.Dimension) -> Iterator[Domain]: + get_domain = domain(dim) + domains = ( + get_domain(zone) + for zone in _get_zones_for_dim(dim) + if zone.is_lateral_boundary() or zone.is_nudging() + ) + return domains + + +def get_last_nudging(dim): + zone = Zone.NUDGING if dim in (dims.VertexDim, dims.CellDim) else Zone.NUDGING_LEVEL_2 + return domain(dim)(zone) + + def get_start_end_idx_from_icon_arrays( dim: gtx.Dimension, start_indices: dict[gtx.Dimension, np.ndarray], diff --git a/model/common/tests/common/decomposition/mpi_tests/test_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_halo.py index 60fc057bea..46388b9874 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_halo.py @@ -35,7 +35,6 @@ from icon4py.model.common.grid import base as base_grid, simple - backend = None diff --git a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py index 921f1a7419..23a264d2e7 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py @@ -31,7 +31,7 @@ from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import definitions as test_defs, parallel_helpers, serialbox from icon4py.model.testing.parallel_helpers import check_comm_size -from ..utils import dummy_four_ranks + from ...fixtures import ( backend, data_provider, @@ -45,8 +45,7 @@ processor_props, ranked_data_path, ) - - +from ..utils import dummy_four_ranks _log = logging.getLogger(__name__) @@ -176,28 +175,39 @@ def test_decomposition_info_local_index( @pytest.mark.datatest @pytest.mark.mpi -@pytest.mark.parametrize("dim", (dims.CellDim, dims.EdgeDim, dims.VertexDim) ) +@pytest.mark.parametrize("dim", (dims.CellDim, dims.EdgeDim, dims.VertexDim)) def test_decomposition_info_halo_level_mask( - dim:gtx.Dimension, - experiment:test_defs.Experiment, - decomposition_info:definitions.DecompositionInfo -)->None: - first_halo_level = decomposition_info.halo_level_mask(dim, definitions.DecompositionFlag.FIRST_HALO_LINE) + dim: gtx.Dimension, + experiment: test_defs.Experiment, + decomposition_info: definitions.DecompositionInfo, +) -> None: + first_halo_level = decomposition_info.halo_level_mask( + dim, definitions.DecompositionFlag.FIRST_HALO_LINE + ) assert first_halo_level.ndim == 1 - assert np.count_nonzero(first_halo_level) == decomposition_info.get_halo_size(dim, definitions.DecompositionFlag.FIRST_HALO_LINE) - second_halo_level = decomposition_info.halo_level_mask(dim, definitions.DecompositionFlag.SECOND_HALO_LINE) + assert np.count_nonzero(first_halo_level) == decomposition_info.get_halo_size( + dim, definitions.DecompositionFlag.FIRST_HALO_LINE + ) + second_halo_level = decomposition_info.halo_level_mask( + dim, definitions.DecompositionFlag.SECOND_HALO_LINE + ) assert second_halo_level.ndim == 1 - assert np.count_nonzero(second_halo_level) == decomposition_info.get_halo_size(dim, definitions.DecompositionFlag.SECOND_HALO_LINE) - assert np.count_nonzero(first_halo_level) + np.count_nonzero(second_halo_level) == np.count_nonzero(~ decomposition_info.owner_mask(dim)) + assert np.count_nonzero(second_halo_level) == decomposition_info.get_halo_size( + dim, definitions.DecompositionFlag.SECOND_HALO_LINE + ) + assert np.count_nonzero(first_halo_level) + np.count_nonzero( + second_halo_level + ) == np.count_nonzero(~decomposition_info.owner_mask(dim)) + @pytest.mark.datatest @pytest.mark.mpi -@pytest.mark.parametrize("dim", (dims.CellDim, dims.EdgeDim, dims.VertexDim) ) +@pytest.mark.parametrize("dim", (dims.CellDim, dims.EdgeDim, dims.VertexDim)) def test_decomposition_info_third_level_is_empty( - dim:gtx.Dimension, - experiment:test_defs.Experiment, - decomposition_info:definitions.DecompositionInfo -)->None: + dim: gtx.Dimension, + experiment: test_defs.Experiment, + decomposition_info: definitions.DecompositionInfo, +) -> None: level = decomposition_info.halo_level_mask(dim, definitions.DecompositionFlag.THIRD_HALO_LINE) assert np.count_nonzero(level) == 0 diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index 2a805a5b2d..2dccd72ad0 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -6,10 +6,10 @@ # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause +import gt4py.next as gtx import numpy as np import pytest from gt4py.next import common as gtx_common -import gt4py.next as gtx import icon4py.model.common.dimension as dims import icon4py.model.common.utils.data_allocation as data_alloc @@ -17,8 +17,8 @@ from icon4py.model.common.grid import simple from icon4py.model.testing.fixtures import processor_props -from .. import utils from ...grid import utils as grid_utils +from .. import utils from ..mpi_tests.test_halo import simple_neighbor_tables from ..utils import dummy_four_ranks @@ -125,14 +125,16 @@ def test_horizontal_size(rank): horizontal_size.num_cells == expected_cells ), f"local size mismatch on rank={rank} for {dims.CellDim}: expected {expected_cells}, but was {horizontal_size.num_cells}" + @pytest.mark.datatest @pytest.mark.parametrize("dim", grid_utils.main_horizontal_dims()) -def test_decomposition_info_single_node_empty_halo(dim:gtx.Dimension, decomposition_info: definitions.DecompositionInfo, processor_props: definitions.ProcessProperties)->None: +def test_decomposition_info_single_node_empty_halo( + dim: gtx.Dimension, + decomposition_info: definitions.DecompositionInfo, + processor_props: definitions.ProcessProperties, +) -> None: if not processor_props.single_node(): pytest.xfail() for level in definitions.DecompositionFlag.__values__: assert decomposition_info.get_halo_size(dim, level) == 0 assert np.count_nonzero(decomposition_info.halo_level_mask(dim, level)) == 0 - - - diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index cde54eb79b..0244aed6eb 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -40,7 +40,16 @@ from ...decomposition import utils as decomp_utils from .. import utils -from ..fixtures import backend, experiment, grid_savepoint, icon_grid, processor_props, data_provider, download_ser_data, ranked_data_path +from ..fixtures import ( + backend, + data_provider, + download_ser_data, + experiment, + grid_savepoint, + icon_grid, + processor_props, + ranked_data_path, +) try: @@ -85,48 +94,6 @@ def run_grid_manager_for_singlenode( return manager -#@pytest.mark.mpi -@pytest.mark.parametrize("processor_props", [True], indirect=True) -@pytest.mark.parametrize( - "experiment", - [ - (test_defs.Experiments.EXCLAIM_APE), - # (test_defs.Experiments.MCH_CH_R04B09) - ], -) -@pytest.mark.parametrize("dim", utils.horizontal_dims()) -def test_start_end_index( - caplog: Any, - backend: gtx_typing.Backend | None, - processor_props: defs.ProcessProperties, - experiment: definitions.Experiment, - dim: gtx.Dimension, - icon_grid: base.Grid, -) -> None: - #decomp_utils.dummy_four_ranks(3) - caplog.set_level(logging.INFO) - grid_file = experiment.grid - file = grid_utils.resolve_full_grid_file_name(grid_file) - - partitioner = halo.SimpleMetisDecomposer() - manager = gm.GridManager( - file, - v_grid.VerticalGridConfig(1), - icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation(), - ) - manager(backend, keep_skip_values=True, decomposer=partitioner, run_properties=processor_props) - grid = manager.grid - - domains = (h_grid.domain(dim)(z) for z in h_grid.VERTEX_AND_CELL_ZONES) - for domain in domains: - assert icon_grid.start_index(domain) == grid.start_index( - domain - ), f"start index wrong for domain {domain}" - assert icon_grid.end_index(domain) == grid.end_index( - domain - ), f"end index wrong for domain {domain}" - - @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi(min_size=2) def test_grid_manager_validate_decomposer(processor_props: defs.ProcessProperties) -> None: @@ -277,6 +244,7 @@ def assert_gathered_field_against_global( # TODO (halungge): fix non contiguous dimension for embedded in gt4py +@pytest.mark.xfail() # non-contiguous dimensions in gt4py embedded: embedded/nd_array_field.py 576 @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_halo_neighbor_access_c2e( diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py index ea83b64543..4cdc895009 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py @@ -1,45 +1,61 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + import pytest + + try: import mpi4py - import mpi4py.MPI except ImportError: pytest.skip("Skipping parallel on single node installation", allow_module_level=True) -import numpy as np import gt4py.next as gtx -from icon4py.model.testing import definitions, serialbox + from icon4py.model.common.decomposition import definitions as decomposition, mpi_decomposition from icon4py.model.common.grid import grid_refinement, horizontal as h_grid -from icon4py.model.common import dimension as dims +from icon4py.model.testing import definitions, serialbox + from .. import utils -from ..fixtures import backend, experiment, grid_savepoint, icon_grid, data_provider, download_ser_data, ranked_data_path, processor_props +from ..fixtures import ( + backend, + data_provider, + download_ser_data, + experiment, + grid_savepoint, + processor_props, + ranked_data_path, +) + + @pytest.mark.parametrize("processor_props", [True], indirect=True) -#@pytest.mark.parametrize("dim", utils.main_horizontal_dims()) -@pytest.mark.parametrize("dim", (dims.EdgeDim,)) -@pytest.mark.parametrize("halos", (h_grid.Zone.HALO,)) -@pytest.mark.parametrize("experiment", (definitions.Experiments.MCH_CH_R04B09,)) +@pytest.mark.parametrize("dim", utils.main_horizontal_dims()) @pytest.mark.mpi -def test_halo_start_end_index(dim: gtx.Dimension, halos: h_grid.Zone, experiment:definitions.Experiment, grid_savepoint: serialbox.IconGridSavepoint, - processor_props:decomposition.ProcessProperties)->None: +def test_start_end_index( + dim: gtx.Dimension, + experiment: definitions.Experiment, + grid_savepoint: serialbox.IconGridSavepoint, + processor_props: decomposition.ProcessProperties, +) -> None: ref_grid = grid_savepoint.construct_icon_grid(None, keep_skip_values=True) - print(f"{processor_props.rank}/{processor_props.comm_size} - ref start {grid_savepoint.edge_start_index()}") - print(f"{processor_props.rank}/{processor_props.comm_size} - ref start {grid_savepoint.edge_end_index()}") - decomposition_info = grid_savepoint.construct_decomposition_info() refin_ctrl = {dim: grid_savepoint.refin_ctrl(dim) for dim in utils.main_horizontal_dims()} - domain = h_grid.domain(dim)(halos) - print(f"rank = {processor_props.rank}/{processor_props.comm_size} - refinement ctrl {refin_ctrl[dim]}") - processor_props.comm.Barrier() - - start_indices, end_indices = grid_refinement.compute_domain_bounds(dim, refin_ctrl, decomposition_info, processor_props.rank) - print(f"rank = {processor_props.rank}/{processor_props.comm_size} - start {start_indices}") - print(f"rank = {processor_props.rank}/{processor_props.comm_size} - end {end_indices}") - ref_start_index = ref_grid.start_index(domain) - ref_end_index = ref_grid.end_index(domain) - computed_start = start_indices[domain] - computed_end = end_indices[domain] - assert computed_start == ref_start_index, f"{processor_props.rank}/{processor_props.comm_size} - experiment = {experiment.name}: start_index for {domain} does not match: is {computed_start}, expected {ref_start_index}" - assert computed_end == ref_end_index, f"{processor_props.rank}/{processor_props.comm_size} - experiment = {experiment.name}: end_index for {domain} does not match: is {computed_end}, expected {ref_end_index}" - - + start_indices, end_indices = grid_refinement.compute_domain_bounds( + dim, refin_ctrl, decomposition_info + ) + for domain in h_grid.get_domains_for_dim(dim): + ref_start_index = ref_grid.start_index(domain) + ref_end_index = ref_grid.end_index(domain) + computed_start = start_indices[domain] + computed_end = end_indices[domain] + assert ( + computed_start == ref_start_index + ), f"rank={processor_props.rank}/{processor_props.comm_size} - experiment = {experiment.name}: start_index for {domain} does not match: is {computed_start}, expected {ref_start_index}" + assert ( + computed_end == ref_end_index + ), f"rank={processor_props.rank}/{processor_props.comm_size} - experiment = {experiment.name}: end_index for {domain} does not match: is {computed_end}, expected {ref_end_index}" diff --git a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py index a8fa0bed1d..eb83316129 100644 --- a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py +++ b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py @@ -15,7 +15,15 @@ from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import definitions, grid_utils, serialbox -from ..fixtures import backend, experiment, grid_savepoint, data_provider, download_ser_data, processor_props, ranked_data_path +from ..fixtures import ( + backend, + data_provider, + download_ser_data, + experiment, + grid_savepoint, + processor_props, + ranked_data_path, +) @pytest.mark.level("unit") diff --git a/model/common/tests/common/grid/unit_tests/test_grid_refinement.py b/model/common/tests/common/grid/unit_tests/test_grid_refinement.py index d1f7cae5d7..e25586a64d 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_refinement.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_refinement.py @@ -15,10 +15,18 @@ from icon4py.model.common import dimension as dims, model_backends from icon4py.model.common.grid import grid_refinement as refinement, horizontal as h_grid from icon4py.model.common.utils import data_allocation as data_alloc, device_utils -from icon4py.model.testing import definitions as test_defs, grid_utils +from icon4py.model.testing import definitions as test_defs, grid_utils, serialbox from icon4py.model.testing.fixtures import backend, cpu_allocator from .. import utils +from ..fixtures import ( + data_provider, + download_ser_data, + experiment, + grid_savepoint, + processor_props, + ranked_data_path, +) _FALLBACK_FAIL = (-10, -10) @@ -100,11 +108,15 @@ def test_compute_start_index_for_limited_area_grid( expected: dict[h_grid.Zone, tuple[int, int]], cpu_allocator: gtx_typing.FieldBufferAllocationUtil, ) -> None: - grid = grid_utils.get_grid_manager_from_identifier( + grid_manager = grid_utils.get_grid_manager_from_identifier( test_defs.Grids.MCH_OPR_R04B07_DOMAIN01, 1, True, cpu_allocator - ).grid + ) + grid = grid_manager.grid refinement_field = grid.refinement_control - start_index, end_index = refinement.compute_domain_bounds(dim, refinement_field, array_ns=np) + decomposition_info = grid_manager.decomposition_info + start_index, end_index = refinement.compute_domain_bounds( + dim, refinement_field, decomposition_info=decomposition_info, array_ns=np + ) for d, v in start_index.items(): expected_value = expected.get(d.zone, _FALLBACK_FAIL)[0] @@ -126,9 +138,13 @@ def test_compute_domain_bounds_for_global_grid( dim: gtx.Dimension, cpu_allocator: gtx_typing.FieldBufferAllocationUtil, ) -> None: - grid = grid_utils.get_grid_manager_from_identifier(file, 1, True, cpu_allocator).grid + grid_manager = grid_utils.get_grid_manager_from_identifier(file, 1, True, cpu_allocator) + grid = grid_manager.grid refinement_fields = grid.refinement_control - start_index, end_index = refinement.compute_domain_bounds(dim, refinement_fields, array_ns=np) + decomposition_info = grid_manager.decomposition_info + start_index, end_index = refinement.compute_domain_bounds( + dim, refinement_fields, decomposition_info, array_ns=np + ) for k, v in start_index.items(): assert isinstance(v, gtx.int32) if k.zone.is_halo() or k.zone is h_grid.Zone.END: @@ -146,3 +162,29 @@ def test_compute_domain_bounds_for_global_grid( assert ( v == grid.size[k.dim] ), f"Expected end index '{grid.size[k.dim]}' for {dim} in {k.zone}, but got '{v}'" + + +@pytest.mark.parametrize("dim", utils.main_horizontal_dims()) +@pytest.mark.datatest +def test_start_end_index( + dim: gtx.Dimension, + experiment: test_defs.Experiment, + grid_savepoint: serialbox.IconGridSavepoint, +) -> None: + ref_grid = grid_savepoint.construct_icon_grid(None, keep_skip_values=True) + decomposition_info = grid_savepoint.construct_decomposition_info() + refin_ctrl = {dim: grid_savepoint.refin_ctrl(dim) for dim in utils.main_horizontal_dims()} + start_indices, end_indices = refinement.compute_domain_bounds( + dim, refin_ctrl, decomposition_info + ) + for domain in h_grid.get_domains_for_dim(dim): + ref_start_index = ref_grid.start_index(domain) + ref_end_index = ref_grid.end_index(domain) + computed_start = start_indices[domain] + computed_end = end_indices[domain] + assert ( + computed_start == ref_start_index + ), f" experiment = {experiment.name}: start_index for {domain} does not match: is {computed_start}, expected {ref_start_index}" + assert ( + computed_end == ref_end_index + ), f"experiment = {experiment.name}: end_index for {domain} does not match: is {computed_end}, expected {ref_end_index}" diff --git a/model/common/tests/common/grid/unit_tests/test_horizontal.py b/model/common/tests/common/grid/unit_tests/test_horizontal.py index 8d138c5dcd..6fa4fbab9d 100644 --- a/model/common/tests/common/grid/unit_tests/test_horizontal.py +++ b/model/common/tests/common/grid/unit_tests/test_horizontal.py @@ -61,6 +61,8 @@ def test_halo_zones(zone: h_grid.Zone) -> None: assert not zone.is_halo() -@pytest.mark.parametrize("dim, expected", [(dims.CellDim, 4), (dims.VertexDim, 4), (dims.EdgeDim, 8)]) -def test_max_boundary_level(dim:gtx.Dimension, expected)->None: +@pytest.mark.parametrize( + "dim, expected", [(dims.CellDim, 4), (dims.VertexDim, 4), (dims.EdgeDim, 8)] +) +def test_max_boundary_level(dim: gtx.Dimension, expected: int) -> None: assert expected == h_grid.max_boundary_level(dim) diff --git a/model/testing/src/icon4py/model/testing/fixtures/benchmark.py b/model/testing/src/icon4py/model/testing/fixtures/benchmark.py index f230f43e82..47c4c990ac 100644 --- a/model/testing/src/icon4py/model/testing/fixtures/benchmark.py +++ b/model/testing/src/icon4py/model/testing/fixtures/benchmark.py @@ -64,9 +64,8 @@ def interpolation_field_source( ) -> Generator[interpolation_factory.InterpolationFieldsFactory, None, None]: mesh = grid_manager.grid - allocator = model_backends.get_allocator(backend_like) generic_concrete_backend = model_options.customize_backend(None, backend_like) - decomposition_info = grid_utils.construct_decomposition_info(mesh, allocator) + decomposition_info = grid_manager.decomposition_info interpolation_field_source = interpolation_factory.InterpolationFieldsFactory( grid=mesh, @@ -91,7 +90,7 @@ def metrics_field_source( allocator = model_backends.get_allocator(backend_like) generic_concrete_backend = model_options.customize_backend(None, backend_like) - decomposition_info = grid_utils.construct_decomposition_info(mesh, allocator) + decomposition_info = grid_manager.decomposition_info vertical_config = v_grid.VerticalGridConfig( mesh.num_levels, diff --git a/model/testing/src/icon4py/model/testing/serialbox.py b/model/testing/src/icon4py/model/testing/serialbox.py index a1a9c74de4..0b2541ad87 100644 --- a/model/testing/src/icon4py/model/testing/serialbox.py +++ b/model/testing/src/icon4py/model/testing/serialbox.py @@ -456,7 +456,7 @@ def global_index(self, dim: gtx.Dimension): def decomp_domain(self, dim): return self._read_field_for_dim("decomp_domain", self._read_int32, dim) - def construct_decomposition_info(self)->decomposition.DecompositionInfo: + def construct_decomposition_info(self) -> decomposition.DecompositionInfo: return ( decomposition.DecompositionInfo() .set_dimension(*self._get_decomposition_fields(dims.CellDim)) From 464302a5558316b9fe630edd4b5a0b9e45bc57c2 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 26 Nov 2025 08:53:28 +0100 Subject: [PATCH 110/240] add tests WIP(1) --- .../icon4py/model/common/grid/grid_manager.py | 60 +---- .../src/icon4py/model/common/grid/gridfile.py | 63 +++-- .../mpi_tests/test_parallel_grid_manager.py | 246 +++++++++++++++--- .../common/grid/unit_tests/test_gridfile.py | 15 +- .../src/icon4py/model/testing/serialbox.py | 1 + 5 files changed, 264 insertions(+), 121 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 04ffebdc86..3ff841cf82 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -291,60 +291,6 @@ def _read_grid_refinement_fields( } return refinement_control_fields - def _read_start_end_indices( - self, - ) -> tuple[ - dict[gtx.Dimension, data_alloc.NDArray], - dict[gtx.Dimension, data_alloc.NDArray], - ]: - """ " - Read the start/end indices from the grid file. - - This should be used for a single node run. In the case of a multi node distributed run the start and end indices need to be reconstructed from the decomposed grid. - """ - _child_dom = 0 - grid_refinement_dimensions = { - dims.CellDim: gridfile.DimensionName.CELL_GRF, - dims.EdgeDim: gridfile.DimensionName.EDGE_GRF, - dims.VertexDim: gridfile.DimensionName.VERTEX_GRF, - } - max_refinement_control_values = { - dim: self._reader.dimension(name) for dim, name in grid_refinement_dimensions.items() - } - start_index_names = { - dims.CellDim: gridfile.GridRefinementName.START_INDEX_CELLS, - dims.EdgeDim: gridfile.GridRefinementName.START_INDEX_EDGES, - dims.VertexDim: gridfile.GridRefinementName.START_INDEX_VERTICES, - } - - start_indices = { - dim: self._get_index_field(name, transpose=False, apply_offset=True)[_child_dom] - for dim, name in start_index_names.items() - } - for dim in grid_refinement_dimensions: - assert start_indices[dim].shape == ( - max_refinement_control_values[dim], - ), f"start index array for {dim} has wrong shape" - - end_index_names = { - dims.CellDim: gridfile.GridRefinementName.END_INDEX_CELLS, - dims.EdgeDim: gridfile.GridRefinementName.END_INDEX_EDGES, - dims.VertexDim: gridfile.GridRefinementName.END_INDEX_VERTICES, - } - end_indices = { - dim: self._get_index_field(name, transpose=False, apply_offset=False)[_child_dom] - for dim, name in end_index_names.items() - } - for dim in grid_refinement_dimensions: - assert start_indices[dim].shape == ( - max_refinement_control_values[dim], - ), f"start index array for {dim} has wrong shape" - assert end_indices[dim].shape == ( - max_refinement_control_values[dim], - ), f"start index array for {dim} has wrong shape" - - return start_indices, end_indices - @property def grid(self) -> icon.IconGrid: return self._grid @@ -515,9 +461,9 @@ def _read_full_grid_size(self) -> base.HorizontalGridSize: As the grid file contains the _full_ (non-distributed) grid, these are the sizes of prior to distribution. """ - num_cells = self._reader.dimension(gridfile.DimensionName.CELL_NAME) - num_edges = self._reader.dimension(gridfile.DimensionName.EDGE_NAME) - num_vertices = self._reader.dimension(gridfile.DimensionName.VERTEX_NAME) + num_cells = self._reader.dimension(gridfile.DynamicDimension.CELL_NAME) + num_edges = self._reader.dimension(gridfile.DynamicDimension.EDGE_NAME) + num_vertices = self._reader.dimension(gridfile.DynamicDimension.VERTEX_NAME) full_grid_size = base.HorizontalGridSize( num_vertices=num_vertices, num_edges=num_edges, num_cells=num_cells ) diff --git a/model/common/src/icon4py/model/common/grid/gridfile.py b/model/common/src/icon4py/model/common/grid/gridfile.py index e9403bcfe9..602b0253bd 100644 --- a/model/common/src/icon4py/model/common/grid/gridfile.py +++ b/model/common/src/icon4py/model/common/grid/gridfile.py @@ -8,7 +8,7 @@ import enum import logging -from typing import Protocol +from typing import Protocol, Any import numpy as np from gt4py import next as gtx @@ -134,37 +134,62 @@ class MandatoryPropertyName(PropertyName): LEVEL = "grid_level" ROOT = "grid_root" - class DimensionName(GridFileName): + ... + +class DynamicDimension(DimensionName): """Dimension values (sizes) used in grid file.""" #: number of vertices VERTEX_NAME = "vertex" - #: number of edges EDGE_NAME = "edge" #: number of cells CELL_NAME = "cell" + #: number of child domains (for nesting) + MAX_CHILD_DOMAINS = "max_chdom" + + + +class FixedSizeDimension(DimensionName): + size: int + + def __new__(cls, value:str, size_:int): + obj = str.__new__(cls) + obj._value_ = value + obj.size = size_ + return obj + #: number of edges in a diamond: 4 - DIAMOND_EDGE_SIZE = "no" + DIAMOND_EDGE_SIZE = ("no",4) #: number of edges/cells neighboring one vertex: 6 (for regular, non pentagons) - NEIGHBORS_TO_VERTEX_SIZE = "ne" + NEIGHBORS_TO_VERTEX_SIZE = ("ne",6) #: number of cells edges, vertices and cells neighboring a cell: 3 - NEIGHBORS_TO_CELL_SIZE = "nv" + NEIGHBORS_TO_CELL_SIZE = ("nv",3) #: number of vertices/cells neighboring an edge: 2 - NEIGHBORS_TO_EDGE_SIZE = "nc" - - #: number of child domains (for nesting) - MAX_CHILD_DOMAINS = "max_chdom" + NEIGHBORS_TO_EDGE_SIZE = ("nc",2) #: Grid refinement: maximal number in grid-refinement (refin_ctl) array for each dimension - CELL_GRF = "cell_grf" - EDGE_GRF = "edge_grf" - VERTEX_GRF = "vert_grf" + CELL_GRF = ("cell_grf", 14) + EDGE_GRF = ("edge_grf", 28) + VERTEX_GRF = ("vert_grf",14) + + + def __str__(self): + return f"{self.name}({self.name}: {self.size})" + + def __hash__(self): + return hash((self.name, self.size)) + + def __eq__(self, other: Any) -> bool: + """Check equality based on zone name and level.""" + if not isinstance(other, FixedSizeDimension): + return False + return (self.name, self.size) == (other.name, other.size) class FieldName(GridFileName): ... @@ -327,7 +352,7 @@ def variable( ) -> np.ndarray: """Read a field from the grid file. - If a index array is given it only reads the values at those positions. + If an index array is given it only reads the values at those positions. Args: name: name of the field to read indices: indices to read if requesting a restricted set of indices. We assume this be a 1d array it will be applied to the 1. dimension (after transposition) @@ -340,12 +365,18 @@ def variable( try: variable = self._dataset.variables[name] - slicer = [slice(None) for _ in range(variable.ndim)] + variable_size = variable.ndim + n = (variable.shape[0],) if variable_size > 1 else () + target_shape = n + (-1,) + + + slicer = [slice(None) for _ in range(variable_size)] if indices is not None and indices.size > 0: + # apply the slicing to the correct dimension slicer[(1 if transpose else 0)] = indices _log.debug(f"reading {name}: transposing = {transpose}") data = variable[tuple(slicer)] - data = np.array(data, dtype=dtype) + data = np.array(data, dtype=dtype).ravel(order="K").reshape(target_shape) return np.transpose(data) if transpose else data except KeyError as err: msg = f"{name} does not exist in dataset" diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 0244aed6eb..7fe90625a7 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -17,19 +17,20 @@ from gt4py import next as gtx from gt4py.next import typing as gtx_typing -import icon4py.model.common.grid.gridfile + from icon4py.model.common import dimension as dims, exceptions from icon4py.model.common.decomposition import definitions as defs, halo, mpi_decomposition from icon4py.model.common.grid import ( base, geometry, +geometry_stencils, geometry_attributes, grid_manager as gm, gridfile, horizontal as h_grid, vertical as v_grid, ) -from icon4py.model.common.interpolation.interpolation_fields import compute_geofac_div +from icon4py.model.common.interpolation import interpolation_fields from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import ( definitions, @@ -94,6 +95,7 @@ def run_grid_manager_for_singlenode( return manager + @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi(min_size=2) def test_grid_manager_validate_decomposer(processor_props: defs.ProcessProperties) -> None: @@ -109,6 +111,50 @@ def test_grid_manager_validate_decomposer(processor_props: defs.ProcessPropertie assert "Need a Decomposer for multi" in e.value.args[0] +@pytest.mark.mpi +@pytest.mark.parametrize( + "field_offset", + [dims.C2V, dims.E2V, dims.V2C, dims.E2C, dims.C2E, dims.V2E, dims.C2E2C, dims.V2E2V], +) +def test_local_connectivities( + processor_props: defs.ProcessProperties, + caplog: Iterator, + field_offset: gtx.FieldOffset, +) -> None: + caplog.set_level(logging.INFO) # type: ignore [attr-defined] + grid = utils.run_grid_manager( + test_defs.Grids.R02B04_GLOBAL, keep_skip_values=True, backend=None + ).grid + partitioner = halo.SimpleMetisDecomposer() + face_face_connectivity = grid.get_connectivity(dims.C2E2C).ndarray + neighbor_tables = grid.get_neighbor_tables() + labels = partitioner(face_face_connectivity, num_partitions=processor_props.comm_size) + halo_generator = halo.IconLikeHaloConstructor( + connectivities=neighbor_tables, + run_properties=processor_props, + num_levels=1, + ) + + decomposition_info = halo_generator(labels) + + connectivity = gm.construct_local_connectivity( + field_offset, decomposition_info, connectivity=grid.get_connectivity(field_offset).ndarray + ) + # there is an neighbor list for each index of the target dimension on the node + assert ( + connectivity.shape[0] + == decomposition_info.global_index( + field_offset.target[0], defs.DecompositionInfo.EntryType.ALL + ).size + ) + # all neighbor indices are valid local indices + assert np.max(connectivity) == np.max( + decomposition_info.local_index(field_offset.source, defs.DecompositionInfo.EntryType.ALL) + ) + # TODO what else to assert? + # - outer halo entries have SKIP_VALUE neighbors (depends on offsets) + + @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @@ -244,14 +290,15 @@ def assert_gathered_field_against_global( # TODO (halungge): fix non contiguous dimension for embedded in gt4py -@pytest.mark.xfail() # non-contiguous dimensions in gt4py embedded: embedded/nd_array_field.py 576 +#@pytest.mark.xfail() # non-contiguous dimensions in gt4py embedded: embedded/nd_array_field.py 576 @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) +@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL, )) def test_halo_neighbor_access_c2e( - processor_props: defs.ProcessProperties, backend: gtx_typing.Backend | None + processor_props: defs.ProcessProperties, backend: gtx_typing.Backend | None, grid:definitions.GridDescription ) -> None: # processor_props = decomp_utils.DummyProps(1) - file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) + file = grid_utils.resolve_full_grid_file_name(grid) print(f"running on {processor_props.comm}") single_node = run_grid_manager_for_singlenode(file, vertical_config) single_node_grid = single_node.grid @@ -273,7 +320,8 @@ def test_halo_neighbor_access_c2e( single_node_edge_orientation = single_node_geometry.get( geometry_attributes.CELL_NORMAL_ORIENTATION ) - compute_geofac_div.with_backend(None)( + # has to be computed in gt4py-embedded + interpolation_fields.compute_geofac_div.with_backend(None)( primal_edge_length=single_node_edge_length, area=single_node_cell_area, edge_orientation=single_node_edge_orientation, @@ -312,7 +360,8 @@ def test_halo_neighbor_access_c2e( edge_orientation = distributed_geometry.get(geometry_attributes.CELL_NORMAL_ORIENTATION) geofac_div = data_alloc.zero_field(distributed_grid, dims.CellDim, dims.C2EDim) - compute_geofac_div.with_backend(None)( + #has to be computed in gt4py-embedded + interpolation_fields.compute_geofac_div.with_backend(None)( primal_edge_length=edge_length, area=cell_area, edge_orientation=edge_orientation, @@ -332,44 +381,161 @@ def test_halo_neighbor_access_c2e( @pytest.mark.mpi -@pytest.mark.parametrize( - "field_offset", - [dims.C2V, dims.E2V, dims.V2C, dims.E2C, dims.C2E, dims.V2E, dims.C2E2C, dims.V2E2V], -) -def test_local_connectivities( - processor_props: defs.ProcessProperties, - caplog: Iterator, - field_offset: gtx.FieldOffset, +@pytest.mark.parametrize("processor_props", [True], indirect=True) +#@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL, definitions.Grids.MCH_CH_R04B09_DSL)) +@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL, )) +def test_halo_neighbor_access_e2v( + processor_props: defs.ProcessProperties, backend: gtx_typing.Backend | None, grid: definitions.GridDescription ) -> None: - caplog.set_level(logging.INFO) # type: ignore [attr-defined] - grid = utils.run_grid_manager( - test_defs.Grids.R02B04_GLOBAL, keep_skip_values=True, backend=None - ).grid - partitioner = halo.SimpleMetisDecomposer() - face_face_connectivity = grid.get_connectivity(dims.C2E2C).ndarray - neighbor_tables = grid.get_neighbor_tables() - labels = partitioner(face_face_connectivity, num_partitions=processor_props.comm_size) - halo_generator = halo.IconLikeHaloConstructor( - connectivities=neighbor_tables, + print(f"running on {processor_props.comm}") + file = grid_utils.resolve_full_grid_file_name(grid) + single_node = run_grid_manager_for_singlenode(file, vertical_config) + single_node_grid = single_node.grid + single_node_geometry = geometry.GridGeometry( + backend=backend, + grid=single_node_grid, + coordinates=single_node.coordinates, + decomposition_info=single_node.decomposition_info, + extra_fields=single_node.geometry_fields, + metadata=geometry_attributes.attrs, + ) + reference_tangent_x = single_node_geometry.get(geometry_attributes.EDGE_TANGENT_X).asnumpy() + reference_tangent_y = single_node_geometry.get(geometry_attributes.EDGE_TANGENT_Y).asnumpy() + print( + f"rank = {processor_props.rank} : single node computed field reference has size {reference_tangent_x.shape}" + ) + multinode_grid_manager = run_gridmananger_for_multinode( + file=file, + vertical_config=vertical_config, run_properties=processor_props, - num_levels=1, + decomposer=halo.SimpleMetisDecomposer(), ) + distributed_grid = multinode_grid_manager.grid + decomposition_info = multinode_grid_manager.decomposition_info - decomposition_info = halo_generator(labels) + print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") + print( + f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + ) + distributed_coordinates = multinode_grid_manager.coordinates + vertex_lat = distributed_coordinates.get(dims.VertexDim)["lat"] + vertex_lon = distributed_coordinates.get(dims.VertexDim)["lon"] + tangent_orientation = multinode_grid_manager.geometry_fields.get(gridfile.GeometryName.TANGENT_ORIENTATION) - connectivity = gm.construct_local_connectivity( - field_offset, decomposition_info, connectivity=grid.get_connectivity(field_offset).ndarray + #output fields + tangent_x = data_alloc.zero_field(distributed_grid, dims.EdgeDim) + tangent_y = data_alloc.zero_field(distributed_grid, dims.EdgeDim) + tangent_z = data_alloc.zero_field(distributed_grid, dims.EdgeDim) + + geometry_stencils.cartesian_coordinates_of_edge_tangent.with_backend(backend)( + vertex_lat = vertex_lat, + vertex_lon=vertex_lon, + edge_orientation = tangent_orientation, + domain = {dims.EdgeDim:(0, distributed_grid.num_edges)}, + offset_provider=distributed_grid.connectivities, + out=(tangent_x, tangent_y, tangent_z)) + + + # only the computation of the tangent uses neighbor access + assert_gathered_field_against_global( + decomposition_info, + processor_props, + dims.EdgeDim, + global_reference_field=reference_tangent_x, + local_field=tangent_x.asnumpy(), ) - # there is an neighbor list for each index of the target dimension on the node - assert ( - connectivity.shape[0] - == decomposition_info.global_index( - field_offset.target[0], defs.DecompositionInfo.EntryType.ALL - ).size + assert_gathered_field_against_global( + decomposition_info, + processor_props, + dims.EdgeDim, + global_reference_field=reference_tangent_y, + local_field=tangent_y.asnumpy(), ) - # all neighbor indices are valid local indices - assert np.max(connectivity) == np.max( - decomposition_info.local_index(field_offset.source, defs.DecompositionInfo.EntryType.ALL) + + print(f"rank = {processor_props.rank} - DONE") + +@pytest.mark.skip +@pytest.mark.mpi +@pytest.mark.parametrize("processor_props", [True], indirect=True) +def test_halo_neighbor_access_v2e( + processor_props: defs.ProcessProperties, backend: gtx_typing.Backend | None, experiment: definitions.Experiment +) -> None: + file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) + print(f"running on {processor_props.comm}") + single_node = run_grid_manager_for_singlenode(file, vertical_config) + single_node_grid = single_node.grid + single_node_geometry = geometry.GridGeometry( + backend=backend, + grid=single_node_grid, + coordinates=single_node.coordinates, + decomposition_info=single_node.decomposition_info, + extra_fields=single_node.geometry_fields, + metadata=geometry_attributes.attrs, ) - # TODO what else to assert? - # - outer halo entries have SKIP_VALUE neighbors (depends on offsets) + dual_edge_length = single_node_geometry.get(geometry_attributes.DUAL_EDGE_LENGTH) + edge_orientation = single_node_geometry.get(geometry_attributes.TANGENT_ORIENTATION) + dual_area = single_node_geometry.get(geometry_attributes.DUAL_AREA) + edge_owner_mask = single_node.decomposition_info.owner_mask(dims.EdgeDim) + reference = data_alloc.zero_field(single_node_grid, dims.EdgeDim, dims.V2EDim) + interpolation_fields.compute_geofac_rot.with_backend(None)(dual_edge_length=dual_edge_length, + edge_orientation=edge_orientation, + dual_area=dual_area, + owner_mask=edge_owner_mask, + out=reference, + offset_provider = single_node_grid.connectivities) + + + print( + f"rank = {processor_props.rank} : single node computed field reference has size {reference.asnumpy().shape}" + ) + multinode_grid_manager = run_gridmananger_for_multinode( + file=file, + vertical_config=vertical_config, + run_properties=processor_props, + decomposer=halo.SimpleMetisDecomposer(), + ) + distributed_grid = multinode_grid_manager.grid + decomposition_info = multinode_grid_manager.decomposition_info + + print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") + print( + f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + ) + distributed_coordinates = multinode_grid_manager.coordinates + extra_geometry_fields = multinode_grid_manager.geometry_fields + distributed_geometry = geometry.GridGeometry( + backend=backend, + grid=distributed_grid, + coordinates=distributed_coordinates, + decomposition_info=decomposition_info, + extra_fields=extra_geometry_fields, + metadata=geometry_attributes.attrs, + ) + + dual_edge_length = distributed_geometry.get(geometry_attributes.DUAL_EDGE_LENGTH) + edge_orientation = distributed_geometry.get(geometry_attributes.CELL_NORMAL_ORIENTATION) + dual_area = distributed_geometry.get(geometry_attributes.DUAL_AREA) + + #output fields + geofac_rot = data_alloc.zero_field(distributed_grid, dims.VertexDim, dims.V2EDim) + + interpolation_fields.compute_geofac_rot.with_backend(None)(dual_edge_length=dual_edge_length, + edge_orientation=edge_orientation, + dual_area=dual_area, + owner_mask=edge_owner_mask, + out=geofac_rot, + offset_provider=single_node_grid.connectivities) + + + # only the computation of the tangent uses neighbor access + assert_gathered_field_against_global( + decomposition_info, + processor_props, + dims.VertexDim, + global_reference_field=reference.asnumpy(), + local_field=geofac_rot.asnumpy(), + ) + + print(f"rank = {processor_props.rank} - DONE") + +def test_halo_access_c2v(): diff --git a/model/common/tests/common/grid/unit_tests/test_gridfile.py b/model/common/tests/common/grid/unit_tests/test_gridfile.py index 030cb52ee8..25797bbec1 100644 --- a/model/common/tests/common/grid/unit_tests/test_gridfile.py +++ b/model/common/tests/common/grid/unit_tests/test_gridfile.py @@ -38,11 +38,11 @@ def test_grid_file_dimension() -> None: parser = gridfile.GridFile(global_grid_file, transformation=gridfile.NoTransformation()) try: parser.open() - assert parser.dimension(gridfile.DimensionName.CELL_NAME) == grid_descriptor.sizes["cell"] + assert parser.dimension(gridfile.DynamicDimension.CELL_NAME) == grid_descriptor.sizes["cell"] assert ( - parser.dimension(gridfile.DimensionName.VERTEX_NAME) == grid_descriptor.sizes["vertex"] + parser.dimension(gridfile.DynamicDimension.VERTEX_NAME) == grid_descriptor.sizes["vertex"] ) - assert parser.dimension(gridfile.DimensionName.EDGE_NAME) == grid_descriptor.sizes["edge"] + assert parser.dimension(gridfile.DynamicDimension.EDGE_NAME) == grid_descriptor.sizes["edge"] except Exception: pytest.fail() finally: @@ -58,13 +58,13 @@ def test_grid_file_vertex_cell_edge_dimensions( parser = gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) try: parser.open() - assert parser.dimension(gridfile.DimensionName.CELL_NAME) == grid_savepoint.num( + assert parser.dimension(gridfile.DynamicDimension.CELL_NAME) == grid_savepoint.num( dims.CellDim ) - assert parser.dimension(gridfile.DimensionName.VERTEX_NAME) == grid_savepoint.num( + assert parser.dimension(gridfile.DynamicDimension.VERTEX_NAME) == grid_savepoint.num( dims.VertexDim ) - assert parser.dimension(gridfile.DimensionName.EDGE_NAME) == grid_savepoint.num( + assert parser.dimension(gridfile.DynamicDimension.EDGE_NAME) == grid_savepoint.num( dims.EdgeDim ) except Exception as error: @@ -80,7 +80,7 @@ def test_int_variable( ) -> None: file = gridtest_utils.resolve_full_grid_file_name(grid_descriptor) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: - edge_dim = parser.dimension(gridfile.DimensionName.EDGE_NAME) + edge_dim = parser.dimension(gridfile.DynamicDimension.EDGE_NAME) # use a test field that does not contain Pentagons test_field = parser.int_variable( gridfile.ConnectivityName.C2E, apply_transformation=apply_transformation @@ -140,7 +140,6 @@ def test_index_read_for_2d_connectivity( file = gridtest_utils.resolve_full_grid_file_name(grid_descriptor) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: indices_to_read = np.asarray(selection) if len(selection) > 0 else None - # TODO(halungge): grid_file.ConnectivityName.V2E:P 2 D fields full_field = parser.int_variable(field, transpose=True, apply_transformation=apply_offset) selective_field = parser.int_variable( field, indices=indices_to_read, transpose=True, apply_transformation=apply_offset diff --git a/model/testing/src/icon4py/model/testing/serialbox.py b/model/testing/src/icon4py/model/testing/serialbox.py index 0b2541ad87..ec744ebde0 100644 --- a/model/testing/src/icon4py/model/testing/serialbox.py +++ b/model/testing/src/icon4py/model/testing/serialbox.py @@ -557,6 +557,7 @@ def c_intp(self): return self._get_field("c_intp", dims.VertexDim, dims.V2CDim) def c_lin_e(self): + return self._get_field("c_lin_e", dims.EdgeDim, dims.E2CDim) def e_bln_c_s(self): From b0746c3c1e101d6420bbea1a74e4a22aecd09121 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Mon, 1 Dec 2025 18:15:06 +0100 Subject: [PATCH 111/240] fix typing --- .../stencils/compute_cell_2_vertex_interpolation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/model/common/src/icon4py/model/common/interpolation/stencils/compute_cell_2_vertex_interpolation.py b/model/common/src/icon4py/model/common/interpolation/stencils/compute_cell_2_vertex_interpolation.py index c729c08d7f..902bd1f949 100644 --- a/model/common/src/icon4py/model/common/interpolation/stencils/compute_cell_2_vertex_interpolation.py +++ b/model/common/src/icon4py/model/common/interpolation/stencils/compute_cell_2_vertex_interpolation.py @@ -15,8 +15,8 @@ @gtx.field_operator def _compute_cell_2_vertex_interpolation( - cell_in: gtx.Field[[dims.CellDim, dims.KDim], types.wpfloat], - c_int: gtx.Field[[dims.VertexDim, V2CDim], types.wpfloat], + cell_in: gtx.Field[gtx.Dims[dims.CellDim, dims.KDim], types.wpfloat], + c_int: gtx.Field[gtx.Dims[dims.VertexDim, dims.V2CDim], types.wpfloat], ) -> gtx.Field[[dims.VertexDim, dims.KDim], types.wpfloat]: vert_out = neighbor_sum(c_int * cell_in(V2C), axis=V2CDim) return vert_out @@ -25,7 +25,7 @@ def _compute_cell_2_vertex_interpolation( @gtx.program(grid_type=gtx.GridType.UNSTRUCTURED) def compute_cell_2_vertex_interpolation( cell_in: gtx.Field[[dims.CellDim, dims.KDim], types.wpfloat], - c_int: gtx.Field[[dims.VertexDim, V2CDim], types.wpfloat], + c_int: gtx.Field[[dims.VertexDim, dims.V2CDim], types.wpfloat], vert_out: gtx.Field[[dims.VertexDim, dims.KDim], types.wpfloat], horizontal_start: gtx.int32, horizontal_end: gtx.int32, From 12dbe47556cd8291fa8e74897585b7c53b123415 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Mon, 1 Dec 2025 18:15:30 +0100 Subject: [PATCH 112/240] pre-commit --- .../src/icon4py/model/common/grid/gridfile.py | 22 +++++++++---------- .../common/grid/unit_tests/test_gridfile.py | 11 +++++++--- .../src/icon4py/model/testing/serialbox.py | 1 - 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/gridfile.py b/model/common/src/icon4py/model/common/grid/gridfile.py index 602b0253bd..128eaf2903 100644 --- a/model/common/src/icon4py/model/common/grid/gridfile.py +++ b/model/common/src/icon4py/model/common/grid/gridfile.py @@ -8,7 +8,7 @@ import enum import logging -from typing import Protocol, Any +from typing import Any, Protocol import numpy as np from gt4py import next as gtx @@ -134,8 +134,9 @@ class MandatoryPropertyName(PropertyName): LEVEL = "grid_level" ROOT = "grid_root" -class DimensionName(GridFileName): - ... + +class DimensionName(GridFileName): ... + class DynamicDimension(DimensionName): """Dimension values (sizes) used in grid file.""" @@ -151,33 +152,31 @@ class DynamicDimension(DimensionName): MAX_CHILD_DOMAINS = "max_chdom" - class FixedSizeDimension(DimensionName): size: int - def __new__(cls, value:str, size_:int): + def __new__(cls, value: str, size_: int): obj = str.__new__(cls) obj._value_ = value obj.size = size_ return obj #: number of edges in a diamond: 4 - DIAMOND_EDGE_SIZE = ("no",4) + DIAMOND_EDGE_SIZE = ("no", 4) #: number of edges/cells neighboring one vertex: 6 (for regular, non pentagons) - NEIGHBORS_TO_VERTEX_SIZE = ("ne",6) + NEIGHBORS_TO_VERTEX_SIZE = ("ne", 6) #: number of cells edges, vertices and cells neighboring a cell: 3 - NEIGHBORS_TO_CELL_SIZE = ("nv",3) + NEIGHBORS_TO_CELL_SIZE = ("nv", 3) #: number of vertices/cells neighboring an edge: 2 - NEIGHBORS_TO_EDGE_SIZE = ("nc",2) + NEIGHBORS_TO_EDGE_SIZE = ("nc", 2) #: Grid refinement: maximal number in grid-refinement (refin_ctl) array for each dimension CELL_GRF = ("cell_grf", 14) EDGE_GRF = ("edge_grf", 28) - VERTEX_GRF = ("vert_grf",14) - + VERTEX_GRF = ("vert_grf", 14) def __str__(self): return f"{self.name}({self.name}: {self.size})" @@ -369,7 +368,6 @@ def variable( n = (variable.shape[0],) if variable_size > 1 else () target_shape = n + (-1,) - slicer = [slice(None) for _ in range(variable_size)] if indices is not None and indices.size > 0: # apply the slicing to the correct dimension diff --git a/model/common/tests/common/grid/unit_tests/test_gridfile.py b/model/common/tests/common/grid/unit_tests/test_gridfile.py index 25797bbec1..800c1daad3 100644 --- a/model/common/tests/common/grid/unit_tests/test_gridfile.py +++ b/model/common/tests/common/grid/unit_tests/test_gridfile.py @@ -38,11 +38,16 @@ def test_grid_file_dimension() -> None: parser = gridfile.GridFile(global_grid_file, transformation=gridfile.NoTransformation()) try: parser.open() - assert parser.dimension(gridfile.DynamicDimension.CELL_NAME) == grid_descriptor.sizes["cell"] assert ( - parser.dimension(gridfile.DynamicDimension.VERTEX_NAME) == grid_descriptor.sizes["vertex"] + parser.dimension(gridfile.DynamicDimension.CELL_NAME) == grid_descriptor.sizes["cell"] + ) + assert ( + parser.dimension(gridfile.DynamicDimension.VERTEX_NAME) + == grid_descriptor.sizes["vertex"] + ) + assert ( + parser.dimension(gridfile.DynamicDimension.EDGE_NAME) == grid_descriptor.sizes["edge"] ) - assert parser.dimension(gridfile.DynamicDimension.EDGE_NAME) == grid_descriptor.sizes["edge"] except Exception: pytest.fail() finally: diff --git a/model/testing/src/icon4py/model/testing/serialbox.py b/model/testing/src/icon4py/model/testing/serialbox.py index ec744ebde0..0b2541ad87 100644 --- a/model/testing/src/icon4py/model/testing/serialbox.py +++ b/model/testing/src/icon4py/model/testing/serialbox.py @@ -557,7 +557,6 @@ def c_intp(self): return self._get_field("c_intp", dims.VertexDim, dims.V2CDim) def c_lin_e(self): - return self._get_field("c_lin_e", dims.EdgeDim, dims.E2CDim) def e_bln_c_s(self): From 88ae0eca77a029657bf87c1b7ed95a42e81a9322 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Mon, 1 Dec 2025 18:16:27 +0100 Subject: [PATCH 113/240] add tests (WIP) --- .../mpi_tests/test_parallel_grid_manager.py | 460 ++++++++++++++++-- 1 file changed, 416 insertions(+), 44 deletions(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 7fe90625a7..38f67c9d43 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -17,20 +17,22 @@ from gt4py import next as gtx from gt4py.next import typing as gtx_typing - from icon4py.model.common import dimension as dims, exceptions from icon4py.model.common.decomposition import definitions as defs, halo, mpi_decomposition from icon4py.model.common.grid import ( base, geometry, -geometry_stencils, geometry_attributes, + geometry_stencils, grid_manager as gm, gridfile, horizontal as h_grid, vertical as v_grid, ) from icon4py.model.common.interpolation import interpolation_fields +from icon4py.model.common.interpolation.stencils.compute_cell_2_vertex_interpolation import ( + _compute_cell_2_vertex_interpolation, +) from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import ( definitions, @@ -61,7 +63,7 @@ pytest.skip("Skipping parallel on single node installation", allow_module_level=True) log = logging.getLogger(__file__) -vertical_config = v_grid.VerticalGridConfig(num_levels=1) +vertical_config = v_grid.VerticalGridConfig(num_levels=10) def run_gridmananger_for_multinode( @@ -95,7 +97,6 @@ def run_grid_manager_for_singlenode( return manager - @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi(min_size=2) def test_grid_manager_validate_decomposer(processor_props: defs.ProcessProperties) -> None: @@ -111,6 +112,7 @@ def test_grid_manager_validate_decomposer(processor_props: defs.ProcessPropertie assert "Need a Decomposer for multi" in e.value.args[0] + @pytest.mark.mpi @pytest.mark.parametrize( "field_offset", @@ -155,7 +157,6 @@ def test_local_connectivities( # - outer halo entries have SKIP_VALUE neighbors (depends on offsets) - @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_fields_distribute_and_gather(processor_props: defs.ProcessProperties, caplog: Any) -> None: @@ -290,12 +291,14 @@ def assert_gathered_field_against_global( # TODO (halungge): fix non contiguous dimension for embedded in gt4py -#@pytest.mark.xfail() # non-contiguous dimensions in gt4py embedded: embedded/nd_array_field.py 576 +# @pytest.mark.xfail() # non-contiguous dimensions in gt4py embedded: embedded/nd_array_field.py 576 @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL, )) +@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL,)) def test_halo_neighbor_access_c2e( - processor_props: defs.ProcessProperties, backend: gtx_typing.Backend | None, grid:definitions.GridDescription + processor_props: defs.ProcessProperties, + backend: gtx_typing.Backend | None, + grid: definitions.GridDescription, ) -> None: # processor_props = decomp_utils.DummyProps(1) file = grid_utils.resolve_full_grid_file_name(grid) @@ -360,7 +363,7 @@ def test_halo_neighbor_access_c2e( edge_orientation = distributed_geometry.get(geometry_attributes.CELL_NORMAL_ORIENTATION) geofac_div = data_alloc.zero_field(distributed_grid, dims.CellDim, dims.C2EDim) - #has to be computed in gt4py-embedded + # has to be computed in gt4py-embedded interpolation_fields.compute_geofac_div.with_backend(None)( primal_edge_length=edge_length, area=cell_area, @@ -382,10 +385,192 @@ def test_halo_neighbor_access_c2e( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -#@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL, definitions.Grids.MCH_CH_R04B09_DSL)) -@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL, )) +@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL,)) +def test_halo_access_e2c2v( + processor_props: defs.ProcessProperties, + backend: gtx_typing.Backend | None, + grid: definitions.GridDescription, +) -> None: + file = grid_utils.resolve_full_grid_file_name(grid) + print(f"running on {processor_props.comm}") + single_node = run_grid_manager_for_singlenode(file, vertical_config) + single_node_grid = single_node.grid + single_node_geometry = geometry.GridGeometry( + backend=backend, + grid=single_node_grid, + coordinates=single_node.coordinates, + decomposition_info=single_node.decomposition_info, + extra_fields=single_node.geometry_fields, + metadata=geometry_attributes.attrs, + ) + print( + f"rank = {processor_props.rank} : single node grid has size {single_node.decomposition_info.get_horizontal_size()!r}" + ) + reference_u = single_node_geometry.get(geometry_attributes.EDGE_NORMAL_VERTEX_U).asnumpy() + reference_v = single_node_geometry.get(geometry_attributes.EDGE_NORMAL_VERTEX_V).asnumpy() + multinode_grid_manager = run_gridmananger_for_multinode( + file=file, + vertical_config=vertical_config, + run_properties=processor_props, + decomposer=halo.SimpleMetisDecomposer(), + ) + distributed_grid = multinode_grid_manager.grid + extra_geometry_fields = multinode_grid_manager.geometry_fields + decomposition_info = multinode_grid_manager.decomposition_info + + print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") + print( + f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + ) + distributed_coordinates = multinode_grid_manager.coordinates + distributed_geometry = geometry.GridGeometry( + backend=backend, + grid=distributed_grid, + coordinates=distributed_coordinates, + decomposition_info=decomposition_info, + extra_fields=extra_geometry_fields, + metadata=geometry_attributes.attrs, + ) + vertex_lat = distributed_geometry.get(geometry_attributes.VERTEX_LAT) + vertex_lon = distributed_geometry.get(geometry_attributes.VERTEX_LON) + x = distributed_geometry.get(geometry_attributes.EDGE_NORMAL_X) + y = distributed_geometry.get(geometry_attributes.EDGE_NORMAL_Y) + z = distributed_geometry.get(geometry_attributes.EDGE_NORMAL_Z) + + u0 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) + v0 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) + u1 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) + v1 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) + u2 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) + v2 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) + u3 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) + v3 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) + + geometry_stencils.zonal_and_meridional_component_of_edge_field_at_vertex.with_backend(backend)( + vertex_lat, + vertex_lon, + x, + y, + z, + out=(u0, v0, u1, v1, u2, v2, u3, v3), + offset_provider={"E2C2V": distributed_grid.get_connectivity(dims.E2C2V)}, + ) + u_component = np.vstack((u0.asnumpy(), u1.asnumpy(), u2.asnumpy(), u3.asnumpy())).T + v_component = np.vstack((v0.asnumpy(), v1.asnumpy(), v2.asnumpy(), v3.asnumpy())).T + + assert_gathered_field_against_global( + decomposition_info, + processor_props, + dims.EdgeDim, + global_reference_field=reference_u, + local_field=u_component, + ) + assert_gathered_field_against_global( + decomposition_info, + processor_props, + dims.EdgeDim, + global_reference_field=reference_v, + local_field=v_component, + ) + + +@pytest.mark.mpi +@pytest.mark.parametrize("processor_props", [True], indirect=True) +@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL,)) +def test_halo_access_e2c( + processor_props: defs.ProcessProperties, + backend: gtx_typing.Backend | None, + grid: definitions.GridDescription, +) -> None: + file = grid_utils.resolve_full_grid_file_name(grid) + print(f"running on {processor_props.comm}") + single_node = run_grid_manager_for_singlenode(file, vertical_config) + single_node_grid = single_node.grid + single_node_geometry = geometry.GridGeometry( + backend=backend, + grid=single_node_grid, + coordinates=single_node.coordinates, + decomposition_info=single_node.decomposition_info, + extra_fields=single_node.geometry_fields, + metadata=geometry_attributes.attrs, + ) + print( + f"rank = {processor_props.rank} : single node grid has size {single_node.decomposition_info.get_horizontal_size()!r}" + ) + reference_u = single_node_geometry.get(geometry_attributes.EDGE_NORMAL_CELL_U).asnumpy() + reference_v = single_node_geometry.get(geometry_attributes.EDGE_NORMAL_CELL_V).asnumpy() + multinode_grid_manager = run_gridmananger_for_multinode( + file=file, + vertical_config=vertical_config, + run_properties=processor_props, + decomposer=halo.SimpleMetisDecomposer(), + ) + distributed_grid = multinode_grid_manager.grid + extra_geometry_fields = multinode_grid_manager.geometry_fields + decomposition_info = multinode_grid_manager.decomposition_info + + print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") + print( + f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + ) + distributed_coordinates = multinode_grid_manager.coordinates + distributed_geometry = geometry.GridGeometry( + backend=backend, + grid=distributed_grid, + coordinates=distributed_coordinates, + decomposition_info=decomposition_info, + extra_fields=extra_geometry_fields, + metadata=geometry_attributes.attrs, + ) + cell_lat = distributed_geometry.get(geometry_attributes.CELL_LAT) + cell_lon = distributed_geometry.get(geometry_attributes.CELL_LON) + x = distributed_geometry.get(geometry_attributes.EDGE_NORMAL_X) + y = distributed_geometry.get(geometry_attributes.EDGE_NORMAL_Y) + z = distributed_geometry.get(geometry_attributes.EDGE_NORMAL_Z) + + u0 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) + v0 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) + u1 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) + v1 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) + + geometry_stencils.zonal_and_meridional_component_of_edge_field_at_cell_center.with_backend( + backend + )( + cell_lat, + cell_lon, + x, + y, + z, + out=(u0, v0, u1, v1), + offset_provider={"E2C": distributed_grid.get_connectivity(dims.E2C)}, + ) + u_component = np.vstack((u0.asnumpy(), u1.asnumpy())).T + v_component = np.vstack((v0.asnumpy(), v1.asnumpy())).T + + assert_gathered_field_against_global( + decomposition_info, + processor_props, + dims.EdgeDim, + global_reference_field=reference_u, + local_field=u_component, + ) + assert_gathered_field_against_global( + decomposition_info, + processor_props, + dims.EdgeDim, + global_reference_field=reference_v, + local_field=v_component, + ) + + +@pytest.mark.mpi +@pytest.mark.parametrize("processor_props", [True], indirect=True) +# @pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL, definitions.Grids.MCH_CH_R04B09_DSL)) +@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL,)) def test_halo_neighbor_access_e2v( - processor_props: defs.ProcessProperties, backend: gtx_typing.Backend | None, grid: definitions.GridDescription + processor_props: defs.ProcessProperties, + backend: gtx_typing.Backend | None, + grid: definitions.GridDescription, ) -> None: print(f"running on {processor_props.comm}") file = grid_utils.resolve_full_grid_file_name(grid) @@ -420,23 +605,24 @@ def test_halo_neighbor_access_e2v( distributed_coordinates = multinode_grid_manager.coordinates vertex_lat = distributed_coordinates.get(dims.VertexDim)["lat"] vertex_lon = distributed_coordinates.get(dims.VertexDim)["lon"] - tangent_orientation = multinode_grid_manager.geometry_fields.get(gridfile.GeometryName.TANGENT_ORIENTATION) + tangent_orientation = multinode_grid_manager.geometry_fields.get( + gridfile.GeometryName.TANGENT_ORIENTATION + ) - #output fields tangent_x = data_alloc.zero_field(distributed_grid, dims.EdgeDim) tangent_y = data_alloc.zero_field(distributed_grid, dims.EdgeDim) tangent_z = data_alloc.zero_field(distributed_grid, dims.EdgeDim) geometry_stencils.cartesian_coordinates_of_edge_tangent.with_backend(backend)( - vertex_lat = vertex_lat, + vertex_lat=vertex_lat, vertex_lon=vertex_lon, - edge_orientation = tangent_orientation, - domain = {dims.EdgeDim:(0, distributed_grid.num_edges)}, + edge_orientation=tangent_orientation, + domain={dims.EdgeDim: (0, distributed_grid.num_edges)}, offset_provider=distributed_grid.connectivities, - out=(tangent_x, tangent_y, tangent_z)) - + out=(tangent_x, tangent_y, tangent_z), + ) - # only the computation of the tangent uses neighbor access + # only the computation of the tangent uses neighbor access assert_gathered_field_against_global( decomposition_info, processor_props, @@ -454,13 +640,16 @@ def test_halo_neighbor_access_e2v( print(f"rank = {processor_props.rank} - DONE") -@pytest.mark.skip + @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) +@pytest.mark.parametrize("experiment", (definitions.Grids.R02B04_GLOBAL,)) def test_halo_neighbor_access_v2e( - processor_props: defs.ProcessProperties, backend: gtx_typing.Backend | None, experiment: definitions.Experiment + processor_props: defs.ProcessProperties, + backend: gtx_typing.Backend | None, + experiment: definitions.Experiment, ) -> None: - file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) + file = grid_utils.resolve_full_grid_file_name(experiment) print(f"running on {processor_props.comm}") single_node = run_grid_manager_for_singlenode(file, vertical_config) single_node_grid = single_node.grid @@ -473,17 +662,23 @@ def test_halo_neighbor_access_v2e( metadata=geometry_attributes.attrs, ) dual_edge_length = single_node_geometry.get(geometry_attributes.DUAL_EDGE_LENGTH) - edge_orientation = single_node_geometry.get(geometry_attributes.TANGENT_ORIENTATION) + edge_orientation = single_node_geometry.get(geometry_attributes.VERTEX_EDGE_ORIENTATION) dual_area = single_node_geometry.get(geometry_attributes.DUAL_AREA) - edge_owner_mask = single_node.decomposition_info.owner_mask(dims.EdgeDim) - reference = data_alloc.zero_field(single_node_grid, dims.EdgeDim, dims.V2EDim) - interpolation_fields.compute_geofac_rot.with_backend(None)(dual_edge_length=dual_edge_length, - edge_orientation=edge_orientation, - dual_area=dual_area, - owner_mask=edge_owner_mask, - out=reference, - offset_provider = single_node_grid.connectivities) + owner_mask = single_node.decomposition_info.owner_mask(dims.VertexDim) + reference = data_alloc.zero_field(single_node_grid, dims.VertexDim, dims.V2EDim) + lateral_boundary_start = single_node_grid.start_index( + h_grid.vertex_domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2) + ) + interpolation_fields.compute_geofac_rot.with_backend(None)( + dual_edge_length, + edge_orientation, + dual_area, + owner_mask, + out=reference, + domain={dims.VertexDim: (lateral_boundary_start, single_node_grid.num_vertices)}, + offset_provider={"V2E": single_node_grid.get_connectivity(dims.V2E)}, + ) print( f"rank = {processor_props.rank} : single node computed field reference has size {reference.asnumpy().shape}" @@ -513,21 +708,20 @@ def test_halo_neighbor_access_v2e( ) dual_edge_length = distributed_geometry.get(geometry_attributes.DUAL_EDGE_LENGTH) - edge_orientation = distributed_geometry.get(geometry_attributes.CELL_NORMAL_ORIENTATION) + edge_orientation = distributed_geometry.get(geometry_attributes.VERTEX_EDGE_ORIENTATION) dual_area = distributed_geometry.get(geometry_attributes.DUAL_AREA) - #output fields geofac_rot = data_alloc.zero_field(distributed_grid, dims.VertexDim, dims.V2EDim) - interpolation_fields.compute_geofac_rot.with_backend(None)(dual_edge_length=dual_edge_length, - edge_orientation=edge_orientation, - dual_area=dual_area, - owner_mask=edge_owner_mask, - out=geofac_rot, - offset_provider=single_node_grid.connectivities) - + interpolation_fields.compute_geofac_rot.with_backend(None)( + dual_edge_length=dual_edge_length, + edge_orientation=edge_orientation, + dual_area=dual_area, + owner_mask=decomposition_info.owner_mask(dims.VertexDim), + out=geofac_rot, + offset_provider={"V2E": distributed_grid.get_connectivity(dims.V2E)}, + ) - # only the computation of the tangent uses neighbor access assert_gathered_field_against_global( decomposition_info, processor_props, @@ -536,6 +730,184 @@ def test_halo_neighbor_access_v2e( local_field=geofac_rot.asnumpy(), ) - print(f"rank = {processor_props.rank} - DONE") + print(f"rank = {processor_props.rank}/{processor_props.comm_size} - DONE") -def test_halo_access_c2v(): + +@pytest.mark.mpi +@pytest.mark.parametrize("processor_props", [True], indirect=True) +@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL,)) +def test_halo_neighbor_access_c2e2c( + processor_props: defs.ProcessProperties, + backend: gtx_typing.Backend | None, + grid: definitions.GridDescription, +) -> None: + file = grid_utils.resolve_full_grid_file_name(grid) + center_weight = 0.3 + xp = data_alloc.import_array_ns(allocator=backend) + start_zone = h_grid.cell_domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2) + print(f"running on {processor_props.comm}") + single_node = run_grid_manager_for_singlenode(file, vertical_config) + single_node_grid = single_node.grid + single_node_geometry = geometry.GridGeometry( + backend=backend, + grid=single_node_grid, + coordinates=single_node.coordinates, + decomposition_info=single_node.decomposition_info, + extra_fields=single_node.geometry_fields, + metadata=geometry_attributes.attrs, + ) + reference = interpolation_fields._compute_c_bln_avg( + single_node_grid.get_connectivity(dims.C2E2C).ndarray, + single_node_geometry.get(geometry_attributes.CELL_LAT).ndarray, + single_node_geometry.get(geometry_attributes.CELL_LON).ndarray, + center_weight, + horizontal_start=single_node_grid.start_index(start_zone), + array_ns=xp, + ) + + print( + f"rank = {processor_props.rank} : single node computed field reference has size {reference.shape}" + ) + multinode_grid_manager = run_gridmananger_for_multinode( + file=file, + vertical_config=vertical_config, + run_properties=processor_props, + decomposer=halo.SimpleMetisDecomposer(), + ) + distributed_grid = multinode_grid_manager.grid + decomposition_info = multinode_grid_manager.decomposition_info + + print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") + print( + f"rank = {processor_props.rank}: halo size for 'CellDim' (1 : {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + ) + distributed_coordinates = multinode_grid_manager.coordinates + extra_geometry_fields = multinode_grid_manager.geometry_fields + distributed_geometry = geometry.GridGeometry( + backend=backend, + grid=distributed_grid, + coordinates=distributed_coordinates, + decomposition_info=decomposition_info, + extra_fields=extra_geometry_fields, + metadata=geometry_attributes.attrs, + ) + + c_bln_avg = interpolation_fields._compute_c_bln_avg( + distributed_grid.get_connectivity(dims.C2E2C).ndarray, + distributed_geometry.get(geometry_attributes.CELL_LAT).ndarray, + distributed_geometry.get(geometry_attributes.CELL_LON).ndarray, + center_weight, + horizontal_start=distributed_grid.start_index(start_zone), + array_ns=xp, + ) + + print( + f"rank = {processor_props.rank}/{processor_props.comm_size} - computed field has shape =({c_bln_avg.shape})" + ) + + assert_gathered_field_against_global( + decomposition_info, + processor_props, + dims.CellDim, + global_reference_field=reference, + local_field=c_bln_avg, + ) + + print(f"rank = {processor_props.rank}/{processor_props.comm_size} - DONE") + + +@pytest.mark.mpi +@pytest.mark.parametrize("processor_props", [True], indirect=True) +def test_halo_access_v2c(processor_props, backend): + file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) + print(f"running on {processor_props.comm}") + single_node = run_grid_manager_for_singlenode(file, vertical_config) + single_node_grid = single_node.grid + + full_cell_k_field = data_alloc.random_field( + single_node.grid, dims.CellDim, dims.KDim, low=1.0, high=10.0, allocator=backend + ) + print( + f"rank = {processor_props.rank} / {processor_props.comm_size}: single node input field has size {full_cell_k_field.asnumpy().shape}" + ) + full_coef = data_alloc.random_field( + single_node_grid, dims.VertexDim, dims.V2CDim, low=-1.0, high=1.0, allocator=backend + ) + reference = data_alloc.zero_field( + single_node_grid, + dims.VertexDim, + dims.KDim, + dtype=full_cell_k_field.dtype, + allocator=backend, + ) + _compute_cell_2_vertex_interpolation( + full_cell_k_field, + full_coef, + out=reference, + offset_provider={"V2C": single_node_grid.get_connectivity(dims.V2C)}, + ) + + print( + f"rank = {processor_props.rank}/ {processor_props.comm_size} : single node computed field reference has size {reference.asnumpy().shape}" + ) + multinode_grid_manager = run_gridmananger_for_multinode( + file=file, + vertical_config=vertical_config, + run_properties=processor_props, + decomposer=halo.SimpleMetisDecomposer(), + ) + distributed_grid = multinode_grid_manager.grid + decomposition_info = multinode_grid_manager.decomposition_info + + print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") + print( + f"rank = {processor_props.rank}: halo size for 'CellDim' (1 : {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + ) + print( + f"rank = {processor_props.rank}: halo size for 'VertexDim' (1 : {decomposition_info.get_halo_size(dims.VertexDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.VertexDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + ) + global_cell_index = decomposition_info.global_index(dims.CellDim) + cell_k_buffer = ( + full_cell_k_field.ndarray[global_cell_index, :] + .ravel(order="K") + .reshape((global_cell_index.shape[0], vertical_config.num_levels)) + ) + print( + f"rank={processor_props.rank}/{processor_props.comm_size}: input field shape = ([{cell_k_buffer.shape})" + ) + cell_k_field = gtx.as_field( + (dims.CellDim, dims.KDim), data=cell_k_buffer, dtype=cell_k_buffer.dtype, allocator=backend + ) + assert_gathered_field_against_global( + decomposition_info, + processor_props, + dims.CellDim, + global_reference_field=cell_k_field.ndarray, + local_field=cell_k_buffer, + ) + global_vertex_index = decomposition_info.global_index(dims.VertexDim) + + coef = ( + full_coef.ndarray[global_vertex_index, :] + .ravel(order="K") + .reshape((global_vertex_index.shape[0], 6)) + ) + print( + f"rank={processor_props.rank}/{processor_props.comm_size}: coefficient shape = ([{coef.shape})" + ) + coef_field = gtx.as_field((dims.VertexDim, dims.V2CDim), data=coef, allocator=backend) + output = data_alloc.zero_field(distributed_grid, dims.VertexDim, dims.KDim, allocator=backend) + _compute_cell_2_vertex_interpolation( + cell_k_field, + coef_field, + out=output, + offset_provider={"V2C": distributed_grid.get_connectivity(dims.V2C)}, + ) + + assert_gathered_field_against_global( + decomposition_info, + processor_props, + dim=dims.VertexDim, + global_reference_field=reference.asnumpy(), + local_field=output.asnumpy(), + ) From 3c35b619a61e166c322286dea177f7a8b92aa015 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 3 Dec 2025 09:45:08 +0100 Subject: [PATCH 114/240] simple fixes --- .../common/decomposition/mpi_decomposition.py | 2 +- .../mpi_tests/test_parallel_grid_manager.py | 23 ++++++++++--------- .../grid/mpi_tests/test_parallel_icon.py | 15 ++++++------ 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py index d5dca2353d..0bad373dbd 100644 --- a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py +++ b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py @@ -92,7 +92,7 @@ def filter(self, record: logging.LogRecord) -> bool: @definitions.get_processor_properties.register(definitions.MultiNodeRun) def get_multinode_properties( - s: definitions.RunType, comm_id: CommId = None + s: definitions.MultiNodeRun, comm_id: CommId = None ) -> definitions.ProcessProperties: return _get_processor_properties(with_mpi=True, comm_id=comm_id) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 38f67c9d43..0e2667af0c 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -284,10 +284,7 @@ def assert_gathered_field_against_global( ) -# TODO add test including halo access: -# Will uses -# - geofac_div -# - geofac_n2s + # TODO (halungge): fix non contiguous dimension for embedded in gt4py @@ -643,13 +640,13 @@ def test_halo_neighbor_access_e2v( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -@pytest.mark.parametrize("experiment", (definitions.Grids.R02B04_GLOBAL,)) +@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL,)) def test_halo_neighbor_access_v2e( processor_props: defs.ProcessProperties, backend: gtx_typing.Backend | None, - experiment: definitions.Experiment, + grid: definitions.GridDescription, ) -> None: - file = grid_utils.resolve_full_grid_file_name(experiment) + file = grid_utils.resolve_full_grid_file_name(grid) print(f"running on {processor_props.comm}") single_node = run_grid_manager_for_singlenode(file, vertical_config) single_node_grid = single_node.grid @@ -664,11 +661,14 @@ def test_halo_neighbor_access_v2e( dual_edge_length = single_node_geometry.get(geometry_attributes.DUAL_EDGE_LENGTH) edge_orientation = single_node_geometry.get(geometry_attributes.VERTEX_EDGE_ORIENTATION) dual_area = single_node_geometry.get(geometry_attributes.DUAL_AREA) - owner_mask = single_node.decomposition_info.owner_mask(dims.VertexDim) + owner_mask = gtx.as_field((dims.VertexDim,),single_node.decomposition_info.owner_mask(dims.VertexDim)) reference = data_alloc.zero_field(single_node_grid, dims.VertexDim, dims.V2EDim) lateral_boundary_start = single_node_grid.start_index( h_grid.vertex_domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2) ) + horizontal_end = single_node_grid.start_index( + h_grid.vertex_domain(h_grid.Zone.END) + ) interpolation_fields.compute_geofac_rot.with_backend(None)( dual_edge_length, @@ -676,7 +676,7 @@ def test_halo_neighbor_access_v2e( dual_area, owner_mask, out=reference, - domain={dims.VertexDim: (lateral_boundary_start, single_node_grid.num_vertices)}, + domain={dims.VertexDim: (lateral_boundary_start,horizontal_end)}, offset_provider={"V2E": single_node_grid.get_connectivity(dims.V2E)}, ) @@ -713,11 +713,12 @@ def test_halo_neighbor_access_v2e( geofac_rot = data_alloc.zero_field(distributed_grid, dims.VertexDim, dims.V2EDim) + onwner_mask = gtx.as_field((dims.VertexDim,), multinode_grid_manager.decomposition_info.owner_mask(dims.VertexDim)) interpolation_fields.compute_geofac_rot.with_backend(None)( dual_edge_length=dual_edge_length, edge_orientation=edge_orientation, dual_area=dual_area, - owner_mask=decomposition_info.owner_mask(dims.VertexDim), + owner_mask=onwner_mask, out=geofac_rot, offset_provider={"V2E": distributed_grid.get_connectivity(dims.V2E)}, ) @@ -818,7 +819,7 @@ def test_halo_neighbor_access_c2e2c( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_halo_access_v2c(processor_props, backend): +def test_halo_neighbor_access_v2c(processor_props, backend): file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) print(f"running on {processor_props.comm}") single_node = run_grid_manager_for_singlenode(file, vertical_config) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py index ce8174ce33..4c11a7ef36 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py @@ -8,13 +8,15 @@ from __future__ import annotations -from typing import TYPE_CHECKING import pytest +import gt4py.next as gtx import icon4py.model.common.dimension as dims import icon4py.model.common.grid.horizontal as h_grid from icon4py.model.testing import definitions as test_defs, parallel_helpers +from icon4py.model.common.decomposition import definitions as decomposition, mpi_decomposition +from icon4py.model.common.grid import base as base_grid from ...fixtures import ( backend, @@ -28,11 +30,8 @@ from .. import utils -if TYPE_CHECKING: - import gt4py.next as gtx - from icon4py.model.common.decomposition import definitions as decomp_defs - from icon4py.model.common.grid import base as base_grid + try: @@ -43,7 +42,7 @@ @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_props(processor_props: decomp_defs.ProcessProperties) -> None: +def test_props(processor_props: decomposition.ProcessProperties) -> None: """dummy test to check whether the MPI initialization and GHEX setup works.""" import ghex.context as ghex # type: ignore[import-not-found] @@ -81,7 +80,7 @@ def test_props(processor_props: decomp_defs.ProcessProperties) -> None: ) @pytest.mark.parametrize("dim", utils.main_horizontal_dims()) def test_distributed_local( - processor_props: decomp_defs.ProcessProperties, + processor_props: decomposition.ProcessProperties, dim: gtx.Dimension, icon_grid: base_grid.Grid, experiment: test_defs.Experiment, @@ -152,7 +151,7 @@ def test_distributed_local( ) @pytest.mark.parametrize("zone, level", [(h_grid.Zone.HALO, 1), (h_grid.Zone.HALO_LEVEL_2, 2)]) def test_distributed_halo( - processor_props: decomp_defs.ProcessProperties, + processor_props: decomposition.ProcessProperties, dim: gtx.Dimension, zone: h_grid.Zone, icon_grid: base_grid.Grid, From def79a578b97e681936ed354fdc627035af695f0 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 10 Dec 2025 12:57:52 +0100 Subject: [PATCH 115/240] fix reading of additional edge fields --- model/common/src/icon4py/model/common/grid/grid_manager.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 3ff841cf82..be6f5062da 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -184,7 +184,6 @@ def _read_geometry_fields(self, allocator: gtx_typing.FieldBufferAllocationUtil) my_cell_indices = self._decomposition_info.global_index(dims.CellDim) my_edge_indices = self._decomposition_info.global_index(dims.EdgeDim) my_vertex_indices = self._decomposition_info.global_index(dims.VertexDim) - my_vertex_indices = self._decomposition_info.global_index(dims.VertexDim) return { # TODO(halungge): still needs to ported, values from "our" grid files contains (wrong) values: # based on bug in generator fixed with this [PR40](https://gitlab.dkrz.de/dwd-sw/dwd_icon_tools/-/merge_requests/40) . @@ -196,17 +195,17 @@ def _read_geometry_fields(self, allocator: gtx_typing.FieldBufferAllocationUtil) # TODO(halungge): easily computed from a neighbor_sum V2C over the cell areas? gridfile.GeometryName.DUAL_AREA.value: gtx.as_field( (dims.VertexDim,), - self._reader.variable(gridfile.GeometryName.DUAL_AREA), + self._reader.variable(gridfile.GeometryName.DUAL_AREA, indices=my_vertex_indices), allocator=allocator, ), gridfile.GeometryName.EDGE_LENGTH.value: gtx.as_field( (dims.EdgeDim,), - self._reader.variable(gridfile.GeometryName.EDGE_LENGTH), + self._reader.variable(gridfile.GeometryName.EDGE_LENGTH, indices=my_edge_indices), allocator=allocator, ), gridfile.GeometryName.DUAL_EDGE_LENGTH.value: gtx.as_field( (dims.EdgeDim,), - self._reader.variable(gridfile.GeometryName.DUAL_EDGE_LENGTH), + self._reader.variable(gridfile.GeometryName.DUAL_EDGE_LENGTH, indices=my_edge_indices), allocator=allocator, ), gridfile.GeometryName.EDGE_CELL_DISTANCE.value: gtx.as_field( From b418ac24ff34b225e3c5edc2a871e586d3ed85e2 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 10 Dec 2025 13:31:44 +0100 Subject: [PATCH 116/240] make all tests run --- .../src/icon4py/model/common/grid/icon.py | 12 +- .../mpi_tests/test_parallel_grid_manager.py | 104 ++++++++++-------- 2 files changed, 64 insertions(+), 52 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/icon.py b/model/common/src/icon4py/model/common/grid/icon.py index 2eb9923724..6ab4b14ed3 100644 --- a/model/common/src/icon4py/model/common/grid/icon.py +++ b/model/common/src/icon4py/model/common/grid/icon.py @@ -24,8 +24,11 @@ log = logging.getLogger(__name__) CONNECTIVITIES_ON_BOUNDARIES = ( + dims.C2EDim, + dims.C2VDim, # should be removed by includein all vertices on level 2 edges... + dims.E2VDim, dims.C2E2C2EDim, - dims.E2CDim, + dims.E2CDim, # non on halos because of "open halo cells" dims.C2E2CDim, dims.C2E2CODim, dims.E2C2VDim, @@ -201,7 +204,7 @@ class IconGrid(base.Grid): ) -def _has_skip_values(offset: gtx.FieldOffset, limited_area: bool) -> bool: +def _has_skip_values(offset: gtx.FieldOffset, limited_area: bool, distributed: bool) -> bool: """ For the icosahedral global grid skip values are only present for the pentagon points. @@ -210,7 +213,7 @@ def _has_skip_values(offset: gtx.FieldOffset, limited_area: bool) -> bool: """ dimension = offset.target[1] assert dimension.kind == gtx.DimensionKind.LOCAL, "only local dimensions can have skip values" - value = dimension in CONNECTIVITIES_ON_PENTAGONS or ( + value = dimension in CONNECTIVITIES_ON_PENTAGONS or (distributed or limited_area and dimension in CONNECTIVITIES_ON_BOUNDARIES ) @@ -255,11 +258,12 @@ def icon_grid( global_properties: GlobalGridParams, refinement_control: dict[gtx.Dimension, gtx.Field] | None = None, ) -> IconGrid: + distributed = config.num_cells < global_properties.global_num_cells connectivities = { offset.value: base.construct_connectivity( offset, data_alloc.import_array_ns(allocator).asarray(table), - skip_value=-1 if _has_skip_values(offset, config.limited_area) else None, + skip_value=-1 if _has_skip_values(offset, config.limited_area, distributed) else None, allocator=allocator, replace_skip_values=_should_replace_skip_values( offset, config.keep_skip_values, config.limited_area diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 0e2667af0c..75e2742acf 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -249,6 +249,7 @@ def assert_gathered_field_against_global( global_reference_field: np.ndarray, local_field: np.ndarray, ) -> None: + print(f" rank= {processor_props.rank}/{processor_props.comm_size}----exchanging field of main dim {dim}") assert ( local_field.shape[0] == decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.ALL).shape[0] @@ -267,28 +268,29 @@ def assert_gathered_field_against_global( gathered_sizes == global_index_sizes ), f"gathered field sizes do not match {gathered_sizes}" print( - f"rank = {processor_props.rank}: Checking field size: --- gathered sizes {gathered_sizes}" + f"rank = {processor_props.rank}: Checking field size: --- gathered sizes {gathered_sizes} = {sum(gathered_sizes)}" ) print( f"rank = {processor_props.rank}: --- gathered field has size {gathered_sizes}" ) sorted_ = np.zeros(global_reference_field.shape, dtype=gtx.float64) # type: ignore [attr-defined] sorted_[gathered_global_indices] = gathered_field - assert test_helpers.dallclose( - sorted_, global_reference_field - ), f"Gathered field values do not match for dim {dim}.- " - print( - f"rank = {processor_props.rank}: comparing fields (samples) " - f"\n -- gathered {sorted_[:6]} " - f"\n -- global ref {global_reference_field[:6]}" - ) - - + print(f" global reference field {global_reference_field.shape} gathered = {gathered_field.shape}") + np.testing.assert_allclose(sorted_, global_reference_field, rtol=1e-12, verbose=True) # TODO (halungge): fix non contiguous dimension for embedded in gt4py # @pytest.mark.xfail() # non-contiguous dimensions in gt4py embedded: embedded/nd_array_field.py 576 +# the problem should not be -1 but edge indices in the C2E that are not in the local domain. +# from def _hyperslice print slice, hcube, (full connectivity)? + +# something like +# Example: +# index_array = 0 1 -1 +# 3 13 -1 +# -1 -1 -1 +# skip_value = -1 @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL,)) @@ -297,9 +299,8 @@ def test_halo_neighbor_access_c2e( backend: gtx_typing.Backend | None, grid: definitions.GridDescription, ) -> None: - # processor_props = decomp_utils.DummyProps(1) file = grid_utils.resolve_full_grid_file_name(grid) - print(f"running on {processor_props.comm}") + print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") single_node = run_grid_manager_for_singlenode(file, vertical_config) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( @@ -646,6 +647,7 @@ def test_halo_neighbor_access_v2e( backend: gtx_typing.Backend | None, grid: definitions.GridDescription, ) -> None: + #processor_props = decomp_utils.DummyProps(1) file = grid_utils.resolve_full_grid_file_name(grid) print(f"running on {processor_props.comm}") single_node = run_grid_manager_for_singlenode(file, vertical_config) @@ -658,10 +660,10 @@ def test_halo_neighbor_access_v2e( extra_fields=single_node.geometry_fields, metadata=geometry_attributes.attrs, ) - dual_edge_length = single_node_geometry.get(geometry_attributes.DUAL_EDGE_LENGTH) - edge_orientation = single_node_geometry.get(geometry_attributes.VERTEX_EDGE_ORIENTATION) - dual_area = single_node_geometry.get(geometry_attributes.DUAL_AREA) - owner_mask = gtx.as_field((dims.VertexDim,),single_node.decomposition_info.owner_mask(dims.VertexDim)) + single_node_dual_edge_length = single_node_geometry.get(geometry_attributes.DUAL_EDGE_LENGTH) + single_node_edge_orientation = single_node_geometry.get(geometry_attributes.VERTEX_EDGE_ORIENTATION) + single_node_dual_area = single_node_geometry.get(geometry_attributes.DUAL_AREA) + single_node_owner_mask = gtx.as_field((dims.VertexDim,), single_node.decomposition_info.owner_mask(dims.VertexDim)) reference = data_alloc.zero_field(single_node_grid, dims.VertexDim, dims.V2EDim) lateral_boundary_start = single_node_grid.start_index( h_grid.vertex_domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2) @@ -671,12 +673,12 @@ def test_halo_neighbor_access_v2e( ) interpolation_fields.compute_geofac_rot.with_backend(None)( - dual_edge_length, - edge_orientation, - dual_area, - owner_mask, + single_node_dual_edge_length, + single_node_edge_orientation, + single_node_dual_area, + single_node_owner_mask, out=reference, - domain={dims.VertexDim: (lateral_boundary_start,horizontal_end)}, + domain={dims.VertexDim: (lateral_boundary_start, horizontal_end)}, offset_provider={"V2E": single_node_grid.get_connectivity(dims.V2E)}, ) @@ -694,8 +696,8 @@ def test_halo_neighbor_access_v2e( print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" - ) + f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + ) distributed_coordinates = multinode_grid_manager.coordinates extra_geometry_fields = multinode_grid_manager.geometry_fields distributed_geometry = geometry.GridGeometry( @@ -710,10 +712,8 @@ def test_halo_neighbor_access_v2e( dual_edge_length = distributed_geometry.get(geometry_attributes.DUAL_EDGE_LENGTH) edge_orientation = distributed_geometry.get(geometry_attributes.VERTEX_EDGE_ORIENTATION) dual_area = distributed_geometry.get(geometry_attributes.DUAL_AREA) - geofac_rot = data_alloc.zero_field(distributed_grid, dims.VertexDim, dims.V2EDim) - - onwner_mask = gtx.as_field((dims.VertexDim,), multinode_grid_manager.decomposition_info.owner_mask(dims.VertexDim)) + onwner_mask = gtx.as_field((dims.VertexDim,), decomposition_info.owner_mask(dims.VertexDim)) interpolation_fields.compute_geofac_rot.with_backend(None)( dual_edge_length=dual_edge_length, edge_orientation=edge_orientation, @@ -820,20 +820,20 @@ def test_halo_neighbor_access_c2e2c( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_halo_neighbor_access_v2c(processor_props, backend): + file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) print(f"running on {processor_props.comm}") single_node = run_grid_manager_for_singlenode(file, vertical_config) single_node_grid = single_node.grid - full_cell_k_field = data_alloc.random_field( - single_node.grid, dims.CellDim, dims.KDim, low=1.0, high=10.0, allocator=backend - ) + full_cell_k_field = gtx.as_field((dims.CellDim, dims.KDim), data=np.repeat(single_node.coordinates[dims.CellDim]["lat"].ndarray[:, None], vertical_config.num_levels, axis=1), dtype=float, allocator=backend) print( f"rank = {processor_props.rank} / {processor_props.comm_size}: single node input field has size {full_cell_k_field.asnumpy().shape}" ) - full_coef = data_alloc.random_field( - single_node_grid, dims.VertexDim, dims.V2CDim, low=-1.0, high=1.0, allocator=backend - ) + vertex_data = single_node.coordinates[dims.VertexDim]["lat"].ndarray/np.max(single_node.coordinates[dims.VertexDim]["lat"].ndarray) + full_coef = gtx.as_field( + (dims.VertexDim, dims.V2CDim), data= np.repeat(vertex_data[:, None], 6, axis=1), dtype=float, allocator=backend) + reference = data_alloc.zero_field( single_node_grid, dims.VertexDim, @@ -860,6 +860,7 @@ def test_halo_neighbor_access_v2c(processor_props, backend): distributed_grid = multinode_grid_manager.grid decomposition_info = multinode_grid_manager.decomposition_info + print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( f"rank = {processor_props.rank}: halo size for 'CellDim' (1 : {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" @@ -867,31 +868,38 @@ def test_halo_neighbor_access_v2c(processor_props, backend): print( f"rank = {processor_props.rank}: halo size for 'VertexDim' (1 : {decomposition_info.get_halo_size(dims.VertexDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.VertexDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" ) - global_cell_index = decomposition_info.global_index(dims.CellDim) - cell_k_buffer = ( - full_cell_k_field.ndarray[global_cell_index, :] - .ravel(order="K") - .reshape((global_cell_index.shape[0], vertical_config.num_levels)) + my_global_cells = decomposition_info.global_index(dims.CellDim) + cell_k_buffer = full_cell_k_field.ndarray[my_global_cells, :].ravel(order="K").reshape(distributed_grid.num_cells, + 10) + assert_gathered_field_against_global( + decomposition_info, + processor_props, + dims.CellDim, + global_reference_field=full_cell_k_field.ndarray, + local_field=cell_k_buffer, ) print( f"rank={processor_props.rank}/{processor_props.comm_size}: input field shape = ([{cell_k_buffer.shape})" ) + cell_k_field = gtx.as_field( (dims.CellDim, dims.KDim), data=cell_k_buffer, dtype=cell_k_buffer.dtype, allocator=backend ) - assert_gathered_field_against_global( - decomposition_info, - processor_props, - dims.CellDim, - global_reference_field=cell_k_field.ndarray, - local_field=cell_k_buffer, - ) - global_vertex_index = decomposition_info.global_index(dims.VertexDim) + + my_global_vertices = decomposition_info.global_index(dims.VertexDim) coef = ( - full_coef.ndarray[global_vertex_index, :] + full_coef.ndarray[my_global_vertices, :] .ravel(order="K") - .reshape((global_vertex_index.shape[0], 6)) + .reshape((distributed_grid.num_vertices, 6)) + ) + + assert_gathered_field_against_global( + decomposition_info, + processor_props, + dims.VertexDim, + global_reference_field=full_coef.ndarray, + local_field=coef, ) print( f"rank={processor_props.rank}/{processor_props.comm_size}: coefficient shape = ([{coef.shape})" From 7f5973d1d7949f01ffd15697a84c6d3e9bfd3308 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 10 Dec 2025 14:49:16 +0100 Subject: [PATCH 117/240] fixing some things --- .../model/common/decomposition/definitions.py | 42 ++++++---- .../model/common/decomposition/halo.py | 47 +++++------- .../common/decomposition/mpi_decomposition.py | 1 + .../icon4py/model/common/grid/grid_manager.py | 13 ++-- .../src/icon4py/model/common/grid/icon.py | 8 +- .../decomposition/mpi_tests/test_halo.py | 11 +-- .../mpi_tests/test_mpi_decomposition.py | 1 + .../unit_tests/test_definitions.py | 13 ++-- .../mpi_tests/test_parallel_grid_manager.py | 76 +++++++++++-------- .../grid/mpi_tests/test_parallel_icon.py | 9 +-- .../tests/common/grid/unit_tests/test_icon.py | 4 +- model/common/tests/common/grid/utils.py | 1 - .../src/icon4py/model/testing/grid_utils.py | 8 +- 13 files changed, 121 insertions(+), 113 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index cfec030904..d58c4f26d9 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -8,6 +8,7 @@ from __future__ import annotations +import dataclasses import enum import functools import logging @@ -18,6 +19,7 @@ import dace # type: ignore[import-untyped] import gt4py.next as gtx +import mpi4py.MPI import numpy as np from icon4py.model.common import dimension as dims, utils @@ -41,9 +43,17 @@ class ProcessProperties(Protocol): comm: Any - rank: int - comm_name: str - comm_size: int + + @property + def rank(self)->int: + ... + @property + def comm_name(self)->str: + ... + + @property + def comm_size(self) -> int: + ... def single_node(self) -> bool: return self.comm_size == 1 @@ -54,16 +64,22 @@ def __str__(self) -> str: @dataclass(frozen=True, init=False) class SingleNodeProcessProperties(ProcessProperties): - comm: Any - comm_name: str - comm_size: int - rank: int - - def __init__(self): # type: ignore [no-untyped-def] - object.__setattr__(self, "comm", None) - object.__setattr__(self, "rank", 0) - object.__setattr__(self, "comm_name", "") - object.__setattr__(self, "comm_size", 1) + @property + def comm(self)->Any: + return None + + @property + def rank(self)->int: + return 0 + + @property + def comm_name(self)->str: + return "" + + @property + def comm_size(self) -> int: + return 1 + class DomainDescriptorIdGenerator: diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 5f3e6ec232..9747567491 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -11,7 +11,7 @@ from typing import Protocol, runtime_checkable import gt4py.next as gtx -import gt4py.next.backend as gtx_backend +import gt4py.next.typing as gtx_typing import numpy as np from icon4py.model.common import dimension as dims, exceptions @@ -23,7 +23,7 @@ log = logging.getLogger(__name__) -def _value(k: gtx.FieldOffset | str): +def _value(k: gtx.FieldOffset | str) -> str: return k.value if isinstance(k, gtx.FieldOffset) else k @@ -38,15 +38,13 @@ class NoHalos(HaloConstructor): def __init__( self, horizontal_size: base.HorizontalGridSize, - num_levels: int, - backend: gtx_backend.Backend | None = None, + allocator: gtx.Field | None = None, ): self._size = horizontal_size - self._num_levels = num_levels - self._backend = backend + self._allocator = allocator def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: - xp = data_alloc.import_array_ns(self._backend) + xp = data_alloc.import_array_ns(self._allocator) create_arrays = functools.partial(_create_dummy_decomposition_arrays, array_ns=xp) decomposition_info = defs.DecompositionInfo() @@ -56,7 +54,9 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: return decomposition_info -def _create_dummy_decomposition_arrays(size: int, array_ns: ModuleType = np): +def _create_dummy_decomposition_arrays( + size: int, array_ns: ModuleType = np +) -> tuple[data_alloc.NDArray, data_alloc.NDArray, data_alloc.NDArray]: indices = array_ns.arange(size, dtype=gtx.int32) owner_mask = array_ns.ones((size,), dtype=bool) halo_levels = array_ns.ones((size,), dtype=gtx.int32) * defs.DecompositionFlag.OWNED @@ -70,48 +70,45 @@ def __init__( self, run_properties: defs.ProcessProperties, connectivities: dict[gtx.FieldOffset | str, data_alloc.NDArray], - num_levels, - backend: gtx_backend.Backend | None = None, + allocator: gtx_typing.FieldBufferAllocationUtil | None = None, ): """ Args: run_properties: contains information on the communicator and local compute node. connectivities: connectivity arrays needed to construct the halos - num_levels: number of vertical levels, TODO(halungge):: should be removed, it is needed in GHEX that why we have it no the DecompotionInfo backend: GT4Py (used to determine the array ns import) """ - self._xp = data_alloc.import_array_ns(backend) - self._num_levels = num_levels + self._xp = data_alloc.import_array_ns(allocator) self._props = run_properties self._connectivities = {_value(k): v for k, v in connectivities.items()} self._assert_all_neighbor_tables() @property - def face_face_connectivity(self): + def face_face_connectivity(self) -> data_alloc.NDArray: return self._connectivity(dims.C2E2C.value) @property - def edge_face_connectivity(self): + def edge_face_connectivity(self) -> data_alloc.NDArray: return self._connectivity(dims.E2C) @property - def face_edge_connectivity(self): + def face_edge_connectivity(self) -> data_alloc.NDArray: return self._connectivity(dims.C2E) @property - def node_edge_connectivity(self): + def node_edge_connectivity(self) -> data_alloc.NDArray: return self._connectivity(dims.V2E) @property - def node_face_connectivity(self): + def node_face_connectivity(self) -> data_alloc.NDArray: return self._connectivity(dims.V2C) @property - def face_node_connectivity(self): + def face_node_connectivity(self) -> data_alloc.NDArray: return self._connectivity(dims.C2V) - def _validate_mapping(self, face_to_rank_mapping: data_alloc.NDArray): + def _validate_mapping(self, face_to_rank_mapping: data_alloc.NDArray) -> None: # validate the distribution mapping: num_cells = self.face_face_connectivity.shape[0] expected_shape = (num_cells,) @@ -432,10 +429,9 @@ def __call__( def halo_constructor( run_properties: defs.ProcessProperties, - num_levels: int, full_grid_size: base.HorizontalGridSize, connectivities: dict[gtx.FieldOffset, data_alloc.NDArray], - backend=gtx_backend.Backend | None, + allocator=gtx_typing.FieldBufferAllocationUtil | None, ) -> HaloConstructor: """ Factory method to create the halo constructor. We need some input data from the global grid and from @@ -446,7 +442,6 @@ def halo_constructor( parameter Args: processor_props: - num_levels: full_grid_size connectivities: backend: @@ -456,14 +451,12 @@ def halo_constructor( """ if run_properties.single_node(): return NoHalos( - num_levels=num_levels, horizontal_size=full_grid_size, - backend=backend, + allocator=allocator, ) else: return IconLikeHaloConstructor( - num_levels=num_levels, run_properties=run_properties, connectivities=connectivities, - backend=backend, + allocator=allocator, ) diff --git a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py index 0bad373dbd..3ffa17228e 100644 --- a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py +++ b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py @@ -101,6 +101,7 @@ def get_multinode_properties( class MPICommProcessProperties(definitions.ProcessProperties): comm: mpi4py.MPI.Comm + @functools.cached_property def rank(self) -> int: return self.comm.Get_rank() diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index be6f5062da..ba758cb8eb 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -59,12 +59,12 @@ class GridManager: def __init__( self, grid_file: pathlib.Path | str, - config: v_grid.VerticalGridConfig, + num_levels: int, transformation: gridfile.IndexTransformation = _fortan_to_python_transformer, ): self._transformation = transformation self._file_name = str(grid_file) - self._vertical_config = config + self._num_levels = num_levels self._halo_constructor: halo.HaloConstructor | None = None # Output self._grid: icon.IconGrid | None = None @@ -205,7 +205,9 @@ def _read_geometry_fields(self, allocator: gtx_typing.FieldBufferAllocationUtil) ), gridfile.GeometryName.DUAL_EDGE_LENGTH.value: gtx.as_field( (dims.EdgeDim,), - self._reader.variable(gridfile.GeometryName.DUAL_EDGE_LENGTH, indices=my_edge_indices), + self._reader.variable( + gridfile.GeometryName.DUAL_EDGE_LENGTH, indices=my_edge_indices + ), allocator=allocator, ), gridfile.GeometryName.EDGE_CELL_DISTANCE.value: gtx.as_field( @@ -346,10 +348,9 @@ def _construct_decomposed_grid( neighbor_tables_for_halo_construction = neighbor_tables halo_constructor = halo.halo_constructor( run_properties=run_properties, - num_levels=self._vertical_config.num_levels, full_grid_size=global_size, connectivities=neighbor_tables_for_halo_construction, - backend=allocator, + allocator=allocator, ) self._decomposition_info = halo_constructor(cells_to_rank_mapping) @@ -378,7 +379,7 @@ def _construct_decomposed_grid( grid_config = base.GridConfig( horizontal_size=distributed_size, - vertical_size=self._vertical_config.num_levels, + vertical_size=self._num_levels, limited_area=limited_area, keep_skip_values=with_skip_values, ) diff --git a/model/common/src/icon4py/model/common/grid/icon.py b/model/common/src/icon4py/model/common/grid/icon.py index 6ab4b14ed3..ca38755c63 100644 --- a/model/common/src/icon4py/model/common/grid/icon.py +++ b/model/common/src/icon4py/model/common/grid/icon.py @@ -25,7 +25,7 @@ CONNECTIVITIES_ON_BOUNDARIES = ( dims.C2EDim, - dims.C2VDim, # should be removed by includein all vertices on level 2 edges... + dims.C2VDim, # should be removed by includein all vertices on level 2 edges... dims.E2VDim, dims.C2E2C2EDim, dims.E2CDim, # non on halos because of "open halo cells" @@ -203,7 +203,7 @@ class IconGrid(base.Grid): default=None, kw_only=True ) - +# TODO (halungge): combine the last to args "into single_node_global" def _has_skip_values(offset: gtx.FieldOffset, limited_area: bool, distributed: bool) -> bool: """ For the icosahedral global grid skip values are only present for the pentagon points. @@ -213,8 +213,8 @@ def _has_skip_values(offset: gtx.FieldOffset, limited_area: bool, distributed: b """ dimension = offset.target[1] assert dimension.kind == gtx.DimensionKind.LOCAL, "only local dimensions can have skip values" - value = dimension in CONNECTIVITIES_ON_PENTAGONS or (distributed or - limited_area and dimension in CONNECTIVITIES_ON_BOUNDARIES + value = dimension in CONNECTIVITIES_ON_PENTAGONS or ( + distributed or (limited_area and dimension in CONNECTIVITIES_ON_BOUNDARIES) ) return value diff --git a/model/common/tests/common/decomposition/mpi_tests/test_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_halo.py index 46388b9874..c5a55b20f2 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_halo.py @@ -48,12 +48,12 @@ def simple_neighbor_tables(): @pytest.mark.mpi(min_size=4) +@pytest.mark.parametrize("processor_props", [True], indirect=True) def test_halo_constructor_owned_cells(processor_props, simple_neighbor_tables): # F811 # fixture halo_generator = halo.IconLikeHaloConstructor( connectivities=simple_neighbor_tables, run_properties=processor_props, - num_levels=1, - backend=backend, + allocator=backend, ) my_owned_cells = halo_generator.owned_cells(utils.SIMPLE_DISTRIBUTION) @@ -70,7 +70,6 @@ def test_halo_constructor_validate_number_of_node_mismatch(processor_props, simp halo_generator = halo.IconLikeHaloConstructor( connectivities=simple_neighbor_tables, run_properties=processor_props, - num_levels=1, ) halo_generator(distribution) assert "The distribution assumes more nodes than the current run" in e.value.args[0] @@ -85,7 +84,6 @@ def test_halo_constructor_validate_rank_mapping_wrong_shape( halo_generator = halo.IconLikeHaloConstructor( connectivities=simple_neighbor_tables, run_properties=processor_props, - num_levels=1, ) halo_generator(np.zeros((num_cells, 3), dtype=int)) assert f"should have shape ({num_cells},)" in e.value.args[0] @@ -107,7 +105,6 @@ def test_element_ownership_is_unique( halo_generator = halo.IconLikeHaloConstructor( connectivities=simple_neighbor_tables, run_properties=processor_props, - num_levels=1, backend=backend, ) @@ -153,7 +150,6 @@ def test_halo_constructor_decomposition_info_global_indices( halo_generator = halo.IconLikeHaloConstructor( connectivities=simple_neighbor_tables, run_properties=processor_props, - num_levels=1, ) decomp_info = halo_generator(utils.SIMPLE_DISTRIBUTION) @@ -177,7 +173,6 @@ def test_halo_constructor_decomposition_info_halo_levels( halo_generator = halo.IconLikeHaloConstructor( connectivities=simple_neighbor_tables, run_properties=processor_props, - num_levels=1, ) decomp_info = halo_generator(utils.SIMPLE_DISTRIBUTION) my_halo_levels = decomp_info.halo_levels(dim) @@ -225,7 +220,7 @@ def decompose(grid: base_grid.Grid, processor_props): # F811 # fixture def test_no_halo(): grid_size = base_grid.HorizontalGridSize(num_cells=9, num_edges=14, num_vertices=6) - halo_generator = halo.NoHalos(horizontal_size=grid_size, num_levels=10, backend=None) + halo_generator = halo.NoHalos(horizontal_size=grid_size, allocator=None) decomposition = halo.SingleNodeDecomposer() decomposition_info = halo_generator(decomposition(np.arange(grid_size.num_cells), 1)) # cells diff --git a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py index 23a264d2e7..ed905f337f 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py @@ -61,6 +61,7 @@ """ +@pytest.mark.mpi(min_size= 2) @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_props(processor_props: definitions.ProcessProperties) -> None: diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index 2dccd72ad0..5021ad5a43 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -14,6 +14,7 @@ import icon4py.model.common.dimension as dims import icon4py.model.common.utils.data_allocation as data_alloc from icon4py.model.common.decomposition import definitions, halo +from icon4py.model.testing import definitions as test_defs from icon4py.model.common.grid import simple from icon4py.model.testing.fixtures import processor_props @@ -54,7 +55,7 @@ def test_global_to_local_index(offset, rank): if gtx_common.is_neighbor_connectivity(v) } props = dummy_four_ranks(rank) - halo_constructor = halo.IconLikeHaloConstructor(props, neighbor_tables, 1) + halo_constructor = halo.IconLikeHaloConstructor(props, neighbor_tables) decomposition_info = halo_constructor(utils.SIMPLE_DISTRIBUTION) source_indices_on_local_grid = decomposition_info.global_index(offset.target[0]) @@ -88,7 +89,6 @@ def test_halo_constructor_decomposition_info_global_indices(dim, rank): halo_generator = halo.IconLikeHaloConstructor( connectivities=simple_neighbor_tables, run_properties=props, - num_levels=1, ) decomp_info = halo_generator(utils.SIMPLE_DISTRIBUTION) @@ -108,7 +108,6 @@ def test_horizontal_size(rank): halo_generator = halo.IconLikeHaloConstructor( connectivities=simple_neighbor_tables, run_properties=props, - num_levels=1, ) decomp_info = halo_generator(utils.SIMPLE_DISTRIBUTION) horizontal_size = decomp_info.get_horizontal_size() @@ -126,15 +125,17 @@ def test_horizontal_size(rank): ), f"local size mismatch on rank={rank} for {dims.CellDim}: expected {expected_cells}, but was {horizontal_size.num_cells}" -@pytest.mark.datatest @pytest.mark.parametrize("dim", grid_utils.main_horizontal_dims()) def test_decomposition_info_single_node_empty_halo( dim: gtx.Dimension, - decomposition_info: definitions.DecompositionInfo, processor_props: definitions.ProcessProperties, ) -> None: if not processor_props.single_node(): pytest.xfail() - for level in definitions.DecompositionFlag.__values__: + + manager = grid_utils.run_grid_manager(test_defs.Grids.MCH_CH_R04B09_DSL, keep_skip_values=True, backend=None) + + decomposition_info = manager.decomposition_info + for level in definitions.DecompositionFlag: assert decomposition_info.get_halo_size(dim, level) == 0 assert np.count_nonzero(decomposition_info.halo_level_mask(dim, level)) == 0 diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 75e2742acf..b7241073c8 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -249,7 +249,9 @@ def assert_gathered_field_against_global( global_reference_field: np.ndarray, local_field: np.ndarray, ) -> None: - print(f" rank= {processor_props.rank}/{processor_props.comm_size}----exchanging field of main dim {dim}") + print( + f" rank= {processor_props.rank}/{processor_props.comm_size}----exchanging field of main dim {dim}" + ) assert ( local_field.shape[0] == decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.ALL).shape[0] @@ -275,22 +277,12 @@ def assert_gathered_field_against_global( ) sorted_ = np.zeros(global_reference_field.shape, dtype=gtx.float64) # type: ignore [attr-defined] sorted_[gathered_global_indices] = gathered_field - print(f" global reference field {global_reference_field.shape} gathered = {gathered_field.shape}") + print( + f" global reference field {global_reference_field.shape} gathered = {gathered_field.shape}" + ) np.testing.assert_allclose(sorted_, global_reference_field, rtol=1e-12, verbose=True) - -# TODO (halungge): fix non contiguous dimension for embedded in gt4py -# @pytest.mark.xfail() # non-contiguous dimensions in gt4py embedded: embedded/nd_array_field.py 576 -# the problem should not be -1 but edge indices in the C2E that are not in the local domain. -# from def _hyperslice print slice, hcube, (full connectivity)? - -# something like -# Example: -# index_array = 0 1 -1 -# 3 13 -1 -# -1 -1 -1 -# skip_value = -1 @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL,)) @@ -601,8 +593,8 @@ def test_halo_neighbor_access_e2v( f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" ) distributed_coordinates = multinode_grid_manager.coordinates - vertex_lat = distributed_coordinates.get(dims.VertexDim)["lat"] - vertex_lon = distributed_coordinates.get(dims.VertexDim)["lon"] + vertex_lat = distributed_coordinates[dims.VertexDim]["lat"] + vertex_lon = distributed_coordinates.get[dims.VertexDim]["lon"] tangent_orientation = multinode_grid_manager.geometry_fields.get( gridfile.GeometryName.TANGENT_ORIENTATION ) @@ -647,7 +639,7 @@ def test_halo_neighbor_access_v2e( backend: gtx_typing.Backend | None, grid: definitions.GridDescription, ) -> None: - #processor_props = decomp_utils.DummyProps(1) + # processor_props = decomp_utils.DummyProps(1) file = grid_utils.resolve_full_grid_file_name(grid) print(f"running on {processor_props.comm}") single_node = run_grid_manager_for_singlenode(file, vertical_config) @@ -661,16 +653,20 @@ def test_halo_neighbor_access_v2e( metadata=geometry_attributes.attrs, ) single_node_dual_edge_length = single_node_geometry.get(geometry_attributes.DUAL_EDGE_LENGTH) - single_node_edge_orientation = single_node_geometry.get(geometry_attributes.VERTEX_EDGE_ORIENTATION) + single_node_edge_orientation = single_node_geometry.get( + geometry_attributes.VERTEX_EDGE_ORIENTATION + ) single_node_dual_area = single_node_geometry.get(geometry_attributes.DUAL_AREA) - single_node_owner_mask = gtx.as_field((dims.VertexDim,), single_node.decomposition_info.owner_mask(dims.VertexDim)) + single_node_owner_mask = gtx.as_field( + (dims.VertexDim,), + data=single_node.decomposition_info.owner_mask(dims.VertexDim), + dtype=bool, + ) reference = data_alloc.zero_field(single_node_grid, dims.VertexDim, dims.V2EDim) lateral_boundary_start = single_node_grid.start_index( h_grid.vertex_domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2) ) - horizontal_end = single_node_grid.start_index( - h_grid.vertex_domain(h_grid.Zone.END) - ) + horizontal_end = single_node_grid.start_index(h_grid.vertex_domain(h_grid.Zone.END)) interpolation_fields.compute_geofac_rot.with_backend(None)( single_node_dual_edge_length, @@ -696,8 +692,8 @@ def test_halo_neighbor_access_v2e( print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" - ) + f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + ) distributed_coordinates = multinode_grid_manager.coordinates extra_geometry_fields = multinode_grid_manager.geometry_fields distributed_geometry = geometry.GridGeometry( @@ -820,19 +816,33 @@ def test_halo_neighbor_access_c2e2c( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_halo_neighbor_access_v2c(processor_props, backend): - file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) print(f"running on {processor_props.comm}") single_node = run_grid_manager_for_singlenode(file, vertical_config) single_node_grid = single_node.grid - full_cell_k_field = gtx.as_field((dims.CellDim, dims.KDim), data=np.repeat(single_node.coordinates[dims.CellDim]["lat"].ndarray[:, None], vertical_config.num_levels, axis=1), dtype=float, allocator=backend) + full_cell_k_field = gtx.as_field( + (dims.CellDim, dims.KDim), + data=np.repeat( + single_node.coordinates[dims.CellDim]["lat"].ndarray[:, None], + vertical_config.num_levels, + axis=1, + ), + dtype=float, + allocator=backend, + ) print( f"rank = {processor_props.rank} / {processor_props.comm_size}: single node input field has size {full_cell_k_field.asnumpy().shape}" ) - vertex_data = single_node.coordinates[dims.VertexDim]["lat"].ndarray/np.max(single_node.coordinates[dims.VertexDim]["lat"].ndarray) + vertex_data = single_node.coordinates[dims.VertexDim]["lat"].ndarray / np.max( + single_node.coordinates[dims.VertexDim]["lat"].ndarray + ) full_coef = gtx.as_field( - (dims.VertexDim, dims.V2CDim), data= np.repeat(vertex_data[:, None], 6, axis=1), dtype=float, allocator=backend) + (dims.VertexDim, dims.V2CDim), + data=np.repeat(vertex_data[:, None], 6, axis=1), + dtype=float, + allocator=backend, + ) reference = data_alloc.zero_field( single_node_grid, @@ -860,7 +870,6 @@ def test_halo_neighbor_access_v2c(processor_props, backend): distributed_grid = multinode_grid_manager.grid decomposition_info = multinode_grid_manager.decomposition_info - print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( f"rank = {processor_props.rank}: halo size for 'CellDim' (1 : {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" @@ -869,8 +878,11 @@ def test_halo_neighbor_access_v2c(processor_props, backend): f"rank = {processor_props.rank}: halo size for 'VertexDim' (1 : {decomposition_info.get_halo_size(dims.VertexDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.VertexDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" ) my_global_cells = decomposition_info.global_index(dims.CellDim) - cell_k_buffer = full_cell_k_field.ndarray[my_global_cells, :].ravel(order="K").reshape(distributed_grid.num_cells, - 10) + cell_k_buffer = ( + full_cell_k_field.ndarray[my_global_cells, :] + .ravel(order="K") + .reshape(distributed_grid.num_cells, 10) + ) assert_gathered_field_against_global( decomposition_info, processor_props, @@ -891,7 +903,7 @@ def test_halo_neighbor_access_v2c(processor_props, backend): coef = ( full_coef.ndarray[my_global_vertices, :] .ravel(order="K") - .reshape((distributed_grid.num_vertices, 6)) + .reshape((distributed_grid.num_vertices, 6)) ) assert_gathered_field_against_global( diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py index 4c11a7ef36..30148e67b1 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py @@ -8,15 +8,14 @@ from __future__ import annotations - -import pytest import gt4py.next as gtx +import pytest import icon4py.model.common.dimension as dims import icon4py.model.common.grid.horizontal as h_grid -from icon4py.model.testing import definitions as test_defs, parallel_helpers from icon4py.model.common.decomposition import definitions as decomposition, mpi_decomposition from icon4py.model.common.grid import base as base_grid +from icon4py.model.testing import definitions as test_defs, parallel_helpers from ...fixtures import ( backend, @@ -30,10 +29,6 @@ from .. import utils - - - - try: import mpi4py except ImportError: diff --git a/model/common/tests/common/grid/unit_tests/test_icon.py b/model/common/tests/common/grid/unit_tests/test_icon.py index 84656a9faf..71a62d02b6 100644 --- a/model/common/tests/common/grid/unit_tests/test_icon.py +++ b/model/common/tests/common/grid/unit_tests/test_icon.py @@ -194,8 +194,8 @@ def test_when_keep_skip_value_then_neighbor_table_matches_config( assert ( np.any(connectivity.asnumpy() == gridfile.GridFile.INVALID_INDEX).item() - ) == icon._has_skip_values(offset, grid.config.limited_area) - if not icon._has_skip_values(offset, grid.config.limited_area): + ) == icon._has_skip_values(offset, grid.config.limited_area, distributed=False) + if not icon._has_skip_values(offset, grid.config.limited_area, distributed=False): assert connectivity.skip_value is None else: assert connectivity.skip_value == gridfile.GridFile.INVALID_INDEX diff --git a/model/common/tests/common/grid/utils.py b/model/common/tests/common/grid/utils.py index 00e76414ec..65f5daec23 100644 --- a/model/common/tests/common/grid/utils.py +++ b/model/common/tests/common/grid/utils.py @@ -87,7 +87,6 @@ def run_grid_manager( manager = gridtest_utils.get_grid_manager_from_identifier( grid, keep_skip_values=keep_skip_values, - num_levels=1, allocator=model_backends.get_allocator(backend), ) managers[key] = manager diff --git a/model/testing/src/icon4py/model/testing/grid_utils.py b/model/testing/src/icon4py/model/testing/grid_utils.py index 975a008832..392eae19c3 100644 --- a/model/testing/src/icon4py/model/testing/grid_utils.py +++ b/model/testing/src/icon4py/model/testing/grid_utils.py @@ -15,7 +15,6 @@ geometry_attributes as geometry_attrs, grid_manager as gm, gridfile, - vertical as v_grid, ) from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import config, data_handling, definitions, locking @@ -31,7 +30,6 @@ def get_grid_manager_from_experiment( ) -> gm.GridManager: return get_grid_manager_from_identifier( experiment.grid, - num_levels=experiment.num_levels, keep_skip_values=keep_skip_values, allocator=allocator, ) @@ -39,19 +37,17 @@ def get_grid_manager_from_experiment( def get_grid_manager_from_identifier( grid: definitions.GridDescription, - num_levels: int, keep_skip_values: bool, allocator: gtx_typing.FieldBufferAllocationUtil, ) -> gm.GridManager: grid_file = _download_grid_file(grid) return get_grid_manager( - grid_file, num_levels=num_levels, keep_skip_values=keep_skip_values, allocator=allocator + grid_file, keep_skip_values=keep_skip_values, allocator=allocator ) def get_grid_manager( filename: pathlib.Path, - num_levels: int, keep_skip_values: bool, allocator: gtx_typing.FieldBufferAllocationUtil, ) -> gm.GridManager: @@ -66,7 +62,6 @@ def get_grid_manager( """ manager = gm.GridManager( filename, - v_grid.VerticalGridConfig(num_levels=num_levels), gridfile.ToZeroBasedIndexTransformation(), ) manager(allocator=allocator, keep_skip_values=keep_skip_values) @@ -108,7 +103,6 @@ def _construct_grid_geometry() -> geometry.GridGeometry: gm = get_grid_manager_from_identifier( experiment.grid, keep_skip_values=True, - num_levels=experiment.num_levels, allocator=model_backends.get_allocator(backend), ) grid = gm.grid From 73e6a08891945175bee311522cbd603d7df18829 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Wed, 10 Dec 2025 16:15:20 +0100 Subject: [PATCH 118/240] fix typings --- .../test_benchmark_solve_nonhydro.py | 2 +- .../model/common/decomposition/definitions.py | 9 +- .../model/common/decomposition/halo.py | 29 ++++--- .../common/decomposition/mpi_decomposition.py | 50 +++++------ .../icon4py/model/common/grid/grid_manager.py | 10 +-- .../src/icon4py/model/common/grid/icon.py | 1 + .../mpi_tests/test_mpi_decomposition.py | 3 +- .../unit_tests/test_definitions.py | 6 +- .../mpi_tests/test_parallel_grid_manager.py | 84 ++++++++----------- .../common/grid/unit_tests/test_geometry.py | 2 +- .../grid/unit_tests/test_geometry_stencils.py | 2 +- .../grid/unit_tests/test_grid_manager.py | 4 +- model/common/tests/common/grid/utils.py | 1 + .../model/testing/fixtures/benchmark.py | 4 +- .../src/icon4py/model/testing/grid_utils.py | 9 +- 15 files changed, 110 insertions(+), 106 deletions(-) diff --git a/model/atmosphere/dycore/tests/dycore/integration_tests/test_benchmark_solve_nonhydro.py b/model/atmosphere/dycore/tests/dycore/integration_tests/test_benchmark_solve_nonhydro.py index 95d055b22f..f857bfc2f8 100644 --- a/model/atmosphere/dycore/tests/dycore/integration_tests/test_benchmark_solve_nonhydro.py +++ b/model/atmosphere/dycore/tests/dycore/integration_tests/test_benchmark_solve_nonhydro.py @@ -201,7 +201,7 @@ def solve_nonhydro( vertical_params=vertical_grid, edge_geometry=edge_geometry, cell_geometry=cell_geometry, - owner_mask=grid_geometry.get("cell_owner_mask"), + owner_mask=geometry_field_source.get("cell_owner_mask"), backend=backend_like, ) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 98efbf281d..cd44592b89 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -21,8 +21,6 @@ from icon4py.model.common import dimension as dims, utils from icon4py.model.common.grid import base, gridfile - - from icon4py.model.common.orchestration.halo_exchange import DummyNestedSDFG from icon4py.model.common.utils import data_allocation as data_alloc @@ -36,6 +34,9 @@ class ProcessProperties(Protocol): comm_name: str comm_size: int + def single_node(self) -> bool: + return self.comm_size == 1 + @dataclasses.dataclass(frozen=True, init=False) class SingleNodeProcessProperties(ProcessProperties): @@ -74,7 +75,7 @@ def __call__(self) -> int: class DecompositionInfo: def __init__( self, - ): + ) -> None: self._global_index: dict[gtx.Dimension, data_alloc.NDArray] = {} self._halo_levels: dict[gtx.Dimension, data_alloc.NDArray] = {} self._owner_mask: dict[gtx.Dimension, data_alloc.NDArray] = {} @@ -96,6 +97,7 @@ def set_dimension( self._owner_mask[dim] = owner_mask self._halo_levels[dim] = halo_levels + def local_index( self, dim: gtx.Dimension, entry_type: EntryType = EntryType.ALL ) -> data_alloc.NDArray: @@ -430,4 +432,5 @@ class DecompositionFlag(int, Enum): - edges that are only on the cell(SECOND_HALO_LINE) """ + single_node_default = SingleNodeExchange() diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 9747567491..ea4bc15934 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -24,7 +24,7 @@ def _value(k: gtx.FieldOffset | str) -> str: - return k.value if isinstance(k, gtx.FieldOffset) else k + return str(k.value) if isinstance(k, gtx.FieldOffset) else k @runtime_checkable @@ -38,7 +38,7 @@ class NoHalos(HaloConstructor): def __init__( self, horizontal_size: base.HorizontalGridSize, - allocator: gtx.Field | None = None, + allocator: gtx_typing.FieldBufferAllocationUtil | None = None, ): self._size = horizontal_size self._allocator = allocator @@ -86,7 +86,7 @@ def __init__( @property def face_face_connectivity(self) -> data_alloc.NDArray: - return self._connectivity(dims.C2E2C.value) + return self._connectivity(dims.C2E2C) @property def edge_face_connectivity(self) -> data_alloc.NDArray: @@ -149,7 +149,9 @@ def _connectivity(self, offset: gtx.FieldOffset | str) -> data_alloc.NDArray: f"Connectivity for offset {offset} is not available" ) from err - def next_halo_line(self, cells: data_alloc.NDArray, depot: data_alloc.NDArray | None = None): + def next_halo_line( + self, cells: data_alloc.NDArray, depot: data_alloc.NDArray | None = None + ) -> data_alloc.NDArray: """Returns the full-grid indices of the next halo line. If a depot is given the function only return indices that are not in the depot @@ -181,7 +183,7 @@ def _find_neighbors( unique_neighbors = self._xp.unique(neighbors.reshape(shp[0] * shp[1])) return unique_neighbors - def _find_cell_neighbors(self, cells: data_alloc.NDArray): + def _find_cell_neighbors(self, cells: data_alloc.NDArray) -> data_alloc.NDArray: """Find all neighboring cells of a list of cells.""" return self._find_neighbors(cells, connectivity=self.face_face_connectivity) @@ -202,8 +204,13 @@ def owned_cells(self, face_to_rank: data_alloc.NDArray) -> data_alloc.NDArray: return self._xp.asarray(owned_cells).nonzero()[0] def _update_owner_mask_by_max_rank_convention( - self, face_to_rank, owner_mask, all_indices, indices_on_cutting_line, target_connectivity - ): + self, + face_to_rank: data_alloc.NDArray, + owner_mask: data_alloc.NDArray, + all_indices: data_alloc.NDArray, + indices_on_cutting_line: data_alloc.NDArray, + target_connectivity: data_alloc.NDArray, + ) -> data_alloc.NDArray: """ In order to have unique ownership of edges (and vertices) among nodes there needs to be a convention as to where those elements on the cutting line go: @@ -413,7 +420,7 @@ def __call__( Returns: np.ndarray: array with partition label (int, rank number) for each cell """ - import pymetis + import pymetis # type: ignore [import-not-found] _, partition_index = pymetis.part_graph(nparts=num_partitions, adjacency=adjacency_matrix) return np.array(partition_index) @@ -421,7 +428,7 @@ def __call__( class SingleNodeDecomposer(Decomposer): def __call__( - self, adjacency_matrix: data_alloc.NDArray, num_partitions=1 + self, adjacency_matrix: data_alloc.NDArray, num_partitions: int = 1 ) -> data_alloc.NDArray: """Dummy decomposer for single node: assigns all cells to rank = 0""" return np.zeros(adjacency_matrix.shape[0], dtype=gtx.int32) @@ -430,8 +437,8 @@ def __call__( def halo_constructor( run_properties: defs.ProcessProperties, full_grid_size: base.HorizontalGridSize, - connectivities: dict[gtx.FieldOffset, data_alloc.NDArray], - allocator=gtx_typing.FieldBufferAllocationUtil | None, + connectivities: dict[gtx.FieldOffset | str, data_alloc.NDArray], + allocator: gtx_typing.FieldBufferAllocationUtil | None, ) -> HaloConstructor: """ Factory method to create the halo constructor. We need some input data from the global grid and from diff --git a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py index 3f3aef3c2e..2a4b3a6e51 100644 --- a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py +++ b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py @@ -25,10 +25,17 @@ try: - import ghex # type: ignore [no-untyped-def] - import mpi4py - from ghex import unstructured, util + import ghex # type: ignore [import-not-found] + import mpi4py # type: ignore [import-not-found] from ghex.context import make_context # type: ignore [import-not-found] + from ghex.unstructured import ( # type: ignore [import-not-found] + DomainDescriptor, + HaloGenerator, + make_communication_object, + make_field_descriptor, + make_pattern, + ) + from ghex.util import Architecture # type: ignore [import-not-found] mpi4py.rc.initialize = False mpi4py.rc.finalize = True @@ -39,7 +46,7 @@ unstructured = None if TYPE_CHECKING: - import mpi4py.MPI + import mpi4py.MPI # type: ignore [import-not-found] CommId = Union[int, "mpi4py.MPI.Comm", None] log = logging.getLogger(__name__) @@ -98,15 +105,14 @@ def get_multinode_properties( @dataclass(frozen=True) class MPICommProcessProperties(definitions.ProcessProperties): - comm: mpi4py.MPI.Comm - + comm: mpi4py.MPI.Comm = None @functools.cached_property - def rank(self) -> int: + def rank(self) -> int: # type: ignore [override] return self.comm.Get_rank() @functools.cached_property - def comm_name(self) -> str: + def comm_name(self) -> str: # type: ignore [override] return self.comm.Get_name() @functools.cached_property @@ -131,7 +137,7 @@ def __init__( dim: self._create_domain_descriptor(dim) for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values() } - self._field_size: dict[gtx.Dimension : int] = { + self._field_size: dict[gtx.Dimension, int] = { dim: self._decomposition_info.global_index(dim).shape[0] for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values() } @@ -140,7 +146,7 @@ def __init__( dim: self._create_pattern(dim) for dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values() } log.info(f"patterns for dimensions {self._patterns.keys()} initialized ") - self._comm = unstructured.make_communication_object(self._context) + self._comm = make_communication_object(self._context) # DaCe SDFGConvertible interface self.num_of_halo_tasklets = ( @@ -151,7 +157,7 @@ def __init__( log.info("communication object initialized") - def _domain_descriptor_info(self, descr: unstructured.DomainDescriptor) -> str: + def _domain_descriptor_info(self, descr: DomainDescriptor) -> str: return f" domain_descriptor=[id='{descr.domain_id()}', size='{descr.size()}', inner_size='{descr.inner_size()}' (halo size='{descr.size() - descr.inner_size()}')" def get_size(self) -> int: @@ -160,7 +166,7 @@ def get_size(self) -> int: def my_rank(self) -> int: return self._context.rank() - def _create_domain_descriptor(self, dim: gtx.Dimension) -> unstructured.DomainDescriptor: + def _create_domain_descriptor(self, dim: gtx.Dimension) -> DomainDescriptor: all_global = self._decomposition_info.global_index( dim, definitions.DecompositionInfo.EntryType.ALL ) @@ -170,7 +176,7 @@ def _create_domain_descriptor(self, dim: gtx.Dimension) -> unstructured.DomainDe # first arg is the domain ID which builds up an MPI Tag. # if those ids are not different for all domain descriptors the system might deadlock # if two parallel exchanges with the same domain id are done - domain_desc = unstructured.DomainDescriptor( + domain_desc = DomainDescriptor( self._domain_id_gen(), all_global.tolist(), local_halo.tolist() ) log.debug( @@ -178,15 +184,15 @@ def _create_domain_descriptor(self, dim: gtx.Dimension) -> unstructured.DomainDe ) return domain_desc - def _create_pattern(self, horizontal_dim: gtx.Dimension) -> unstructured.DomainDescriptor: + def _create_pattern(self, horizontal_dim: gtx.Dimension) -> DomainDescriptor: assert horizontal_dim.kind == gtx.DimensionKind.HORIZONTAL global_halo_idx = self._decomposition_info.global_index( horizontal_dim, definitions.DecompositionInfo.EntryType.HALO ) - halo_generator = unstructured.HaloGenerator.from_gids(global_halo_idx) + halo_generator = HaloGenerator.from_gids(global_halo_idx) log.debug(f"halo generator for dim='{horizontal_dim.value}' created") - pattern = unstructured.make_pattern( + pattern = make_pattern( self._context, halo_generator, [self._domain_descriptors[horizontal_dim]], @@ -203,20 +209,16 @@ def _slice_field_based_on_dim(self, field: gtx.Field, dim: gtx.Dimension) -> dat This operation is *necessary* for the use inside FORTRAN as there fields are larger than the grid (nproma size). where it does not do anything in a purely Python setup. the granule context where fields otherwise have length nproma. """ - if dim == dims.VertexDim: - return field.ndarray[: self._decomposition_info.num_vertices] - elif dim == dims.EdgeDim: - return field.ndarray[: self._decomposition_info.num_edges] - elif dim == dims.CellDim: - return field.ndarray[: self._decomposition_info.num_cells] + if dim == dims.VertexDim or dim == dims.EdgeDim or dim == dims.CellDim: + return field.ndarray[: self._field_size[dim]] else: raise ValueError(f"Unknown dimension {dim}") def _make_field_descriptor(self, dim: gtx.Dimension, array: data_alloc.NDArray) -> Any: - return unstructured.make_field_descriptor( + return make_field_descriptor( self._domain_descriptors[dim], array, - arch=unstructured.Architecture.CPU if isinstance(array, np.ndarray) else Architecture.GPU, + arch=Architecture.CPU if isinstance(array, np.ndarray) else Architecture.GPU, ) def _get_applied_pattern(self, dim: gtx.Dimension, f: gtx.Field | data_alloc.NDArray) -> str: diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index ba758cb8eb..f4823d9f0d 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -19,13 +19,7 @@ from icon4py.model.common import dimension as dims, type_alias as ta from icon4py.model.common.decomposition import definitions as decomposition, halo from icon4py.model.common.exceptions import InvalidConfigError -from icon4py.model.common.grid import ( - base, - grid_refinement as refinement, - gridfile, - icon, - vertical as v_grid, -) +from icon4py.model.common.grid import base, grid_refinement as refinement, gridfile, icon from icon4py.model.common.utils import data_allocation as data_alloc @@ -97,7 +91,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): def __call__( self, - allocator: gtx_typing.FieldBufferAllocationUtil, + allocator: gtx_typing.FieldBufferAllocationUtil | None, keep_skip_values: bool, decomposer: halo.Decomposer = _single_node_decomposer, run_properties=_single_process_props, diff --git a/model/common/src/icon4py/model/common/grid/icon.py b/model/common/src/icon4py/model/common/grid/icon.py index ca38755c63..0c6f365284 100644 --- a/model/common/src/icon4py/model/common/grid/icon.py +++ b/model/common/src/icon4py/model/common/grid/icon.py @@ -203,6 +203,7 @@ class IconGrid(base.Grid): default=None, kw_only=True ) + # TODO (halungge): combine the last to args "into single_node_global" def _has_skip_values(offset: gtx.FieldOffset, limited_area: bool, distributed: bool) -> bool: """ diff --git a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py index 74310eb07c..e8f8710993 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py @@ -62,7 +62,8 @@ """ -@pytest.mark.mpi(min_size= 2) + +@pytest.mark.mpi(min_size=2) @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_props(processor_props: definitions.ProcessProperties) -> None: assert processor_props.comm diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index 5021ad5a43..6e88ebee4f 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -14,8 +14,8 @@ import icon4py.model.common.dimension as dims import icon4py.model.common.utils.data_allocation as data_alloc from icon4py.model.common.decomposition import definitions, halo -from icon4py.model.testing import definitions as test_defs from icon4py.model.common.grid import simple +from icon4py.model.testing import definitions as test_defs from icon4py.model.testing.fixtures import processor_props from ...grid import utils as grid_utils @@ -133,7 +133,9 @@ def test_decomposition_info_single_node_empty_halo( if not processor_props.single_node(): pytest.xfail() - manager = grid_utils.run_grid_manager(test_defs.Grids.MCH_CH_R04B09_DSL, keep_skip_values=True, backend=None) + manager = grid_utils.run_grid_manager( + test_defs.Grids.MCH_CH_R04B09_DSL, keep_skip_values=True, backend=None + ) decomposition_info = manager.decomposition_info for level in definitions.DecompositionFlag: diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index b7241073c8..5827ff75fc 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -55,6 +55,8 @@ ) +NUM_LEVELS = 10 + try: import mpi4py @@ -63,31 +65,27 @@ pytest.skip("Skipping parallel on single node installation", allow_module_level=True) log = logging.getLogger(__file__) -vertical_config = v_grid.VerticalGridConfig(num_levels=10) def run_gridmananger_for_multinode( file: pathlib.Path, - vertical_config: v_grid.VerticalGridConfig, run_properties: defs.ProcessProperties, decomposer: halo.Decomposer, ) -> gm.GridManager: - manager = _grid_manager(file, vertical_config) + manager = _grid_manager(file, num_levels=NUM_LEVELS) manager( keep_skip_values=True, allocator=None, run_properties=run_properties, decomposer=decomposer ) return manager -def _grid_manager(file: pathlib.Path, vertical_config: v_grid.VerticalGridConfig) -> gm.GridManager: - manager = gm.GridManager(str(file), vertical_config) +def _grid_manager(file: pathlib.Path, num_levels: int) -> gm.GridManager: + manager = gm.GridManager(str(file), num_levels=num_levels) return manager -def run_grid_manager_for_singlenode( - file: pathlib.Path, vertical_config: v_grid.VerticalGridConfig -) -> gm.GridManager: - manager = _grid_manager(file, vertical_config) +def run_grid_manager_for_singlenode(file: pathlib.Path) -> gm.GridManager: + manager = _grid_manager(file, NUM_LEVELS) manager( keep_skip_values=True, run_properties=defs.SingleNodeProcessProperties(), @@ -101,7 +99,7 @@ def run_grid_manager_for_singlenode( @pytest.mark.mpi(min_size=2) def test_grid_manager_validate_decomposer(processor_props: defs.ProcessProperties) -> None: file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) - manager = gm.GridManager(file, vertical_config, gridfile.ToZeroBasedIndexTransformation()) + manager = gm.GridManager(file, NUM_LEVELS, gridfile.ToZeroBasedIndexTransformation()) with pytest.raises(exceptions.InvalidConfigError) as e: manager( keep_skip_values=True, @@ -134,7 +132,6 @@ def test_local_connectivities( halo_generator = halo.IconLikeHaloConstructor( connectivities=neighbor_tables, run_properties=processor_props, - num_levels=1, ) decomposition_info = halo_generator(labels) @@ -163,7 +160,7 @@ def test_fields_distribute_and_gather(processor_props: defs.ProcessProperties, c caplog.set_level(logging.INFO) print(f"myrank - {processor_props.rank}: running with processor_props = {processor_props}") file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) - single_node = run_grid_manager_for_singlenode(file, vertical_config) + single_node = run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid global_cell_area = single_node.geometry_fields[gridfile.GeometryName.CELL_AREA] global_edge_lat = single_node.coordinates[dims.EdgeDim]["lat"] @@ -171,7 +168,6 @@ def test_fields_distribute_and_gather(processor_props: defs.ProcessProperties, c multinode = run_gridmananger_for_multinode( file=file, - vertical_config=vertical_config, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), ) @@ -293,7 +289,7 @@ def test_halo_neighbor_access_c2e( ) -> None: file = grid_utils.resolve_full_grid_file_name(grid) print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") - single_node = run_grid_manager_for_singlenode(file, vertical_config) + single_node = run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( backend=backend, @@ -326,7 +322,6 @@ def test_halo_neighbor_access_c2e( ) multinode_grid_manager = run_gridmananger_for_multinode( file=file, - vertical_config=vertical_config, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), ) @@ -383,7 +378,7 @@ def test_halo_access_e2c2v( ) -> None: file = grid_utils.resolve_full_grid_file_name(grid) print(f"running on {processor_props.comm}") - single_node = run_grid_manager_for_singlenode(file, vertical_config) + single_node = run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( backend=backend, @@ -400,7 +395,6 @@ def test_halo_access_e2c2v( reference_v = single_node_geometry.get(geometry_attributes.EDGE_NORMAL_VERTEX_V).asnumpy() multinode_grid_manager = run_gridmananger_for_multinode( file=file, - vertical_config=vertical_config, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), ) @@ -474,7 +468,7 @@ def test_halo_access_e2c( ) -> None: file = grid_utils.resolve_full_grid_file_name(grid) print(f"running on {processor_props.comm}") - single_node = run_grid_manager_for_singlenode(file, vertical_config) + single_node = run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( backend=backend, @@ -491,7 +485,6 @@ def test_halo_access_e2c( reference_v = single_node_geometry.get(geometry_attributes.EDGE_NORMAL_CELL_V).asnumpy() multinode_grid_manager = run_gridmananger_for_multinode( file=file, - vertical_config=vertical_config, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), ) @@ -564,7 +557,7 @@ def test_halo_neighbor_access_e2v( ) -> None: print(f"running on {processor_props.comm}") file = grid_utils.resolve_full_grid_file_name(grid) - single_node = run_grid_manager_for_singlenode(file, vertical_config) + single_node = run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( backend=backend, @@ -581,7 +574,6 @@ def test_halo_neighbor_access_e2v( ) multinode_grid_manager = run_gridmananger_for_multinode( file=file, - vertical_config=vertical_config, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), ) @@ -594,7 +586,7 @@ def test_halo_neighbor_access_e2v( ) distributed_coordinates = multinode_grid_manager.coordinates vertex_lat = distributed_coordinates[dims.VertexDim]["lat"] - vertex_lon = distributed_coordinates.get[dims.VertexDim]["lon"] + vertex_lon = distributed_coordinates[dims.VertexDim]["lon"] tangent_orientation = multinode_grid_manager.geometry_fields.get( gridfile.GeometryName.TANGENT_ORIENTATION ) @@ -639,10 +631,9 @@ def test_halo_neighbor_access_v2e( backend: gtx_typing.Backend | None, grid: definitions.GridDescription, ) -> None: - # processor_props = decomp_utils.DummyProps(1) file = grid_utils.resolve_full_grid_file_name(grid) print(f"running on {processor_props.comm}") - single_node = run_grid_manager_for_singlenode(file, vertical_config) + single_node = run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( backend=backend, @@ -659,7 +650,7 @@ def test_halo_neighbor_access_v2e( single_node_dual_area = single_node_geometry.get(geometry_attributes.DUAL_AREA) single_node_owner_mask = gtx.as_field( (dims.VertexDim,), - data=single_node.decomposition_info.owner_mask(dims.VertexDim), + data=single_node.decomposition_info.owner_mask(dims.VertexDim), # type: ignore [arg-type] dtype=bool, ) reference = data_alloc.zero_field(single_node_grid, dims.VertexDim, dims.V2EDim) @@ -683,7 +674,6 @@ def test_halo_neighbor_access_v2e( ) multinode_grid_manager = run_gridmananger_for_multinode( file=file, - vertical_config=vertical_config, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), ) @@ -709,7 +699,7 @@ def test_halo_neighbor_access_v2e( edge_orientation = distributed_geometry.get(geometry_attributes.VERTEX_EDGE_ORIENTATION) dual_area = distributed_geometry.get(geometry_attributes.DUAL_AREA) geofac_rot = data_alloc.zero_field(distributed_grid, dims.VertexDim, dims.V2EDim) - onwner_mask = gtx.as_field((dims.VertexDim,), decomposition_info.owner_mask(dims.VertexDim)) + onwner_mask = gtx.as_field((dims.VertexDim,), decomposition_info.owner_mask(dims.VertexDim)) # type: ignore [arg-type] interpolation_fields.compute_geofac_rot.with_backend(None)( dual_edge_length=dual_edge_length, edge_orientation=edge_orientation, @@ -743,7 +733,7 @@ def test_halo_neighbor_access_c2e2c( xp = data_alloc.import_array_ns(allocator=backend) start_zone = h_grid.cell_domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2) print(f"running on {processor_props.comm}") - single_node = run_grid_manager_for_singlenode(file, vertical_config) + single_node = run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( backend=backend, @@ -767,7 +757,6 @@ def test_halo_neighbor_access_c2e2c( ) multinode_grid_manager = run_gridmananger_for_multinode( file=file, - vertical_config=vertical_config, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), ) @@ -815,31 +804,30 @@ def test_halo_neighbor_access_c2e2c( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_halo_neighbor_access_v2c(processor_props, backend): +def test_halo_neighbor_access_v2c( + processor_props: defs.ProcessProperties, backend: gtx_typing.Backend +) -> None: file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) print(f"running on {processor_props.comm}") - single_node = run_grid_manager_for_singlenode(file, vertical_config) + single_node = run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid + data = np.repeat( + single_node.coordinates[dims.CellDim]["lat"].asnumpy()[:, None], repeats=NUM_LEVELS, axis=1 + ) full_cell_k_field = gtx.as_field( (dims.CellDim, dims.KDim), - data=np.repeat( - single_node.coordinates[dims.CellDim]["lat"].ndarray[:, None], - vertical_config.num_levels, - axis=1, - ), + data=data, # type: ignore [arg-type] dtype=float, allocator=backend, ) print( f"rank = {processor_props.rank} / {processor_props.comm_size}: single node input field has size {full_cell_k_field.asnumpy().shape}" ) - vertex_data = single_node.coordinates[dims.VertexDim]["lat"].ndarray / np.max( - single_node.coordinates[dims.VertexDim]["lat"].ndarray - ) + buffer = single_node.coordinates[dims.VertexDim]["lat"].asnumpy() full_coef = gtx.as_field( (dims.VertexDim, dims.V2CDim), - data=np.repeat(vertex_data[:, None], 6, axis=1), + data=np.repeat((buffer / np.max(buffer))[:, None], 6, axis=1), # type: ignore [arg-type] dtype=float, allocator=backend, ) @@ -863,7 +851,6 @@ def test_halo_neighbor_access_v2c(processor_props, backend): ) multinode_grid_manager = run_gridmananger_for_multinode( file=file, - vertical_config=vertical_config, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), ) @@ -879,7 +866,7 @@ def test_halo_neighbor_access_v2c(processor_props, backend): ) my_global_cells = decomposition_info.global_index(dims.CellDim) cell_k_buffer = ( - full_cell_k_field.ndarray[my_global_cells, :] + full_cell_k_field.asnumpy()[my_global_cells, :] .ravel(order="K") .reshape(distributed_grid.num_cells, 10) ) @@ -887,7 +874,7 @@ def test_halo_neighbor_access_v2c(processor_props, backend): decomposition_info, processor_props, dims.CellDim, - global_reference_field=full_cell_k_field.ndarray, + global_reference_field=full_cell_k_field.asnumpy(), local_field=cell_k_buffer, ) print( @@ -895,13 +882,16 @@ def test_halo_neighbor_access_v2c(processor_props, backend): ) cell_k_field = gtx.as_field( - (dims.CellDim, dims.KDim), data=cell_k_buffer, dtype=cell_k_buffer.dtype, allocator=backend + (dims.CellDim, dims.KDim), + data=cell_k_buffer, # type: ignore [arg-type] + dtype=cell_k_buffer.dtype, + allocator=backend, ) my_global_vertices = decomposition_info.global_index(dims.VertexDim) coef = ( - full_coef.ndarray[my_global_vertices, :] + full_coef.asnumpy()[my_global_vertices, :] .ravel(order="K") .reshape((distributed_grid.num_vertices, 6)) ) @@ -910,13 +900,13 @@ def test_halo_neighbor_access_v2c(processor_props, backend): decomposition_info, processor_props, dims.VertexDim, - global_reference_field=full_coef.ndarray, + global_reference_field=full_coef.asnumpy(), local_field=coef, ) print( f"rank={processor_props.rank}/{processor_props.comm_size}: coefficient shape = ([{coef.shape})" ) - coef_field = gtx.as_field((dims.VertexDim, dims.V2CDim), data=coef, allocator=backend) + coef_field = gtx.as_field((dims.VertexDim, dims.V2CDim), data=coef, allocator=backend) # type: ignore [arg-type] output = data_alloc.zero_field(distributed_grid, dims.VertexDim, dims.KDim, allocator=backend) _compute_cell_2_vertex_interpolation( cell_k_field, diff --git a/model/common/tests/common/grid/unit_tests/test_geometry.py b/model/common/tests/common/grid/unit_tests/test_geometry.py index ce82fb4bbe..593418f715 100644 --- a/model/common/tests/common/grid/unit_tests/test_geometry.py +++ b/model/common/tests/common/grid/unit_tests/test_geometry.py @@ -374,7 +374,7 @@ def test_create_auxiliary_orientation_coordinates( ) -> None: gm = grid_utils.get_grid_manager_from_identifier( experiment.grid, - num_levels=1, + num_levels=experiment.num_levels, keep_skip_values=True, allocator=backend, ) diff --git a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py index eb83316129..eab198cbad 100644 --- a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py +++ b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py @@ -36,7 +36,7 @@ def test_edge_length( keep = True grid_file = experiment.grid gm = grid_utils.get_grid_manager_from_identifier( - grid_file, keep_skip_values=keep, num_levels=1, allocator=backend + grid_file, num_levels=experiment.num_levels, keep_skip_values=keep, allocator=backend ) grid = gm.grid coordinates = gm.coordinates[dims.VertexDim] diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index 8c6a03cd0e..11cf30c5af 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -336,8 +336,8 @@ def test_gridmanager_given_file_not_found_then_abort( with pytest.raises(FileNotFoundError) as error: manager = gm.GridManager( fname, - v_grid.VerticalGridConfig(num_levels=80), - icon4py.model.common.grid.gridfile.NoTransformation(), + num_levels=80, + transformation=icon4py.model.common.grid.gridfile.NoTransformation(), ) manager(allocator=cpu_allocator, keep_skip_values=True) assert error.value == 1 diff --git a/model/common/tests/common/grid/utils.py b/model/common/tests/common/grid/utils.py index 65f5daec23..00e76414ec 100644 --- a/model/common/tests/common/grid/utils.py +++ b/model/common/tests/common/grid/utils.py @@ -87,6 +87,7 @@ def run_grid_manager( manager = gridtest_utils.get_grid_manager_from_identifier( grid, keep_skip_values=keep_skip_values, + num_levels=1, allocator=model_backends.get_allocator(backend), ) managers[key] = manager diff --git a/model/testing/src/icon4py/model/testing/fixtures/benchmark.py b/model/testing/src/icon4py/model/testing/fixtures/benchmark.py index 47c4c990ac..08b91147a4 100644 --- a/model/testing/src/icon4py/model/testing/fixtures/benchmark.py +++ b/model/testing/src/icon4py/model/testing/fixtures/benchmark.py @@ -25,7 +25,6 @@ from icon4py.model.common.interpolation import interpolation_attributes, interpolation_factory from icon4py.model.common.metrics import metrics_attributes, metrics_factory from icon4py.model.common.utils import data_allocation as data_alloc -from icon4py.model.testing import grid_utils @pytest.fixture( @@ -39,9 +38,8 @@ def geometry_field_source( pytest.skip("Incomplete grid Information for test, are you running with `simple_grid`?") mesh = grid_manager.grid - allocator = model_backends.get_allocator(backend_like) generic_concrete_backend = model_options.customize_backend(None, backend_like) - decomposition_info = grid_utils.construct_decomposition_info(mesh, allocator) + decomposition_info = grid_manager.decomposition_info geometry_field_source = grid_geometry.GridGeometry( grid=mesh, diff --git a/model/testing/src/icon4py/model/testing/grid_utils.py b/model/testing/src/icon4py/model/testing/grid_utils.py index 392eae19c3..0b8df3daa2 100644 --- a/model/testing/src/icon4py/model/testing/grid_utils.py +++ b/model/testing/src/icon4py/model/testing/grid_utils.py @@ -30,6 +30,7 @@ def get_grid_manager_from_experiment( ) -> gm.GridManager: return get_grid_manager_from_identifier( experiment.grid, + num_levels=experiment.num_levels, keep_skip_values=keep_skip_values, allocator=allocator, ) @@ -37,17 +38,19 @@ def get_grid_manager_from_experiment( def get_grid_manager_from_identifier( grid: definitions.GridDescription, + num_levels: int, keep_skip_values: bool, allocator: gtx_typing.FieldBufferAllocationUtil, ) -> gm.GridManager: grid_file = _download_grid_file(grid) return get_grid_manager( - grid_file, keep_skip_values=keep_skip_values, allocator=allocator + grid_file, num_levels=num_levels, keep_skip_values=keep_skip_values, allocator=allocator ) def get_grid_manager( filename: pathlib.Path, + num_levels: int, keep_skip_values: bool, allocator: gtx_typing.FieldBufferAllocationUtil, ) -> gm.GridManager: @@ -62,7 +65,8 @@ def get_grid_manager( """ manager = gm.GridManager( filename, - gridfile.ToZeroBasedIndexTransformation(), + num_levels=num_levels, + transformation=gridfile.ToZeroBasedIndexTransformation(), ) manager(allocator=allocator, keep_skip_values=keep_skip_values) return manager @@ -102,6 +106,7 @@ def get_grid_geometry( def _construct_grid_geometry() -> geometry.GridGeometry: gm = get_grid_manager_from_identifier( experiment.grid, + num_levels=experiment.num_levels, keep_skip_values=True, allocator=model_backends.get_allocator(backend), ) From 378d269ceb510d7da0d17e36579b545c77ad815a Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Thu, 11 Dec 2025 21:06:47 +0100 Subject: [PATCH 119/240] use dummy props on test to make them simple unit tests add third level edges --- .../model/common/decomposition/definitions.py | 9 +- .../model/common/decomposition/halo.py | 119 +++++++++----- .../common/decomposition/mpi_decomposition.py | 10 +- .../model/common/grid/grid_refinement.py | 4 +- .../src/icon4py/model/common/grid/gridfile.py | 2 +- .../src/icon4py/model/common/grid/icon.py | 14 +- .../interpolation/interpolation_factory.py | 2 +- .../tests/common/decomposition/fixtures.py | 21 +++ .../decomposition/mpi_tests/test_halo.py | 137 ++--------------- .../mpi_tests/test_mpi_decomposition.py | 10 +- .../unit_tests/test_definitions.py | 2 +- .../decomposition/unit_tests/test_halo.py | 145 ++++++++++++++++++ .../tests/common/decomposition/utils.py | 13 +- .../mpi_tests/test_parallel_grid_manager.py | 17 +- .../tests/common/grid/unit_tests/test_icon.py | 4 +- .../icon4py/model/testing/parallel_helpers.py | 21 +-- 16 files changed, 307 insertions(+), 223 deletions(-) create mode 100644 model/common/tests/common/decomposition/fixtures.py create mode 100644 model/common/tests/common/decomposition/unit_tests/test_halo.py diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index cd44592b89..81a495695f 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -97,7 +97,6 @@ def set_dimension( self._owner_mask[dim] = owner_mask self._halo_levels[dim] = halo_levels - def local_index( self, dim: gtx.Dimension, entry_type: EntryType = EntryType.ALL ) -> data_alloc.NDArray: @@ -407,7 +406,7 @@ class DecompositionFlag(int, Enum): OWNED = 0 """used for locally owned cells, vertices, edges""" - FIRST_HALO_LINE = 1 + FIRST_HALO_LEVEL = 1 """ used for: - cells that share 1 edge with an OWNED cell @@ -415,15 +414,15 @@ class DecompositionFlag(int, Enum): - edges that are on OWNED cell, but not owned """ - SECOND_HALO_LINE = 2 + SECOND_HALO_LEVEL = 2 """ used for: - - cells that share a vertex with an OWNED cell + - cells that share one vertex with an OWNED cell - vertices that are on a cell(FIRST_HALO_LINE) but not on an owned cell - edges that have _exactly_ one vertex shared with and OWNED Cell """ - THIRD_HALO_LINE = 3 + THIRD_HALO_LEVEL = 3 """ This type does not exist in ICON. It denotes the "closing/far" edges of the SECOND_HALO_LINE cells used for: diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index ea4bc15934..07b19d8938 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -57,9 +57,9 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: def _create_dummy_decomposition_arrays( size: int, array_ns: ModuleType = np ) -> tuple[data_alloc.NDArray, data_alloc.NDArray, data_alloc.NDArray]: - indices = array_ns.arange(size, dtype=gtx.int32) + indices = array_ns.arange(size, dtype=gtx.int32) # type: ignore [attr-defined] owner_mask = array_ns.ones((size,), dtype=bool) - halo_levels = array_ns.ones((size,), dtype=gtx.int32) * defs.DecompositionFlag.OWNED + halo_levels = array_ns.ones((size,), dtype=gtx.int32) * defs.DecompositionFlag.OWNED # type: ignore [attr-defined] return indices, owner_mask, halo_levels @@ -77,7 +77,7 @@ def __init__( Args: run_properties: contains information on the communicator and local compute node. connectivities: connectivity arrays needed to construct the halos - backend: GT4Py (used to determine the array ns import) + allocator: GT4Py buffer allocator """ self._xp = data_alloc.import_array_ns(allocator) self._props = run_properties @@ -244,29 +244,67 @@ def _update_owner_mask_by_max_rank_convention( def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: """ - Constructs the DecompositionInfo for the current rank. - - The DecompositionInfo object is constructed for all horizontal dimension starting from the - cell distribution. Edges and vertices are then handled through their connectivity to the distributed cells. + Constructs the DecompositionInfo for the current rank. + + Args: + face_to_rank: a mapping of cells to a rank + + The DecompositionInfo object is constructed for all horizontal dimension starting from the + cell distribution. + Edges and vertices are then handled through their connectivity to the distributed cells. + + This constructs a halo similar to ICON which consists of **exactly** 2 cell-halo lines + + | /| /| /| + | / | / | / | + | / | / | / | + | / | / | / | + | / | / | / | rank 0 + |/ |/ |/ | + ----------e0--------e1--------e3------------ cutting line + | /| /| /| + | c0 / | c1 / | c2 / | + | / | / | / | rank 1 + e4 e5 e6 e7 e8 e9 e10 + | / | / | / | / + | / c3 | / c4 | / c5 | / + |/ |/ |/ |/ + ------e11------e12-------e13------------ + | /| /| /| + | / | / | / | + | / | / | / | + | / | / | / | + | / | / | / | / + |/ |/ |/ |/ + + + Cells: + The "numbered" cells and edges are relevant for the halo construction from the point of view of rank 0 + Cells (c0, c1, c2) are the 1. HALO LEVEL: these are cells that are neighbors of an owned cell + Cells (c3, c4. c5) are the 2. HALO LEVEL: these are cells that are neighbors of a cell of line 1 + + Note that this definition of 1. and 2. line differs from the definition of boundary line counting used in [grid refinement](grid_refinement.py), in terms + of "distance" to the cutting line all halo cells have a distance of 1. + + Edges: + Edges (e0, e1, e2) are on the cutting line. + For both ranks the edges on the cutting line sit on **owned cells**. As all elements need to have a unique owner, they are assigned by convention to the rank with the higher number, here rank 1. + From the point of view of rank 0 they are 1. HALO LINE edges. This conventional assignement has as an effect that there ranks (essentially rank 0) that have an *empty* first edge HALO LINE, + even thought they have elements in the 2. HALO LEVEL (e4, e5, e6, e7, e8, e9, e10) which are the edges that share exactly one vertex with an owned cell. + The edges (e11, e12, e13) that "close" the halo cells (share exactly 2 vertices with a halo cell, but none with an owned cell) are **not** included in the halo in ICON. We include them as 3. HALO LINE which + makes the C2E connectivity complete (= without skip value) for a distributed setup. + + + + # TODO(halungge): make number of halo lines (in terms of cells) a parameter + # icon does hard coding of 2 halo lines for cells, make this dynamic! """ - #: icon does hard coding of 2 halo lines for cells, make this dynamic! - - # TODO(halungge): make number of halo lines a parameter self._validate_mapping(face_to_rank) - #: cells + #: cells owned_cells = self.owned_cells(face_to_rank) # global indices of owned cells - # cell_halos = [] - # current = owned_cells - # depot = None - # for i in range(num_cell_halo_lines): - # cell_halos[i] = self.next_halo_line(current, depot) - # depot = self._xp.union1d(depot, current) - # current = cell_halos[i] - # first_halo_cells = cell_halos[0] - # second_halo_cells = cell_halos[1] first_halo_cells = self.next_halo_line(owned_cells) second_halo_cells = self.next_halo_line(first_halo_cells, owned_cells) @@ -280,10 +318,10 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: ) cell_halo_levels[cell_owner_mask] = defs.DecompositionFlag.OWNED cell_halo_levels[self._xp.isin(all_cells, first_halo_cells)] = ( - defs.DecompositionFlag.FIRST_HALO_LINE + defs.DecompositionFlag.FIRST_HALO_LEVEL ) cell_halo_levels[self._xp.isin(all_cells, second_halo_cells)] = ( - defs.DecompositionFlag.SECOND_HALO_LINE + defs.DecompositionFlag.SECOND_HALO_LEVEL ) decomp_info = defs.DecompositionInfo().set_dimension( dims.CellDim, all_cells, cell_owner_mask, cell_halo_levels @@ -292,9 +330,6 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: #: vertices vertex_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) vertex_on_first_halo_line = self.find_vertex_neighbors_for_cells(first_halo_cells) - vertex_on_second_halo_line = self.find_vertex_neighbors_for_cells( - second_halo_cells - ) # TODO(halungge): do we need that at all? vertex_on_cutting_line = self._xp.intersect1d( vertex_on_owned_cells, vertex_on_first_halo_line @@ -322,9 +357,9 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: self._xp.logical_not(vertex_owner_mask), self._xp.isin(all_vertices, vertex_on_cutting_line), ) - ] = defs.DecompositionFlag.FIRST_HALO_LINE + ] = defs.DecompositionFlag.FIRST_HALO_LEVEL vertex_halo_levels[self._xp.isin(all_vertices, vertex_second_level)] = ( - defs.DecompositionFlag.SECOND_HALO_LINE + defs.DecompositionFlag.SECOND_HALO_LEVEL ) decomp_info.set_dimension( dims.VertexDim, all_vertices, vertex_owner_mask, vertex_halo_levels @@ -334,17 +369,16 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: edges_on_owned_cells = self.find_edge_neighbors_for_cells(owned_cells) edges_on_first_halo_line = self.find_edge_neighbors_for_cells(first_halo_cells) edges_on_cutting_line = self._xp.intersect1d(edges_on_owned_cells, edges_on_first_halo_line) - level_two_edges = self._xp.setdiff1d( + edges_on_halo_cells = self.find_edge_neighbors_for_cells(total_halo_cells) + + edge_second_level = self._xp.setdiff1d( self.find_edge_neighbors_for_vertices(vertex_on_cutting_line), edges_on_owned_cells ) + edge_second_and_third_level = self._xp.setdiff1d(edges_on_halo_cells, edges_on_cutting_line) + edge_third_level = self._xp.setdiff1d(edge_second_and_third_level, edge_second_level) all_edges = self._xp.unique( - self._xp.hstack( - ( - edges_on_owned_cells, - level_two_edges, - ) - ) + self._xp.hstack((edges_on_owned_cells, edge_second_level, edge_third_level)) ) # construct the owner mask @@ -361,21 +395,26 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: all_edges.shape, dtype=int ) edge_halo_levels[edge_owner_mask] = defs.DecompositionFlag.OWNED - # LEVEL_ONE edges are on a owned cell but are not owned: these are all edges on the cutting line that are not owned (by the convention) + # LEVEL_ONE edges are on an owned cell but are not owned: these are all edges on the cutting line that are not owned (by the convention) edge_halo_levels[ self._xp.logical_and( self._xp.logical_not(edge_owner_mask), self._xp.isin(all_edges, edges_on_cutting_line), ) - ] = defs.DecompositionFlag.FIRST_HALO_LINE + ] = defs.DecompositionFlag.FIRST_HALO_LEVEL - # LEVEL_TWO edges share exactly one vertext with an owned cell, they are on the first halo-line cells, but not on the cutting line - edge_halo_levels[self._xp.isin(all_edges, level_two_edges)] = ( - defs.DecompositionFlag.SECOND_HALO_LINE + # LEVEL_TWO edges share exactly one vertex with an owned cell, they are on the first halo-line cells, but not on the cutting line + edge_halo_levels[self._xp.isin(all_edges, edge_second_level)] = ( + defs.DecompositionFlag.SECOND_HALO_LEVEL ) decomp_info.set_dimension(dims.EdgeDim, all_edges, edge_owner_mask, edge_halo_levels) + # LEVEL_THREE edges + # LEVEL_TWO edges share exactly one vertex with an owned cell, they are on the first halo-line cells, but not on the cutting line + edge_halo_levels[self._xp.isin(all_edges, edge_third_level)] = ( + defs.DecompositionFlag.THIRD_HALO_LEVEL + ) return decomp_info @@ -431,7 +470,7 @@ def __call__( self, adjacency_matrix: data_alloc.NDArray, num_partitions: int = 1 ) -> data_alloc.NDArray: """Dummy decomposer for single node: assigns all cells to rank = 0""" - return np.zeros(adjacency_matrix.shape[0], dtype=gtx.int32) + return np.zeros(adjacency_matrix.shape[0], dtype=gtx.int32) # type: ignore [attr-defined] def halo_constructor( diff --git a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py index 2a4b3a6e51..044624e549 100644 --- a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py +++ b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py @@ -26,7 +26,7 @@ try: import ghex # type: ignore [import-not-found] - import mpi4py # type: ignore [import-not-found] + import mpi4py from ghex.context import make_context # type: ignore [import-not-found] from ghex.unstructured import ( # type: ignore [import-not-found] DomainDescriptor, @@ -41,12 +41,12 @@ mpi4py.rc.finalize = True except ImportError: - mpi4py = None + mpi4py = None # type: ignore [assignment] ghex = None unstructured = None if TYPE_CHECKING: - import mpi4py.MPI # type: ignore [import-not-found] + import mpi4py.MPI CommId = Union[int, "mpi4py.MPI.Comm", None] log = logging.getLogger(__name__) @@ -105,7 +105,7 @@ def get_multinode_properties( @dataclass(frozen=True) class MPICommProcessProperties(definitions.ProcessProperties): - comm: mpi4py.MPI.Comm = None + comm: mpi4py.MPI.Comm @functools.cached_property def rank(self) -> int: # type: ignore [override] @@ -209,7 +209,7 @@ def _slice_field_based_on_dim(self, field: gtx.Field, dim: gtx.Dimension) -> dat This operation is *necessary* for the use inside FORTRAN as there fields are larger than the grid (nproma size). where it does not do anything in a purely Python setup. the granule context where fields otherwise have length nproma. """ - if dim == dims.VertexDim or dim == dims.EdgeDim or dim == dims.CellDim: + if dim in dims.MAIN_HORIZONTAL_DIMENSIONS: return field.ndarray[: self._field_size[dim]] else: raise ValueError(f"Unknown dimension {dim}") diff --git a/model/common/src/icon4py/model/common/grid/grid_refinement.py b/model/common/src/icon4py/model/common/grid/grid_refinement.py index e280255b5d..9b84b93e10 100644 --- a/model/common/src/icon4py/model/common/grid/grid_refinement.py +++ b/model/common/src/icon4py/model/common/grid/grid_refinement.py @@ -184,10 +184,10 @@ def compute_domain_bounds( ) owned = decomposition_info.owner_mask(dim) halo_level_1 = decomposition_info.halo_level_mask( - dim, decomposition.DecompositionFlag.FIRST_HALO_LINE + dim, decomposition.DecompositionFlag.FIRST_HALO_LEVEL ) halo_level_2 = decomposition_info.halo_level_mask( - dim, decomposition.DecompositionFlag.SECOND_HALO_LINE + dim, decomposition.DecompositionFlag.SECOND_HALO_LEVEL ) start_indices = {} diff --git a/model/common/src/icon4py/model/common/grid/gridfile.py b/model/common/src/icon4py/model/common/grid/gridfile.py index 128eaf2903..6c3db3f01a 100644 --- a/model/common/src/icon4py/model/common/grid/gridfile.py +++ b/model/common/src/icon4py/model/common/grid/gridfile.py @@ -366,7 +366,7 @@ def variable( variable = self._dataset.variables[name] variable_size = variable.ndim n = (variable.shape[0],) if variable_size > 1 else () - target_shape = n + (-1,) + target_shape = (*n, -1) slicer = [slice(None) for _ in range(variable_size)] if indices is not None and indices.size > 0: diff --git a/model/common/src/icon4py/model/common/grid/icon.py b/model/common/src/icon4py/model/common/grid/icon.py index 0c6f365284..a4616af2a6 100644 --- a/model/common/src/icon4py/model/common/grid/icon.py +++ b/model/common/src/icon4py/model/common/grid/icon.py @@ -82,6 +82,8 @@ def __init__( _T = TypeVar("_T") +# TODO (@halungge): fields should be removed from this object mean values computed in the geometry factory +# as it needs global reduction... also consider _not_making_everything_ optional that causes troubles at runtime @dataclasses.dataclass(kw_only=True, frozen=True) class GlobalGridParams: grid_shape: Final[GridShape | None] = None @@ -204,18 +206,17 @@ class IconGrid(base.Grid): ) -# TODO (halungge): combine the last to args "into single_node_global" -def _has_skip_values(offset: gtx.FieldOffset, limited_area: bool, distributed: bool) -> bool: +def _has_skip_values(offset: gtx.FieldOffset, limited_area_or_distributed: bool) -> bool: """ For the icosahedral global grid skip values are only present for the pentagon points. - In the local area model there are also skip values at the boundaries when + In the local area model or a distributed grid there are also skip values at the boundaries or halos when accessing neighbouring cells or edges from vertices. """ dimension = offset.target[1] assert dimension.kind == gtx.DimensionKind.LOCAL, "only local dimensions can have skip values" value = dimension in CONNECTIVITIES_ON_PENTAGONS or ( - distributed or (limited_area and dimension in CONNECTIVITIES_ON_BOUNDARIES) + limited_area_or_distributed and dimension in CONNECTIVITIES_ON_BOUNDARIES ) return value @@ -259,12 +260,13 @@ def icon_grid( global_properties: GlobalGridParams, refinement_control: dict[gtx.Dimension, gtx.Field] | None = None, ) -> IconGrid: - distributed = config.num_cells < global_properties.global_num_cells + distributed = config.num_cells < global_properties.num_cells + limited_area_or_distributed = config.limited_area or distributed connectivities = { offset.value: base.construct_connectivity( offset, data_alloc.import_array_ns(allocator).asarray(table), - skip_value=-1 if _has_skip_values(offset, config.limited_area, distributed) else None, + skip_value=-1 if _has_skip_values(offset, limited_area_or_distributed) else None, allocator=allocator, replace_skip_values=_should_replace_skip_values( offset, config.keep_skip_values, config.limited_area diff --git a/model/common/src/icon4py/model/common/interpolation/interpolation_factory.py b/model/common/src/icon4py/model/common/interpolation/interpolation_factory.py index 79cd0d7823..9a2114c2c9 100644 --- a/model/common/src/icon4py/model/common/interpolation/interpolation_factory.py +++ b/model/common/src/icon4py/model/common/interpolation/interpolation_factory.py @@ -101,7 +101,7 @@ def _sources(self) -> factory.FieldSource: def _register_computed_fields(self) -> None: nudging_coefficients_for_edges = factory.ProgramFieldProvider( - func=nudgecoeffs.compute_nudgecoeffs.with_backend(None), + func=nudgecoeffs.compute_nudgecoeffs, domain={ dims.EdgeDim: ( edge_domain(h_grid.Zone.NUDGING_LEVEL_2), diff --git a/model/common/tests/common/decomposition/fixtures.py b/model/common/tests/common/decomposition/fixtures.py new file mode 100644 index 0000000000..ab78d81e9e --- /dev/null +++ b/model/common/tests/common/decomposition/fixtures.py @@ -0,0 +1,21 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +import pytest +from gt4py.next import common as gtx_common + +from icon4py.model.common.grid import simple + + +@pytest.fixture(scope="session") +def simple_neighbor_tables(): + grid = simple.simple_grid() + neighbor_tables = { + k: v.ndarray for k, v in grid.connectivities.items() if gtx_common.is_neighbor_table(v) + } + return neighbor_tables diff --git a/model/common/tests/common/decomposition/mpi_tests/test_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_halo.py index c5a55b20f2..afc93ce392 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_halo.py @@ -16,7 +16,7 @@ from icon4py.model.testing import parallel_helpers from icon4py.model.testing.fixtures import processor_props -from .. import utils +from ..fixtures import simple_neighbor_tables try: @@ -29,22 +29,13 @@ except ImportError: pytest.skip("Skipping parallel on single node installation", allow_module_level=True) -from gt4py.next import common as gtx_common - from icon4py.model.common.decomposition import halo from icon4py.model.common.grid import base as base_grid, simple - -backend = None +from .. import utils -@pytest.fixture(scope="session") -def simple_neighbor_tables(): - grid = simple.simple_grid() - neighbor_tables = { - k: v.ndarray for k, v in grid.connectivities.items() if gtx_common.is_neighbor_table(v) - } - return neighbor_tables +backend = None @pytest.mark.mpi(min_size=4) @@ -75,20 +66,6 @@ def test_halo_constructor_validate_number_of_node_mismatch(processor_props, simp assert "The distribution assumes more nodes than the current run" in e.value.args[0] -@pytest.mark.parametrize("processor_props", [True, False], indirect=True) -def test_halo_constructor_validate_rank_mapping_wrong_shape( - processor_props, simple_neighbor_tables -): - num_cells = simple_neighbor_tables["C2E2C"].shape[0] - with pytest.raises(exceptions.ValidationError) as e: - halo_generator = halo.IconLikeHaloConstructor( - connectivities=simple_neighbor_tables, - run_properties=processor_props, - ) - halo_generator(np.zeros((num_cells, 3), dtype=int)) - assert f"should have shape ({num_cells},)" in e.value.args[0] - - def global_indices(dim: gtx.Dimension) -> np.ndarray: mesh = simple.simple_grid() return np.arange(mesh.size[dim], dtype=gtx.int32) @@ -98,14 +75,16 @@ def global_indices(dim: gtx.Dimension) -> np.ndarray: @pytest.mark.mpi(min_size=4) @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_element_ownership_is_unique( - dim, processor_props, simple_neighbor_tables -): # F811 # fixture + dim, + processor_props, + simple_neighbor_tables, +): parallel_helpers.check_comm_size(processor_props, sizes=[4]) halo_generator = halo.IconLikeHaloConstructor( connectivities=simple_neighbor_tables, run_properties=processor_props, - backend=backend, + allocator=backend, ) decomposition_info = halo_generator(utils.SIMPLE_DISTRIBUTION) @@ -137,107 +116,9 @@ def test_element_ownership_is_unique( assert np.all(np.sort(values) == global_indices(dim)) -@pytest.mark.mpi(min_size=4) -@pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) -@pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_halo_constructor_decomposition_info_global_indices( - processor_props, simple_neighbor_tables, dim -): # F811 # fixture - if processor_props.comm_size != 4: - pytest.skip( - f"This test requires exactly 4 MPI ranks, current run has {processor_props.comm_size}" - ) - halo_generator = halo.IconLikeHaloConstructor( - connectivities=simple_neighbor_tables, - run_properties=processor_props, - ) - - decomp_info = halo_generator(utils.SIMPLE_DISTRIBUTION) - my_halo = decomp_info.global_index(dim, defs.DecompositionInfo.EntryType.HALO) - print(f"rank {processor_props.rank} has halo {dim} : {my_halo}") - assert my_halo.size == len(utils.HALO[dim][processor_props.rank]) - assert ( - np.setdiff1d(my_halo, utils.HALO[dim][processor_props.rank], assume_unique=True).size == 0 - ) - my_owned = decomp_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED) - print(f"rank {processor_props.rank} owns {dim} : {my_owned} ") - utils.assert_same_entries(dim, my_owned, utils.OWNED, processor_props.rank) - - -@pytest.mark.mpi(min_size=4) -@pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) -@pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_halo_constructor_decomposition_info_halo_levels( - processor_props, dim, simple_neighbor_tables -): # F811 # fixture - halo_generator = halo.IconLikeHaloConstructor( - connectivities=simple_neighbor_tables, - run_properties=processor_props, - ) - decomp_info = halo_generator(utils.SIMPLE_DISTRIBUTION) - my_halo_levels = decomp_info.halo_levels(dim) - print(f"{dim.value}: rank {processor_props.rank} has halo levels {my_halo_levels} ") - if dim != dims.EdgeDim: - assert np.all( - my_halo_levels != defs.DecompositionFlag.UNDEFINED - ), ( - "All indices should have a defined DecompositionFlag" - ) # THIS WILL CURRENTLY FAIL FOR EDGES - assert np.where(my_halo_levels == defs.DecompositionFlag.OWNED)[0].size == len( - utils.OWNED[dim][processor_props.rank] - ) - owned_local_indices = decomp_info.local_index(dim, defs.DecompositionInfo.EntryType.OWNED) - assert np.all( - my_halo_levels[owned_local_indices] == defs.DecompositionFlag.OWNED - ), "owned local indices should have DecompositionFlag.OWNED" - first_halo_line_local_index = np.where( - my_halo_levels == defs.DecompositionFlag.FIRST_HALO_LINE - )[0] - first_halo_line_global_index = decomp_info.global_index( - dim, defs.DecompositionInfo.EntryType.ALL - )[first_halo_line_local_index] - utils.assert_same_entries( - dim, first_halo_line_global_index, utils.FIRST_HALO_LINE, processor_props.rank - ) - second_halo_line_local_index = np.where( - my_halo_levels == defs.DecompositionFlag.SECOND_HALO_LINE - )[0] - second_halo_line_global_index = decomp_info.global_index( - dim, defs.DecompositionInfo.EntryType.ALL - )[second_halo_line_local_index] - utils.assert_same_entries( - dim, second_halo_line_global_index, utils.SECOND_HALO_LINE, processor_props.rank - ) - - -def decompose(grid: base_grid.Grid, processor_props): # F811 # fixture +def decompose(grid: base_grid.Grid, processor_props): partitioner = halo.SimpleMetisDecomposer() labels = partitioner( grid.connectivities[dims.C2E2C].asnumpy(), n_part=processor_props.comm_size ) return labels - - -def test_no_halo(): - grid_size = base_grid.HorizontalGridSize(num_cells=9, num_edges=14, num_vertices=6) - halo_generator = halo.NoHalos(horizontal_size=grid_size, allocator=None) - decomposition = halo.SingleNodeDecomposer() - decomposition_info = halo_generator(decomposition(np.arange(grid_size.num_cells), 1)) - # cells - np.testing.assert_allclose( - np.arange(grid_size.num_cells), decomposition_info.global_index(dims.CellDim) - ) - assert np.all(decomposition_info.owner_mask(dims.CellDim)) - assert np.all(decomposition_info.halo_levels(dims.CellDim) == defs.DecompositionFlag.OWNED) - # edges - np.testing.assert_allclose( - np.arange(grid_size.num_edges), decomposition_info.global_index(dims.EdgeDim) - ) - assert np.all(decomposition_info.halo_levels(dims.EdgeDim) == defs.DecompositionFlag.OWNED) - assert np.all(decomposition_info.owner_mask(dims.EdgeDim)) - # vertices - np.testing.assert_allclose( - np.arange(grid_size.num_vertices), decomposition_info.global_index(dims.VertexDim) - ) - assert np.all(decomposition_info.halo_levels(dims.VertexDim) == defs.DecompositionFlag.OWNED) - assert np.all(decomposition_info.owner_mask(dims.VertexDim)) diff --git a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py index e8f8710993..2bad92d4cc 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py @@ -184,18 +184,18 @@ def test_decomposition_info_halo_level_mask( decomposition_info: definitions.DecompositionInfo, ) -> None: first_halo_level = decomposition_info.halo_level_mask( - dim, definitions.DecompositionFlag.FIRST_HALO_LINE + dim, definitions.DecompositionFlag.FIRST_HALO_LEVEL ) assert first_halo_level.ndim == 1 assert np.count_nonzero(first_halo_level) == decomposition_info.get_halo_size( - dim, definitions.DecompositionFlag.FIRST_HALO_LINE + dim, definitions.DecompositionFlag.FIRST_HALO_LEVEL ) second_halo_level = decomposition_info.halo_level_mask( - dim, definitions.DecompositionFlag.SECOND_HALO_LINE + dim, definitions.DecompositionFlag.SECOND_HALO_LEVEL ) assert second_halo_level.ndim == 1 assert np.count_nonzero(second_halo_level) == decomposition_info.get_halo_size( - dim, definitions.DecompositionFlag.SECOND_HALO_LINE + dim, definitions.DecompositionFlag.SECOND_HALO_LEVEL ) assert np.count_nonzero(first_halo_level) + np.count_nonzero( second_halo_level @@ -210,7 +210,7 @@ def test_decomposition_info_third_level_is_empty( experiment: test_defs.Experiment, decomposition_info: definitions.DecompositionInfo, ) -> None: - level = decomposition_info.halo_level_mask(dim, definitions.DecompositionFlag.THIRD_HALO_LINE) + level = decomposition_info.halo_level_mask(dim, definitions.DecompositionFlag.THIRD_HALO_LEVEL) assert np.count_nonzero(level) == 0 diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index 6e88ebee4f..cf391f2047 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -20,7 +20,7 @@ from ...grid import utils as grid_utils from .. import utils -from ..mpi_tests.test_halo import simple_neighbor_tables +from ..fixtures import simple_neighbor_tables from ..utils import dummy_four_ranks diff --git a/model/common/tests/common/decomposition/unit_tests/test_halo.py b/model/common/tests/common/decomposition/unit_tests/test_halo.py new file mode 100644 index 0000000000..8d5f931688 --- /dev/null +++ b/model/common/tests/common/decomposition/unit_tests/test_halo.py @@ -0,0 +1,145 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +import numpy as np +import pytest + +from icon4py.model.common import dimension as dims, exceptions +from icon4py.model.common.decomposition import definitions, halo +from icon4py.model.common.grid import base as base_grid + +from .. import utils +from ..fixtures import simple_neighbor_tables + + +@pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) +@pytest.mark.parametrize("rank", [0, 1, 2, 4]) +def test_halo_constructor_decomposition_info_global_indices(rank, simple_neighbor_tables, dim): + processor_props = utils.DummyProps(rank=rank) + if processor_props.comm_size != 4: + pytest.skip( + f"This test requires exactly 4 MPI ranks, current run has {processor_props.comm_size}" + ) + + halo_generator = halo.IconLikeHaloConstructor( + connectivities=simple_neighbor_tables, + run_properties=processor_props, + ) + + decomp_info = halo_generator(utils.SIMPLE_DISTRIBUTION) + my_halo = decomp_info.global_index(dim, definitions.DecompositionInfo.EntryType.HALO) + print(f"rank {processor_props.rank} has halo {dim} : {my_halo}") + expected = len(utils.HALO[dim][processor_props.rank]) + assert ( + my_halo.size == expected + ), f"total halo size does not match for dim {dim}- expected {expected} bot was {my_halo.size}" + assert ( + missing := np.setdiff1d( + my_halo, utils.HALO[dim][processor_props.rank], assume_unique=True + ).size + == 0 + ), f"missing halo elements are {missing}" + my_owned = decomp_info.global_index(dim, definitions.DecompositionInfo.EntryType.OWNED) + print(f"rank {processor_props.rank} owns {dim} : {my_owned} ") + utils.assert_same_entries(dim, my_owned, utils.OWNED, processor_props.rank) + + +@pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) +@pytest.mark.parametrize("rank", [0, 1, 2, 3]) +def test_halo_constructor_definitions_info_halo_levels(rank, dim, simple_neighbor_tables): + processor_props = utils.DummyProps(rank=rank) + halo_generator = halo.IconLikeHaloConstructor( + connectivities=simple_neighbor_tables, + run_properties=processor_props, + ) + decomp_info = halo_generator(utils.SIMPLE_DISTRIBUTION) + my_halo_levels = decomp_info.halo_levels(dim) + print(f"{dim.value}: rank {processor_props.rank} has halo levels {my_halo_levels} ") + assert np.all( + my_halo_levels != definitions.DecompositionFlag.UNDEFINED + ), "All indices should have a defined DecompositionFlag" + + assert np.where(my_halo_levels == definitions.DecompositionFlag.OWNED)[0].size == len( + utils.OWNED[dim][processor_props.rank] + ) + owned_local_indices = decomp_info.local_index( + dim, definitions.DecompositionInfo.EntryType.OWNED + ) + assert np.all( + my_halo_levels[owned_local_indices] == definitions.DecompositionFlag.OWNED + ), "owned local indices should have DecompositionFlag.OWNED" + first_halo_level_local_index = np.where( + my_halo_levels == definitions.DecompositionFlag.FIRST_HALO_LEVEL + )[0] + first_halo_level_global_index = decomp_info.global_index( + dim, definitions.DecompositionInfo.EntryType.ALL + )[first_halo_level_local_index] + utils.assert_same_entries( + dim, first_halo_level_global_index, utils.FIRST_HALO_LINE, processor_props.rank + ) + second_halo_level_local_index = np.where( + my_halo_levels == definitions.DecompositionFlag.SECOND_HALO_LEVEL + )[0] + second_halo_level_global_index = decomp_info.global_index( + dim, definitions.DecompositionInfo.EntryType.ALL + )[second_halo_level_local_index] + utils.assert_same_entries( + dim, second_halo_level_global_index, utils.SECOND_HALO_LINE, processor_props.rank + ) + third_halo_level_index = np.where( + my_halo_levels == definitions.DecompositionFlag.THIRD_HALO_LEVEL + )[0] + third_halo_level_global_index = decomp_info.global_index( + dim, definitions.DecompositionInfo.EntryType.ALL + )[third_halo_level_index] + utils.assert_same_entries( + dim, third_halo_level_global_index, utils.THIRD_HALO_INE, processor_props.rank + ) + + +def test_no_halo(): + grid_size = base_grid.HorizontalGridSize(num_cells=9, num_edges=14, num_vertices=6) + halo_generator = halo.NoHalos(horizontal_size=grid_size, allocator=None) + decomposition = halo.SingleNodeDecomposer() + decomposition_info = halo_generator(decomposition(np.arange(grid_size.num_cells), 1)) + # cells + np.testing.assert_allclose( + np.arange(grid_size.num_cells), decomposition_info.global_index(dims.CellDim) + ) + assert np.all(decomposition_info.owner_mask(dims.CellDim)) + assert np.all( + decomposition_info.halo_levels(dims.CellDim) == definitions.DecompositionFlag.OWNED + ) + # edges + np.testing.assert_allclose( + np.arange(grid_size.num_edges), decomposition_info.global_index(dims.EdgeDim) + ) + assert np.all( + decomposition_info.halo_levels(dims.EdgeDim) == definitions.DecompositionFlag.OWNED + ) + assert np.all(decomposition_info.owner_mask(dims.EdgeDim)) + # vertices + np.testing.assert_allclose( + np.arange(grid_size.num_vertices), decomposition_info.global_index(dims.VertexDim) + ) + assert np.all( + decomposition_info.halo_levels(dims.VertexDim) == definitions.DecompositionFlag.OWNED + ) + assert np.all(decomposition_info.owner_mask(dims.VertexDim)) + + +def test_halo_constructor_validate_rank_mapping_wrong_shape(simple_neighbor_tables): + processor_props = utils.DummyProps(rank=2) + num_cells = simple_neighbor_tables["C2E2C"].shape[0] + with pytest.raises(exceptions.ValidationError) as e: + halo_generator = halo.IconLikeHaloConstructor( + connectivities=simple_neighbor_tables, + run_properties=processor_props, + ) + halo_generator(np.zeros((num_cells, 3), dtype=int)) + assert f"should have shape ({num_cells},)" in e.value.args[0] diff --git a/model/common/tests/common/decomposition/utils.py b/model/common/tests/common/decomposition/utils.py index cf249f9725..bfff327e6e 100644 --- a/model/common/tests/common/decomposition/utils.py +++ b/model/common/tests/common/decomposition/utils.py @@ -98,10 +98,10 @@ 3: [9, 12, 15], } _EDGE_HALO = { - 0: _EDGE_FIRST_HALO_LINE[0] + _EDGE_SECOND_HALO_LINE[0], - 1: _EDGE_FIRST_HALO_LINE[1] + _EDGE_SECOND_HALO_LINE[1], - 2: _EDGE_FIRST_HALO_LINE[2] + _EDGE_SECOND_HALO_LINE[2], - 3: _EDGE_FIRST_HALO_LINE[3] + _EDGE_SECOND_HALO_LINE[3], + 0: _EDGE_FIRST_HALO_LINE[0] + _EDGE_SECOND_HALO_LINE[0] + _EDGE_THIRD_HALO_LINE[0], + 1: _EDGE_FIRST_HALO_LINE[1] + _EDGE_SECOND_HALO_LINE[1] + _EDGE_THIRD_HALO_LINE[1], + 2: _EDGE_FIRST_HALO_LINE[2] + _EDGE_SECOND_HALO_LINE[2] + _EDGE_THIRD_HALO_LINE[2], + 3: _EDGE_FIRST_HALO_LINE[3] + _EDGE_SECOND_HALO_LINE[3] + _EDGE_THIRD_HALO_LINE[3], } _VERTEX_OWN = { 0: [4], @@ -150,6 +150,11 @@ dims.VertexDim: _VERTEX_SECOND_HALO_LINE, dims.EdgeDim: _EDGE_SECOND_HALO_LINE, } +THIRD_HALO_INE = { + dims.CellDim: {0: [], 1: [], 2: [], 3: []}, + dims.VertexDim: {0: [], 1: [], 2: [], 3: []}, + dims.EdgeDim: _EDGE_THIRD_HALO_LINE, +} def assert_same_entries( diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 5827ff75fc..8e1b2bbbaf 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -111,6 +111,7 @@ def test_grid_manager_validate_decomposer(processor_props: defs.ProcessPropertie assert "Need a Decomposer for multi" in e.value.args[0] +# TODO (halungge): is this used??? @pytest.mark.mpi @pytest.mark.parametrize( "field_offset", @@ -331,7 +332,7 @@ def test_halo_neighbor_access_c2e( print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'CellDim' (1 : {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + f"rank = {processor_props.rank}: halo size for 'CellDim' (1 : {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.FIRST_HALO_LEVEL)}), (2: {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) distributed_coordinates = multinode_grid_manager.coordinates distributed_geometry = geometry.GridGeometry( @@ -404,7 +405,7 @@ def test_halo_access_e2c2v( print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LEVEL)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) distributed_coordinates = multinode_grid_manager.coordinates distributed_geometry = geometry.GridGeometry( @@ -494,7 +495,7 @@ def test_halo_access_e2c( print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LEVEL)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) distributed_coordinates = multinode_grid_manager.coordinates distributed_geometry = geometry.GridGeometry( @@ -582,7 +583,7 @@ def test_halo_neighbor_access_e2v( print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LEVEL)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) distributed_coordinates = multinode_grid_manager.coordinates vertex_lat = distributed_coordinates[dims.VertexDim]["lat"] @@ -682,7 +683,7 @@ def test_halo_neighbor_access_v2e( print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LEVEL)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) distributed_coordinates = multinode_grid_manager.coordinates extra_geometry_fields = multinode_grid_manager.geometry_fields @@ -765,7 +766,7 @@ def test_halo_neighbor_access_c2e2c( print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'CellDim' (1 : {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + f"rank = {processor_props.rank}: halo size for 'CellDim' (1 : {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.FIRST_HALO_LEVEL)}), (2: {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) distributed_coordinates = multinode_grid_manager.coordinates extra_geometry_fields = multinode_grid_manager.geometry_fields @@ -859,10 +860,10 @@ def test_halo_neighbor_access_v2c( print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'CellDim' (1 : {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + f"rank = {processor_props.rank}: halo size for 'CellDim' (1 : {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.FIRST_HALO_LEVEL)}), (2: {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) print( - f"rank = {processor_props.rank}: halo size for 'VertexDim' (1 : {decomposition_info.get_halo_size(dims.VertexDim, defs.DecompositionFlag.FIRST_HALO_LINE)}), (2: {decomposition_info.get_halo_size(dims.VertexDim, defs.DecompositionFlag.SECOND_HALO_LINE)})" + f"rank = {processor_props.rank}: halo size for 'VertexDim' (1 : {decomposition_info.get_halo_size(dims.VertexDim, defs.DecompositionFlag.FIRST_HALO_LEVEL)}), (2: {decomposition_info.get_halo_size(dims.VertexDim, defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) my_global_cells = decomposition_info.global_index(dims.CellDim) cell_k_buffer = ( diff --git a/model/common/tests/common/grid/unit_tests/test_icon.py b/model/common/tests/common/grid/unit_tests/test_icon.py index 71a62d02b6..84656a9faf 100644 --- a/model/common/tests/common/grid/unit_tests/test_icon.py +++ b/model/common/tests/common/grid/unit_tests/test_icon.py @@ -194,8 +194,8 @@ def test_when_keep_skip_value_then_neighbor_table_matches_config( assert ( np.any(connectivity.asnumpy() == gridfile.GridFile.INVALID_INDEX).item() - ) == icon._has_skip_values(offset, grid.config.limited_area, distributed=False) - if not icon._has_skip_values(offset, grid.config.limited_area, distributed=False): + ) == icon._has_skip_values(offset, grid.config.limited_area) + if not icon._has_skip_values(offset, grid.config.limited_area): assert connectivity.skip_value is None else: assert connectivity.skip_value == gridfile.GridFile.INVALID_INDEX diff --git a/model/testing/src/icon4py/model/testing/parallel_helpers.py b/model/testing/src/icon4py/model/testing/parallel_helpers.py index 4837d1c711..b0ad1b0465 100644 --- a/model/testing/src/icon4py/model/testing/parallel_helpers.py +++ b/model/testing/src/icon4py/model/testing/parallel_helpers.py @@ -6,12 +6,11 @@ # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause import logging -from collections.abc import Iterable import pytest +from icon4py.model.common import dimension as dims from icon4py.model.common.decomposition import definitions -from icon4py.model.common.decomposition.mpi_decomposition import get_multinode_properties log = logging.getLogger(__file__) @@ -24,21 +23,13 @@ def check_comm_size( pytest.xfail(f"wrong comm size: {props.comm_size}: test only works for comm-sizes: {sizes}") -@pytest.fixture(scope="session") -def processor_props(request: pytest.FixtureRequest) -> Iterable[definitions.ProcessProperties]: - runtype = definitions.get_runtype(with_mpi=True) - yield get_multinode_properties(runtype) - - -def log_process_properties( - props: definitions.ProcessProperties, level: int = logging.DEBUG -) -> None: +def log_process_properties(props: definitions.ProcessProperties) -> None: log.info(f"rank={props.rank}/{props.comm_size}") -def log_local_field_size( - decomposition_info: definitions.DecompositionInfo, level: int = logging.DEBUG -) -> None: +def log_local_field_size(decomposition_info: definitions.DecompositionInfo) -> None: log.info( - f"local grid size: cells={decomposition_info.num_cells}, edges={decomposition_info.num_edges}, vertices={decomposition_info.num_vertices}" + f"local grid size: cells={decomposition_info.global_index(dims.CellDim).size}, " + f"edges={decomposition_info.global_index(dims.EdgeDim).size}, " + f"vertices={decomposition_info.global_index(dims.VertexDim).size}" ) From 927e7d129430c5e8d8c6b300d7cba46b88a6ef8c Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 12 Dec 2025 13:43:51 +0100 Subject: [PATCH 120/240] add test to check skip value configuration --- .../src/icon4py/model/common/grid/icon.py | 5 +- .../mpi_tests/test_parallel_grid_manager.py | 140 ++++++++++++------ 2 files changed, 93 insertions(+), 52 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/icon.py b/model/common/src/icon4py/model/common/grid/icon.py index a4616af2a6..df9e9f2d11 100644 --- a/model/common/src/icon4py/model/common/grid/icon.py +++ b/model/common/src/icon4py/model/common/grid/icon.py @@ -24,11 +24,10 @@ log = logging.getLogger(__name__) CONNECTIVITIES_ON_BOUNDARIES = ( - dims.C2EDim, - dims.C2VDim, # should be removed by includein all vertices on level 2 edges... + dims.C2VDim, dims.E2VDim, dims.C2E2C2EDim, - dims.E2CDim, # non on halos because of "open halo cells" + dims.E2CDim, dims.C2E2CDim, dims.C2E2CODim, dims.E2C2VDim, diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 8e1b2bbbaf..09299bca9c 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -15,31 +15,25 @@ import numpy as np import pytest from gt4py import next as gtx -from gt4py.next import typing as gtx_typing +from gt4py.next import common as gtx_common, typing as gtx_typing from icon4py.model.common import dimension as dims, exceptions -from icon4py.model.common.decomposition import definitions as defs, halo, mpi_decomposition +from icon4py.model.common.decomposition import definitions as decomp_defs, halo, mpi_decomposition from icon4py.model.common.grid import ( - base, geometry, geometry_attributes, geometry_stencils, grid_manager as gm, gridfile, horizontal as h_grid, - vertical as v_grid, + icon, ) from icon4py.model.common.interpolation import interpolation_fields from icon4py.model.common.interpolation.stencils.compute_cell_2_vertex_interpolation import ( _compute_cell_2_vertex_interpolation, ) from icon4py.model.common.utils import data_allocation as data_alloc -from icon4py.model.testing import ( - definitions, - definitions as test_defs, - grid_utils, - test_utils as test_helpers, -) +from icon4py.model.testing import definitions as test_defs, grid_utils from ...decomposition import utils as decomp_utils from .. import utils @@ -69,7 +63,7 @@ def run_gridmananger_for_multinode( file: pathlib.Path, - run_properties: defs.ProcessProperties, + run_properties: decomp_defs.ProcessProperties, decomposer: halo.Decomposer, ) -> gm.GridManager: manager = _grid_manager(file, num_levels=NUM_LEVELS) @@ -88,7 +82,7 @@ def run_grid_manager_for_singlenode(file: pathlib.Path) -> gm.GridManager: manager = _grid_manager(file, NUM_LEVELS) manager( keep_skip_values=True, - run_properties=defs.SingleNodeProcessProperties(), + run_properties=decomp_defs.SingleNodeProcessProperties(), decomposer=halo.SingleNodeDecomposer(), allocator=None, ) @@ -97,7 +91,7 @@ def run_grid_manager_for_singlenode(file: pathlib.Path) -> gm.GridManager: @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi(min_size=2) -def test_grid_manager_validate_decomposer(processor_props: defs.ProcessProperties) -> None: +def test_grid_manager_validate_decomposer(processor_props: decomp_defs.ProcessProperties) -> None: file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) manager = gm.GridManager(file, NUM_LEVELS, gridfile.ToZeroBasedIndexTransformation()) with pytest.raises(exceptions.InvalidConfigError) as e: @@ -118,7 +112,7 @@ def test_grid_manager_validate_decomposer(processor_props: defs.ProcessPropertie [dims.C2V, dims.E2V, dims.V2C, dims.E2C, dims.C2E, dims.V2E, dims.C2E2C, dims.V2E2V], ) def test_local_connectivities( - processor_props: defs.ProcessProperties, + processor_props: decomp_defs.ProcessProperties, caplog: Iterator, field_offset: gtx.FieldOffset, ) -> None: @@ -144,12 +138,14 @@ def test_local_connectivities( assert ( connectivity.shape[0] == decomposition_info.global_index( - field_offset.target[0], defs.DecompositionInfo.EntryType.ALL + field_offset.target[0], decomp_defs.DecompositionInfo.EntryType.ALL ).size ) # all neighbor indices are valid local indices assert np.max(connectivity) == np.max( - decomposition_info.local_index(field_offset.source, defs.DecompositionInfo.EntryType.ALL) + decomposition_info.local_index( + field_offset.source, decomp_defs.DecompositionInfo.EntryType.ALL + ) ) # TODO what else to assert? # - outer halo entries have SKIP_VALUE neighbors (depends on offsets) @@ -157,7 +153,9 @@ def test_local_connectivities( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_fields_distribute_and_gather(processor_props: defs.ProcessProperties, caplog: Any) -> None: +def test_fields_distribute_and_gather( + processor_props: decomp_defs.ProcessProperties, caplog: Any +) -> None: caplog.set_level(logging.INFO) print(f"myrank - {processor_props.rank}: running with processor_props = {processor_props}") file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) @@ -240,8 +238,8 @@ def gather_field(field: np.ndarray, comm: mpi4py.MPI.Comm) -> tuple: def assert_gathered_field_against_global( - decomposition_info: defs.DecompositionInfo, - processor_props: defs.ProcessProperties, # F811 # fixture + decomposition_info: decomp_defs.DecompositionInfo, + processor_props: decomp_defs.ProcessProperties, # F811 # fixture dim: gtx.Dimension, global_reference_field: np.ndarray, local_field: np.ndarray, @@ -251,14 +249,16 @@ def assert_gathered_field_against_global( ) assert ( local_field.shape[0] - == decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.ALL).shape[0] + == decomposition_info.global_index(dim, decomp_defs.DecompositionInfo.EntryType.ALL).shape[ + 0 + ] ) owned_entries = local_field[ - decomposition_info.local_index(dim, defs.DecompositionInfo.EntryType.OWNED) + decomposition_info.local_index(dim, decomp_defs.DecompositionInfo.EntryType.OWNED) ] gathered_sizes, gathered_field = gather_field(owned_entries, processor_props.comm) global_index_sizes, gathered_global_indices = gather_field( - decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED), + decomposition_info.global_index(dim, decomp_defs.DecompositionInfo.EntryType.OWNED), processor_props.comm, ) if processor_props.rank == 0: @@ -282,11 +282,11 @@ def assert_gathered_field_against_global( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL,)) +@pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) def test_halo_neighbor_access_c2e( - processor_props: defs.ProcessProperties, + processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, - grid: definitions.GridDescription, + grid: test_defs.GridDescription, ) -> None: file = grid_utils.resolve_full_grid_file_name(grid) print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") @@ -332,7 +332,9 @@ def test_halo_neighbor_access_c2e( print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'CellDim' (1 : {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.FIRST_HALO_LEVEL)}), (2: {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.SECOND_HALO_LEVEL)})" + f"rank = {processor_props.rank}: halo size for 'CellDim' " + f"(1 : {decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " + f"(2: {decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) distributed_coordinates = multinode_grid_manager.coordinates distributed_geometry = geometry.GridGeometry( @@ -371,11 +373,11 @@ def test_halo_neighbor_access_c2e( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL,)) +@pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) def test_halo_access_e2c2v( - processor_props: defs.ProcessProperties, + processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, - grid: definitions.GridDescription, + grid: test_defs.GridDescription, ) -> None: file = grid_utils.resolve_full_grid_file_name(grid) print(f"running on {processor_props.comm}") @@ -405,7 +407,9 @@ def test_halo_access_e2c2v( print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LEVEL)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LEVEL)})" + f"rank = {processor_props.rank}: halo size for 'EdgeDim' " + f"(1 : {decomposition_info.get_halo_size(dims.EdgeDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " + f"(2: {decomposition_info.get_halo_size(dims.EdgeDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) distributed_coordinates = multinode_grid_manager.coordinates distributed_geometry = geometry.GridGeometry( @@ -461,11 +465,11 @@ def test_halo_access_e2c2v( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL,)) +@pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) def test_halo_access_e2c( - processor_props: defs.ProcessProperties, + processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, - grid: definitions.GridDescription, + grid: test_defs.GridDescription, ) -> None: file = grid_utils.resolve_full_grid_file_name(grid) print(f"running on {processor_props.comm}") @@ -495,7 +499,9 @@ def test_halo_access_e2c( print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LEVEL)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LEVEL)})" + f"rank = {processor_props.rank}: halo size for 'EdgeDim' " + f"(1 : {decomposition_info.get_halo_size(dims.EdgeDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " + f"(2: {decomposition_info.get_halo_size(dims.EdgeDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) distributed_coordinates = multinode_grid_manager.coordinates distributed_geometry = geometry.GridGeometry( @@ -550,11 +556,11 @@ def test_halo_access_e2c( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) # @pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL, definitions.Grids.MCH_CH_R04B09_DSL)) -@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL,)) +@pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) def test_halo_neighbor_access_e2v( - processor_props: defs.ProcessProperties, + processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, - grid: definitions.GridDescription, + grid: test_defs.GridDescription, ) -> None: print(f"running on {processor_props.comm}") file = grid_utils.resolve_full_grid_file_name(grid) @@ -583,7 +589,9 @@ def test_halo_neighbor_access_e2v( print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LEVEL)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LEVEL)})" + f"rank = {processor_props.rank}: halo size for 'EdgeDim' " + f"(1 : {decomposition_info.get_halo_size(dims.EdgeDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " + f"(2: {decomposition_info.get_halo_size(dims.EdgeDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) distributed_coordinates = multinode_grid_manager.coordinates vertex_lat = distributed_coordinates[dims.VertexDim]["lat"] @@ -626,11 +634,11 @@ def test_halo_neighbor_access_e2v( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL,)) +@pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) def test_halo_neighbor_access_v2e( - processor_props: defs.ProcessProperties, + processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, - grid: definitions.GridDescription, + grid: test_defs.GridDescription, ) -> None: file = grid_utils.resolve_full_grid_file_name(grid) print(f"running on {processor_props.comm}") @@ -683,7 +691,9 @@ def test_halo_neighbor_access_v2e( print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'EdgeDim' (1 : {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.FIRST_HALO_LEVEL)}), (2: {decomposition_info.get_halo_size(dims.EdgeDim, defs.DecompositionFlag.SECOND_HALO_LEVEL)})" + f"rank = {processor_props.rank}: halo size for 'EdgeDim' " + f"(1 : {decomposition_info.get_halo_size(dims.EdgeDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)})," + f" (2: {decomposition_info.get_halo_size(dims.EdgeDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) distributed_coordinates = multinode_grid_manager.coordinates extra_geometry_fields = multinode_grid_manager.geometry_fields @@ -723,11 +733,11 @@ def test_halo_neighbor_access_v2e( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -@pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL,)) +@pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) def test_halo_neighbor_access_c2e2c( - processor_props: defs.ProcessProperties, + processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, - grid: definitions.GridDescription, + grid: test_defs.GridDescription, ) -> None: file = grid_utils.resolve_full_grid_file_name(grid) center_weight = 0.3 @@ -766,7 +776,9 @@ def test_halo_neighbor_access_c2e2c( print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'CellDim' (1 : {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.FIRST_HALO_LEVEL)}), (2: {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.SECOND_HALO_LEVEL)})" + f"rank = {processor_props.rank}: halo size for 'CellDim' " + f"(1 : {decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " + f"(2: {decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) distributed_coordinates = multinode_grid_manager.coordinates extra_geometry_fields = multinode_grid_manager.geometry_fields @@ -806,7 +818,7 @@ def test_halo_neighbor_access_c2e2c( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_halo_neighbor_access_v2c( - processor_props: defs.ProcessProperties, backend: gtx_typing.Backend + processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend ) -> None: file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) print(f"running on {processor_props.comm}") @@ -860,10 +872,14 @@ def test_halo_neighbor_access_v2c( print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'CellDim' (1 : {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.FIRST_HALO_LEVEL)}), (2: {decomposition_info.get_halo_size(dims.CellDim, defs.DecompositionFlag.SECOND_HALO_LEVEL)})" + f"rank = {processor_props.rank}: halo size for 'CellDim' " + f"(1 : {decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " + f"(2: {decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) print( - f"rank = {processor_props.rank}: halo size for 'VertexDim' (1 : {decomposition_info.get_halo_size(dims.VertexDim, defs.DecompositionFlag.FIRST_HALO_LEVEL)}), (2: {decomposition_info.get_halo_size(dims.VertexDim, defs.DecompositionFlag.SECOND_HALO_LEVEL)})" + f"rank = {processor_props.rank}: halo size for 'VertexDim' " + f"(1 : {decomposition_info.get_halo_size(dims.VertexDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " + f"(2: {decomposition_info.get_halo_size(dims.VertexDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) my_global_cells = decomposition_info.global_index(dims.CellDim) cell_k_buffer = ( @@ -923,3 +939,29 @@ def test_halo_neighbor_access_v2c( global_reference_field=reference.asnumpy(), local_field=output.asnumpy(), ) + + +@pytest.mark.mpi +@pytest.mark.parametrize("processor_props", [True], indirect=True) +@pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) +def test_validate_skip_values_in_distributed_connectivities( + processor_props: decomp_defs.ProcessProperties, grid: test_defs.GridDescription +) -> None: + file = grid_utils.resolve_full_grid_file_name(grid) + multinode_grid_manager = run_gridmananger_for_multinode( + file=file, + run_properties=processor_props, + decomposer=halo.SimpleMetisDecomposer(), + ) + distributed_grid = multinode_grid_manager.grid + for k, c in distributed_grid.connectivities.items(): + if gtx_common.is_neighbor_connectivity(c): + skip_values_in_table = np.count_nonzero(c.asnumpy() == c.skip_value) + found_skips = skip_values_in_table > 0 + assert ( + found_skips == (c.skip_value is not None) + ), f"rank={processor_props.rank} / {processor_props.comm_size}: {k} - # of skip values found in table = {skip_values_in_table}, skip value is {c.skip_value}" + if skip_values_in_table > 0: + assert ( + c in icon.CONNECTIVITIES_ON_BOUNDARIES or icon.CONNECTIVITIES_ON_PENTAGONS + ), f"rank={processor_props.rank} / {processor_props.comm_size}: {k} has skip found in table" From 3f64b2acd2a2c61a6c6f4256260ed5ceff84a067 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 12 Dec 2025 14:36:59 +0100 Subject: [PATCH 121/240] simple clean ups in PR --- .../muphys/tests/muphys/fixtures.py | 7 ++++ .../icon4py/model/common/grid/grid_manager.py | 1 + .../model/common/grid/grid_refinement.py | 9 +++--- .../icon4py/model/common/states/factory.py | 32 +++++++++++++++---- .../mpi_tests/test_parallel_grid_manager.py | 6 ++-- 5 files changed, 39 insertions(+), 16 deletions(-) create mode 100644 model/atmosphere/subgrid_scale_physics/muphys/tests/muphys/fixtures.py diff --git a/model/atmosphere/subgrid_scale_physics/muphys/tests/muphys/fixtures.py b/model/atmosphere/subgrid_scale_physics/muphys/tests/muphys/fixtures.py new file mode 100644 index 0000000000..de9850de36 --- /dev/null +++ b/model/atmosphere/subgrid_scale_physics/muphys/tests/muphys/fixtures.py @@ -0,0 +1,7 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index f4823d9f0d..d3a3311515 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -695,6 +695,7 @@ def _patch_with_dummy_lastline(ar, array_ns: ModuleType = np): return patched_ar +# TODO (halungge): is this function used at all?? def construct_local_connectivity( field_offset: gtx.FieldOffset, decomposition_info: decomposition.DecompositionInfo, diff --git a/model/common/src/icon4py/model/common/grid/grid_refinement.py b/model/common/src/icon4py/model/common/grid/grid_refinement.py index 9b84b93e10..95c4a5830e 100644 --- a/model/common/src/icon4py/model/common/grid/grid_refinement.py +++ b/model/common/src/icon4py/model/common/grid/grid_refinement.py @@ -12,9 +12,9 @@ import numpy as np from gt4py import next as gtx -import icon4py.model.common.grid.horizontal as h_grid from icon4py.model.common import dimension as dims from icon4py.model.common.decomposition import definitions as decomposition +from icon4py.model.common.grid import gridfile, horizontal as h_grid from icon4py.model.common.utils import data_allocation as data_alloc @@ -35,11 +35,10 @@ """ _log = logging.getLogger(__name__) -# TODO(halungge): get these from grid file cell_grf, edge_grf, vertex_grf _MAX_ORDERED: Final[dict[gtx.Dimension, int]] = { - dims.CellDim: 14, - dims.EdgeDim: 28, - dims.VertexDim: 14, + dims.CellDim: gridfile.FixedSizeDimension.CELL_GRF.size, + dims.EdgeDim: gridfile.FixedSizeDimension.EDGE_GRF.size, + dims.VertexDim: gridfile.FixedSizeDimension.VERTEX_GRF.size, } """ Grid points in the grid refinement fields are labeled with their distance to the lateral boundary. diff --git a/model/common/src/icon4py/model/common/states/factory.py b/model/common/src/icon4py/model/common/states/factory.py index 74239404f5..626c6950a1 100644 --- a/model/common/src/icon4py/model/common/states/factory.py +++ b/model/common/src/icon4py/model/common/states/factory.py @@ -533,10 +533,28 @@ def _map_dim(dim: gtx.Dimension) -> gtx.Dimension: field_domain = {_map_dim(dim): (0, _map_size(dim, grid)) for dim in self._dims} return {k: allocate(field_domain, dtype=dtype[k]) for k in self._fields} - def _grid_connectivities( - self, grid: icon_grid.IconGrid - ) -> dict[str, gtx.Connectivity | gtx.Dimension]: - return grid.connectivities + # TODO(halungge): this can be simplified when completely disentangling vertical and horizontal grid. + # the IconGrid should then only contain horizontal connectivities and no longer any Koff which should be moved to the VerticalGrid + def _get_offset_providers(self, grid: icon_grid.IconGrid) -> dict[str, gtx.FieldOffset]: + offset_providers = {} + for dim in self._compute_domain: + if dim.kind == gtx.DimensionKind.HORIZONTAL: + horizontal_offsets = { + k: v + for k, v in grid.connectivities.items() + # TODO(halungge): review this workaround, as the fix should be available in the gt4py baseline + if isinstance(v, gtx.Connectivity) + and v.domain.dims[0].kind == gtx.DimensionKind.HORIZONTAL + } + offset_providers.update(horizontal_offsets) + if dim.kind == gtx.DimensionKind.VERTICAL: + vertical_offsets = { + k: v + for k, v in grid.connectivities.items() + if isinstance(v, gtx.Dimension) and v.kind == gtx.DimensionKind.VERTICAL + } + offset_providers.update(vertical_offsets) + return offset_providers def _domain_args( self, grid: icon_grid.IconGrid, vertical_grid: v_grid.VerticalGrid @@ -595,9 +613,9 @@ def _compute( deps = {k: factory.get(v) for k, v in self._dependencies.items()} deps.update(self._params) deps.update({k: self._fields[v] for k, v in self._output.items()}) - domain_bounds = self._domain_args(grid_provider.grid, grid_provider.vertical_grid) - deps.update(domain_bounds) - offset_providers = self._grid_connectivities(grid_provider.grid) + dims = self._domain_args(grid_provider.grid, grid_provider.vertical_grid) + offset_providers = self._get_offset_providers(grid_provider.grid) + deps.update(dims) self._func.with_backend(backend)(**deps, offset_provider=offset_providers) @property diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 09299bca9c..b8d86b444e 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -105,13 +105,13 @@ def test_grid_manager_validate_decomposer(processor_props: decomp_defs.ProcessPr assert "Need a Decomposer for multi" in e.value.args[0] -# TODO (halungge): is this used??? +# TODO (halungge): is this function used at all?? @pytest.mark.mpi @pytest.mark.parametrize( "field_offset", [dims.C2V, dims.E2V, dims.V2C, dims.E2C, dims.C2E, dims.V2E, dims.C2E2C, dims.V2E2V], ) -def test_local_connectivities( +def test_construct_local_connectivity( processor_props: decomp_defs.ProcessProperties, caplog: Iterator, field_offset: gtx.FieldOffset, @@ -147,7 +147,6 @@ def test_local_connectivities( field_offset.source, decomp_defs.DecompositionInfo.EntryType.ALL ) ) - # TODO what else to assert? # - outer halo entries have SKIP_VALUE neighbors (depends on offsets) @@ -555,7 +554,6 @@ def test_halo_access_e2c( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -# @pytest.mark.parametrize("grid", (definitions.Grids.R02B04_GLOBAL, definitions.Grids.MCH_CH_R04B09_DSL)) @pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) def test_halo_neighbor_access_e2v( processor_props: decomp_defs.ProcessProperties, From 799ac8ab2b1197963a35fb3be06e306d57596209 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 12 Dec 2025 14:41:19 +0100 Subject: [PATCH 122/240] move function only used in test --- model/common/src/icon4py/model/common/grid/base.py | 7 ------- .../grid/mpi_tests/test_parallel_grid_manager.py | 11 ++++++++++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/base.py b/model/common/src/icon4py/model/common/grid/base.py index 5154a9f24a..c96d1b3ff3 100644 --- a/model/common/src/icon4py/model/common/grid/base.py +++ b/model/common/src/icon4py/model/common/grid/base.py @@ -166,13 +166,6 @@ def get_connectivity(self, offset: str | gtx.FieldOffset) -> gtx_common.Neighbor assert gtx_common.is_neighbor_table(connectivity) return connectivity - def get_neighbor_tables(self): - return { - k: v.ndarray - for k, v in self.connectivities.items() - if gtx_common.is_neighbor_connectivity(v) - } - def construct_connectivity( offset: gtx.FieldOffset, diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index b8d86b444e..3ad9f5fce5 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -20,6 +20,7 @@ from icon4py.model.common import dimension as dims, exceptions from icon4py.model.common.decomposition import definitions as decomp_defs, halo, mpi_decomposition from icon4py.model.common.grid import ( + base, geometry, geometry_attributes, geometry_stencils, @@ -105,6 +106,14 @@ def test_grid_manager_validate_decomposer(processor_props: decomp_defs.ProcessPr assert "Need a Decomposer for multi" in e.value.args[0] +def _get_neighbor_tables(grid: base.Grid) -> dict: + return { + k: v.ndarray + for k, v in grid.connectivities.items() + if gtx_common.is_neighbor_connectivity(v) + } + + # TODO (halungge): is this function used at all?? @pytest.mark.mpi @pytest.mark.parametrize( @@ -122,7 +131,7 @@ def test_construct_local_connectivity( ).grid partitioner = halo.SimpleMetisDecomposer() face_face_connectivity = grid.get_connectivity(dims.C2E2C).ndarray - neighbor_tables = grid.get_neighbor_tables() + neighbor_tables = _get_neighbor_tables(grid) labels = partitioner(face_face_connectivity, num_partitions=processor_props.comm_size) halo_generator = halo.IconLikeHaloConstructor( connectivities=neighbor_tables, From e796b74894520ccef50ed03e06c848abdb59609a Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Fri, 12 Dec 2025 16:05:35 +0100 Subject: [PATCH 123/240] fix imports in parallel tests --- .../model/common/decomposition/halo.py | 5 ++-- .../common/decomposition/mpi_decomposition.py | 2 +- .../icon4py/model/common/grid/grid_manager.py | 30 ------------------- .../src/icon4py/model/common/grid/gridfile.py | 10 +++---- .../grid/mpi_tests/test_parallel_geometry.py | 5 ++++ .../test_parallel_grid_refinement.py | 4 ++- .../grid/mpi_tests/test_parallel_icon.py | 4 ++- .../mpi_tests/test_parallel_interpolation.py | 6 ++++ .../mpi_tests/test_parallel_metrics.py | 6 ++++ 9 files changed, 31 insertions(+), 41 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 07b19d8938..40024e7bc8 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -480,8 +480,7 @@ def halo_constructor( allocator: gtx_typing.FieldBufferAllocationUtil | None, ) -> HaloConstructor: """ - Factory method to create the halo constructor. We need some input data from the global grid and from - Run parameters, hence this method is called during grid construction. + Factory method to create the halo constructor. Currently there is only one halo type (except for single node dummy). If in the future we want to experiment with different halo types we should add an extra selection @@ -489,8 +488,8 @@ def halo_constructor( Args: processor_props: full_grid_size + allocator: connectivities: - backend: Returns: a HaloConstructor suitable for the run_properties diff --git a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py index 044624e549..38795fb243 100644 --- a/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py +++ b/model/common/src/icon4py/model/common/decomposition/mpi_decomposition.py @@ -209,7 +209,7 @@ def _slice_field_based_on_dim(self, field: gtx.Field, dim: gtx.Dimension) -> dat This operation is *necessary* for the use inside FORTRAN as there fields are larger than the grid (nproma size). where it does not do anything in a purely Python setup. the granule context where fields otherwise have length nproma. """ - if dim in dims.MAIN_HORIZONTAL_DIMENSIONS: + if dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values(): return field.ndarray[: self._field_size[dim]] else: raise ValueError(f"Unknown dimension {dim}") diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index d3a3311515..767891474e 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -743,33 +743,3 @@ def construct_local_connectivity( indices = sorted_index_of_global_idx[positions] local_connectivity[i, valid_neighbor_mask] = indices return local_connectivity - - -def _single_node_decomposition_info( - grid_config: base.GridConfig, xp: ModuleType -) -> decomposition.DecompositionInfo: - cell_size = (grid_config.num_cells,) - edge_size = (grid_config.num_edges,) - vertex_size = (grid_config.num_vertices,) - info = ( - decomposition.DecompositionInfo(grid_config.num_levels) - .set_dimension( - dims.CellDim, - xp.arange(cell_size[0], dtype=gtx.int32), - xp.ones(cell_size, dtype=bool), - xp.zeros(cell_size), - ) - .set_dimension( - dims.EdgeDim, - xp.arange(edge_size[0], dtype=gtx.int32), - xp.ones(edge_size, dtype=bool), - xp.zeros(edge_size), - ) - .set_dimension( - dims.VertexDim, - xp.arange(vertex_size[0], dtype=gtx.int32), - xp.ones(vertex_size, dtype=bool), - xp.zeros(vertex_size), - ) - ) - return info diff --git a/model/common/src/icon4py/model/common/grid/gridfile.py b/model/common/src/icon4py/model/common/grid/gridfile.py index 6c3db3f01a..ab1fec3dab 100644 --- a/model/common/src/icon4py/model/common/grid/gridfile.py +++ b/model/common/src/icon4py/model/common/grid/gridfile.py @@ -228,15 +228,15 @@ class ConnectivityName(FieldName): class GeometryName(FieldName): + CELL_NORMAL_ORIENTATION = "orientation_of_normal" + TANGENT_ORIENTATION = "edge_system_orientation" + EDGE_ORIENTATION_ON_VERTEX = "edge_orientation" + # TODO(halungge): compute from coordinates CELL_AREA = "cell_area" - # TODO(halungge): compute from coordinates DUAL_AREA = "dual_area" EDGE_LENGTH = "edge_length" DUAL_EDGE_LENGTH = "dual_edge_length" - CELL_NORMAL_ORIENTATION = "orientation_of_normal" - TANGENT_ORIENTATION = "edge_system_orientation" - EDGE_ORIENTATION_ON_VERTEX = "edge_orientation" # TODO(halungge): compute from coordinates EDGE_CELL_DISTANCE = "edge_cell_distance" EDGE_VERTEX_DISTANCE = "edge_vert_distance" @@ -245,7 +245,7 @@ class GeometryName(FieldName): class CoordinateName(FieldName): """ Coordinates of cell centers, edge midpoints and vertices. - Units: radianfor both MPI-M and DWD + Units: radian for both MPI-M and DWD """ CELL_LONGITUDE = "clon" diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_geometry.py b/model/common/tests/common/grid/mpi_tests/test_parallel_geometry.py index 313c44c11f..fd8896bfc2 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_geometry.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_geometry.py @@ -36,6 +36,11 @@ ) +try: + from icon4py.model.common.decomposition import mpi_decomposition +except ImportError: + pytest.skip("Skipping parallel on single node installation", allow_module_level=True) + if TYPE_CHECKING: from icon4py.model.testing import serialbox as sb diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py index 4cdc895009..8386ee779f 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py @@ -12,12 +12,14 @@ try: import mpi4py import mpi4py.MPI + + from icon4py.model.common.decomposition import mpi_decomposition except ImportError: pytest.skip("Skipping parallel on single node installation", allow_module_level=True) import gt4py.next as gtx -from icon4py.model.common.decomposition import definitions as decomposition, mpi_decomposition +from icon4py.model.common.decomposition import definitions as decomposition from icon4py.model.common.grid import grid_refinement, horizontal as h_grid from icon4py.model.testing import definitions, serialbox diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py index 30148e67b1..571abfdd7a 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py @@ -13,7 +13,7 @@ import icon4py.model.common.dimension as dims import icon4py.model.common.grid.horizontal as h_grid -from icon4py.model.common.decomposition import definitions as decomposition, mpi_decomposition +from icon4py.model.common.decomposition import definitions as decomposition from icon4py.model.common.grid import base as base_grid from icon4py.model.testing import definitions as test_defs, parallel_helpers @@ -31,6 +31,8 @@ try: import mpi4py + + from icon4py.model.common.decomposition import mpi_decomposition except ImportError: pytest.skip("Skipping parallel on single node installation", allow_module_level=True) diff --git a/model/common/tests/common/interpolation/mpi_tests/test_parallel_interpolation.py b/model/common/tests/common/interpolation/mpi_tests/test_parallel_interpolation.py index e74da2a64f..9f11b7a648 100644 --- a/model/common/tests/common/interpolation/mpi_tests/test_parallel_interpolation.py +++ b/model/common/tests/common/interpolation/mpi_tests/test_parallel_interpolation.py @@ -12,6 +12,12 @@ import pytest + +try: + from icon4py.model.common.decomposition import mpi_decomposition +except ImportError: + pytest.skip("Skipping parallel on single node installation", allow_module_level=True) + from icon4py.model.common.decomposition import definitions as decomposition from icon4py.model.common.grid import horizontal as h_grid from icon4py.model.common.interpolation import ( diff --git a/model/common/tests/common/metrics/mpi_tests/test_parallel_metrics.py b/model/common/tests/common/metrics/mpi_tests/test_parallel_metrics.py index fca8ef6dd7..67311f39a7 100644 --- a/model/common/tests/common/metrics/mpi_tests/test_parallel_metrics.py +++ b/model/common/tests/common/metrics/mpi_tests/test_parallel_metrics.py @@ -12,6 +12,12 @@ import pytest + +try: + from icon4py.model.common.decomposition import mpi_decomposition +except ImportError: + pytest.skip("Skipping parallel on single node installation", allow_module_level=True) + from icon4py.model.common.decomposition import definitions as decomposition from icon4py.model.common.metrics import metrics_attributes as attrs, metrics_factory from icon4py.model.testing import definitions as test_defs, parallel_helpers, test_utils From 7f0dedf36c11f604ba28d1f2d8644e19cbf33b86 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Sun, 14 Dec 2025 18:03:16 +0100 Subject: [PATCH 124/240] add doc strings for compute_domain bounds --- .../model/common/grid/grid_refinement.py | 44 ++++++-- .../src/icon4py/model/common/grid/icon.py | 2 - .../test_parallel_grid_refinement.py | 46 ++++---- .../grid/unit_tests/test_grid_refinement.py | 102 +++++++++++------- .../tests/common/grid/unit_tests/test_icon.py | 6 +- .../src/icon4py/model/testing/serialbox.py | 6 +- 6 files changed, 137 insertions(+), 69 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_refinement.py b/model/common/src/icon4py/model/common/grid/grid_refinement.py index 95c4a5830e..9f8fcf7316 100644 --- a/model/common/src/icon4py/model/common/grid/grid_refinement.py +++ b/model/common/src/icon4py/model/common/grid/grid_refinement.py @@ -154,7 +154,16 @@ } -def _refinement_level_in_halo(domain: h_grid.Domain) -> int: +def _refinement_level_placed_with_halo(domain: h_grid.Domain) -> int: + """There is a speciality in the setup of the ICON halos: generally halo points are located at the end of the arrays after all + points owned by a node. This is true for global grids and for a local area model for all points with a + refinement control value larger than (6, 2) for HALO level 1 and (4,1) for HALO_LEVEL_2. + + That is for the local area grid some (but not all!) halo points that are lateral boundary points are placed with the lateral boundary + domains rather than the halo. The reason for this is mysterious (to me) as well as what advantage it might have. + + Disadvantage clearly is that in the LAM case the halos are **not contigous**. + """ assert domain.zone.is_halo(), "Domain must be a halo Zone." dim = domain.dim match dim: @@ -175,9 +184,30 @@ def compute_domain_bounds( """ Compute the domain bounds (start_index, end_index) based on a grid Domain. - Icon orders the field arrays according to their + In a local area model, ICON orders the field arrays according to their distance from the boundary. For each + dimension (cell, vertex, edge) points are "ordered" (moved to the beginning of the array) up to + the values defined in _GRID_REFINEMENT_BOUNDARY_WIDTH. We call these distance a Grid Zone. The `Dimension` + and the `Zone` determine a Grid `Domain`. For a given field values for a domain are located + in a contiguous section of the field array. + + This function can be used to deterine the start_index and end_index of a `Domain`in the arrays. + + For a global model grid all points are unordered. `Zone` that do not exist for a global model grid + return empty domains. + + For distributed grids halo points build their own `Domain`and are located at the end of the field arrays, with the exception of + some points in the lateral boundary as described in (_refinement_level_placed_with_halo) + + Args: + dim: Dimension one of `CellDim`. `VertexDim`, `EdgeDim` + refinement_fields: dict[Dimension, ndarray] containing the refinement_control values for each dimension + decomposition_info: DecompositionInfo needed to determine the HALO `Zone`s + array_ns: numpy or cupy """ + assert ( + dim in dims.MAIN_HORIZONTAL_DIMENSIONS.values() + ), f"Dimension must be one of {dims.MAIN_HORIZONTAL_DIMENSIONS.values()}" refinement_ctrl = convert_to_non_nested_refinement_values( refinement_fields[dim].ndarray, dim, array_ns ) @@ -199,14 +229,14 @@ def compute_domain_bounds( halo_domains = h_grid.get_halo_domains(dim) for domain in halo_domains: my_flag = decomposition.DecompositionFlag(domain.zone.level) - upper_boundary_level_1 = _refinement_level_in_halo(h_grid.domain(dim)(h_grid.Zone.HALO)) - not_lateral_boundary_1 = (refinement_ctrl < 1) | ( - refinement_ctrl > upper_boundary_level_1 - ) # edge 6 + upper_boundary_level_1 = _refinement_level_placed_with_halo( + h_grid.domain(dim)(h_grid.Zone.HALO) + ) + not_lateral_boundary_1 = (refinement_ctrl < 1) | (refinement_ctrl > upper_boundary_level_1) halo_region_1 = array_ns.where(halo_level_1 & not_lateral_boundary_1)[0] not_lateral_boundary_2 = (refinement_ctrl < 1) | ( refinement_ctrl - > _refinement_level_in_halo(h_grid.domain(dim)(h_grid.Zone.HALO_LEVEL_2)) + > _refinement_level_placed_with_halo(h_grid.domain(dim)(h_grid.Zone.HALO_LEVEL_2)) ) halo_region_2 = array_ns.where(halo_level_2 & not_lateral_boundary_2)[0] diff --git a/model/common/src/icon4py/model/common/grid/icon.py b/model/common/src/icon4py/model/common/grid/icon.py index df9e9f2d11..0d26b866ca 100644 --- a/model/common/src/icon4py/model/common/grid/icon.py +++ b/model/common/src/icon4py/model/common/grid/icon.py @@ -24,8 +24,6 @@ log = logging.getLogger(__name__) CONNECTIVITIES_ON_BOUNDARIES = ( - dims.C2VDim, - dims.E2VDim, dims.C2E2C2EDim, dims.E2CDim, dims.C2E2CDim, diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py index 8386ee779f..d2047e2b38 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py @@ -37,27 +37,35 @@ @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize("dim", utils.main_horizontal_dims()) -@pytest.mark.mpi -def test_start_end_index( +def test_compute_domain_bounds( dim: gtx.Dimension, experiment: definitions.Experiment, grid_savepoint: serialbox.IconGridSavepoint, processor_props: decomposition.ProcessProperties, ) -> None: - ref_grid = grid_savepoint.construct_icon_grid(None, keep_skip_values=True) - decomposition_info = grid_savepoint.construct_decomposition_info() - refin_ctrl = {dim: grid_savepoint.refin_ctrl(dim) for dim in utils.main_horizontal_dims()} - start_indices, end_indices = grid_refinement.compute_domain_bounds( - dim, refin_ctrl, decomposition_info - ) - for domain in h_grid.get_domains_for_dim(dim): - ref_start_index = ref_grid.start_index(domain) - ref_end_index = ref_grid.end_index(domain) - computed_start = start_indices[domain] - computed_end = end_indices[domain] - assert ( - computed_start == ref_start_index - ), f"rank={processor_props.rank}/{processor_props.comm_size} - experiment = {experiment.name}: start_index for {domain} does not match: is {computed_start}, expected {ref_start_index}" - assert ( - computed_end == ref_end_index - ), f"rank={processor_props.rank}/{processor_props.comm_size} - experiment = {experiment.name}: end_index for {domain} does not match: is {computed_end}, expected {ref_end_index}" + if processor_props.single_node() and experiment == definitions.Experiments.EXCLAIM_APE: + pytest.mark.xfail( + "end index data for single node APE are all 0 - re- serialization should fix that (patch%cells%end_index vs patch%cells%end_idx)" + ) + + else: + ref_grid = grid_savepoint.construct_icon_grid(backend=None, keep_skip_values=True) + decomposition_info = grid_savepoint.construct_decomposition_info() + refin_ctrl = {dim: grid_savepoint.refin_ctrl(dim) for dim in utils.main_horizontal_dims()} + start_indices, end_indices = grid_refinement.compute_domain_bounds( + dim, refin_ctrl, decomposition_info + ) + for domain in h_grid.get_domains_for_dim(dim): + ref_start_index = ref_grid.start_index(domain) + ref_end_index = ref_grid.end_index(domain) + computed_start = start_indices[domain] + computed_end = end_indices[domain] + print( + f"rank = {processor_props.rank}/{processor_props.comm_size}: domain={domain} : start = {computed_start} end = {computed_end} " + ) + assert ( + computed_start == ref_start_index + ), f"rank={processor_props.rank}/{processor_props.comm_size} - experiment = {experiment.name}: start_index for {domain} does not match: is {computed_start}, expected {ref_start_index}" + assert ( + computed_end == ref_end_index + ), f"rank={processor_props.rank}/{processor_props.comm_size} - experiment = {experiment.name}: end_index for {domain} does not match: is {computed_end}, expected {ref_end_index}" diff --git a/model/common/tests/common/grid/unit_tests/test_grid_refinement.py b/model/common/tests/common/grid/unit_tests/test_grid_refinement.py index e25586a64d..4dc94da143 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_refinement.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_refinement.py @@ -27,9 +27,7 @@ processor_props, ranked_data_path, ) - - -_FALLBACK_FAIL = (-10, -10) +from ..utils import main_horizontal_dims @pytest.mark.parametrize("dim", utils.main_horizontal_dims()) @@ -56,7 +54,7 @@ def test_is_local_area_grid_for_grid_files( assert expected == limited_area -cell_bounds: dict[h_grid.Zone, tuple[int, int]] = { +MCH_OPR_R04B07_CELL_BOUNDS: dict[h_grid.Zone, tuple[int, int]] = { h_grid.Zone.LATERAL_BOUNDARY: (0, 629), h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2: (629, 1244), h_grid.Zone.LATERAL_BOUNDARY_LEVEL_3: (1244, 1843), @@ -68,7 +66,7 @@ def test_is_local_area_grid_for_grid_files( h_grid.Zone.HALO: (10700, 10700), h_grid.Zone.HALO_LEVEL_2: (10700, 10700), } -edge_bounds: dict[h_grid.Zone, tuple[int, int]] = { +MCH_OPR_R04B07_EDGE_BOUNDS: dict[h_grid.Zone, tuple[int, int]] = { h_grid.Zone.LATERAL_BOUNDARY: (0, 318), h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2: (318, 947), h_grid.Zone.LATERAL_BOUNDARY_LEVEL_3: (947, 1258), @@ -85,7 +83,7 @@ def test_is_local_area_grid_for_grid_files( h_grid.Zone.HALO: (16209, 16209), h_grid.Zone.HALO_LEVEL_2: (16209, 16209), } -vertex_bounds: dict[h_grid.Zone, tuple[int, int]] = { +MCH_OPR_R04B07_VERTEX_BOUNDS: dict[h_grid.Zone, tuple[int, int]] = { h_grid.Zone.LATERAL_BOUNDARY: (0, 318), h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2: (318, 629), h_grid.Zone.LATERAL_BOUNDARY_LEVEL_3: (629, 933), @@ -99,21 +97,75 @@ def test_is_local_area_grid_for_grid_files( } +MCH_CH_R04B09_CELL_BOUNDS: dict[h_grid.Zone, tuple[int, int]] = { + h_grid.Zone.LATERAL_BOUNDARY: (0, 850), + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2: (850, 1688), + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_3: (1688, 2511), + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_4: (2511, 3316), + h_grid.Zone.NUDGING: (3316, 4104), + h_grid.Zone.INTERIOR: (4104, 20896), + h_grid.Zone.LOCAL: (0, 20896), + h_grid.Zone.END: (20896, 20896), + h_grid.Zone.HALO: (20896, 20896), + h_grid.Zone.HALO_LEVEL_2: (20896, 20896), +} +MCH_CH_R04B09_EDGE_BOUNDS: dict[h_grid.Zone, tuple[int, int]] = { + h_grid.Zone.LATERAL_BOUNDARY: (0, 428), + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2: (428, 1278), + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_3: (1278, 1700), + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_4: (1700, 2538), + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_5: (2538, 2954), + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_6: (2954, 3777), + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_7: (3777, 4184), + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_8: (4184, 4989), + h_grid.Zone.NUDGING: (4989, 5387), + h_grid.Zone.NUDGING_LEVEL_2: (5387, 6176), + h_grid.Zone.INTERIOR: (6176, 31558), + h_grid.Zone.LOCAL: (0, 31558), + h_grid.Zone.END: (31558, 31558), + h_grid.Zone.HALO: (31558, 31558), + h_grid.Zone.HALO_LEVEL_2: (31558, 31558), +} +MCH_CH_R04B09_VERTEX_BOUNDS: dict[h_grid.Zone, tuple[int, int]] = { + h_grid.Zone.LATERAL_BOUNDARY: (0, 428), + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2: (428, 850), + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_3: (850, 1266), + h_grid.Zone.LATERAL_BOUNDARY_LEVEL_4: (1266, 1673), + h_grid.Zone.NUDGING: (1673, 2071), + h_grid.Zone.INTERIOR: (2071, 10663), + h_grid.Zone.LOCAL: (0, 10663), + h_grid.Zone.END: (10663, 10663), + h_grid.Zone.HALO: (10663, 10663), + h_grid.Zone.HALO_LEVEL_2: (10663, 10663), +} +#: random invalid values to mark failure for start_/end_index tuple +_FALLBACK_FAIL = (-10, -10) + + @pytest.mark.parametrize( - "dim, expected", - [(dims.CellDim, cell_bounds), (dims.EdgeDim, edge_bounds), (dims.VertexDim, vertex_bounds)], + "grid_description, dim, expected", + [ + (test_defs.Grids.MCH_OPR_R04B07_DOMAIN01, dims.CellDim, MCH_OPR_R04B07_CELL_BOUNDS), + (test_defs.Grids.MCH_OPR_R04B07_DOMAIN01, dims.EdgeDim, MCH_OPR_R04B07_EDGE_BOUNDS), + (test_defs.Grids.MCH_OPR_R04B07_DOMAIN01, dims.VertexDim, MCH_OPR_R04B07_VERTEX_BOUNDS), + (test_defs.Grids.MCH_CH_R04B09_DSL, dims.CellDim, MCH_CH_R04B09_CELL_BOUNDS), + (test_defs.Grids.MCH_CH_R04B09_DSL, dims.EdgeDim, MCH_CH_R04B09_EDGE_BOUNDS), + (test_defs.Grids.MCH_CH_R04B09_DSL, dims.VertexDim, MCH_CH_R04B09_VERTEX_BOUNDS), + ], ) -def test_compute_start_index_for_limited_area_grid( +def test_compute_domain_bounds_for_limited_area_grid( + grid_description: test_defs.GridDescription, dim: gtx.Dimension, expected: dict[h_grid.Zone, tuple[int, int]], cpu_allocator: gtx_typing.FieldBufferAllocationUtil, ) -> None: - grid_manager = grid_utils.get_grid_manager_from_identifier( - test_defs.Grids.MCH_OPR_R04B07_DOMAIN01, 1, True, cpu_allocator - ) + grid_manager = grid_utils.get_grid_manager_from_identifier(grid_description, 1, True, cpu_allocator) + grid = grid_manager.grid + assert grid.limited_area == True, "Test expects limited area grid" refinement_field = grid.refinement_control decomposition_info = grid_manager.decomposition_info + start_index, end_index = refinement.compute_domain_bounds( dim, refinement_field, decomposition_info=decomposition_info, array_ns=np ) @@ -162,29 +214,3 @@ def test_compute_domain_bounds_for_global_grid( assert ( v == grid.size[k.dim] ), f"Expected end index '{grid.size[k.dim]}' for {dim} in {k.zone}, but got '{v}'" - - -@pytest.mark.parametrize("dim", utils.main_horizontal_dims()) -@pytest.mark.datatest -def test_start_end_index( - dim: gtx.Dimension, - experiment: test_defs.Experiment, - grid_savepoint: serialbox.IconGridSavepoint, -) -> None: - ref_grid = grid_savepoint.construct_icon_grid(None, keep_skip_values=True) - decomposition_info = grid_savepoint.construct_decomposition_info() - refin_ctrl = {dim: grid_savepoint.refin_ctrl(dim) for dim in utils.main_horizontal_dims()} - start_indices, end_indices = refinement.compute_domain_bounds( - dim, refin_ctrl, decomposition_info - ) - for domain in h_grid.get_domains_for_dim(dim): - ref_start_index = ref_grid.start_index(domain) - ref_end_index = ref_grid.end_index(domain) - computed_start = start_indices[domain] - computed_end = end_indices[domain] - assert ( - computed_start == ref_start_index - ), f" experiment = {experiment.name}: start_index for {domain} does not match: is {computed_start}, expected {ref_start_index}" - assert ( - computed_end == ref_end_index - ), f"experiment = {experiment.name}: end_index for {domain} does not match: is {computed_end}, expected {ref_end_index}" diff --git a/model/common/tests/common/grid/unit_tests/test_icon.py b/model/common/tests/common/grid/unit_tests/test_icon.py index 84656a9faf..78642c27f6 100644 --- a/model/common/tests/common/grid/unit_tests/test_icon.py +++ b/model/common/tests/common/grid/unit_tests/test_icon.py @@ -196,9 +196,11 @@ def test_when_keep_skip_value_then_neighbor_table_matches_config( np.any(connectivity.asnumpy() == gridfile.GridFile.INVALID_INDEX).item() ) == icon._has_skip_values(offset, grid.config.limited_area) if not icon._has_skip_values(offset, grid.config.limited_area): - assert connectivity.skip_value is None + assert connectivity.skip_value is None, f"skip value for offset {offset} should be None" else: - assert connectivity.skip_value == gridfile.GridFile.INVALID_INDEX + assert ( + connectivity.skip_value == gridfile.GridFile.INVALID_INDEX + ), f"skip for offset {offset} value should be {gridfile.GridFile.INVALID_INDEX}" @pytest.mark.parametrize( diff --git a/model/testing/src/icon4py/model/testing/serialbox.py b/model/testing/src/icon4py/model/testing/serialbox.py index 16c5957041..eb5ae92d80 100644 --- a/model/testing/src/icon4py/model/testing/serialbox.py +++ b/model/testing/src/icon4py/model/testing/serialbox.py @@ -152,7 +152,11 @@ def __init__( super().__init__(sp, ser, size, backend) self._grid_id = grid_id self.global_grid_params = icon.GlobalGridParams( - mean_cell_area=self.mean_cell_area(), grid_shape=grid_shape + mean_cell_area=self.mean_cell_area(), + grid_shape=grid_shape, + num_cells=size[dims.CellDim] + if grid_shape.geometry_type == base.GeometryType.TORUS + else None, ) def verts_vertex_lat(self): From b50691e7adca8afa2c2a1cae5ddb02a13bc0e7f8 Mon Sep 17 00:00:00 2001 From: halungge Date: Sun, 14 Dec 2025 22:49:45 +0100 Subject: [PATCH 125/240] pre-commit --- .../tests/common/grid/unit_tests/test_grid_refinement.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/model/common/tests/common/grid/unit_tests/test_grid_refinement.py b/model/common/tests/common/grid/unit_tests/test_grid_refinement.py index 4dc94da143..21149b90f4 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_refinement.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_refinement.py @@ -159,10 +159,12 @@ def test_compute_domain_bounds_for_limited_area_grid( expected: dict[h_grid.Zone, tuple[int, int]], cpu_allocator: gtx_typing.FieldBufferAllocationUtil, ) -> None: - grid_manager = grid_utils.get_grid_manager_from_identifier(grid_description, 1, True, cpu_allocator) + grid_manager = grid_utils.get_grid_manager_from_identifier( + grid_description, 1, True, cpu_allocator + ) grid = grid_manager.grid - assert grid.limited_area == True, "Test expects limited area grid" + assert grid.limited_area, "Test expects limited area grid" refinement_field = grid.refinement_control decomposition_info = grid_manager.decomposition_info From 09d4aaf740d7a23b0881afd94aa2060b8a5784f4 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Mon, 15 Dec 2025 07:56:50 +0100 Subject: [PATCH 126/240] move serializable tests to unit_tests --- .../{test_halo.py => test_parallel_halo.py} | 29 --------------- .../decomposition/unit_tests/test_halo.py | 35 +++++++++++++++++-- 2 files changed, 33 insertions(+), 31 deletions(-) rename model/common/tests/common/decomposition/mpi_tests/{test_halo.py => test_parallel_halo.py} (69%) diff --git a/model/common/tests/common/decomposition/mpi_tests/test_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py similarity index 69% rename from model/common/tests/common/decomposition/mpi_tests/test_halo.py rename to model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py index afc93ce392..636e0a6f3f 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py @@ -11,7 +11,6 @@ import pytest import icon4py.model.common.dimension as dims -from icon4py.model.common import exceptions from icon4py.model.common.decomposition import definitions as defs from icon4py.model.testing import parallel_helpers from icon4py.model.testing.fixtures import processor_props @@ -38,34 +37,6 @@ backend = None -@pytest.mark.mpi(min_size=4) -@pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_halo_constructor_owned_cells(processor_props, simple_neighbor_tables): # F811 # fixture - halo_generator = halo.IconLikeHaloConstructor( - connectivities=simple_neighbor_tables, - run_properties=processor_props, - allocator=backend, - ) - my_owned_cells = halo_generator.owned_cells(utils.SIMPLE_DISTRIBUTION) - - print(f"rank {processor_props.rank} owns {my_owned_cells} ") - assert my_owned_cells.size == len(utils._CELL_OWN[processor_props.rank]) - assert np.setdiff1d(my_owned_cells, utils._CELL_OWN[processor_props.rank]).size == 0 - - -@pytest.mark.parametrize("processor_props", [True, False], indirect=True) -def test_halo_constructor_validate_number_of_node_mismatch(processor_props, simple_neighbor_tables): - num_cells = simple_neighbor_tables["C2E2C"].shape[0] - distribution = (processor_props.comm_size + 1) * np.ones((num_cells,), dtype=int) - with pytest.raises(expected_exception=exceptions.ValidationError) as e: - halo_generator = halo.IconLikeHaloConstructor( - connectivities=simple_neighbor_tables, - run_properties=processor_props, - ) - halo_generator(distribution) - assert "The distribution assumes more nodes than the current run" in e.value.args[0] - - def global_indices(dim: gtx.Dimension) -> np.ndarray: mesh = simple.simple_grid() return np.arange(mesh.size[dim], dtype=gtx.int32) diff --git a/model/common/tests/common/decomposition/unit_tests/test_halo.py b/model/common/tests/common/decomposition/unit_tests/test_halo.py index 8d5f931688..fe721a8932 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_halo.py +++ b/model/common/tests/common/decomposition/unit_tests/test_halo.py @@ -9,14 +9,31 @@ import numpy as np import pytest -from icon4py.model.common import dimension as dims, exceptions +from icon4py.model.common import dimension as dims, exceptions, model_backends from icon4py.model.common.decomposition import definitions, halo from icon4py.model.common.grid import base as base_grid +from ...fixtures import backend_like, processor_props from .. import utils from ..fixtures import simple_neighbor_tables +@pytest.mark.parametrize("rank", [0, 1, 2, 4]) +def test_halo_constructor_owned_cells(rank, simple_neighbor_tables, backend_like): + processor_props = utils.DummyProps(rank=rank) + allocator = model_backends.get_allocator(backend_like) + halo_generator = halo.IconLikeHaloConstructor( + connectivities=simple_neighbor_tables, + run_properties=processor_props, + allocator=allocator, + ) + my_owned_cells = halo_generator.owned_cells(utils.SIMPLE_DISTRIBUTION) + + print(f"rank {processor_props.rank} owns {my_owned_cells} ") + assert my_owned_cells.size == len(utils._CELL_OWN[processor_props.rank]) + assert np.setdiff1d(my_owned_cells, utils._CELL_OWN[processor_props.rank]).size == 0 + + @pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) @pytest.mark.parametrize("rank", [0, 1, 2, 4]) def test_halo_constructor_decomposition_info_global_indices(rank, simple_neighbor_tables, dim): @@ -51,7 +68,7 @@ def test_halo_constructor_decomposition_info_global_indices(rank, simple_neighbo @pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) @pytest.mark.parametrize("rank", [0, 1, 2, 3]) -def test_halo_constructor_definitions_info_halo_levels(rank, dim, simple_neighbor_tables): +def test_halo_constructor_decomposition_info_halo_levels(rank, dim, simple_neighbor_tables): processor_props = utils.DummyProps(rank=rank) halo_generator = halo.IconLikeHaloConstructor( connectivities=simple_neighbor_tables, @@ -143,3 +160,17 @@ def test_halo_constructor_validate_rank_mapping_wrong_shape(simple_neighbor_tabl ) halo_generator(np.zeros((num_cells, 3), dtype=int)) assert f"should have shape ({num_cells},)" in e.value.args[0] + + +@pytest.mark.parametrize("rank", (0, 1, 2, 3)) +def test_halo_constructor_validate_number_of_node_mismatch(rank, simple_neighbor_tables): + processor_props = utils.DummyProps(rank=rank) + num_cells = simple_neighbor_tables["C2E2C"].shape[0] + distribution = (processor_props.comm_size + 1) * np.ones((num_cells,), dtype=int) + with pytest.raises(expected_exception=exceptions.ValidationError) as e: + halo_generator = halo.IconLikeHaloConstructor( + connectivities=simple_neighbor_tables, + run_properties=processor_props, + ) + halo_generator(distribution) + assert "The distribution assumes more nodes than the current run" in e.value.args[0] From 5df9adc546bc80fec7f61623861552622ccb0a7a Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Mon, 15 Dec 2025 09:26:40 +0100 Subject: [PATCH 127/240] doc string for IconLikeHaloConstructor --- .../model/common/decomposition/halo.py | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 40024e7bc8..eb9a802a41 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -162,9 +162,7 @@ def next_halo_line( Returns: next_halo_cells: full-grid indices of the next halo line """ - assert ( - cells.ndim == 1 - ), "input should be 1d array" # TODO(halungge): otherwise reshape instead + assert cells.ndim == 1, "input should be 1d array" cell_neighbors = self._find_cell_neighbors(cells) cells_so_far = self._xp.hstack((depot, cells)) if depot is not None else cells @@ -261,7 +259,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: | / | / | / | | / | / | / | rank 0 |/ |/ |/ | - ----------e0--------e1--------e3------------ cutting line + -----v0---e0----v1---e1----v2---e2----v3---e3-- cutting line | /| /| /| | c0 / | c1 / | c2 / | | / | / | / | rank 1 @@ -269,7 +267,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: | / | / | / | / | / c3 | / c4 | / c5 | / |/ |/ |/ |/ - ------e11------e12-------e13------------ + -v4---e11---v5---e12---v6----e13--v7--------- | /| /| /| | / | / | / | | / | / | / | @@ -286,18 +284,31 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: Note that this definition of 1. and 2. line differs from the definition of boundary line counting used in [grid refinement](grid_refinement.py), in terms of "distance" to the cutting line all halo cells have a distance of 1. - Edges: - Edges (e0, e1, e2) are on the cutting line. - For both ranks the edges on the cutting line sit on **owned cells**. As all elements need to have a unique owner, they are assigned by convention to the rank with the higher number, here rank 1. - From the point of view of rank 0 they are 1. HALO LINE edges. This conventional assignement has as an effect that there ranks (essentially rank 0) that have an *empty* first edge HALO LINE, - even thought they have elements in the 2. HALO LEVEL (e4, e5, e6, e7, e8, e9, e10) which are the edges that share exactly one vertex with an owned cell. - The edges (e11, e12, e13) that "close" the halo cells (share exactly 2 vertices with a halo cell, but none with an owned cell) are **not** included in the halo in ICON. We include them as 3. HALO LINE which - makes the C2E connectivity complete (= without skip value) for a distributed setup. + Vertices: + - 1. HALO LEVEL: are vertices on the cutting line that are not owned, or put in a different wording: all vertices on owned cells that ar not + owned. + In ICON every element in an array needs **exactly one owner**. For elements on the cutting line (vertices and edges) there is no clear + indication which rank should own it, ICON uses the rank with the higher rank (see (_update_owner_mask_by_max_rank_convention)) + In the example above (v0, v1, v2, v3) are in the 1. HALO LEVEL or rank 0 and owend by rank 1. Consequently there ranks that have no + 1.HALO LEVEL cells. + + - 2. HALO LEVEL: are vertices that are on halo 1. HALO LEVEL cells, but not on owned. For rank 0 these are (v4, v5, v6, v7) - # TODO(halungge): make number of halo lines (in terms of cells) a parameter - # icon does hard coding of 2 halo lines for cells, make this dynamic! + Edges: + For edges a similar pattern is used as for the vertices. + - 1. HALO LEVEl: edges that are on owned cells but not owned themselves (these are edges that share 2 vertices with and owned cell). + In terms of ownership the same convention is applied as for the vertices: (e0, e1, e2, e3) are in the HALO LEVEL 1 of rank 0, and owned by rank 1 + - 2. HALO LEVEL: edges that share exactly one vertex with an owned cell. The definition via vertices is important: TODD (halungge): EXAMPLE??? + For rank 0 above these are the edges (e4, e5, e6, e7, e8, e9, e10) + + - 3. HALO LEVEL: + We flag the edges (e11, e12, e13) that "close" the halo cells (share exactly 2 vertices with a HALO LEVEL 2 cell, but none with + an owned cell). These edges are **not** included in the halo in ICON. We include them as 3. HALO LINE which + makes the C2E connectivity complete (= without skip value) for a distributed setup. + + # TODO(halungge): make number of halo lines (in terms of cells) a parameter: icon does hard coding of 2 halo lines for cells, make this dynamic! """ From 4c8adf3dade8af4921ba08f15ab8585342aec53a Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Mon, 15 Dec 2025 16:47:07 +0100 Subject: [PATCH 128/240] fix test_local_connectivity to run on grid_manager --- .../icon4py/model/common/grid/grid_manager.py | 51 ------------- .../mpi_tests/test_parallel_grid_manager.py | 59 +-------------- .../grid/unit_tests/test_grid_manager.py | 75 ++++++++++++++++++- 3 files changed, 73 insertions(+), 112 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 767891474e..036b7a8be3 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -11,7 +11,6 @@ from types import ModuleType from typing import Literal, TypeAlias -import gt4py._core.definitions as gtx_core_defs import gt4py.next as gtx import gt4py.next.typing as gtx_typing import numpy as np @@ -693,53 +692,3 @@ def _patch_with_dummy_lastline(ar, array_ns: ModuleType = np): axis=0, ) return patched_ar - - -# TODO (halungge): is this function used at all?? -def construct_local_connectivity( - field_offset: gtx.FieldOffset, - decomposition_info: decomposition.DecompositionInfo, - connectivity: np.ndarray | gtx_core_defs.NDArrayObject, -) -> np.ndarray: - """ - Construct a connectivity table for use on a given rank: it maps from source to target dimension in _local_ indices. - - Starting from the connectivity table on the global grid - - we reduce it to the lines for the locally present entries of the the target dimension - - the reduced connectivity then still maps to global source dimension indices: - we replace those source dimension indices not present on the node to SKIP_VALUE and replace the rest with the local indices - - Args: - field_offset: FieldOffset for which we want to construct the local connectivity table - decomposition_info: DecompositionInfo for the current rank. - connectivity: - - Returns: - connectivity are for the same FieldOffset but mapping from local target dimension indices to local source dimension indices. - """ - source_dim = field_offset.source - target_dim = field_offset.target[0] - sliced_connectivity = connectivity[ - decomposition_info.global_index(target_dim, decomposition.DecompositionInfo.EntryType.ALL) - ] - - global_idx = decomposition_info.global_index( - source_dim, decomposition.DecompositionInfo.EntryType.ALL - ) - - # replace indices in the connectivity that do not exist on the local node by the SKIP_VALUE (those are for example neighbors of the outermost halo points) - local_connectivity = np.where( - np.isin(sliced_connectivity, global_idx), - sliced_connectivity, - gridfile.GridFile.INVALID_INDEX, - ) - - # map to local source indices - sorted_index_of_global_idx = np.argsort(global_idx) - global_idx_sorted = global_idx[sorted_index_of_global_idx] - for i in np.arange(local_connectivity.shape[0]): - valid_neighbor_mask = local_connectivity[i, :] != gridfile.GridFile.INVALID_INDEX - positions = np.searchsorted(global_idx_sorted, local_connectivity[i, valid_neighbor_mask]) - indices = sorted_index_of_global_idx[positions] - local_connectivity[i, valid_neighbor_mask] = indices - return local_connectivity diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 3ad9f5fce5..55b150831a 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -9,7 +9,6 @@ import logging import operator import pathlib -from collections.abc import Iterator from typing import Any import numpy as np @@ -36,18 +35,7 @@ from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import definitions as test_defs, grid_utils -from ...decomposition import utils as decomp_utils -from .. import utils -from ..fixtures import ( - backend, - data_provider, - download_ser_data, - experiment, - grid_savepoint, - icon_grid, - processor_props, - ranked_data_path, -) +from ..fixtures import backend, processor_props NUM_LEVELS = 10 @@ -114,51 +102,6 @@ def _get_neighbor_tables(grid: base.Grid) -> dict: } -# TODO (halungge): is this function used at all?? -@pytest.mark.mpi -@pytest.mark.parametrize( - "field_offset", - [dims.C2V, dims.E2V, dims.V2C, dims.E2C, dims.C2E, dims.V2E, dims.C2E2C, dims.V2E2V], -) -def test_construct_local_connectivity( - processor_props: decomp_defs.ProcessProperties, - caplog: Iterator, - field_offset: gtx.FieldOffset, -) -> None: - caplog.set_level(logging.INFO) # type: ignore [attr-defined] - grid = utils.run_grid_manager( - test_defs.Grids.R02B04_GLOBAL, keep_skip_values=True, backend=None - ).grid - partitioner = halo.SimpleMetisDecomposer() - face_face_connectivity = grid.get_connectivity(dims.C2E2C).ndarray - neighbor_tables = _get_neighbor_tables(grid) - labels = partitioner(face_face_connectivity, num_partitions=processor_props.comm_size) - halo_generator = halo.IconLikeHaloConstructor( - connectivities=neighbor_tables, - run_properties=processor_props, - ) - - decomposition_info = halo_generator(labels) - - connectivity = gm.construct_local_connectivity( - field_offset, decomposition_info, connectivity=grid.get_connectivity(field_offset).ndarray - ) - # there is an neighbor list for each index of the target dimension on the node - assert ( - connectivity.shape[0] - == decomposition_info.global_index( - field_offset.target[0], decomp_defs.DecompositionInfo.EntryType.ALL - ).size - ) - # all neighbor indices are valid local indices - assert np.max(connectivity) == np.max( - decomposition_info.local_index( - field_offset.source, decomp_defs.DecompositionInfo.EntryType.ALL - ) - ) - # - outer halo entries have SKIP_VALUE neighbors (depends on offsets) - - @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_fields_distribute_and_gather( diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index 11cf30c5af..f0fac7dc80 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -17,16 +17,21 @@ import pytest import icon4py.model.common.grid.gridfile -from icon4py.model.common import dimension as dims -from icon4py.model.common.decomposition import definitions as decomposition, halo +from icon4py.model.common import dimension as dims, model_backends +from icon4py.model.common.decomposition import ( + definitions as decomp_defs, + definitions as decomposition, + halo, +) from icon4py.model.common.grid import ( grid_manager as gm, grid_refinement as refin, gridfile, horizontal as h_grid, + icon, vertical as v_grid, ) -from icon4py.model.testing import definitions, test_utils +from icon4py.model.testing import definitions, definitions as test_defs, grid_utils, test_utils if typing.TYPE_CHECKING: @@ -42,6 +47,7 @@ from icon4py.model.testing.fixtures import ( backend, + backend_like, cpu_allocator, data_provider, download_ser_data, @@ -51,6 +57,7 @@ ranked_data_path, ) +from ...decomposition import utils as decomp_utils from .. import utils @@ -564,3 +571,65 @@ def test_decomposition_info_single_node( assert np.all(result.global_index(dim) == expected.global_index(dim)) assert np.all(result.owner_mask(dim) == expected.owner_mask(dim)) assert np.all(result.halo_levels(dim) == expected.halo_levels(dim)) + + +@pytest.mark.parametrize("rank", (0, 1, 2, 3)) +@pytest.mark.parametrize( + "field_offset", + [dims.C2V, dims.E2V, dims.V2C, dims.E2C, dims.C2E, dims.V2E, dims.C2E2C, dims.V2E2V], +) +def test_local_connectivity( + rank: int, + caplog: Iterator, + field_offset: gtx.FieldOffset, + backend_like: model_backends.BackendLike, +) -> None: + processor_props = decomp_utils.DummyProps(rank=rank) + caplog.set_level(logging.INFO) # type: ignore [attr-defined] + partitioner = halo.SimpleMetisDecomposer() + allocator = model_backends.get_allocator(backend_like) + file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) + manager = gm.GridManager(num_levels=10, grid_file=file) + manager( + decomposer=partitioner, + allocator=allocator, + keep_skip_values=True, + run_properties=processor_props, + ) + grid = manager.grid + + decomposition_info = manager.decomposition_info + connectivity = grid.get_connectivity(field_offset).asnumpy() + + assert ( + connectivity.shape[0] + == decomposition_info.global_index( + field_offset.target[0], decomp_defs.DecompositionInfo.EntryType.ALL + ).size + ), "connectivity shapes do not match" + + # all neighbor indices are valid local indices + max_local_index = np.max( + decomposition_info.local_index( + field_offset.source, decomp_defs.DecompositionInfo.EntryType.ALL + ) + ) + assert ( + np.max(connectivity) == max_local_index + ), f"max value in the connectivity is {np.max(connectivity)} is larger than the local patch size {max_local_index}" + # - outer halo entries have SKIP_VALUE neighbors (depends on offsets) + neighbor_dim = field_offset.target[1] # type: ignore [misc] + if ( + neighbor_dim in icon.CONNECTIVITIES_ON_BOUNDARIES + or neighbor_dim in icon.CONNECTIVITIES_ON_PENTAGONS + ): + dim = field_offset.target[0] + last_halo_level = ( + decomp_defs.DecompositionFlag.THIRD_HALO_LEVEL + if neighbor_dim == dims.E2CDim + else decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL + ) + level_index = np.where(decomposition_info.halo_levels(dim) == last_halo_level) + assert np.count_nonzero( + (connectivity[level_index] == gridfile.GridFile.INVALID_INDEX) > 0 + ), f"missing invalid index in {dim} - offset {field_offset}" From 34c61c443e8250847bc8d8159c6ed51270f93909 Mon Sep 17 00:00:00 2001 From: Magdalena Luz Date: Tue, 16 Dec 2025 18:41:01 +0100 Subject: [PATCH 129/240] move global_to_local function --- .../model/common/decomposition/definitions.py | 31 ++------- .../model/common/decomposition/halo.py | 25 ++++++- .../icon4py/model/common/grid/grid_manager.py | 41 +++++++---- .../unit_tests/test_definitions.py | 68 ++++++++----------- .../decomposition/unit_tests/test_halo.py | 42 +++++++++++- .../common/grid/unit_tests/test_geometry.py | 41 +++++++++-- .../src/icon4py/model/testing/grid_utils.py | 3 +- 7 files changed, 162 insertions(+), 89 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 81a495695f..e917d493f5 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -20,7 +20,7 @@ import numpy as np from icon4py.model.common import dimension as dims, utils -from icon4py.model.common.grid import base, gridfile +from icon4py.model.common.grid import base from icon4py.model.common.orchestration.halo_exchange import DummyNestedSDFG from icon4py.model.common.utils import data_allocation as data_alloc @@ -97,6 +97,9 @@ def set_dimension( self._owner_mask[dim] = owner_mask self._halo_levels[dim] = halo_levels + def is_distributed(self) -> bool: + return max(self._halo_levels[dims.CellDim]) > DecompositionFlag.OWNED + def local_index( self, dim: gtx.Dimension, entry_type: EntryType = EntryType.ALL ) -> data_alloc.NDArray: @@ -123,32 +126,6 @@ def _to_local_index(self, dim: gtx.Dimension) -> data_alloc.NDArray: xp.arange(data.shape[0]) return xp.arange(data.shape[0]) - def global_to_local( - self, dim: gtx.Dimension, indices_to_translate: data_alloc.NDArray - ) -> data_alloc.NDArray: - global_indices = self.global_index(dim) - sorter = np.argsort(global_indices) - - mask = np.isin(indices_to_translate, global_indices) - positions = np.searchsorted(global_indices, indices_to_translate, sorter=sorter) - local_neighbors = np.full_like(indices_to_translate, gridfile.GridFile.INVALID_INDEX) - local_neighbors[mask] = sorter[positions[mask]] - return local_neighbors - - # TODO (halungge): use for test reference? in test_definitions.py - - def global_to_local_ref( - self, dim: gtx.Dimension, indices_to_translate: data_alloc.NDArray - ) -> data_alloc.NDArray: - global_indices = self.global_index(dim) - local_neighbors = np.full_like(indices_to_translate, gridfile.GridFile.INVALID_INDEX) - for i in range(indices_to_translate.shape[0]): - for j in range(indices_to_translate.shape[1]): - if np.isin(indices_to_translate[i, j], global_indices): - pos = np.where(indices_to_translate[i, j] == global_indices)[0] - local_neighbors[i, j] = pos - return local_neighbors - def owner_mask(self, dim: gtx.Dimension) -> data_alloc.NDArray: return self._owner_mask[dim] diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index eb9a802a41..13505d6713 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -16,7 +16,7 @@ from icon4py.model.common import dimension as dims, exceptions from icon4py.model.common.decomposition import definitions as defs -from icon4py.model.common.grid import base +from icon4py.model.common.grid import base, gridfile from icon4py.model.common.utils import data_allocation as data_alloc @@ -484,7 +484,7 @@ def __call__( return np.zeros(adjacency_matrix.shape[0], dtype=gtx.int32) # type: ignore [attr-defined] -def halo_constructor( +def get_halo_constructor( run_properties: defs.ProcessProperties, full_grid_size: base.HorizontalGridSize, connectivities: dict[gtx.FieldOffset | str, data_alloc.NDArray], @@ -516,3 +516,24 @@ def halo_constructor( connectivities=connectivities, allocator=allocator, ) + + +def global_to_local( + global_indices: data_alloc.NDArray, + indices_to_translate: data_alloc.NDArray, + array_ns: ModuleType = np, +) -> data_alloc.NDArray: + """Translate an array of global indices into rank-local ones. + + Args: + global_indices: global indices owned on the rank: this is the implicit mapping encoding the local to global + indices_to_translate: the array to map to local indices + + """ + sorter = array_ns.argsort(global_indices) + + mask = array_ns.isin(indices_to_translate, global_indices) + positions = array_ns.searchsorted(global_indices, indices_to_translate, sorter=sorter) + local_neighbors = array_ns.full_like(indices_to_translate, gridfile.GridFile.INVALID_INDEX) + local_neighbors[mask] = sorter[positions[mask]] + return local_neighbors diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 036b7a8be3..f7962a3268 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -58,7 +58,6 @@ def __init__( self._transformation = transformation self._file_name = str(grid_file) self._num_levels = num_levels - self._halo_constructor: halo.HaloConstructor | None = None # Output self._grid: icon.IconGrid | None = None self._decomposition_info: decomposition.DecompositionInfo | None = None @@ -316,7 +315,6 @@ def _construct_decomposed_grid( """ xp = data_alloc.import_array_ns(allocator) ## FULL GRID PROPERTIES - cell_refinement = self._reader.variable(gridfile.GridRefinementName.CONTROL_CELLS) global_size = self._read_full_grid_size() global_params = self._construct_global_params(allocator, global_size) @@ -339,7 +337,7 @@ def _construct_decomposed_grid( # TODO(halungge): reduce the set of neighbor tables used in the halo construction # TODO(halungge): figure out where to do the host to device copies (xp.asarray...) neighbor_tables_for_halo_construction = neighbor_tables - halo_constructor = halo.halo_constructor( + halo_constructor = halo.get_halo_constructor( run_properties=run_properties, full_grid_size=global_size, connectivities=neighbor_tables_for_halo_construction, @@ -349,13 +347,9 @@ def _construct_decomposed_grid( self._decomposition_info = halo_constructor(cells_to_rank_mapping) distributed_size = self._decomposition_info.get_horizontal_size() - # TODO(halungge): run this only for distrbuted grids otherwise to nothing internally - neighbor_tables = { - k: self._decomposition_info.global_to_local( - k.source, v[self._decomposition_info.global_index(k.target[0])] - ) - for k, v in neighbor_tables_for_halo_construction.items() - } + neighbor_tables = self._get_local_connectivities( + neighbor_tables_for_halo_construction, array_ns=xp + ) # COMPUTE remaining derived connectivities neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) @@ -389,6 +383,23 @@ def _construct_decomposed_grid( ) self._grid = grid + def _get_local_connectivities( + self, + neighbor_tables_for_halo_construction: dict[gtx.FieldOffset, data_alloc.NDArray], + array_ns, + ) -> dict[gtx.FieldOffset, data_alloc.NDArray]: + global_to_local = functools.partial(halo.global_to_local, array_ns=array_ns) + if self.decomposition_info.is_distributed(): + return { + k: global_to_local( + self._decomposition_info.global_index(k.source), + v[self._decomposition_info.global_index(k.target[0])], + ) + for k, v in neighbor_tables_for_halo_construction.items() + } + else: + return neighbor_tables_for_halo_construction + def _construct_global_params( self, allocator: gtx_typing.FieldBufferAllocationUtil, global_size: base.HorizontalGridSize ): @@ -399,14 +410,16 @@ def _construct_global_params( sphere_radius = self._reader.try_attribute(gridfile.MPIMPropertyName.SPHERE_RADIUS) domain_length = self._reader.try_attribute(gridfile.MPIMPropertyName.DOMAIN_LENGTH) domain_height = self._reader.try_attribute(gridfile.MPIMPropertyName.DOMAIN_HEIGHT) - mean_edge_length = self._reader.try_attribute(gridfile.MPIMPropertyName.MEAN_EDGE_LENGTH) - mean_dual_edge_length = self._reader.try_attribute( - gridfile.MPIMPropertyName.MEAN_DUAL_EDGE_LENGTH - ) + + # TODO (@halungge): use global reduction in geometry.py mean_cell_area = self._reader.try_attribute(gridfile.MPIMPropertyName.MEAN_CELL_AREA) mean_dual_cell_area = self._reader.try_attribute( gridfile.MPIMPropertyName.MEAN_DUAL_CELL_AREA ) + mean_edge_length = self._reader.try_attribute(gridfile.MPIMPropertyName.MEAN_EDGE_LENGTH) + mean_dual_edge_length = self._reader.try_attribute( + gridfile.MPIMPropertyName.MEAN_DUAL_EDGE_LENGTH + ) # TODO (@halungge): Fix this: reads the global fields... edge_lengths = self._reader.variable(gridfile.GeometryName.EDGE_LENGTH) dual_edge_lengths = self._reader.variable(gridfile.GeometryName.DUAL_EDGE_LENGTH) diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index cf391f2047..3c0a9766d8 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -20,7 +20,6 @@ from ...grid import utils as grid_utils from .. import utils -from ..fixtures import simple_neighbor_tables from ..utils import dummy_four_ranks @@ -45,42 +44,6 @@ def get_neighbor_tables_for_simple_grid() -> dict[str, data_alloc.NDArray]: offsets = [dims.E2C, dims.E2V, dims.C2E, dims.C2E2C, dims.V2C, dims.V2E, dims.C2V, dims.E2C2V] -@pytest.mark.parametrize("offset", offsets) -@pytest.mark.parametrize("rank", [0, 1, 2, 3]) -def test_global_to_local_index(offset, rank): - grid = simple.simple_grid() - neighbor_tables = { - k: v.ndarray - for k, v in grid.connectivities.items() - if gtx_common.is_neighbor_connectivity(v) - } - props = dummy_four_ranks(rank) - halo_constructor = halo.IconLikeHaloConstructor(props, neighbor_tables) - decomposition_info = halo_constructor(utils.SIMPLE_DISTRIBUTION) - source_indices_on_local_grid = decomposition_info.global_index(offset.target[0]) - - offset_full_grid = grid.connectivities[offset.value].ndarray[source_indices_on_local_grid] - neighbor_dim = offset.source - neighbor_index_full_grid = decomposition_info.global_index(neighbor_dim) - - local_offset = decomposition_info.global_to_local(neighbor_dim, offset_full_grid) - - ## assert by backmapping - - for i in range(local_offset.shape[0]): - for k in range(local_offset.shape[1]): - k_ = local_offset[i][k] - if k_ == -1: - # global index is not on this local patch: - assert not np.isin(offset_full_grid[i][k], neighbor_index_full_grid) - else: - ( - neighbor_index_full_grid[k_] == offset_full_grid[i][k], - f"failed to map [{offset_full_grid[i]}] to local: [{local_offset[i]}]", - ) - - -# TODO this duplicates and serializes a test from mpi_tests/test_halo.py @pytest.mark.parametrize("dim", grid_utils.main_horizontal_dims()) @pytest.mark.parametrize("rank", [0, 1, 2, 3]) def test_halo_constructor_decomposition_info_global_indices(dim, rank): @@ -138,6 +101,35 @@ def test_decomposition_info_single_node_empty_halo( ) decomposition_info = manager.decomposition_info - for level in definitions.DecompositionFlag: + for level in ( + definitions.DecompositionFlag.FIRST_HALO_LEVEL, + definitions.DecompositionFlag.SECOND_HALO_LEVEL, + definitions.DecompositionFlag.THIRD_HALO_LEVEL, + ): assert decomposition_info.get_halo_size(dim, level) == 0 assert np.count_nonzero(decomposition_info.halo_level_mask(dim, level)) == 0 + assert ( + decomposition_info.get_halo_size(dim, definitions.DecompositionFlag.OWNED) + == manager.grid.size[dim] + ) + + +@pytest.mark.parametrize( + "flag, expected", + [ + (definitions.DecompositionFlag.OWNED, False), + (definitions.DecompositionFlag.SECOND_HALO_LEVEL, True), + (definitions.DecompositionFlag.THIRD_HALO_LEVEL, True), + (definitions.DecompositionFlag.FIRST_HALO_LEVEL, True), + (definitions.DecompositionFlag.UNDEFINED, False), + ], +) +def test_decomposition_info_is_distributed(flag, expected): + mesh = simple.simple_grid(allocator=None, num_levels=10) + decomp = definitions.DecompositionInfo().set_dimension( + dims.CellDim, + np.arange(mesh.num_cells), + np.arange(mesh.num_cells), + np.ones((mesh.num_cells,)) * flag, + ) + assert decomp.is_distributed() == expected diff --git a/model/common/tests/common/decomposition/unit_tests/test_halo.py b/model/common/tests/common/decomposition/unit_tests/test_halo.py index fe721a8932..812a7efc49 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_halo.py +++ b/model/common/tests/common/decomposition/unit_tests/test_halo.py @@ -8,14 +8,17 @@ import numpy as np import pytest +from gt4py.next import common as gtx_common from icon4py.model.common import dimension as dims, exceptions, model_backends from icon4py.model.common.decomposition import definitions, halo -from icon4py.model.common.grid import base as base_grid +from icon4py.model.common.grid import base as base_grid, simple from ...fixtures import backend_like, processor_props from .. import utils from ..fixtures import simple_neighbor_tables +from ..utils import dummy_four_ranks +from .test_definitions import offsets @pytest.mark.parametrize("rank", [0, 1, 2, 4]) @@ -174,3 +177,40 @@ def test_halo_constructor_validate_number_of_node_mismatch(rank, simple_neighbor ) halo_generator(distribution) assert "The distribution assumes more nodes than the current run" in e.value.args[0] + + +@pytest.mark.parametrize("offset", offsets) +@pytest.mark.parametrize("rank", [0, 1, 2, 3]) +def test_global_to_local_index(offset, rank): + grid = simple.simple_grid() + neighbor_tables = { + k: v.ndarray + for k, v in grid.connectivities.items() + if gtx_common.is_neighbor_connectivity(v) + } + props = dummy_four_ranks(rank) + halo_constructor = halo.IconLikeHaloConstructor(props, neighbor_tables) + decomposition_info = halo_constructor(utils.SIMPLE_DISTRIBUTION) + source_indices_on_local_grid = decomposition_info.global_index(offset.target[0]) + + offset_full_grid = grid.connectivities[offset.value].ndarray[source_indices_on_local_grid] + neighbor_dim = offset.source + neighbor_index_full_grid = decomposition_info.global_index(neighbor_dim) + + local_offset = halo.global_to_local( + decomposition_info.global_index(neighbor_dim), offset_full_grid + ) + + ## assert by backmapping + + for i in range(local_offset.shape[0]): + for k in range(local_offset.shape[1]): + k_ = local_offset[i][k] + if k_ == -1: + # global index is not on this local patch: + assert not np.isin(offset_full_grid[i][k], neighbor_index_full_grid) + else: + ( + neighbor_index_full_grid[k_] == offset_full_grid[i][k], + f"failed to map [{offset_full_grid[i]}] to local: [{local_offset[i]}]", + ) diff --git a/model/common/tests/common/grid/unit_tests/test_geometry.py b/model/common/tests/common/grid/unit_tests/test_geometry.py index 593418f715..e218ec342e 100644 --- a/model/common/tests/common/grid/unit_tests/test_geometry.py +++ b/model/common/tests/common/grid/unit_tests/test_geometry.py @@ -10,17 +10,17 @@ import functools from typing import TYPE_CHECKING +import gt4py.next as gtx import numpy as np import pytest -from icon4py.model.common import dimension as dims +from icon4py.model.common import dimension as dims, model_backends from icon4py.model.common.grid import ( geometry, geometry_attributes as attrs, horizontal as h_grid, simple, ) -from icon4py.model.common.grid.geometry import as_sparse_field from icon4py.model.common.math import helpers as math_helpers from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import definitions, grid_utils, test_utils @@ -49,6 +49,37 @@ def test_geometry_raises_for_unknown_field(backend: gtx_typing.Backend) -> None: assert "'GridGeometry'" in e.value # type: ignore[operator] +@pytest.mark.parametrize("dim", (dims.CellDim,)) +def test_coordinates( + experiment: definitions.Experiment, + backend: model_backends.BackendLike, + grid_savepoint: sb.IconGridSavepoint, + dim: gtx.Dimension, +) -> None: + allocator = model_backends.get_allocator(backend) + geometry = grid_utils.get_grid_geometry(allocator, experiment) + cell_lat_ref = grid_savepoint.lat(dims.CellDim) + cell_lon_ref = grid_savepoint.lon(dims.CellDim) + edge_lat_ref = grid_savepoint.lat(dims.EdgeDim) + edge_lon_ref = grid_savepoint.lon(dims.EdgeDim) + vertex_lat_ref = grid_savepoint.lat(dims.VertexDim) + vertex_lon_ref = grid_savepoint.lon(dims.VertexDim) + + cell_lon = geometry.get(attrs.CELL_LON) + cell_lat = geometry.get(attrs.CELL_LAT) + edge_lon = geometry.get(attrs.EDGE_LON) + edge_lat = geometry.get(attrs.EDGE_LAT) + vertex_lat = geometry.get(attrs.VERTEX_LAT) + vertex_lon = geometry.get(attrs.VERTEX_LON) + test_utils.dallclose(edge_lat_ref.asnumpy(), edge_lat.asnumpy()) + test_utils.dallclose(edge_lon_ref.asnumpy(), edge_lon.asnumpy()) + + test_utils.dallclose(vertex_lat_ref.asnumpy(), vertex_lat.asnumpy()) + test_utils.dallclose(vertex_lon_ref.asnumpy(), vertex_lon.asnumpy()) + test_utils.dallclose(cell_lon.asnumpy(), cell_lon_ref.asnumpy()) + test_utils.dallclose(cell_lat.asnumpy(), cell_lat_ref.asnumpy()) + + @pytest.mark.parametrize( "experiment, rtol", [ @@ -180,7 +211,7 @@ def test_compute_coordinates_of_edge_tangent_and_normal( assert test_utils.dallclose(x_tangent.asnumpy(), x_tangent_ref.asnumpy(), atol=1e-12) assert test_utils.dallclose(y_tangent.asnumpy(), y_tangent_ref.asnumpy(), atol=1e-12) assert test_utils.dallclose(z_tangent.asnumpy(), z_tangent_ref.asnumpy(), atol=1e-12) - assert test_utils.dallclose(x_normal.asnumpy(), x_normal_ref.asnumpy(), atol=1e-13) # 1e-16 + assert test_utils.dallclose(x_normal.asnumpy(), x_normal_ref.asnumpy(), atol=1e-13) assert test_utils.dallclose(z_normal.asnumpy(), z_normal_ref.asnumpy(), atol=1e-13) assert test_utils.dallclose(y_normal.asnumpy(), y_normal_ref.asnumpy(), atol=1e-12) @@ -360,8 +391,8 @@ def test_sparse_fields_creator() -> None: g1 = data_alloc.random_field(grid, dims.EdgeDim) g2 = data_alloc.random_field(grid, dims.EdgeDim) - sparse = as_sparse_field((dims.EdgeDim, dims.E2CDim), [(f1, f2), (g1, g2)]) - sparse_e2c = functools.partial(as_sparse_field, (dims.EdgeDim, dims.E2CDim)) + sparse = geometry.as_sparse_field((dims.EdgeDim, dims.E2CDim), [(f1, f2), (g1, g2)]) + sparse_e2c = functools.partial(geometry.as_sparse_field, (dims.EdgeDim, dims.E2CDim)) sparse2 = sparse_e2c(((f1, f2), (g1, g2))) assert sparse[0].asnumpy().shape == (grid.num_edges, 2) assert test_utils.dallclose(sparse[0].asnumpy(), sparse2[0].asnumpy()) diff --git a/model/testing/src/icon4py/model/testing/grid_utils.py b/model/testing/src/icon4py/model/testing/grid_utils.py index 0b8df3daa2..1816a79d13 100644 --- a/model/testing/src/icon4py/model/testing/grid_utils.py +++ b/model/testing/src/icon4py/model/testing/grid_utils.py @@ -110,9 +110,8 @@ def _construct_grid_geometry() -> geometry.GridGeometry: keep_skip_values=True, allocator=model_backends.get_allocator(backend), ) - grid = gm.grid geometry_source = geometry.GridGeometry( - grid, + gm.grid, gm.decomposition_info, backend, gm.coordinates, From 0a79fdfc90767a1a4298fb66be6adc99b5382f35 Mon Sep 17 00:00:00 2001 From: halungge Date: Tue, 16 Dec 2025 22:17:18 +0100 Subject: [PATCH 130/240] add pymetis to standard common dependencies, fix old gt4py version in standalone driver --- model/common/pyproject.toml | 3 +- .../decomposition/two_ranks_distribution.py | 31 ------------------- model/standalone_driver/pyproject.toml | 2 +- uv.lock | 20 ++++++------ 4 files changed, 13 insertions(+), 43 deletions(-) delete mode 100644 model/common/tests/common/decomposition/two_ranks_distribution.py diff --git a/model/common/pyproject.toml b/model/common/pyproject.toml index fe24af8a75..fe16a502c1 100644 --- a/model/common/pyproject.toml +++ b/model/common/pyproject.toml @@ -42,7 +42,7 @@ version = "0.0.6" all = ["icon4py-common[distributed,io]"] cuda11 = ['cupy-cuda11x>=13.0', 'gt4py[cuda11]'] cuda12 = ['cupy-cuda12x>=13.0', 'gt4py[cuda12]'] -distributed = ["ghex>=0.5.0", "mpi4py>=3.1.5", "pymetis>2022.1"] +distributed = ["ghex>=0.5.0", "mpi4py>=3.1.5"] io = [ # external dependencies "cartopy>=0.22.0", @@ -52,6 +52,7 @@ io = [ "netcdf4>=1.6.1", "numpy>=1.23.3", "scikit-learn>=1.4.0", + "pymetis>2022.1", # TODO(halungge): there are failing tests starting from uxarray==2024.4.0: when a data file does not have # fields of a given dimension (eg 'edge') then something in uxarray goes wrong with the dimension # mapping. It is not yet clear whether this is a uxarray bug or on our side. diff --git a/model/common/tests/common/decomposition/two_ranks_distribution.py b/model/common/tests/common/decomposition/two_ranks_distribution.py deleted file mode 100644 index 44097f6e88..0000000000 --- a/model/common/tests/common/decomposition/two_ranks_distribution.py +++ /dev/null @@ -1,31 +0,0 @@ -# ICON4Py - ICON inspired code in Python and GT4Py -# -# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss -# All rights reserved. -# -# Please, refer to the LICENSE file in the root directory. -# SPDX-License-Identifier: BSD-3-Clause -from typing import Final - -import numpy as np - - -TWO_RANKS_DISTRIBUTION: np.ndarray = np.ones(10) -TWO_RANKS_DISTRIBUTION[5, 6, 10] = 0 - - -# TODO (@halungge): define all the rest or delete -CELL_OWN: Final[dict[int, list[int]]] = { - 0: [6, 7, 10], - 1: [0, 1, 2, 3, 4, 5, 8, 9, 11, 12, 13, 14, 15, 16, 17], -} -EDGE_OWN: Final[dict[int, list[int]]] = { - 0: [13, 14], - 1: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26], -} -VERTEX_OWN: Final[dict[int, list[int]]] = {0: [], 1: [0, 1, 2, 3, 4, 5, 6, 7, 8]} - -CELL_FIRST_HALO_LINE = {0: [3, 4, 11, 13, 9], 1: [6, 7, 14]} -CELL_SECOND_HALO_LINE = {0: [0, 1, 5, 8, 14, 17, 16], 1: []} -EDGE_FIRST_HALO_LINE = {0: [9, 12, 17, 21, 10], 1: []} -EDGE_SECOND_HALO_LINE = {0: [1, 2, 5, 4, 15, 16, 24, 25, 26, 22, 23, 18, 11], 1: [14, 13]} diff --git a/model/standalone_driver/pyproject.toml b/model/standalone_driver/pyproject.toml index c8a9130a59..137913dedb 100644 --- a/model/standalone_driver/pyproject.toml +++ b/model/standalone_driver/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ # external dependencies "typer>=0.20.0", "devtools>=0.12", - "gt4py==1.1.0", + "gt4py==1.1.2", "packaging>=20.0", "numpy>=1.23.3" ] diff --git a/uv.lock b/uv.lock index 9ebed37c16..d0fdc7534e 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version < '3.11'", @@ -175,8 +175,8 @@ dependencies = [ { name = "pathspec" }, { name = "platformdirs" }, { name = "pytokens" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "tomli", marker = "python_full_version < '3.11' or (extra == 'extra-7-icon4py-cuda11' and extra == 'extra-7-icon4py-cuda12')" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-7-icon4py-cuda11' and extra == 'extra-7-icon4py-cuda12')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/8c/ad/33adf4708633d047950ff2dfdea2e215d84ac50ef95aff14a614e4b6e9b2/black-25.11.0.tar.gz", hash = "sha256:9a323ac32f5dc75ce7470501b887250be5005a01602e931a15e45593f70f6e08", size = 655669, upload-time = "2025-11-10T01:53:50.558Z" } wheels = [ @@ -568,7 +568,7 @@ name = "click" version = "8.1.7" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-7-icon4py-cuda11' and extra == 'extra-7-icon4py-cuda12')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de", size = 336121, upload-time = "2023-08-17T17:29:11.868Z" } wheels = [ @@ -923,7 +923,7 @@ dependencies = [ { name = "numpy" }, { name = "packaging" }, { name = "ply" }, - { name = "pyreadline", marker = "sys_platform == 'win32'" }, + { name = "pyreadline", marker = "sys_platform == 'win32' or (extra == 'extra-7-icon4py-cuda11' and extra == 'extra-7-icon4py-cuda12')" }, { name = "pyyaml" }, { name = "sympy" }, ] @@ -1868,7 +1868,6 @@ cuda12 = [ distributed = [ { name = "ghex" }, { name = "mpi4py" }, - { name = "pymetis" }, ] io = [ { name = "cartopy" }, @@ -1877,6 +1876,7 @@ io = [ { name = "holoviews" }, { name = "netcdf4" }, { name = "numpy" }, + { name = "pymetis" }, { name = "scikit-learn" }, { name = "uxarray" }, { name = "xarray", extra = ["complete"] }, @@ -1901,7 +1901,7 @@ requires-dist = [ { name = "numpy", specifier = ">=1.23.3" }, { name = "numpy", marker = "extra == 'io'", specifier = ">=1.23.3" }, { name = "packaging", specifier = ">=20.0" }, - { name = "pymetis", marker = "extra == 'distributed'", specifier = ">2022.1" }, + { name = "pymetis", marker = "extra == 'io'", specifier = ">2022.1" }, { name = "scikit-learn", marker = "extra == 'io'", specifier = ">=1.4.0" }, { name = "scipy", specifier = ">=1.14.1" }, { name = "typing-extensions", specifier = ">=4.11.0" }, @@ -1958,7 +1958,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "devtools", specifier = ">=0.12" }, - { name = "gt4py", specifier = "==1.1.0" }, + { name = "gt4py", specifier = "==1.1.2" }, { name = "icon4py-atmosphere-diffusion", editable = "model/atmosphere/diffusion" }, { name = "icon4py-atmosphere-dycore", editable = "model/atmosphere/dycore" }, { name = "icon4py-common", editable = "model/common" }, @@ -4023,7 +4023,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "packaging" }, { name = "setuptools" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "tomli", marker = "python_full_version < '3.11' or (extra == 'extra-7-icon4py-cuda11' and extra == 'extra-7-icon4py-cuda12')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4f/a4/00a9ac1b555294710d4a68d2ce8dfdf39d72aa4d769a7395d05218d88a42/setuptools_scm-8.1.0.tar.gz", hash = "sha256:42dea1b65771cba93b7a515d65a65d8246e560768a66b9106a592c8e7f26c8a7", size = 76465, upload-time = "2024-05-06T15:07:56.934Z" } wheels = [ @@ -4648,7 +4648,7 @@ version = "3.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "packaging" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "tomli", marker = "python_full_version < '3.11' or (extra == 'extra-7-icon4py-cuda11' and extra == 'extra-7-icon4py-cuda12')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5c/9b/941647e9e3616b5da7bbc4601ed9920f44a886704100fa8151406c07c149/versioningit-3.1.2.tar.gz", hash = "sha256:4db83ed99f56b07d83940bee3445ca46ca120d13b6b304cdb5fb44e5aa4edec0", size = 213047, upload-time = "2024-07-20T12:41:07.927Z" } wheels = [ From 1fb0c2518220f984dfad8f3f947b8378ca8986e9 Mon Sep 17 00:00:00 2001 From: halungge Date: Tue, 16 Dec 2025 22:29:37 +0100 Subject: [PATCH 131/240] mypy --- model/common/src/icon4py/model/common/decomposition/halo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 13505d6713..58e5d29a9e 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -470,7 +470,7 @@ def __call__( Returns: np.ndarray: array with partition label (int, rank number) for each cell """ - import pymetis # type: ignore [import-not-found] + import pymetis # type: ignore [import-untyped] _, partition_index = pymetis.part_graph(nparts=num_partitions, adjacency=adjacency_matrix) return np.array(partition_index) From 305d7a034acc3784faa115742235286b4f39dd38 Mon Sep 17 00:00:00 2001 From: halungge Date: Tue, 16 Dec 2025 22:38:57 +0100 Subject: [PATCH 132/240] add missing datatest annotation --- .../common/grid/unit_tests/test_geometry.py | 31 ------------------- .../grid/unit_tests/test_grid_manager.py | 1 + 2 files changed, 1 insertion(+), 31 deletions(-) diff --git a/model/common/tests/common/grid/unit_tests/test_geometry.py b/model/common/tests/common/grid/unit_tests/test_geometry.py index e218ec342e..cecd5c7d38 100644 --- a/model/common/tests/common/grid/unit_tests/test_geometry.py +++ b/model/common/tests/common/grid/unit_tests/test_geometry.py @@ -49,37 +49,6 @@ def test_geometry_raises_for_unknown_field(backend: gtx_typing.Backend) -> None: assert "'GridGeometry'" in e.value # type: ignore[operator] -@pytest.mark.parametrize("dim", (dims.CellDim,)) -def test_coordinates( - experiment: definitions.Experiment, - backend: model_backends.BackendLike, - grid_savepoint: sb.IconGridSavepoint, - dim: gtx.Dimension, -) -> None: - allocator = model_backends.get_allocator(backend) - geometry = grid_utils.get_grid_geometry(allocator, experiment) - cell_lat_ref = grid_savepoint.lat(dims.CellDim) - cell_lon_ref = grid_savepoint.lon(dims.CellDim) - edge_lat_ref = grid_savepoint.lat(dims.EdgeDim) - edge_lon_ref = grid_savepoint.lon(dims.EdgeDim) - vertex_lat_ref = grid_savepoint.lat(dims.VertexDim) - vertex_lon_ref = grid_savepoint.lon(dims.VertexDim) - - cell_lon = geometry.get(attrs.CELL_LON) - cell_lat = geometry.get(attrs.CELL_LAT) - edge_lon = geometry.get(attrs.EDGE_LON) - edge_lat = geometry.get(attrs.EDGE_LAT) - vertex_lat = geometry.get(attrs.VERTEX_LAT) - vertex_lon = geometry.get(attrs.VERTEX_LON) - test_utils.dallclose(edge_lat_ref.asnumpy(), edge_lat.asnumpy()) - test_utils.dallclose(edge_lon_ref.asnumpy(), edge_lon.asnumpy()) - - test_utils.dallclose(vertex_lat_ref.asnumpy(), vertex_lat.asnumpy()) - test_utils.dallclose(vertex_lon_ref.asnumpy(), vertex_lon.asnumpy()) - test_utils.dallclose(cell_lon.asnumpy(), cell_lon_ref.asnumpy()) - test_utils.dallclose(cell_lat.asnumpy(), cell_lat_ref.asnumpy()) - - @pytest.mark.parametrize( "experiment, rtol", [ diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index f0fac7dc80..6789272bda 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -556,6 +556,7 @@ def test_limited_area_on_grid(grid_descriptor: definitions.GridDescription, expe assert expected == grid.limited_area +@pytest.mark.datatest @pytest.mark.parametrize("dim", utils.horizontal_dims()) def test_decomposition_info_single_node( dim: gtx.Dimension, From aa95ce8188272c3471ed77315362753b3fc38b14 Mon Sep 17 00:00:00 2001 From: halungge Date: Thu, 1 Jan 2026 16:31:22 +0100 Subject: [PATCH 133/240] rename test functions --- .../tests/common/grid/mpi_tests/test_parallel_icon.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py index 571abfdd7a..607bf29bbb 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py @@ -76,7 +76,7 @@ def test_props(processor_props: decomposition.ProcessProperties) -> None: ], ) @pytest.mark.parametrize("dim", utils.main_horizontal_dims()) -def test_distributed_local( +def test_start_index_end_index_local_zone_on_distributed_lam_grid( processor_props: decomposition.ProcessProperties, dim: gtx.Dimension, icon_grid: base_grid.Grid, @@ -147,7 +147,7 @@ def test_distributed_local( ], ) @pytest.mark.parametrize("zone, level", [(h_grid.Zone.HALO, 1), (h_grid.Zone.HALO_LEVEL_2, 2)]) -def test_distributed_halo( +def test_start_index_end_index_halo_zones_on_distributed_lam_grid( processor_props: decomposition.ProcessProperties, dim: gtx.Dimension, zone: h_grid.Zone, @@ -167,3 +167,5 @@ def test_distributed_halo( assert start_index == expected, f"expected start index {expected}, but was {start_index}" expected = HALO_IDX[processor_props.comm_size][dim][rank][level] assert end_index == expected, f"expected start index {1}, but was {start_index}" + + From d1234ede2acd6c72136b41d58fcc78b7953c927c Mon Sep 17 00:00:00 2001 From: halungge Date: Thu, 1 Jan 2026 19:04:32 +0100 Subject: [PATCH 134/240] fix halo construction: additional vertices for second level edges --- .../model/common/decomposition/halo.py | 41 ++++----- .../mpi_tests/test_parallel_halo.py | 6 +- .../tests/common/decomposition/utils.py | 6 +- .../mpi_tests/test_parallel_grid_manager.py | 85 +++++++------------ .../grid/mpi_tests/test_parallel_icon.py | 30 ++++++- .../tests/common/grid/mpi_tests/utils.py | 43 ++++++++++ 6 files changed, 129 insertions(+), 82 deletions(-) create mode 100644 model/common/tests/common/grid/mpi_tests/utils.py diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 58e5d29a9e..5859292b33 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -286,14 +286,14 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: Vertices: - - 1. HALO LEVEL: are vertices on the cutting line that are not owned, or put in a different wording: all vertices on owned cells that ar not + - 1. HALO LEVEL: are vertices on the cutting line that are not owned, or in a different wording: all vertices on owned cells that ar not owned. In ICON every element in an array needs **exactly one owner**. For elements on the cutting line (vertices and edges) there is no clear indication which rank should own it, ICON uses the rank with the higher rank (see (_update_owner_mask_by_max_rank_convention)) In the example above (v0, v1, v2, v3) are in the 1. HALO LEVEL or rank 0 and owend by rank 1. Consequently there ranks that have no 1.HALO LEVEL cells. - - 2. HALO LEVEL: are vertices that are on halo 1. HALO LEVEL cells, but not on owned. For rank 0 these are (v4, v5, v6, v7) + - 2. HALO LEVEL: are vertices that are on HALO LEVEL cells, but not on owned. For rank 0 these are (v4, v5, v6, v7) Edges: @@ -311,7 +311,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: # TODO(halungge): make number of halo lines (in terms of cells) a parameter: icon does hard coding of 2 halo lines for cells, make this dynamic! """ - + decomp_info = defs.DecompositionInfo() self._validate_mapping(face_to_rank) #: cells @@ -334,21 +334,17 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: cell_halo_levels[self._xp.isin(all_cells, second_halo_cells)] = ( defs.DecompositionFlag.SECOND_HALO_LEVEL ) - decomp_info = defs.DecompositionInfo().set_dimension( - dims.CellDim, all_cells, cell_owner_mask, cell_halo_levels - ) + decomp_info.set_dimension(dims.CellDim, all_cells, cell_owner_mask, cell_halo_levels) #: vertices vertex_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) - vertex_on_first_halo_line = self.find_vertex_neighbors_for_cells(first_halo_cells) + vertex_on_halo_cells = self.find_vertex_neighbors_for_cells(total_halo_cells) - vertex_on_cutting_line = self._xp.intersect1d( - vertex_on_owned_cells, vertex_on_first_halo_line - ) + vertex_on_cutting_line = self._xp.intersect1d(vertex_on_owned_cells, vertex_on_halo_cells) # create decomposition_info for vertices all_vertices = self._xp.unique( - self._xp.hstack((vertex_on_owned_cells, vertex_on_first_halo_line)) + self._xp.hstack((vertex_on_owned_cells, vertex_on_halo_cells)) ) vertex_owner_mask = self._xp.isin(all_vertices, vertex_on_owned_cells) vertex_owner_mask = self._update_owner_mask_by_max_rank_convention( # icon specific @@ -358,17 +354,16 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: vertex_on_cutting_line, self.node_face_connectivity, ) - vertex_second_level = self._xp.setdiff1d(vertex_on_first_halo_line, vertex_on_owned_cells) + vertex_second_level = self._xp.setdiff1d(vertex_on_halo_cells, vertex_on_owned_cells) vertex_halo_levels = defs.DecompositionFlag.UNDEFINED * self._xp.ones( all_vertices.size, dtype=int ) vertex_halo_levels[vertex_owner_mask] = defs.DecompositionFlag.OWNED vertex_halo_levels[ - self._xp.logical_and( - self._xp.logical_not(vertex_owner_mask), - self._xp.isin(all_vertices, vertex_on_cutting_line), - ) + self._xp.logical_not(vertex_owner_mask) + & self._xp.isin(all_vertices, vertex_on_cutting_line) ] = defs.DecompositionFlag.FIRST_HALO_LEVEL + vertex_halo_levels[self._xp.isin(all_vertices, vertex_second_level)] = ( defs.DecompositionFlag.SECOND_HALO_LEVEL ) @@ -379,14 +374,15 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: # edges edges_on_owned_cells = self.find_edge_neighbors_for_cells(owned_cells) edges_on_first_halo_line = self.find_edge_neighbors_for_cells(first_halo_cells) + edges_on_second_halo_line = self.find_edge_neighbors_for_cells(second_halo_cells) + edges_on_cutting_line = self._xp.intersect1d(edges_on_owned_cells, edges_on_first_halo_line) - edges_on_halo_cells = self.find_edge_neighbors_for_cells(total_halo_cells) + # needs to be defined as vertex neighbor due to "corners" in the cut. edge_second_level = self._xp.setdiff1d( self.find_edge_neighbors_for_vertices(vertex_on_cutting_line), edges_on_owned_cells ) - edge_second_and_third_level = self._xp.setdiff1d(edges_on_halo_cells, edges_on_cutting_line) - edge_third_level = self._xp.setdiff1d(edge_second_and_third_level, edge_second_level) + edge_third_level = self._xp.setdiff1d(edges_on_second_halo_line, edge_second_level) all_edges = self._xp.unique( self._xp.hstack((edges_on_owned_cells, edge_second_level, edge_third_level)) @@ -409,23 +405,20 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: # LEVEL_ONE edges are on an owned cell but are not owned: these are all edges on the cutting line that are not owned (by the convention) edge_halo_levels[ - self._xp.logical_and( - self._xp.logical_not(edge_owner_mask), - self._xp.isin(all_edges, edges_on_cutting_line), - ) + self._xp.logical_not(edge_owner_mask) & self._xp.isin(all_edges, edges_on_cutting_line) ] = defs.DecompositionFlag.FIRST_HALO_LEVEL # LEVEL_TWO edges share exactly one vertex with an owned cell, they are on the first halo-line cells, but not on the cutting line edge_halo_levels[self._xp.isin(all_edges, edge_second_level)] = ( defs.DecompositionFlag.SECOND_HALO_LEVEL ) - decomp_info.set_dimension(dims.EdgeDim, all_edges, edge_owner_mask, edge_halo_levels) # LEVEL_THREE edges # LEVEL_TWO edges share exactly one vertex with an owned cell, they are on the first halo-line cells, but not on the cutting line edge_halo_levels[self._xp.isin(all_edges, edge_third_level)] = ( defs.DecompositionFlag.THIRD_HALO_LEVEL ) + decomp_info.set_dimension(dims.EdgeDim, all_edges, edge_owner_mask, edge_halo_levels) return decomp_info diff --git a/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py index 636e0a6f3f..16e62d76ad 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py @@ -11,7 +11,7 @@ import pytest import icon4py.model.common.dimension as dims -from icon4py.model.common.decomposition import definitions as defs +from icon4py.model.common.decomposition import definitions as decomposition_defs from icon4py.model.testing import parallel_helpers from icon4py.model.testing.fixtures import processor_props @@ -59,7 +59,9 @@ def test_element_ownership_is_unique( ) decomposition_info = halo_generator(utils.SIMPLE_DISTRIBUTION) - owned = decomposition_info.global_index(dim, defs.DecompositionInfo.EntryType.OWNED) + owned = decomposition_info.global_index( + dim, decomposition_defs.DecompositionInfo.EntryType.OWNED + ) print(f"\nrank {processor_props.rank} owns {dim} : {owned} ") # assert that each cell is only owned by one rank comm = processor_props.comm diff --git a/model/common/tests/common/decomposition/utils.py b/model/common/tests/common/decomposition/utils.py index bfff327e6e..65f66fec62 100644 --- a/model/common/tests/common/decomposition/utils.py +++ b/model/common/tests/common/decomposition/utils.py @@ -160,7 +160,11 @@ def assert_same_entries( dim: gtx.Dimension, my_owned: np.ndarray, reference: dict[gtx.Dimension, dict], rank: int ) -> None: - assert my_owned.size == len(reference[dim][rank]) + print(f"myowned {my_owned}") + print(reference[dim][rank]) + assert ( + my_owned.size == len(reference[dim][rank]) + ), f"{dim}(rank = {rank}) : wrong size expected {len(reference[dim][rank])} but was {my_owned.size}" assert np.setdiff1d(my_owned, reference[dim][rank], assume_unique=True).size == 0 diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 55b150831a..76a7905b3f 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -8,7 +8,6 @@ import functools import logging import operator -import pathlib from typing import Any import numpy as np @@ -36,10 +35,9 @@ from icon4py.model.testing import definitions as test_defs, grid_utils from ..fixtures import backend, processor_props +from . import utils -NUM_LEVELS = 10 - try: import mpi4py @@ -50,39 +48,11 @@ log = logging.getLogger(__file__) -def run_gridmananger_for_multinode( - file: pathlib.Path, - run_properties: decomp_defs.ProcessProperties, - decomposer: halo.Decomposer, -) -> gm.GridManager: - manager = _grid_manager(file, num_levels=NUM_LEVELS) - manager( - keep_skip_values=True, allocator=None, run_properties=run_properties, decomposer=decomposer - ) - return manager - - -def _grid_manager(file: pathlib.Path, num_levels: int) -> gm.GridManager: - manager = gm.GridManager(str(file), num_levels=num_levels) - return manager - - -def run_grid_manager_for_singlenode(file: pathlib.Path) -> gm.GridManager: - manager = _grid_manager(file, NUM_LEVELS) - manager( - keep_skip_values=True, - run_properties=decomp_defs.SingleNodeProcessProperties(), - decomposer=halo.SingleNodeDecomposer(), - allocator=None, - ) - return manager - - @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi(min_size=2) def test_grid_manager_validate_decomposer(processor_props: decomp_defs.ProcessProperties) -> None: file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) - manager = gm.GridManager(file, NUM_LEVELS, gridfile.ToZeroBasedIndexTransformation()) + manager = gm.GridManager(file, utils.NUM_LEVELS, gridfile.ToZeroBasedIndexTransformation()) with pytest.raises(exceptions.InvalidConfigError) as e: manager( keep_skip_values=True, @@ -110,13 +80,13 @@ def test_fields_distribute_and_gather( caplog.set_level(logging.INFO) print(f"myrank - {processor_props.rank}: running with processor_props = {processor_props}") file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) - single_node = run_grid_manager_for_singlenode(file) + single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid global_cell_area = single_node.geometry_fields[gridfile.GeometryName.CELL_AREA] global_edge_lat = single_node.coordinates[dims.EdgeDim]["lat"] global_vertex_lon = single_node.coordinates[dims.VertexDim]["lon"] - multinode = run_gridmananger_for_multinode( + multinode = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), @@ -212,6 +182,14 @@ def assert_gathered_field_against_global( decomposition_info.global_index(dim, decomp_defs.DecompositionInfo.EntryType.OWNED), processor_props.comm, ) + if np.any( + decomposition_info.global_index( + dims.VertexDim, decomp_defs.DecompositionInfo.EntryType.OWNED + ) + == 3855 + ): + print(f"owning rank is {processor_props.rank}") + if processor_props.rank == 0: print(f"rank = {processor_props.rank}: asserting gathered fields: ") assert np.all( @@ -228,6 +206,8 @@ def assert_gathered_field_against_global( print( f" global reference field {global_reference_field.shape} gathered = {gathered_field.shape}" ) + print(f"{ np.where(np.abs(sorted_-global_reference_field) > 1e-12)}") + np.testing.assert_allclose(sorted_, global_reference_field, rtol=1e-12, verbose=True) @@ -241,7 +221,7 @@ def test_halo_neighbor_access_c2e( ) -> None: file = grid_utils.resolve_full_grid_file_name(grid) print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") - single_node = run_grid_manager_for_singlenode(file) + single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( backend=backend, @@ -272,7 +252,7 @@ def test_halo_neighbor_access_c2e( print( f"rank = {processor_props.rank} : single node computed field reference has size {reference.asnumpy().shape}" ) - multinode_grid_manager = run_gridmananger_for_multinode( + multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), @@ -332,7 +312,7 @@ def test_halo_access_e2c2v( ) -> None: file = grid_utils.resolve_full_grid_file_name(grid) print(f"running on {processor_props.comm}") - single_node = run_grid_manager_for_singlenode(file) + single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( backend=backend, @@ -347,7 +327,7 @@ def test_halo_access_e2c2v( ) reference_u = single_node_geometry.get(geometry_attributes.EDGE_NORMAL_VERTEX_U).asnumpy() reference_v = single_node_geometry.get(geometry_attributes.EDGE_NORMAL_VERTEX_V).asnumpy() - multinode_grid_manager = run_gridmananger_for_multinode( + multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), @@ -355,7 +335,6 @@ def test_halo_access_e2c2v( distributed_grid = multinode_grid_manager.grid extra_geometry_fields = multinode_grid_manager.geometry_fields decomposition_info = multinode_grid_manager.decomposition_info - print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( f"rank = {processor_props.rank}: halo size for 'EdgeDim' " @@ -424,7 +403,7 @@ def test_halo_access_e2c( ) -> None: file = grid_utils.resolve_full_grid_file_name(grid) print(f"running on {processor_props.comm}") - single_node = run_grid_manager_for_singlenode(file) + single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( backend=backend, @@ -439,7 +418,7 @@ def test_halo_access_e2c( ) reference_u = single_node_geometry.get(geometry_attributes.EDGE_NORMAL_CELL_U).asnumpy() reference_v = single_node_geometry.get(geometry_attributes.EDGE_NORMAL_CELL_V).asnumpy() - multinode_grid_manager = run_gridmananger_for_multinode( + multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), @@ -514,7 +493,7 @@ def test_halo_neighbor_access_e2v( ) -> None: print(f"running on {processor_props.comm}") file = grid_utils.resolve_full_grid_file_name(grid) - single_node = run_grid_manager_for_singlenode(file) + single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( backend=backend, @@ -529,7 +508,7 @@ def test_halo_neighbor_access_e2v( print( f"rank = {processor_props.rank} : single node computed field reference has size {reference_tangent_x.shape}" ) - multinode_grid_manager = run_gridmananger_for_multinode( + multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), @@ -592,7 +571,7 @@ def test_halo_neighbor_access_v2e( ) -> None: file = grid_utils.resolve_full_grid_file_name(grid) print(f"running on {processor_props.comm}") - single_node = run_grid_manager_for_singlenode(file) + single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( backend=backend, @@ -631,7 +610,7 @@ def test_halo_neighbor_access_v2e( print( f"rank = {processor_props.rank} : single node computed field reference has size {reference.asnumpy().shape}" ) - multinode_grid_manager = run_gridmananger_for_multinode( + multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), @@ -694,7 +673,7 @@ def test_halo_neighbor_access_c2e2c( xp = data_alloc.import_array_ns(allocator=backend) start_zone = h_grid.cell_domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2) print(f"running on {processor_props.comm}") - single_node = run_grid_manager_for_singlenode(file) + single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( backend=backend, @@ -716,7 +695,7 @@ def test_halo_neighbor_access_c2e2c( print( f"rank = {processor_props.rank} : single node computed field reference has size {reference.shape}" ) - multinode_grid_manager = run_gridmananger_for_multinode( + multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), @@ -772,11 +751,13 @@ def test_halo_neighbor_access_v2c( ) -> None: file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) print(f"running on {processor_props.comm}") - single_node = run_grid_manager_for_singlenode(file) + single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid data = np.repeat( - single_node.coordinates[dims.CellDim]["lat"].asnumpy()[:, None], repeats=NUM_LEVELS, axis=1 + single_node.coordinates[dims.CellDim]["lat"].asnumpy()[:, None], + repeats=utils.NUM_LEVELS, + axis=1, ) full_cell_k_field = gtx.as_field( (dims.CellDim, dims.KDim), @@ -812,7 +793,7 @@ def test_halo_neighbor_access_v2c( print( f"rank = {processor_props.rank}/ {processor_props.comm_size} : single node computed field reference has size {reference.asnumpy().shape}" ) - multinode_grid_manager = run_gridmananger_for_multinode( + multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), @@ -834,7 +815,7 @@ def test_halo_neighbor_access_v2c( my_global_cells = decomposition_info.global_index(dims.CellDim) cell_k_buffer = ( full_cell_k_field.asnumpy()[my_global_cells, :] - .ravel(order="K") + # .ravel(order="K") .reshape(distributed_grid.num_cells, 10) ) assert_gathered_field_against_global( @@ -898,7 +879,7 @@ def test_validate_skip_values_in_distributed_connectivities( processor_props: decomp_defs.ProcessProperties, grid: test_defs.GridDescription ) -> None: file = grid_utils.resolve_full_grid_file_name(grid) - multinode_grid_manager = run_gridmananger_for_multinode( + multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, decomposer=halo.SimpleMetisDecomposer(), diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py index 607bf29bbb..b21c66b710 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py @@ -9,13 +9,16 @@ from __future__ import annotations import gt4py.next as gtx +import numpy as np import pytest +from gt4py.next.common import is_neighbor_table import icon4py.model.common.dimension as dims -import icon4py.model.common.grid.horizontal as h_grid +import icon4py.model.common.grid.icon from icon4py.model.common.decomposition import definitions as decomposition -from icon4py.model.common.grid import base as base_grid -from icon4py.model.testing import definitions as test_defs, parallel_helpers +from icon4py.model.common.decomposition.halo import SimpleMetisDecomposer +from icon4py.model.common.grid import base as base_grid, gridfile, horizontal as h_grid, icon +from icon4py.model.testing import definitions as test_defs, grid_utils, parallel_helpers from ...fixtures import ( backend, @@ -27,6 +30,7 @@ ranked_data_path, ) from .. import utils +from . import utils as parallel_utils try: @@ -169,3 +173,23 @@ def test_start_index_end_index_halo_zones_on_distributed_lam_grid( assert end_index == expected, f"expected start index {1}, but was {start_index}" +@pytest.mark.parametrize("processor_props", [True], indirect=True) +@pytest.mark.mpi +@pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) +def test_skip_values_on_distributed_grid( + processor_props: decomposition.ProcessProperties, + grid: test_defs.GridDescription, +) -> None: + file = grid_utils.resolve_full_grid_file_name(grid) + grid_manager = parallel_utils.run_gridmananger_for_multinode( + file, processor_props, decomposer=SimpleMetisDecomposer() + ) + mesh = grid_manager.grid + assert not np.any(mesh.get_connectivity(dims.C2V).asnumpy() == gridfile.GridFile.INVALID_INDEX) + assert not np.any(mesh.get_connectivity(dims.E2V).asnumpy() == gridfile.GridFile.INVALID_INDEX) + assert not np.any(mesh.get_connectivity(dims.E2V).asnumpy() == gridfile.GridFile.INVALID_INDEX) + assert not np.any(mesh.get_connectivity(dims.C2E).asnumpy() == gridfile.GridFile.INVALID_INDEX) + assert np.any(mesh.get_connectivity(dims.E2C).asnumpy() == gridfile.GridFile.INVALID_INDEX) + assert np.any(mesh.get_connectivity(dims.C2E2C).asnumpy() == gridfile.GridFile.INVALID_INDEX) + assert np.any(mesh.get_connectivity(dims.V2E).asnumpy() == gridfile.GridFile.INVALID_INDEX) + assert np.any(mesh.get_connectivity(dims.V2C).asnumpy() == gridfile.GridFile.INVALID_INDEX) diff --git a/model/common/tests/common/grid/mpi_tests/utils.py b/model/common/tests/common/grid/mpi_tests/utils.py new file mode 100644 index 0000000000..a20ec444df --- /dev/null +++ b/model/common/tests/common/grid/mpi_tests/utils.py @@ -0,0 +1,43 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +import pathlib + +from icon4py.model.common.decomposition import definitions as decomp_defs, halo +from icon4py.model.common.grid import grid_manager as gm + + +def run_grid_manager_for_singlenode(file: pathlib.Path) -> gm.GridManager: + manager = _grid_manager(file, NUM_LEVELS) + manager( + keep_skip_values=True, + run_properties=decomp_defs.SingleNodeProcessProperties(), + decomposer=halo.SingleNodeDecomposer(), + allocator=None, + ) + return manager + + +def _grid_manager(file: pathlib.Path, num_levels: int) -> gm.GridManager: + manager = gm.GridManager(str(file), num_levels=num_levels) + return manager + + +def run_gridmananger_for_multinode( + file: pathlib.Path, + run_properties: decomp_defs.ProcessProperties, + decomposer: halo.Decomposer, +) -> gm.GridManager: + manager = _grid_manager(file, num_levels=NUM_LEVELS) + manager( + keep_skip_values=True, allocator=None, run_properties=run_properties, decomposer=decomposer + ) + return manager + + +NUM_LEVELS = 10 From 3aaef7db5713b6c791916febdb2d317414d76f40 Mon Sep 17 00:00:00 2001 From: halungge Date: Fri, 2 Jan 2026 14:59:17 +0100 Subject: [PATCH 135/240] fix: add "corner" cell --- .../model/common/decomposition/halo.py | 71 +++++++++---------- .../mpi_tests/test_parallel_grid_manager.py | 64 +++++++++-------- 2 files changed, 71 insertions(+), 64 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 5859292b33..60ce31fd33 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -196,6 +196,11 @@ def find_edge_neighbors_for_vertices( def find_vertex_neighbors_for_cells(self, cell_line: data_alloc.NDArray) -> data_alloc.NDArray: return self._find_neighbors(cell_line, connectivity=self.face_node_connectivity) + def find_cell_neighbors_for_vertices( + self, vertex_line: data_alloc.NDArray + ) -> data_alloc.NDArray: + return self._find_neighbors(vertex_line, connectivity=self.node_face_connectivity) + def owned_cells(self, face_to_rank: data_alloc.NDArray) -> data_alloc.NDArray: """Returns the full-grid indices of the cells owned by this rank""" owned_cells = face_to_rank == self._props.rank @@ -215,7 +220,7 @@ def _update_owner_mask_by_max_rank_convention( according to a remark in `mo_decomposition_tools.f90` ICON puts them to the node with the higher rank. - # TODO(halungge): can we add an assert for the target dimension of the connectivity being cells. + # TODO(halungge): can we add an assert for the target dimension of the connectivity being cells? Args: owner_mask: owner mask for the dimension all_indices: (global) indices of the dimension @@ -311,19 +316,42 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: # TODO(halungge): make number of halo lines (in terms of cells) a parameter: icon does hard coding of 2 halo lines for cells, make this dynamic! """ - decomp_info = defs.DecompositionInfo() + self._validate_mapping(face_to_rank) #: cells owned_cells = self.owned_cells(face_to_rank) # global indices of owned cells - first_halo_cells = self.next_halo_line(owned_cells) second_halo_cells = self.next_halo_line(first_halo_cells, owned_cells) - total_halo_cells = self._xp.hstack((first_halo_cells, second_halo_cells)) - all_cells = self._xp.hstack((owned_cells, total_halo_cells)) + total_halo_cells = self._xp.union1d(first_halo_cells, second_halo_cells) + + #: vertices + vertex_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) + vertex_on_halo_cells = self.find_vertex_neighbors_for_cells(total_halo_cells) + vertex_on_cutting_line = self._xp.intersect1d(vertex_on_owned_cells, vertex_on_halo_cells) + all_vertices = self._xp.union1d(vertex_on_owned_cells, vertex_on_halo_cells) + + #: update cells to include all cells of the "dual cell" (hexagon) for nodes on the cutting line + dual_cells = self.find_cell_neighbors_for_vertices(vertex_on_cutting_line) + total_halo_cells = self._xp.setdiff1d(dual_cells, owned_cells) + all_cells = self._xp.union1d(owned_cells, total_halo_cells) + + #: edges + edges_on_owned_cells = self.find_edge_neighbors_for_cells(owned_cells) + edges_on_any_halo_line = self.find_edge_neighbors_for_cells(total_halo_cells) + + edges_on_cutting_line = self._xp.intersect1d(edges_on_owned_cells, edges_on_any_halo_line) + + # needs to be defined as vertex neighbor due to "corners" in the cut. + edge_second_level = self._xp.setdiff1d( + self.find_edge_neighbors_for_vertices(vertex_on_cutting_line), edges_on_owned_cells + ) + edge_third_level = self._xp.setdiff1d(edges_on_any_halo_line, edge_second_level) + all_edges = self._xp.union1d(edges_on_owned_cells, edges_on_any_halo_line) + #: construct decomposition info + decomp_info = defs.DecompositionInfo() cell_owner_mask = self._xp.isin(all_cells, owned_cells) - # initialize cell halo levels cell_halo_levels = defs.DecompositionFlag.UNDEFINED * self._xp.ones( all_cells.size, dtype=int ) @@ -335,19 +363,8 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: defs.DecompositionFlag.SECOND_HALO_LEVEL ) decomp_info.set_dimension(dims.CellDim, all_cells, cell_owner_mask, cell_halo_levels) - - #: vertices - vertex_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) - vertex_on_halo_cells = self.find_vertex_neighbors_for_cells(total_halo_cells) - - vertex_on_cutting_line = self._xp.intersect1d(vertex_on_owned_cells, vertex_on_halo_cells) - - # create decomposition_info for vertices - all_vertices = self._xp.unique( - self._xp.hstack((vertex_on_owned_cells, vertex_on_halo_cells)) - ) vertex_owner_mask = self._xp.isin(all_vertices, vertex_on_owned_cells) - vertex_owner_mask = self._update_owner_mask_by_max_rank_convention( # icon specific + vertex_owner_mask = self._update_owner_mask_by_max_rank_convention( face_to_rank, vertex_owner_mask, all_vertices, @@ -371,24 +388,6 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: dims.VertexDim, all_vertices, vertex_owner_mask, vertex_halo_levels ) - # edges - edges_on_owned_cells = self.find_edge_neighbors_for_cells(owned_cells) - edges_on_first_halo_line = self.find_edge_neighbors_for_cells(first_halo_cells) - edges_on_second_halo_line = self.find_edge_neighbors_for_cells(second_halo_cells) - - edges_on_cutting_line = self._xp.intersect1d(edges_on_owned_cells, edges_on_first_halo_line) - - # needs to be defined as vertex neighbor due to "corners" in the cut. - edge_second_level = self._xp.setdiff1d( - self.find_edge_neighbors_for_vertices(vertex_on_cutting_line), edges_on_owned_cells - ) - edge_third_level = self._xp.setdiff1d(edges_on_second_halo_line, edge_second_level) - - all_edges = self._xp.unique( - self._xp.hstack((edges_on_owned_cells, edge_second_level, edge_third_level)) - ) - - # construct the owner mask edge_owner_mask = self._xp.isin(all_edges, edges_on_owned_cells) edge_owner_mask = self._update_owner_mask_by_max_rank_convention( face_to_rank, diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 76a7905b3f..03419da80e 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -182,14 +182,7 @@ def assert_gathered_field_against_global( decomposition_info.global_index(dim, decomp_defs.DecompositionInfo.EntryType.OWNED), processor_props.comm, ) - if np.any( - decomposition_info.global_index( - dims.VertexDim, decomp_defs.DecompositionInfo.EntryType.OWNED - ) - == 3855 - ): - print(f"owning rank is {processor_props.rank}") - + rtol = 1e-12 if processor_props.rank == 0: print(f"rank = {processor_props.rank}: asserting gathered fields: ") assert np.all( @@ -204,11 +197,15 @@ def assert_gathered_field_against_global( sorted_ = np.zeros(global_reference_field.shape, dtype=gtx.float64) # type: ignore [attr-defined] sorted_[gathered_global_indices] = gathered_field print( - f" global reference field {global_reference_field.shape} gathered = {gathered_field.shape}" + f" rank = {processor_props.rank}: SHAPES: global reference field {global_reference_field.shape}, gathered = {gathered_field.shape}" ) - print(f"{ np.where(np.abs(sorted_-global_reference_field) > 1e-12)}") - np.testing.assert_allclose(sorted_, global_reference_field, rtol=1e-12, verbose=True) + mismatch = np.where( + np.abs(sorted_ - global_reference_field) / np.abs(global_reference_field) > rtol + ) + print(f"rank = {processor_props.rank}: mismatch found in {mismatch}") + + np.testing.assert_allclose(sorted_, global_reference_field, rtol=rtol, verbose=True) @pytest.mark.mpi @@ -754,21 +751,21 @@ def test_halo_neighbor_access_v2c( single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid - data = np.repeat( + buffer = np.repeat( single_node.coordinates[dims.CellDim]["lat"].asnumpy()[:, None], repeats=utils.NUM_LEVELS, axis=1, ) full_cell_k_field = gtx.as_field( (dims.CellDim, dims.KDim), - data=data, # type: ignore [arg-type] + data=buffer, # type: ignore [arg-type] dtype=float, allocator=backend, ) print( f"rank = {processor_props.rank} / {processor_props.comm_size}: single node input field has size {full_cell_k_field.asnumpy().shape}" ) - buffer = single_node.coordinates[dims.VertexDim]["lat"].asnumpy() + buffer = single_node.coordinates[dims.VertexDim]["lat"].asnumpy() + 1.0 full_coef = gtx.as_field( (dims.VertexDim, dims.V2CDim), data=np.repeat((buffer / np.max(buffer))[:, None], 6, axis=1), # type: ignore [arg-type] @@ -813,52 +810,57 @@ def test_halo_neighbor_access_v2c( f"(2: {decomposition_info.get_halo_size(dims.VertexDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) my_global_cells = decomposition_info.global_index(dims.CellDim) - cell_k_buffer = ( + distributed_cell_k_buffer = ( full_cell_k_field.asnumpy()[my_global_cells, :] - # .ravel(order="K") - .reshape(distributed_grid.num_cells, 10) + .ravel(order="K") + .reshape(distributed_grid.num_cells, utils.NUM_LEVELS) ) + # validate the input data distribution assert_gathered_field_against_global( decomposition_info, processor_props, dims.CellDim, global_reference_field=full_cell_k_field.asnumpy(), - local_field=cell_k_buffer, + local_field=distributed_cell_k_buffer, ) print( - f"rank={processor_props.rank}/{processor_props.comm_size}: input field shape = ([{cell_k_buffer.shape})" + f"rank={processor_props.rank}/{processor_props.comm_size}: input field shape = ([{distributed_cell_k_buffer.shape})" ) - cell_k_field = gtx.as_field( + distributed_cell_k_field = gtx.as_field( (dims.CellDim, dims.KDim), - data=cell_k_buffer, # type: ignore [arg-type] - dtype=cell_k_buffer.dtype, + data=distributed_cell_k_buffer, # type: ignore [arg-type] + dtype=distributed_cell_k_buffer.dtype, allocator=backend, ) my_global_vertices = decomposition_info.global_index(dims.VertexDim) - coef = ( + distributed_coef_buffer = ( full_coef.asnumpy()[my_global_vertices, :] .ravel(order="K") .reshape((distributed_grid.num_vertices, 6)) ) + distributed_coef = gtx.as_field( + (dims.VertexDim, dims.V2CDim), + data=distributed_coef_buffer, # type: ignore [arg-type] + allocator=backend, + ) assert_gathered_field_against_global( decomposition_info, processor_props, dims.VertexDim, global_reference_field=full_coef.asnumpy(), - local_field=coef, + local_field=distributed_coef_buffer, ) print( - f"rank={processor_props.rank}/{processor_props.comm_size}: coefficient shape = ([{coef.shape})" + f"rank={processor_props.rank}/{processor_props.comm_size}: coefficient shape = ([{distributed_coef_buffer.shape})" ) - coef_field = gtx.as_field((dims.VertexDim, dims.V2CDim), data=coef, allocator=backend) # type: ignore [arg-type] output = data_alloc.zero_field(distributed_grid, dims.VertexDim, dims.KDim, allocator=backend) _compute_cell_2_vertex_interpolation( - cell_k_field, - coef_field, + distributed_cell_k_field, + distributed_coef, out=output, offset_provider={"V2C": distributed_grid.get_connectivity(dims.V2C)}, ) @@ -870,6 +872,12 @@ def test_halo_neighbor_access_v2c( global_reference_field=reference.asnumpy(), local_field=output.asnumpy(), ) + nn = np.where(single_node_grid.get_connectivity(dims.V2C).asnumpy() == 3855) + if processor_props.rank == 0: + print(nn) + print(single_node_grid.get_connectivity(dims.V2C).asnumpy()[nn[0]]) + + print(f"rank={processor_props.rank}/{processor_props.comm_size}: ") @pytest.mark.mpi From e5692f1612ffafb5308e8d49ee59f45a67a1be04 Mon Sep 17 00:00:00 2001 From: halungge Date: Fri, 2 Jan 2026 15:13:57 +0100 Subject: [PATCH 136/240] remove duplicate test --- .../model/common/decomposition/halo.py | 6 ++- .../unit_tests/test_definitions.py | 53 ++----------------- .../decomposition/unit_tests/test_halo.py | 29 +++++++++- 3 files changed, 35 insertions(+), 53 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 60ce31fd33..c17263c1df 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -322,8 +322,9 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: #: cells owned_cells = self.owned_cells(face_to_rank) # global indices of owned cells first_halo_cells = self.next_halo_line(owned_cells) - second_halo_cells = self.next_halo_line(first_halo_cells, owned_cells) - total_halo_cells = self._xp.union1d(first_halo_cells, second_halo_cells) + total_halo_cells = self._xp.union1d( + first_halo_cells, self.next_halo_line(first_halo_cells, owned_cells) + ) #: vertices vertex_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) @@ -334,6 +335,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: #: update cells to include all cells of the "dual cell" (hexagon) for nodes on the cutting line dual_cells = self.find_cell_neighbors_for_vertices(vertex_on_cutting_line) total_halo_cells = self._xp.setdiff1d(dual_cells, owned_cells) + second_halo_cells = self._xp.setdiff1d(total_halo_cells, first_halo_cells) all_cells = self._xp.union1d(owned_cells, total_halo_cells) #: edges diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index 3c0a9766d8..e23f0ddd72 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -13,14 +13,12 @@ import icon4py.model.common.dimension as dims import icon4py.model.common.utils.data_allocation as data_alloc -from icon4py.model.common.decomposition import definitions, halo +from icon4py.model.common.decomposition import definitions from icon4py.model.common.grid import simple from icon4py.model.testing import definitions as test_defs from icon4py.model.testing.fixtures import processor_props from ...grid import utils as grid_utils -from .. import utils -from ..utils import dummy_four_ranks @pytest.mark.parametrize("processor_props", [False], indirect=True) @@ -44,50 +42,6 @@ def get_neighbor_tables_for_simple_grid() -> dict[str, data_alloc.NDArray]: offsets = [dims.E2C, dims.E2V, dims.C2E, dims.C2E2C, dims.V2C, dims.V2E, dims.C2V, dims.E2C2V] -@pytest.mark.parametrize("dim", grid_utils.main_horizontal_dims()) -@pytest.mark.parametrize("rank", [0, 1, 2, 3]) -def test_halo_constructor_decomposition_info_global_indices(dim, rank): - simple_neighbor_tables = get_neighbor_tables_for_simple_grid() - props = dummy_four_ranks(rank) - halo_generator = halo.IconLikeHaloConstructor( - connectivities=simple_neighbor_tables, - run_properties=props, - ) - - decomp_info = halo_generator(utils.SIMPLE_DISTRIBUTION) - my_halo = decomp_info.global_index(dim, definitions.DecompositionInfo.EntryType.HALO) - print(f"rank {props.rank} has halo {dim} : {my_halo}") - assert my_halo.size == len(utils.HALO[dim][props.rank]) - assert np.setdiff1d(my_halo, utils.HALO[dim][props.rank], assume_unique=True).size == 0 - my_owned = decomp_info.global_index(dim, definitions.DecompositionInfo.EntryType.OWNED) - print(f"rank {props.rank} owns {dim} : {my_owned} ") - utils.assert_same_entries(dim, my_owned, utils.OWNED, props.rank) - - -@pytest.mark.parametrize("rank", (0, 1, 2, 3)) -def test_horizontal_size(rank): - simple_neighbor_tables = get_neighbor_tables_for_simple_grid() - props = dummy_four_ranks(rank) - halo_generator = halo.IconLikeHaloConstructor( - connectivities=simple_neighbor_tables, - run_properties=props, - ) - decomp_info = halo_generator(utils.SIMPLE_DISTRIBUTION) - horizontal_size = decomp_info.get_horizontal_size() - expected_verts = len(utils.OWNED[dims.VertexDim][rank]) + len(utils.HALO[dims.VertexDim][rank]) - assert ( - horizontal_size.num_vertices == expected_verts - ), f"local size mismatch on rank={rank} for {dims.VertexDim}: expected {expected_verts}, but was {horizontal_size.num_vertices}" - expected_edges = len(utils.OWNED[dims.EdgeDim][rank]) + len(utils.HALO[dims.EdgeDim][rank]) - assert ( - horizontal_size.num_edges == expected_edges - ), f"local size mismatch on rank={rank} for {dims.EdgeDim}: expected {expected_edges}, but was {horizontal_size.num_edges}" - expected_cells = len(utils.OWNED[dims.CellDim][rank]) + len(utils.HALO[dims.CellDim][rank]) - assert ( - horizontal_size.num_cells == expected_cells - ), f"local size mismatch on rank={rank} for {dims.CellDim}: expected {expected_cells}, but was {horizontal_size.num_cells}" - - @pytest.mark.parametrize("dim", grid_utils.main_horizontal_dims()) def test_decomposition_info_single_node_empty_halo( dim: gtx.Dimension, @@ -124,9 +78,10 @@ def test_decomposition_info_single_node_empty_halo( (definitions.DecompositionFlag.UNDEFINED, False), ], ) -def test_decomposition_info_is_distributed(flag, expected): +def test_decomposition_info_is_distributed(flag, expected) -> None: mesh = simple.simple_grid(allocator=None, num_levels=10) - decomp = definitions.DecompositionInfo().set_dimension( + decomp = definitions.DecompositionInfo() + decomp.set_dimension( dims.CellDim, np.arange(mesh.num_cells), np.arange(mesh.num_cells), diff --git a/model/common/tests/common/decomposition/unit_tests/test_halo.py b/model/common/tests/common/decomposition/unit_tests/test_halo.py index 812a7efc49..545269395a 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_halo.py +++ b/model/common/tests/common/decomposition/unit_tests/test_halo.py @@ -15,10 +15,11 @@ from icon4py.model.common.grid import base as base_grid, simple from ...fixtures import backend_like, processor_props +from ...grid import utils as grid_utils from .. import utils from ..fixtures import simple_neighbor_tables from ..utils import dummy_four_ranks -from .test_definitions import offsets +from .test_definitions import get_neighbor_tables_for_simple_grid, offsets @pytest.mark.parametrize("rank", [0, 1, 2, 4]) @@ -40,7 +41,7 @@ def test_halo_constructor_owned_cells(rank, simple_neighbor_tables, backend_like @pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) @pytest.mark.parametrize("rank", [0, 1, 2, 4]) def test_halo_constructor_decomposition_info_global_indices(rank, simple_neighbor_tables, dim): - processor_props = utils.DummyProps(rank=rank) + processor_props = utils.dummy_four_ranks(rank=rank) if processor_props.comm_size != 4: pytest.skip( f"This test requires exactly 4 MPI ranks, current run has {processor_props.comm_size}" @@ -214,3 +215,27 @@ def test_global_to_local_index(offset, rank): neighbor_index_full_grid[k_] == offset_full_grid[i][k], f"failed to map [{offset_full_grid[i]}] to local: [{local_offset[i]}]", ) + + +@pytest.mark.parametrize("rank", (0, 1, 2, 3)) +def test_horizontal_size(rank): + simple_neighbor_tables = get_neighbor_tables_for_simple_grid() + props = dummy_four_ranks(rank) + halo_generator = halo.IconLikeHaloConstructor( + connectivities=simple_neighbor_tables, + run_properties=props, + ) + decomp_info = halo_generator(utils.SIMPLE_DISTRIBUTION) + horizontal_size = decomp_info.get_horizontal_size() + expected_verts = len(utils.OWNED[dims.VertexDim][rank]) + len(utils.HALO[dims.VertexDim][rank]) + assert ( + horizontal_size.num_vertices == expected_verts + ), f"local size mismatch on rank={rank} for {dims.VertexDim}: expected {expected_verts}, but was {horizontal_size.num_vertices}" + expected_edges = len(utils.OWNED[dims.EdgeDim][rank]) + len(utils.HALO[dims.EdgeDim][rank]) + assert ( + horizontal_size.num_edges == expected_edges + ), f"local size mismatch on rank={rank} for {dims.EdgeDim}: expected {expected_edges}, but was {horizontal_size.num_edges}" + expected_cells = len(utils.OWNED[dims.CellDim][rank]) + len(utils.HALO[dims.CellDim][rank]) + assert ( + horizontal_size.num_cells == expected_cells + ), f"local size mismatch on rank={rank} for {dims.CellDim}: expected {expected_cells}, but was {horizontal_size.num_cells}" From cddc72404447453e09f63fb8dc05bf588947bccb Mon Sep 17 00:00:00 2001 From: halungge Date: Fri, 2 Jan 2026 16:08:34 +0100 Subject: [PATCH 137/240] fix edge DecompositionFlag.THIRD --- .../src/icon4py/model/common/decomposition/halo.py | 12 ++++++++---- .../common/decomposition/unit_tests/test_halo.py | 6 +++--- model/common/tests/common/decomposition/utils.py | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index c17263c1df..bccbb3629a 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -260,9 +260,9 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: | /| /| /| | / | / | / | - | / | / | / | - | / | / | / | - | / | / | / | rank 0 + | / | / | cy / | + | / | / cx | / | + | / | / | / cz | rank 0 |/ |/ |/ | -----v0---e0----v1---e1----v2---e2----v3---e3-- cutting line | /| /| /| @@ -284,7 +284,10 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: Cells: The "numbered" cells and edges are relevant for the halo construction from the point of view of rank 0 Cells (c0, c1, c2) are the 1. HALO LEVEL: these are cells that are neighbors of an owned cell - Cells (c3, c4. c5) are the 2. HALO LEVEL: these are cells that are neighbors of a cell of line 1 + Cells (c3, c4. c5) are the 2. HALO LEVEL: cells that "close" the hexagon of a vertex on the cutting line and do not share an edge with an owned cell (that is are not LEVEL 1 cells) + this is _not_ the same as the neighboring cells of LEVEL 1 cells, and the definition might be different from ICON. + In the above picture if the cut was along a corner (e0 -> e1-> e8) then cy is part of the 2. LEVEL because it is part of the hexagon on v2, but it is not + in c2e2c(c2e2c(c4)) Note that this definition of 1. and 2. line differs from the definition of boundary line counting used in [grid refinement](grid_refinement.py), in terms of "distance" to the cutting line all halo cells have a distance of 1. @@ -349,6 +352,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: self.find_edge_neighbors_for_vertices(vertex_on_cutting_line), edges_on_owned_cells ) edge_third_level = self._xp.setdiff1d(edges_on_any_halo_line, edge_second_level) + edge_third_level = self._xp.setdiff1d(edge_third_level, edges_on_cutting_line) all_edges = self._xp.union1d(edges_on_owned_cells, edges_on_any_halo_line) #: construct decomposition info diff --git a/model/common/tests/common/decomposition/unit_tests/test_halo.py b/model/common/tests/common/decomposition/unit_tests/test_halo.py index 545269395a..7f6d0aacc5 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_halo.py +++ b/model/common/tests/common/decomposition/unit_tests/test_halo.py @@ -54,11 +54,11 @@ def test_halo_constructor_decomposition_info_global_indices(rank, simple_neighbo decomp_info = halo_generator(utils.SIMPLE_DISTRIBUTION) my_halo = decomp_info.global_index(dim, definitions.DecompositionInfo.EntryType.HALO) - print(f"rank {processor_props.rank} has halo {dim} : {my_halo}") + print(f"rank = {processor_props.rank}: has halo {dim} : {my_halo}") expected = len(utils.HALO[dim][processor_props.rank]) assert ( my_halo.size == expected - ), f"total halo size does not match for dim {dim}- expected {expected} bot was {my_halo.size}" + ), f" rank = {processor_props.rank}: total halo size does not match for dim {dim}- expected {expected} bot was {my_halo.size}" assert ( missing := np.setdiff1d( my_halo, utils.HALO[dim][processor_props.rank], assume_unique=True @@ -70,7 +70,7 @@ def test_halo_constructor_decomposition_info_global_indices(rank, simple_neighbo utils.assert_same_entries(dim, my_owned, utils.OWNED, processor_props.rank) -@pytest.mark.parametrize("dim", [dims.CellDim, dims.VertexDim, dims.EdgeDim]) +@pytest.mark.parametrize("dim", [dims.EdgeDim]) @pytest.mark.parametrize("rank", [0, 1, 2, 3]) def test_halo_constructor_decomposition_info_halo_levels(rank, dim, simple_neighbor_tables): processor_props = utils.DummyProps(rank=rank) diff --git a/model/common/tests/common/decomposition/utils.py b/model/common/tests/common/decomposition/utils.py index 65f66fec62..f523caeee7 100644 --- a/model/common/tests/common/decomposition/utils.py +++ b/model/common/tests/common/decomposition/utils.py @@ -69,7 +69,7 @@ _CELL_SECOND_HALO_LINE = { 0: [17, 5, 12, 14, 8, 16], 1: [0, 7, 6, 9, 10, 12], - 2: [2, 1, 4, 3, 10, 15, 16, 17], + 2: [2, 1, 4, 3, 10, 15, 16, 17, 13], 3: [6, 7, 8, 2, 3, 4, 5, 11], } _CELL_HALO = { From bb94545af7ddcff49e6a0e7f44715c9d1e221041 Mon Sep 17 00:00:00 2001 From: halungge Date: Fri, 2 Jan 2026 16:39:01 +0100 Subject: [PATCH 138/240] add embedded markers --- .../common/grid/mpi_tests/test_parallel_grid_manager.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 03419da80e..17085bbc8c 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -185,6 +185,7 @@ def assert_gathered_field_against_global( rtol = 1e-12 if processor_props.rank == 0: print(f"rank = {processor_props.rank}: asserting gathered fields: ") + assert np.all( gathered_sizes == global_index_sizes ), f"gathered field sizes do not match {gathered_sizes}" @@ -199,12 +200,6 @@ def assert_gathered_field_against_global( print( f" rank = {processor_props.rank}: SHAPES: global reference field {global_reference_field.shape}, gathered = {gathered_field.shape}" ) - - mismatch = np.where( - np.abs(sorted_ - global_reference_field) / np.abs(global_reference_field) > rtol - ) - print(f"rank = {processor_props.rank}: mismatch found in {mismatch}") - np.testing.assert_allclose(sorted_, global_reference_field, rtol=rtol, verbose=True) @@ -299,6 +294,7 @@ def test_halo_neighbor_access_c2e( print(f"rank = {processor_props.rank} - DONE") +@pytest.mark.embedded_remap_error @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) @@ -390,6 +386,7 @@ def test_halo_access_e2c2v( ) +@pytest.mark.embedded_remap_error @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) From d738ee95d7e539e78f31d952d53dd72d42d1dc5a Mon Sep 17 00:00:00 2001 From: halungge Date: Fri, 2 Jan 2026 19:46:05 +0100 Subject: [PATCH 139/240] add xfail for dace --- .../mpi_tests/test_parallel_grid_manager.py | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 17085bbc8c..e5e8455a8a 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -32,7 +32,7 @@ _compute_cell_2_vertex_interpolation, ) from icon4py.model.common.utils import data_allocation as data_alloc -from icon4py.model.testing import definitions as test_defs, grid_utils +from icon4py.model.testing import definitions as test_defs, grid_utils, test_utils from ..fixtures import backend, processor_props from . import utils @@ -132,25 +132,25 @@ def test_fields_distribute_and_gather( ) -def gather_field(field: np.ndarray, comm: mpi4py.MPI.Comm) -> tuple: - constant_dims = field.shape[1:] +def gather_field(field: np.ndarray, props: decomp_defs.ProcessProperties) -> tuple: + constant_dims = tuple(field.shape[1:]) + print(f"gather_field on rank={props.rank} - gathering field of local shape {field.shape}") constant_length = functools.reduce(operator.mul, constant_dims) if len(constant_dims) > 0 else 1 - - local_sizes = np.array(comm.gather(field.size, root=0)) - - if comm.rank == 0: + local_sizes = np.array(props.comm.gather(field.size, root=0)) + if props.rank == 0: recv_buffer = np.empty(np.sum(local_sizes), dtype=field.dtype) - log.debug( - f"rank:{comm} - {comm.rank} - setup receive buffer with size {sum(local_sizes)} on rank 0" + print( + f"gather_field on rank = {props.rank} - setup receive buffer with size {sum(local_sizes)} on rank 0" ) else: recv_buffer = None - comm.Gatherv(sendbuf=field, recvbuf=(recv_buffer, local_sizes), root=0) - if comm.rank == 0: - log.debug("fields gathered:") - log.debug(f"field sizes {local_sizes}") - local_first_dim = tuple(size / constant_length for size in local_sizes) + props.comm.Gatherv(sendbuf=field, recvbuf=(recv_buffer, local_sizes), root=0) + if props.rank == 0: + local_first_dim = tuple(sz // constant_length for sz in local_sizes) + print( + f" gather_field on rank = 0: computed local dims {local_first_dim} - constant dims {constant_dims}" + ) gathered_field = recv_buffer.reshape((-1, *constant_dims)) # type: ignore [union-attr] else: gathered_field = None @@ -177,10 +177,11 @@ def assert_gathered_field_against_global( owned_entries = local_field[ decomposition_info.local_index(dim, decomp_defs.DecompositionInfo.EntryType.OWNED) ] - gathered_sizes, gathered_field = gather_field(owned_entries, processor_props.comm) + gathered_sizes, gathered_field = gather_field(owned_entries, processor_props) + global_index_sizes, gathered_global_indices = gather_field( decomposition_info.global_index(dim, decomp_defs.DecompositionInfo.EntryType.OWNED), - processor_props.comm, + processor_props, ) rtol = 1e-12 if processor_props.rank == 0: @@ -188,9 +189,9 @@ def assert_gathered_field_against_global( assert np.all( gathered_sizes == global_index_sizes - ), f"gathered field sizes do not match {gathered_sizes}" + ), f"gathered field sizes do not match: {dim} {gathered_sizes} - {global_index_sizes}" print( - f"rank = {processor_props.rank}: Checking field size: --- gathered sizes {gathered_sizes} = {sum(gathered_sizes)}" + f"rank = {processor_props.rank}: Checking field size on dim ={dim}: --- gathered sizes {gathered_sizes} = {sum(gathered_sizes)}" ) print( f"rank = {processor_props.rank}: --- gathered field has size {gathered_sizes}" @@ -303,8 +304,10 @@ def test_halo_access_e2c2v( backend: gtx_typing.Backend | None, grid: test_defs.GridDescription, ) -> None: + if test_utils.is_dace(backend): + pytest.xfail("dace backend make test fail: direct offset access in test function?") + file = grid_utils.resolve_full_grid_file_name(grid) - print(f"running on {processor_props.comm}") single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( @@ -395,8 +398,11 @@ def test_halo_access_e2c( backend: gtx_typing.Backend | None, grid: test_defs.GridDescription, ) -> None: + if test_utils.is_dace(backend): + pytest.xfail( + "dace backend make test fail (and following tests): direct offset access in test function?" + ) file = grid_utils.resolve_full_grid_file_name(grid) - print(f"running on {processor_props.comm}") single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( From 52ddceae27aa3c794af0427d2fbb28b883e36811 Mon Sep 17 00:00:00 2001 From: halungge Date: Fri, 2 Jan 2026 21:47:07 +0100 Subject: [PATCH 140/240] fix ordering --- .../icon4py/model/common/decomposition/halo.py | 17 +++++++++-------- .../mpi_tests/test_parallel_grid_manager.py | 8 ++++++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index bccbb3629a..948628c40f 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -325,21 +325,22 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: #: cells owned_cells = self.owned_cells(face_to_rank) # global indices of owned cells first_halo_cells = self.next_halo_line(owned_cells) - total_halo_cells = self._xp.union1d( - first_halo_cells, self.next_halo_line(first_halo_cells, owned_cells) - ) - #: vertices vertex_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) - vertex_on_halo_cells = self.find_vertex_neighbors_for_cells(total_halo_cells) + vertex_on_halo_cells = self.find_vertex_neighbors_for_cells( + self._xp.hstack( + (first_halo_cells, (self.next_halo_line(first_halo_cells, owned_cells))) + ) + ) vertex_on_cutting_line = self._xp.intersect1d(vertex_on_owned_cells, vertex_on_halo_cells) - all_vertices = self._xp.union1d(vertex_on_owned_cells, vertex_on_halo_cells) + vertex_second_halo = self._xp.setdiff1d(vertex_on_halo_cells, vertex_on_cutting_line) + all_vertices = self._xp.hstack((vertex_on_owned_cells, vertex_second_halo)) #: update cells to include all cells of the "dual cell" (hexagon) for nodes on the cutting line dual_cells = self.find_cell_neighbors_for_vertices(vertex_on_cutting_line) total_halo_cells = self._xp.setdiff1d(dual_cells, owned_cells) second_halo_cells = self._xp.setdiff1d(total_halo_cells, first_halo_cells) - all_cells = self._xp.union1d(owned_cells, total_halo_cells) + all_cells = self._xp.hstack((owned_cells, first_halo_cells, second_halo_cells)) #: edges edges_on_owned_cells = self.find_edge_neighbors_for_cells(owned_cells) @@ -354,7 +355,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: edge_third_level = self._xp.setdiff1d(edges_on_any_halo_line, edge_second_level) edge_third_level = self._xp.setdiff1d(edge_third_level, edges_on_cutting_line) - all_edges = self._xp.union1d(edges_on_owned_cells, edges_on_any_halo_line) + all_edges = self._xp.hstack((edges_on_owned_cells, edge_second_level, edge_third_level)) #: construct decomposition info decomp_info = defs.DecompositionInfo() cell_owner_mask = self._xp.isin(all_cells, owned_cells) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index e5e8455a8a..9f42c93e88 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -299,7 +299,7 @@ def test_halo_neighbor_access_c2e( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) -def test_halo_access_e2c2v( +def test_halo_neighbor_access_e2c2v( processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, grid: test_defs.GridDescription, @@ -360,6 +360,7 @@ def test_halo_access_e2c2v( v2 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) u3 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) v3 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) + horizontal_end = distributed_grid.end_index(h_grid.edge_domain(h_grid.Zone.HALO)) geometry_stencils.zonal_and_meridional_component_of_edge_field_at_vertex.with_backend(backend)( vertex_lat, @@ -367,6 +368,7 @@ def test_halo_access_e2c2v( x, y, z, + domain={dims.EdgeDim: (0, horizontal_end)}, out=(u0, v0, u1, v1, u2, v2, u3, v3), offset_provider={"E2C2V": distributed_grid.get_connectivity(dims.E2C2V)}, ) @@ -393,7 +395,7 @@ def test_halo_access_e2c2v( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) -def test_halo_access_e2c( +def test_halo_neighbor_access_e2c( processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, grid: test_defs.GridDescription, @@ -452,6 +454,7 @@ def test_halo_access_e2c( v0 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) u1 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) v1 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) + horizontal_end = distributed_grid.end_index(h_grid.edge_domain(h_grid.Zone.HALO)) geometry_stencils.zonal_and_meridional_component_of_edge_field_at_cell_center.with_backend( backend @@ -462,6 +465,7 @@ def test_halo_access_e2c( y, z, out=(u0, v0, u1, v1), + domain={dims.EdgeDim: (0, horizontal_end)}, offset_provider={"E2C": distributed_grid.get_connectivity(dims.E2C)}, ) u_component = np.vstack((u0.asnumpy(), u1.asnumpy())).T From f2114731ca47b27ede904b314f3c2183dcee6a8a Mon Sep 17 00:00:00 2001 From: Jacopo Date: Mon, 19 Jan 2026 16:58:38 +0100 Subject: [PATCH 141/240] cleanup of docstring including @msimberg 's review --- .../model/common/decomposition/halo.py | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 948628c40f..351e3f9cb7 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -294,27 +294,26 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: Vertices: - - 1. HALO LEVEL: are vertices on the cutting line that are not owned, or in a different wording: all vertices on owned cells that ar not + - 1. HALO LEVEL: are vertices on the cutting line that are not owned, or in a different wording: all vertices of owned cells that are not owned. - In ICON every element in an array needs **exactly one owner**. For elements on the cutting line (vertices and edges) there is no clear - indication which rank should own it, ICON uses the rank with the higher rank (see (_update_owner_mask_by_max_rank_convention)) - In the example above (v0, v1, v2, v3) are in the 1. HALO LEVEL or rank 0 and owend by rank 1. Consequently there ranks that have no - 1.HALO LEVEL cells. - + In ICON every element in an array needs **exactly one owner** (otherwise there would be duplicates and double-counting). + For elements on the cutting line (vertices and edges) there is no clear indication which rank should own it, + ICON uses the rank with the higher rank value (see (_update_owner_mask_by_max_rank_convention)) + In the example above (v0, v1, v2, v3) are in the 1. HALO LEVEL of rank 0 and owned by rank 1. + As a consequence, there are ranks that have no 1. HALO LEVEL vertices. - 2. HALO LEVEL: are vertices that are on HALO LEVEL cells, but not on owned. For rank 0 these are (v4, v5, v6, v7) Edges: For edges a similar pattern is used as for the vertices. - - 1. HALO LEVEl: edges that are on owned cells but not owned themselves (these are edges that share 2 vertices with and owned cell). - In terms of ownership the same convention is applied as for the vertices: (e0, e1, e2, e3) are in the HALO LEVEL 1 of rank 0, and owned by rank 1 + - 1. HALO LEVEL: edges that are on owned cells but not owned themselves (these are edges that share 2 vertices with a owned cell). + In terms of ownership the same convention is applied as for the vertices: (e0, e1, e2, e3) are in the HALO LEVEL 1 of rank 0, and are owned by rank 1 - 2. HALO LEVEL: edges that share exactly one vertex with an owned cell. The definition via vertices is important: TODD (halungge): EXAMPLE??? - For rank 0 above these are the edges (e4, e5, e6, e7, e8, e9, e10) - + For rank 0 these are the edges (e4, e5, e6, e7, e8, e9, e10) in the example above. - 3. HALO LEVEL: - We flag the edges (e11, e12, e13) that "close" the halo cells (share exactly 2 vertices with a HALO LEVEL 2 cell, but none with - an owned cell). These edges are **not** included in the halo in ICON. We include them as 3. HALO LINE which - makes the C2E connectivity complete (= without skip value) for a distributed setup. + In **ICON4Py ONLY**, edges that "close" the halo cells and share exactly 2 vertices with a HALO LEVEL 2 cell, but none with + an owned cell. These edges are **not** included in the halo in ICON. These are (e11, e12, e13) for rank 0 in the example above. + This is the HALO LINE which makes the C2E connectivity complete (= without skip value) for a distributed setup. # TODO(halungge): make number of halo lines (in terms of cells) a parameter: icon does hard coding of 2 halo lines for cells, make this dynamic! From 336db99fb71f5c49df272a9bb05ba4156f30c0db Mon Sep 17 00:00:00 2001 From: Jacopo Date: Mon, 19 Jan 2026 17:06:15 +0100 Subject: [PATCH 142/240] little cleanup --- model/common/src/icon4py/model/common/decomposition/halo.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 351e3f9cb7..92dfaffc04 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -308,15 +308,12 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: For edges a similar pattern is used as for the vertices. - 1. HALO LEVEL: edges that are on owned cells but not owned themselves (these are edges that share 2 vertices with a owned cell). In terms of ownership the same convention is applied as for the vertices: (e0, e1, e2, e3) are in the HALO LEVEL 1 of rank 0, and are owned by rank 1 - - 2. HALO LEVEL: edges that share exactly one vertex with an owned cell. The definition via vertices is important: TODD (halungge): EXAMPLE??? + - 2. HALO LEVEL: edges that share exactly one vertex with an owned cell. The definition via vertices is important: TODO (halungge): EXAMPLE??? For rank 0 these are the edges (e4, e5, e6, e7, e8, e9, e10) in the example above. - 3. HALO LEVEL: In **ICON4Py ONLY**, edges that "close" the halo cells and share exactly 2 vertices with a HALO LEVEL 2 cell, but none with an owned cell. These edges are **not** included in the halo in ICON. These are (e11, e12, e13) for rank 0 in the example above. This is the HALO LINE which makes the C2E connectivity complete (= without skip value) for a distributed setup. - - # TODO(halungge): make number of halo lines (in terms of cells) a parameter: icon does hard coding of 2 halo lines for cells, make this dynamic! - """ self._validate_mapping(face_to_rank) From a983ed79f076b9831f6225a8c79229eaa021ea36 Mon Sep 17 00:00:00 2001 From: Jacopo Date: Mon, 26 Jan 2026 15:30:44 +0100 Subject: [PATCH 143/240] one more grid for testing --- model/testing/src/icon4py/model/testing/definitions.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/model/testing/src/icon4py/model/testing/definitions.py b/model/testing/src/icon4py/model/testing/definitions.py index 9c78736ac5..775e51d24b 100644 --- a/model/testing/src/icon4py/model/testing/definitions.py +++ b/model/testing/src/icon4py/model/testing/definitions.py @@ -134,6 +134,14 @@ class Grids: file_name="Torus_Triangles_50000m_x_5000m_res500m.nc", uri="https://polybox.ethz.ch/index.php/s/eclzK00TM9nnLtE/download", ) + TORUS_1000X1000_250M: Final = GridDescription( + name="torus_1000x1000_res250", + description="Torus grid with a domain (1000x1000) vertices and a resolution (edge length) of 250m, generated by MPI-M GridGenerator ", + sizes={"cell": 24, "vertex": 12, "edge": 36}, + kind=GridKind.TORUS, + file_name="Torus_Triangles_1000x1000_250m.nc", + uri="https://polybox.ethz.ch/index.php/s/eMDbDbdmKLkDiwp/download", + ) @dataclasses.dataclass From 3dba11f0eb3230b4acb661828a22f41b4e8e3897 Mon Sep 17 00:00:00 2001 From: Jacopo Date: Mon, 26 Jan 2026 15:41:57 +0100 Subject: [PATCH 144/240] fix filename --- model/testing/src/icon4py/model/testing/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/testing/src/icon4py/model/testing/definitions.py b/model/testing/src/icon4py/model/testing/definitions.py index 775e51d24b..e0c1f50676 100644 --- a/model/testing/src/icon4py/model/testing/definitions.py +++ b/model/testing/src/icon4py/model/testing/definitions.py @@ -139,7 +139,7 @@ class Grids: description="Torus grid with a domain (1000x1000) vertices and a resolution (edge length) of 250m, generated by MPI-M GridGenerator ", sizes={"cell": 24, "vertex": 12, "edge": 36}, kind=GridKind.TORUS, - file_name="Torus_Triangles_1000x1000_250m.nc", + file_name="Torus_Triangles_1000m_x_1000m_res250m.nc", uri="https://polybox.ethz.ch/index.php/s/eMDbDbdmKLkDiwp/download", ) From d20f82e3e0503bad9cb78b75311c056f0f1cab8b Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 28 Jan 2026 16:21:45 +0100 Subject: [PATCH 145/240] Remove outdated xfails --- .../common/grid/mpi_tests/test_parallel_grid_manager.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 9f42c93e88..40341ba686 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -304,9 +304,6 @@ def test_halo_neighbor_access_e2c2v( backend: gtx_typing.Backend | None, grid: test_defs.GridDescription, ) -> None: - if test_utils.is_dace(backend): - pytest.xfail("dace backend make test fail: direct offset access in test function?") - file = grid_utils.resolve_full_grid_file_name(grid) single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid @@ -400,10 +397,6 @@ def test_halo_neighbor_access_e2c( backend: gtx_typing.Backend | None, grid: test_defs.GridDescription, ) -> None: - if test_utils.is_dace(backend): - pytest.xfail( - "dace backend make test fail (and following tests): direct offset access in test function?" - ) file = grid_utils.resolve_full_grid_file_name(grid) single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid From be499fe22112abdda0bb41976a0f420b41331f33 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 28 Jan 2026 16:56:23 +0100 Subject: [PATCH 146/240] Remove another xfail --- .../common/tests/common/grid/mpi_tests/test_parallel_geometry.py | 1 - 1 file changed, 1 deletion(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_geometry.py b/model/common/tests/common/grid/mpi_tests/test_parallel_geometry.py index b07618f9e4..edf4312ad1 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_geometry.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_geometry.py @@ -85,7 +85,6 @@ def test_distributed_geometry_attrs( assert test_utils.dallclose(field, field_ref, atol=1e-12) -@pytest.mark.xfail(reason="Wrong results") @pytest.mark.datatest @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) From b1cea9902f11bb45cd9e97ba3676b2b764e203a3 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Thu, 29 Jan 2026 12:05:45 +0100 Subject: [PATCH 147/240] Remove old TODO --- model/common/src/icon4py/model/common/grid/icon.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/icon.py b/model/common/src/icon4py/model/common/grid/icon.py index 774446127c..15ac5c7b84 100644 --- a/model/common/src/icon4py/model/common/grid/icon.py +++ b/model/common/src/icon4py/model/common/grid/icon.py @@ -70,8 +70,6 @@ def __init__( _T = TypeVar("_T") -# TODO (@halungge): fields should be removed from this object mean values computed in the geometry factory -# as it needs global reduction... also consider _not_making_everything_ optional that causes troubles at runtime @dataclasses.dataclass(kw_only=True, frozen=True) class GlobalGridParams: grid_shape: Final[GridShape | None] = None From ff4e7db20410b772cfef29a224135736f782757d Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Thu, 29 Jan 2026 12:09:17 +0100 Subject: [PATCH 148/240] Pass num cells to global grid params --- model/testing/src/icon4py/model/testing/serialbox.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/model/testing/src/icon4py/model/testing/serialbox.py b/model/testing/src/icon4py/model/testing/serialbox.py index e7ee66e86b..b76b9abe5c 100644 --- a/model/testing/src/icon4py/model/testing/serialbox.py +++ b/model/testing/src/icon4py/model/testing/serialbox.py @@ -151,7 +151,10 @@ def __init__( ): super().__init__(sp, ser, size, backend) self._grid_id = grid_id - self.global_grid_params = icon.GlobalGridParams(grid_shape=grid_shape) + self.global_grid_params = icon.GlobalGridParams( + grid_shape=grid_shape, + num_cells=size[dims.CellDim], + ) def verts_vertex_lat(self): """vertex latituted""" From dbbf8298559b1fff6566a49e9ab2edbc74a08252 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Thu, 29 Jan 2026 13:21:40 +0100 Subject: [PATCH 149/240] Revert "refactor: testing infrastructure (#1002)" This reverts commit e30c2f71e668952698fd93e3ce1a1c054029ea6c. --- .../model/common/utils/device_utils.py | 3 --- .../icon4py/model/testing/data_handling.py | 23 +++-------------- .../model/testing/fixtures/datatest.py | 25 +++++++++++++++++-- .../icon4py/model/testing/stencil_tests.py | 17 ++++++------- 4 files changed, 33 insertions(+), 35 deletions(-) diff --git a/model/common/src/icon4py/model/common/utils/device_utils.py b/model/common/src/icon4py/model/common/utils/device_utils.py index 360a53902a..cacfc8eb64 100644 --- a/model/common/src/icon4py/model/common/utils/device_utils.py +++ b/model/common/src/icon4py/model/common/utils/device_utils.py @@ -37,9 +37,6 @@ def sync(allocator: gtx_typing.FieldBufferAllocationUtil | None = None) -> None: Note: this is and ad-hoc interface, maybe the function should get the device to sync for. """ - # Type annotation already describes that only these types are allowed, but mypy coverage is not great. - # The explicit assert avoids critical mistakes in using this function. - assert allocator is None or gtx_allocators.is_field_allocation_tool(allocator) if allocator is not None and is_cupy_device(allocator): cp.cuda.runtime.deviceSynchronize() diff --git a/model/testing/src/icon4py/model/testing/data_handling.py b/model/testing/src/icon4py/model/testing/data_handling.py index 9624c64839..9ecf932335 100644 --- a/model/testing/src/icon4py/model/testing/data_handling.py +++ b/model/testing/src/icon4py/model/testing/data_handling.py @@ -6,13 +6,11 @@ # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause -import pathlib import tarfile +from pathlib import Path -from icon4py.model.testing import config, locking - -def download_and_extract(uri: str, dst: pathlib.Path, data_file: str = "downloaded.tar.gz") -> None: +def download_and_extract(uri: str, dst: Path, data_file: str = "downloaded.tar.gz") -> None: """ Download data archive from remote server. @@ -33,19 +31,4 @@ def download_and_extract(uri: str, dst: pathlib.Path, data_file: str = "download raise OSError(f"{data_file} needs to be a valid tar file") with tarfile.open(data_file, mode="r:*") as tf: tf.extractall(path=dst) - pathlib.Path(data_file).unlink(missing_ok=True) - - -def download_test_data(dst: pathlib.Path, uri: str) -> None: - if config.ENABLE_TESTDATA_DOWNLOAD: - # We create and lock the *parent* directory as we later check for existence of `dst`. - dst.parent.mkdir(parents=True, exist_ok=True) - with locking.lock(dst.parent): - if not dst.exists(): - download_and_extract(uri, dst) - else: - # If test data download is disabled, we check if the directory exists - # without locking. We assume the location is managed by the user - # and avoid locking shared directories (e.g. on CI). - if not dst.exists(): - raise RuntimeError(f"Test data {dst} does not exist, and downloading is disabled.") + Path(data_file).unlink(missing_ok=True) diff --git a/model/testing/src/icon4py/model/testing/fixtures/datatest.py b/model/testing/src/icon4py/model/testing/fixtures/datatest.py index c1d17332e9..28483172a1 100644 --- a/model/testing/src/icon4py/model/testing/fixtures/datatest.py +++ b/model/testing/src/icon4py/model/testing/fixtures/datatest.py @@ -17,7 +17,13 @@ from icon4py.model.common import model_backends, model_options from icon4py.model.common.constants import RayleighType from icon4py.model.common.grid import base as base_grid -from icon4py.model.testing import data_handling as data, datatest_utils as dt_utils, definitions +from icon4py.model.testing import ( + config, + data_handling as data, + datatest_utils as dt_utils, + definitions, + locking, +) if TYPE_CHECKING: @@ -119,7 +125,22 @@ def _download_ser_data( try: destination_path = dt_utils.get_datapath_for_experiment(_ranked_data_path, _experiment) uri = _experiment.partitioned_data[comm_size] - data.download_test_data(destination_path, uri) + + data_file = _ranked_data_path.joinpath(f"{_experiment.name}_mpitask{comm_size}.tar.gz").name + _ranked_data_path.mkdir(parents=True, exist_ok=True) + if config.ENABLE_TESTDATA_DOWNLOAD: + with locking.lock(_ranked_data_path): + # Note: if the lock would be created for `destination_path` it would always exist... + if not destination_path.exists(): + data.download_and_extract(uri, _ranked_data_path, data_file) + else: + # If test data download is disabled, we check if the directory exists + # without locking. We assume the location is managed by the user + # and avoid locking shared directories (e.g. on CI). + if not destination_path.exists(): + raise RuntimeError( + f"Serialization data {data_file} does not exist, and downloading is disabled." + ) except KeyError as err: raise RuntimeError( f"No data for communicator of size {comm_size} exists, use 1, 2 or 4" diff --git a/model/testing/src/icon4py/model/testing/stencil_tests.py b/model/testing/src/icon4py/model/testing/stencil_tests.py index ad1bf5e0ac..f83798f029 100644 --- a/model/testing/src/icon4py/model/testing/stencil_tests.py +++ b/model/testing/src/icon4py/model/testing/stencil_tests.py @@ -21,7 +21,6 @@ config as gtx_config, constructors, metrics as gtx_metrics, - named_collections as gtx_named_collections, typing as gtx_typing, ) @@ -35,15 +34,13 @@ def allocate_data( allocator: gtx_typing.FieldBufferAllocationUtil | None, - input_data: dict[ - str, Any - ], # `Field`s or collection of `Field`s are re-allocated, the rest is passed through -) -> dict[str, Any]: - def _allocate_field(f: gtx.Field) -> gtx.Field: - return constructors.as_field(domain=f.domain, data=f.ndarray, allocator=allocator) - + input_data: dict[str, gtx.Field | tuple[gtx.Field, ...]], +) -> dict[str, gtx.Field | tuple[gtx.Field, ...]]: + _allocate_field = constructors.as_field.partial(allocator=allocator) # type:ignore[attr-defined] # TODO(havogt): check why it doesn't understand the fluid_partial input_data = { - k: gtx_named_collections.tree_map_named_collection(_allocate_field)(v) + k: tuple(_allocate_field(domain=field.domain, data=field.ndarray) for field in v) + if isinstance(v, tuple) + else _allocate_field(domain=v.domain, data=v.ndarray) if not gtx.is_scalar_type(v) and k != "domain" else v for k, v in input_data.items() @@ -210,7 +207,7 @@ def _properly_allocated_input_data( self, input_data: dict[str, gtx.Field | tuple[gtx.Field, ...]], backend_like: model_backends.BackendLike, - ) -> dict[str, Any]: + ) -> dict[str, gtx.Field | tuple[gtx.Field, ...]]: # TODO(havogt): this is a workaround, # because in the `input_data` fixture provided by the user # it does not allocate for the correct device. From eed64ebea2def168b6ff61eb577a52fee4618845 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Thu, 29 Jan 2026 16:57:08 +0100 Subject: [PATCH 150/240] Fix cupy/numpy issues --- .../model/common/decomposition/definitions.py | 17 ++++++------- .../model/common/decomposition/halo.py | 10 ++++---- .../icon4py/model/common/grid/grid_manager.py | 25 ++++++++++--------- .../src/icon4py/model/common/grid/gridfile.py | 17 +++++++------ .../model/common/utils/data_allocation.py | 9 +++++++ 5 files changed, 43 insertions(+), 35 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 46935b837f..ecb32e9c5f 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -100,7 +100,7 @@ def set_dimension( self._halo_levels[dim] = halo_levels def is_distributed(self) -> bool: - return max(self._halo_levels[dims.CellDim]) > DecompositionFlag.OWNED + return max(self._halo_levels[dims.CellDim]).item() > DecompositionFlag.OWNED def local_index( self, dim: gtx.Dimension, entry_type: EntryType = EntryType.ALL @@ -120,13 +120,7 @@ def local_index( def _to_local_index(self, dim: gtx.Dimension) -> data_alloc.NDArray: data = self._global_index[dim] assert data.ndim == 1 - if isinstance(data, np.ndarray): - import numpy as xp - else: - import cupy as xp # type: ignore[import-not-found, no-redef] - - xp.arange(data.shape[0]) - return xp.arange(data.shape[0]) + return data_alloc.array_ns_from_array(data).arange(data.shape[0]) def owner_mask(self, dim: gtx.Dimension) -> data_alloc.NDArray: return self._owner_mask[dim] @@ -154,13 +148,16 @@ def get_horizontal_size(self) -> base.HorizontalGridSize: ) def get_halo_size(self, dim: gtx.Dimension, flag: DecompositionFlag) -> int: - return np.count_nonzero(self.halo_level_mask(dim, flag)) + level_mask = self.halo_level_mask(dim, flag) + # TODO: Just do array_ns? + return data_alloc.array_ns_from_array(level_mask).count_nonzero(level_mask) def halo_levels(self, dim: gtx.Dimension) -> data_alloc.NDArray: return self._halo_levels[dim] def halo_level_mask(self, dim: gtx.Dimension, level: DecompositionFlag) -> data_alloc.NDArray: - return np.where(self._halo_levels[dim] == level, True, False) + levels = self._halo_levels[dim] + return data_alloc.array_ns_from_array(levels).where(levels == level.value, True, False) class ExchangeResult(Protocol): diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 92dfaffc04..9c5abe4655 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -58,8 +58,8 @@ def _create_dummy_decomposition_arrays( size: int, array_ns: ModuleType = np ) -> tuple[data_alloc.NDArray, data_alloc.NDArray, data_alloc.NDArray]: indices = array_ns.arange(size, dtype=gtx.int32) # type: ignore [attr-defined] - owner_mask = array_ns.ones((size,), dtype=bool) - halo_levels = array_ns.ones((size,), dtype=gtx.int32) * defs.DecompositionFlag.OWNED # type: ignore [attr-defined] + owner_mask = array_ns.full((size,), True, dtype=bool) + halo_levels = array_ns.full((size,), defs.DecompositionFlag.OWNED.value, dtype=gtx.int32) # type: ignore [attr-defined] return indices, owner_mask, halo_levels @@ -462,13 +462,13 @@ def __call__( Args: n_part: int, number of partitions to create adjacency_matrix: nd array: neighbor table describing of the main dimension object to be distributed: for example cell -> cell neighbors - Returns: np.ndarray: array with partition label (int, rank number) for each cell + Returns: data_alloc.NDArray: array with partition label (int, rank number) for each cell """ import pymetis # type: ignore [import-untyped] _, partition_index = pymetis.part_graph(nparts=num_partitions, adjacency=adjacency_matrix) - return np.array(partition_index) + return data_alloc.array_ns_from_array(adjacency_matrix).array(partition_index) class SingleNodeDecomposer(Decomposer): @@ -476,7 +476,7 @@ def __call__( self, adjacency_matrix: data_alloc.NDArray, num_partitions: int = 1 ) -> data_alloc.NDArray: """Dummy decomposer for single node: assigns all cells to rank = 0""" - return np.zeros(adjacency_matrix.shape[0], dtype=gtx.int32) # type: ignore [attr-defined] + return data_alloc.array_ns_from_array(adjacency_matrix).zeros(adjacency_matrix.shape[0], dtype=gtx.int32) # type: ignore [attr-defined] def get_halo_constructor( diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 41f8ce152d..5e9973c4d3 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -392,21 +392,21 @@ def _construct_decomposed_grid( """ xp = data_alloc.import_array_ns(allocator) ## FULL GRID PROPERTIES - cell_refinement = self._reader.variable(gridfile.GridRefinementName.CONTROL_CELLS) + cell_refinement = xp.asarray(self._reader.variable(gridfile.GridRefinementName.CONTROL_CELLS)) global_size = self._read_full_grid_size() global_params = self._construct_global_params(allocator, global_size, geometry_type) limited_area = refinement.is_limited_area_grid(cell_refinement, array_ns=xp) - cell_to_cell_neighbors = self._get_index_field(gridfile.ConnectivityName.C2E2C) + cell_to_cell_neighbors = self._get_index_field(gridfile.ConnectivityName.C2E2C, array_ns=xp) neighbor_tables = { dims.C2E2C: cell_to_cell_neighbors, - dims.C2E: self._get_index_field(gridfile.ConnectivityName.C2E), - dims.E2C: self._get_index_field(gridfile.ConnectivityName.E2C), - dims.V2E: self._get_index_field(gridfile.ConnectivityName.V2E), - dims.V2C: self._get_index_field(gridfile.ConnectivityName.V2C), - dims.C2V: self._get_index_field(gridfile.ConnectivityName.C2V), - dims.V2E2V: self._get_index_field(gridfile.ConnectivityName.V2E2V), - dims.E2V: self._get_index_field(gridfile.ConnectivityName.E2V), + dims.C2E: self._get_index_field(gridfile.ConnectivityName.C2E, array_ns=xp), + dims.E2C: self._get_index_field(gridfile.ConnectivityName.E2C, array_ns=xp), + dims.V2E: self._get_index_field(gridfile.ConnectivityName.V2E, array_ns=xp), + dims.V2C: self._get_index_field(gridfile.ConnectivityName.V2C, array_ns=xp), + dims.C2V: self._get_index_field(gridfile.ConnectivityName.C2V, array_ns=xp), + dims.V2E2V: self._get_index_field(gridfile.ConnectivityName.V2E2V, array_ns=xp), + dims.E2V: self._get_index_field(gridfile.ConnectivityName.E2V, array_ns=xp), } cells_to_rank_mapping = decomposer(cell_to_cell_neighbors, run_properties.comm_size) @@ -518,13 +518,14 @@ def _read_full_grid_size(self) -> base.HorizontalGridSize: def _get_index_field( self, field: gridfile.GridFileName, - indices: np.ndarray | None = None, + indices: data_alloc.NDArray | None = None, transpose=True, apply_offset=True, + array_ns: ModuleType = np, ): - return self._reader.int_variable( + return array_ns.asarray(self._reader.int_variable( field, indices=indices, transpose=transpose, apply_transformation=apply_offset - ) + )) def _get_derived_connectivities( diff --git a/model/common/src/icon4py/model/common/grid/gridfile.py b/model/common/src/icon4py/model/common/grid/gridfile.py index d8dad33646..4ca5900c3f 100644 --- a/model/common/src/icon4py/model/common/grid/gridfile.py +++ b/model/common/src/icon4py/model/common/grid/gridfile.py @@ -43,19 +43,20 @@ def __call__( class NoTransformation(IndexTransformation): """Empty implementation of the Protocol. Just return zeros.""" - def __call__(self, array: data_alloc.NDArray): - return np.zeros_like(array) + def __call__(self, array: data_alloc.NDArray) -> data_alloc.NDArray: + return data_alloc.array_ns_from_array(array).zeros_like(array) class ToZeroBasedIndexTransformation(IndexTransformation): - def __call__(self, array: data_alloc.NDArray): + def __call__(self, array: data_alloc.NDArray) -> data_alloc.NDArray: """ Calculate the index offset needed for usage with python. Fortran indices are 1-based, hence the offset is -1 for 0-based ness of python except for INVALID values which are marked with -1 in the grid file and are kept such. """ - return np.asarray(np.where(array == GridFile.INVALID_INDEX, 0, -1), dtype=gtx.int32) + xp = data_alloc.array_ns_from_array(array) + return xp.asarray(xp.where(array == GridFile.INVALID_INDEX, 0, -1), dtype=gtx.int32) class GridFileName(str, enum.Enum): @@ -332,7 +333,7 @@ def try_attribute(self, name: PropertyName) -> str | int | float | None: def int_variable( self, name: FieldName, - indices: np.ndarray | None = None, + indices: data_alloc.NDArray | None = None, transpose: bool = True, apply_transformation: bool = True, ) -> np.ndarray: @@ -361,7 +362,7 @@ def int_variable( def variable( self, name: FieldName, - indices: np.ndarray | None = None, + indices: data_alloc.NDArray | None = None, transpose: bool = False, dtype: np.dtype = gtx.float64, ) -> np.ndarray: @@ -387,9 +388,9 @@ def variable( slicer = [slice(None) for _ in range(variable_size)] if indices is not None and indices.size > 0: # apply the slicing to the correct dimension - slicer[(1 if transpose else 0)] = indices + slicer[(1 if transpose else 0)] = data_alloc.as_numpy(indices) _log.debug(f"reading {name}: transposing = {transpose}") - data = variable[tuple(slicer)] + data = np.asarray(variable[tuple(slicer)]) data = np.array(data, dtype=dtype).ravel(order="K").reshape(target_shape) return np.transpose(data) if transpose else data except KeyError as err: diff --git a/model/common/src/icon4py/model/common/utils/data_allocation.py b/model/common/src/icon4py/model/common/utils/data_allocation.py index c58a8cd827..ae2f58f330 100644 --- a/model/common/src/icon4py/model/common/utils/data_allocation.py +++ b/model/common/src/icon4py/model/common/utils/data_allocation.py @@ -78,6 +78,15 @@ def array_ns(try_cupy: bool) -> ModuleType: return np +def array_ns_from_array(array: NDArray) -> ModuleType: + if isinstance(array, np.ndarray): + import numpy as xp + else: + import cupy as xp # type: ignore[import-not-found, no-redef] + + return xp + + def import_array_ns(allocator: gtx_allocators.FieldBufferAllocationUtil | None) -> ModuleType: """Import cupy or numpy depending on a chosen GT4Py backend DevicType.""" return array_ns(device_utils.is_cupy_device(allocator)) From 5440d2b8809ea2038be3d7ec7157088ac48c4eb2 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 30 Jan 2026 10:08:48 +0100 Subject: [PATCH 151/240] Fix another cupy/numpy issue --- .../tests/common/grid/unit_tests/test_grid_manager.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index fd546a241b..79a7d7598a 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -18,6 +18,7 @@ import icon4py.model.common.grid.gridfile from icon4py.model.common import dimension as dims, model_backends +from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.common.decomposition import ( definitions as decomp_defs, definitions as decomposition, @@ -579,10 +580,10 @@ def test_decomposition_info_single_node( grid_file = experiment.grid gm = utils.run_grid_manager(grid_file, keep_skip_values=True, backend=backend) result = gm.decomposition_info - assert np.all(result.local_index(dim) == expected.local_index(dim)) - assert np.all(result.global_index(dim) == expected.global_index(dim)) - assert np.all(result.owner_mask(dim) == expected.owner_mask(dim)) - assert np.all(result.halo_levels(dim) == expected.halo_levels(dim)) + assert np.all(data_alloc.as_numpy(result.local_index(dim)) == expected.local_index(dim)) + assert np.all(data_alloc.as_numpy(result.global_index(dim)) == expected.global_index(dim)) + assert np.all(data_alloc.as_numpy(result.owner_mask(dim)) == expected.owner_mask(dim)) + assert np.all(data_alloc.as_numpy(result.halo_levels(dim)) == expected.halo_levels(dim)) @pytest.mark.parametrize("rank", (0, 1, 2, 3)) From dbde86bab795543e7bc614c167f5cdbac3fb0787 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 30 Jan 2026 14:46:27 +0100 Subject: [PATCH 152/240] More cupy/numpy --- .../icon4py/model/common/decomposition/halo.py | 18 ++++++++++++------ .../model/common/grid/grid_refinement.py | 4 ++-- .../grid/unit_tests/test_grid_manager.py | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 9c5abe4655..42df3d1642 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -355,8 +355,10 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: #: construct decomposition info decomp_info = defs.DecompositionInfo() cell_owner_mask = self._xp.isin(all_cells, owned_cells) - cell_halo_levels = defs.DecompositionFlag.UNDEFINED * self._xp.ones( - all_cells.size, dtype=int + cell_halo_levels = self._xp.full( + all_cells.size, + defs.DecompositionFlag.UNDEFINED.value, + dtype=gtx.int32, ) cell_halo_levels[cell_owner_mask] = defs.DecompositionFlag.OWNED cell_halo_levels[self._xp.isin(all_cells, first_halo_cells)] = ( @@ -375,8 +377,10 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: self.node_face_connectivity, ) vertex_second_level = self._xp.setdiff1d(vertex_on_halo_cells, vertex_on_owned_cells) - vertex_halo_levels = defs.DecompositionFlag.UNDEFINED * self._xp.ones( - all_vertices.size, dtype=int + vertex_halo_levels = self._xp.full( + all_vertices.size, + defs.DecompositionFlag.UNDEFINED.value, + dtype=gtx.int32, ) vertex_halo_levels[vertex_owner_mask] = defs.DecompositionFlag.OWNED vertex_halo_levels[ @@ -400,8 +404,10 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: self.edge_face_connectivity, ) - edge_halo_levels = defs.DecompositionFlag.UNDEFINED * self._xp.ones( - all_edges.shape, dtype=int + edge_halo_levels = self._xp.full( + all_edges.shape, + defs.DecompositionFlag.UNDEFINED.value, + dtype=gtx.int32, ) edge_halo_levels[edge_owner_mask] = defs.DecompositionFlag.OWNED # LEVEL_ONE edges are on an owned cell but are not owned: these are all edges on the cutting line that are not owned (by the convention) diff --git a/model/common/src/icon4py/model/common/grid/grid_refinement.py b/model/common/src/icon4py/model/common/grid/grid_refinement.py index 9f8fcf7316..c1d2945aca 100644 --- a/model/common/src/icon4py/model/common/grid/grid_refinement.py +++ b/model/common/src/icon4py/model/common/grid/grid_refinement.py @@ -241,12 +241,12 @@ def compute_domain_bounds( halo_region_2 = array_ns.where(halo_level_2 & not_lateral_boundary_2)[0] start_halo_2, end_halo_2 = ( - (array_ns.min(halo_region_2), array_ns.max(halo_region_2) + 1) + (array_ns.min(halo_region_2).item(), array_ns.max(halo_region_2).item() + 1) if halo_region_2.size > 0 else (refinement_ctrl.size, refinement_ctrl.size) ) if my_flag == h_grid.Zone.HALO.level: - start_index = array_ns.min(halo_region_1) if halo_region_1.size > 0 else start_halo_2 + start_index = array_ns.min(halo_region_1).item() if halo_region_1.size > 0 else start_halo_2 end_index = start_halo_2 else: start_index = start_halo_2 diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index 79a7d7598a..26ac5c1fdf 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -642,7 +642,7 @@ def test_local_connectivity( if neighbor_dim == dims.E2CDim else decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL ) - level_index = np.where(decomposition_info.halo_levels(dim) == last_halo_level) + level_index = np.where(data_alloc.as_numpy(decomposition_info.halo_levels(dim)) == last_halo_level.value) assert np.count_nonzero( (connectivity[level_index] == gridfile.GridFile.INVALID_INDEX) > 0 ), f"missing invalid index in {dim} - offset {field_offset}" From 05ba45f22683ac61081058b95856efdac43cca35 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 30 Jan 2026 14:47:30 +0100 Subject: [PATCH 153/240] TODO for distributed check in IconGrid constructor --- model/common/src/icon4py/model/common/grid/icon.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/model/common/src/icon4py/model/common/grid/icon.py b/model/common/src/icon4py/model/common/grid/icon.py index 15ac5c7b84..ced9301eec 100644 --- a/model/common/src/icon4py/model/common/grid/icon.py +++ b/model/common/src/icon4py/model/common/grid/icon.py @@ -191,7 +191,12 @@ def icon_grid( global_properties: GlobalGridParams, refinement_control: dict[gtx.Dimension, gtx.Field] | None = None, ) -> IconGrid: - distributed = config.num_cells < global_properties.num_cells + # TODO: What should we do about this. (The global) num_cells is not + # guaranteed to be set here when used through fortran. Should we: + # 1. Ignore distributed? + # 2. Compute num_cells with a reduction? + # 3. Use a ProcessProperties to detect it? + distributed = config.num_cells < global_properties.num_cells if global_properties.num_cells is not None else False limited_area_or_distributed = config.limited_area or distributed connectivities = { offset.value: base.construct_connectivity( From 0ae367cc1876897b47a0edf76f90dd04ee5f2018 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 30 Jan 2026 14:50:29 +0100 Subject: [PATCH 154/240] Formatting --- .../src/icon4py/model/common/decomposition/halo.py | 4 +++- .../src/icon4py/model/common/grid/grid_manager.py | 12 ++++++++---- .../src/icon4py/model/common/grid/grid_refinement.py | 4 +++- model/common/src/icon4py/model/common/grid/icon.py | 6 +++++- .../icon4py/model/common/utils/data_allocation.py | 10 +++++----- .../common/grid/unit_tests/test_grid_manager.py | 6 ++++-- 6 files changed, 28 insertions(+), 14 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 42df3d1642..a84c3387da 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -482,7 +482,9 @@ def __call__( self, adjacency_matrix: data_alloc.NDArray, num_partitions: int = 1 ) -> data_alloc.NDArray: """Dummy decomposer for single node: assigns all cells to rank = 0""" - return data_alloc.array_ns_from_array(adjacency_matrix).zeros(adjacency_matrix.shape[0], dtype=gtx.int32) # type: ignore [attr-defined] + return data_alloc.array_ns_from_array(adjacency_matrix).zeros( + adjacency_matrix.shape[0], dtype=gtx.int32 + ) # type: ignore [attr-defined] def get_halo_constructor( diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 5e9973c4d3..ef450b8f39 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -392,7 +392,9 @@ def _construct_decomposed_grid( """ xp = data_alloc.import_array_ns(allocator) ## FULL GRID PROPERTIES - cell_refinement = xp.asarray(self._reader.variable(gridfile.GridRefinementName.CONTROL_CELLS)) + cell_refinement = xp.asarray( + self._reader.variable(gridfile.GridRefinementName.CONTROL_CELLS) + ) global_size = self._read_full_grid_size() global_params = self._construct_global_params(allocator, global_size, geometry_type) limited_area = refinement.is_limited_area_grid(cell_refinement, array_ns=xp) @@ -523,9 +525,11 @@ def _get_index_field( apply_offset=True, array_ns: ModuleType = np, ): - return array_ns.asarray(self._reader.int_variable( - field, indices=indices, transpose=transpose, apply_transformation=apply_offset - )) + return array_ns.asarray( + self._reader.int_variable( + field, indices=indices, transpose=transpose, apply_transformation=apply_offset + ) + ) def _get_derived_connectivities( diff --git a/model/common/src/icon4py/model/common/grid/grid_refinement.py b/model/common/src/icon4py/model/common/grid/grid_refinement.py index c1d2945aca..215c69e767 100644 --- a/model/common/src/icon4py/model/common/grid/grid_refinement.py +++ b/model/common/src/icon4py/model/common/grid/grid_refinement.py @@ -246,7 +246,9 @@ def compute_domain_bounds( else (refinement_ctrl.size, refinement_ctrl.size) ) if my_flag == h_grid.Zone.HALO.level: - start_index = array_ns.min(halo_region_1).item() if halo_region_1.size > 0 else start_halo_2 + start_index = ( + array_ns.min(halo_region_1).item() if halo_region_1.size > 0 else start_halo_2 + ) end_index = start_halo_2 else: start_index = start_halo_2 diff --git a/model/common/src/icon4py/model/common/grid/icon.py b/model/common/src/icon4py/model/common/grid/icon.py index ced9301eec..1118a99066 100644 --- a/model/common/src/icon4py/model/common/grid/icon.py +++ b/model/common/src/icon4py/model/common/grid/icon.py @@ -196,7 +196,11 @@ def icon_grid( # 1. Ignore distributed? # 2. Compute num_cells with a reduction? # 3. Use a ProcessProperties to detect it? - distributed = config.num_cells < global_properties.num_cells if global_properties.num_cells is not None else False + distributed = ( + config.num_cells < global_properties.num_cells + if global_properties.num_cells is not None + else False + ) limited_area_or_distributed = config.limited_area or distributed connectivities = { offset.value: base.construct_connectivity( diff --git a/model/common/src/icon4py/model/common/utils/data_allocation.py b/model/common/src/icon4py/model/common/utils/data_allocation.py index ae2f58f330..f27cb0f2ce 100644 --- a/model/common/src/icon4py/model/common/utils/data_allocation.py +++ b/model/common/src/icon4py/model/common/utils/data_allocation.py @@ -79,12 +79,12 @@ def array_ns(try_cupy: bool) -> ModuleType: def array_ns_from_array(array: NDArray) -> ModuleType: - if isinstance(array, np.ndarray): - import numpy as xp - else: - import cupy as xp # type: ignore[import-not-found, no-redef] + if isinstance(array, np.ndarray): + import numpy as xp + else: + import cupy as xp # type: ignore[import-not-found, no-redef] - return xp + return xp def import_array_ns(allocator: gtx_allocators.FieldBufferAllocationUtil | None) -> ModuleType: diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index 26ac5c1fdf..ec8b59d728 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -18,7 +18,6 @@ import icon4py.model.common.grid.gridfile from icon4py.model.common import dimension as dims, model_backends -from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.common.decomposition import ( definitions as decomp_defs, definitions as decomposition, @@ -32,6 +31,7 @@ icon, vertical as v_grid, ) +from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import definitions, definitions as test_defs, grid_utils, test_utils @@ -642,7 +642,9 @@ def test_local_connectivity( if neighbor_dim == dims.E2CDim else decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL ) - level_index = np.where(data_alloc.as_numpy(decomposition_info.halo_levels(dim)) == last_halo_level.value) + level_index = np.where( + data_alloc.as_numpy(decomposition_info.halo_levels(dim)) == last_halo_level.value + ) assert np.count_nonzero( (connectivity[level_index] == gridfile.GridFile.INVALID_INDEX) > 0 ), f"missing invalid index in {dim} - offset {field_offset}" From 221d3640527094f5b5de0d607b5c6cf432ff9b3c Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 30 Jan 2026 14:50:55 +0100 Subject: [PATCH 155/240] Add name to TODO --- model/common/src/icon4py/model/common/grid/icon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/icon.py b/model/common/src/icon4py/model/common/grid/icon.py index 1118a99066..02c64b2e2b 100644 --- a/model/common/src/icon4py/model/common/grid/icon.py +++ b/model/common/src/icon4py/model/common/grid/icon.py @@ -191,8 +191,8 @@ def icon_grid( global_properties: GlobalGridParams, refinement_control: dict[gtx.Dimension, gtx.Field] | None = None, ) -> IconGrid: - # TODO: What should we do about this. (The global) num_cells is not - # guaranteed to be set here when used through fortran. Should we: + # TODO(msimberg): What should we do about this. (The global) num_cells is + # not guaranteed to be set here when used through fortran. Should we: # 1. Ignore distributed? # 2. Compute num_cells with a reduction? # 3. Use a ProcessProperties to detect it? From 6475eadc2088f8417bcf2958a06d5fea5529870f Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 30 Jan 2026 14:53:06 +0100 Subject: [PATCH 156/240] Linting --- .../icon4py/model/common/decomposition/definitions.py | 2 +- .../src/icon4py/model/common/decomposition/halo.py | 11 ++++++----- .../src/icon4py/model/common/utils/data_allocation.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index ecb32e9c5f..d04080f5d1 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -149,7 +149,7 @@ def get_horizontal_size(self) -> base.HorizontalGridSize: def get_halo_size(self, dim: gtx.Dimension, flag: DecompositionFlag) -> int: level_mask = self.halo_level_mask(dim, flag) - # TODO: Just do array_ns? + # TODO(msimberg): Just do array_ns? return data_alloc.array_ns_from_array(level_mask).count_nonzero(level_mask) def halo_levels(self, dim: gtx.Dimension) -> data_alloc.NDArray: diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index a84c3387da..f1321784d9 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -358,7 +358,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: cell_halo_levels = self._xp.full( all_cells.size, defs.DecompositionFlag.UNDEFINED.value, - dtype=gtx.int32, + dtype=gtx.int32, # type: ignore [attr-defined] ) cell_halo_levels[cell_owner_mask] = defs.DecompositionFlag.OWNED cell_halo_levels[self._xp.isin(all_cells, first_halo_cells)] = ( @@ -380,7 +380,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: vertex_halo_levels = self._xp.full( all_vertices.size, defs.DecompositionFlag.UNDEFINED.value, - dtype=gtx.int32, + dtype=gtx.int32, # type: ignore [attr-defined] ) vertex_halo_levels[vertex_owner_mask] = defs.DecompositionFlag.OWNED vertex_halo_levels[ @@ -407,7 +407,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: edge_halo_levels = self._xp.full( all_edges.shape, defs.DecompositionFlag.UNDEFINED.value, - dtype=gtx.int32, + dtype=gtx.int32, # type: ignore [attr-defined] ) edge_halo_levels[edge_owner_mask] = defs.DecompositionFlag.OWNED # LEVEL_ONE edges are on an owned cell but are not owned: these are all edges on the cutting line that are not owned (by the convention) @@ -483,8 +483,9 @@ def __call__( ) -> data_alloc.NDArray: """Dummy decomposer for single node: assigns all cells to rank = 0""" return data_alloc.array_ns_from_array(adjacency_matrix).zeros( - adjacency_matrix.shape[0], dtype=gtx.int32 - ) # type: ignore [attr-defined] + adjacency_matrix.shape[0], + dtype=gtx.int32, # type: ignore [attr-defined] + ) def get_halo_constructor( diff --git a/model/common/src/icon4py/model/common/utils/data_allocation.py b/model/common/src/icon4py/model/common/utils/data_allocation.py index f27cb0f2ce..d059153ee1 100644 --- a/model/common/src/icon4py/model/common/utils/data_allocation.py +++ b/model/common/src/icon4py/model/common/utils/data_allocation.py @@ -82,7 +82,7 @@ def array_ns_from_array(array: NDArray) -> ModuleType: if isinstance(array, np.ndarray): import numpy as xp else: - import cupy as xp # type: ignore[import-not-found, no-redef] + import cupy as xp # type: ignore[no-redef] return xp From 460be4bc7ed0455d2e3d3253a4af32f8f0728fab Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 30 Jan 2026 15:18:12 +0100 Subject: [PATCH 157/240] Unrevert data download changes --- .../icon4py/model/common/utils/device_utils.py | 3 +++ .../src/icon4py/model/testing/data_handling.py | 5 +++-- .../icon4py/model/testing/fixtures/datatest.py | 8 +------- .../src/icon4py/model/testing/stencil_tests.py | 17 ++++++++++------- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/model/common/src/icon4py/model/common/utils/device_utils.py b/model/common/src/icon4py/model/common/utils/device_utils.py index cacfc8eb64..360a53902a 100644 --- a/model/common/src/icon4py/model/common/utils/device_utils.py +++ b/model/common/src/icon4py/model/common/utils/device_utils.py @@ -37,6 +37,9 @@ def sync(allocator: gtx_typing.FieldBufferAllocationUtil | None = None) -> None: Note: this is and ad-hoc interface, maybe the function should get the device to sync for. """ + # Type annotation already describes that only these types are allowed, but mypy coverage is not great. + # The explicit assert avoids critical mistakes in using this function. + assert allocator is None or gtx_allocators.is_field_allocation_tool(allocator) if allocator is not None and is_cupy_device(allocator): cp.cuda.runtime.deviceSynchronize() diff --git a/model/testing/src/icon4py/model/testing/data_handling.py b/model/testing/src/icon4py/model/testing/data_handling.py index c490c8981b..95bc8b8369 100644 --- a/model/testing/src/icon4py/model/testing/data_handling.py +++ b/model/testing/src/icon4py/model/testing/data_handling.py @@ -9,10 +9,11 @@ import os import pathlib import tarfile -from pathlib import Path +from icon4py.model.testing import config, locking -def download_and_extract(uri: str, dst: Path, data_file: str = "downloaded.tar.gz") -> None: + +def download_and_extract(uri: str, dst: pathlib.Path, data_file: str = "downloaded.tar.gz") -> None: """ Download data archive from remote server. diff --git a/model/testing/src/icon4py/model/testing/fixtures/datatest.py b/model/testing/src/icon4py/model/testing/fixtures/datatest.py index 057235b1eb..0727c962ed 100644 --- a/model/testing/src/icon4py/model/testing/fixtures/datatest.py +++ b/model/testing/src/icon4py/model/testing/fixtures/datatest.py @@ -17,13 +17,7 @@ from icon4py.model.common import model_backends, model_options from icon4py.model.common.constants import RayleighType from icon4py.model.common.grid import base as base_grid -from icon4py.model.testing import ( - config, - data_handling as data, - datatest_utils as dt_utils, - definitions, - locking, -) +from icon4py.model.testing import data_handling as data, datatest_utils as dt_utils, definitions if TYPE_CHECKING: diff --git a/model/testing/src/icon4py/model/testing/stencil_tests.py b/model/testing/src/icon4py/model/testing/stencil_tests.py index f83798f029..ad1bf5e0ac 100644 --- a/model/testing/src/icon4py/model/testing/stencil_tests.py +++ b/model/testing/src/icon4py/model/testing/stencil_tests.py @@ -21,6 +21,7 @@ config as gtx_config, constructors, metrics as gtx_metrics, + named_collections as gtx_named_collections, typing as gtx_typing, ) @@ -34,13 +35,15 @@ def allocate_data( allocator: gtx_typing.FieldBufferAllocationUtil | None, - input_data: dict[str, gtx.Field | tuple[gtx.Field, ...]], -) -> dict[str, gtx.Field | tuple[gtx.Field, ...]]: - _allocate_field = constructors.as_field.partial(allocator=allocator) # type:ignore[attr-defined] # TODO(havogt): check why it doesn't understand the fluid_partial + input_data: dict[ + str, Any + ], # `Field`s or collection of `Field`s are re-allocated, the rest is passed through +) -> dict[str, Any]: + def _allocate_field(f: gtx.Field) -> gtx.Field: + return constructors.as_field(domain=f.domain, data=f.ndarray, allocator=allocator) + input_data = { - k: tuple(_allocate_field(domain=field.domain, data=field.ndarray) for field in v) - if isinstance(v, tuple) - else _allocate_field(domain=v.domain, data=v.ndarray) + k: gtx_named_collections.tree_map_named_collection(_allocate_field)(v) if not gtx.is_scalar_type(v) and k != "domain" else v for k, v in input_data.items() @@ -207,7 +210,7 @@ def _properly_allocated_input_data( self, input_data: dict[str, gtx.Field | tuple[gtx.Field, ...]], backend_like: model_backends.BackendLike, - ) -> dict[str, gtx.Field | tuple[gtx.Field, ...]]: + ) -> dict[str, Any]: # TODO(havogt): this is a workaround, # because in the `input_data` fixture provided by the user # it does not allocate for the correct device. From de480b5a9971cc8f8b4ab32e4c296e4b04b41b4a Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 14:02:33 +0100 Subject: [PATCH 158/240] Rename single_node to is_single_rank --- .../src/icon4py/model/common/decomposition/definitions.py | 2 +- model/common/src/icon4py/model/common/decomposition/halo.py | 2 +- model/common/src/icon4py/model/common/grid/grid_manager.py | 4 +++- .../tests/common/decomposition/unit_tests/test_definitions.py | 2 +- .../common/grid/mpi_tests/test_parallel_grid_refinement.py | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index d04080f5d1..590172d2a6 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -36,7 +36,7 @@ class ProcessProperties(Protocol): comm_name: str comm_size: int - def single_node(self) -> bool: + def is_single_rank(self) -> bool: return self.comm_size == 1 diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index f1321784d9..112d46bc54 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -509,7 +509,7 @@ def get_halo_constructor( Returns: a HaloConstructor suitable for the run_properties """ - if run_properties.single_node(): + if run_properties.is_single_rank(): return NoHalos( horizontal_size=full_grid_size, allocator=allocator, diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index ef450b8f39..19340cad27 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -98,7 +98,9 @@ def __call__( decomposer: halo.Decomposer = _single_node_decomposer, run_properties=_single_process_props, ): - if not run_properties.single_node() and isinstance(decomposer, halo.SingleNodeDecomposer): + if not run_properties.is_single_rank() and isinstance( + decomposer, halo.SingleNodeDecomposer + ): raise InvalidConfigError("Need a Decomposer for multi node run") if not self._reader: diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index e23f0ddd72..7cd0fd87fb 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -47,7 +47,7 @@ def test_decomposition_info_single_node_empty_halo( dim: gtx.Dimension, processor_props: definitions.ProcessProperties, ) -> None: - if not processor_props.single_node(): + if not processor_props.is_single_rank(): pytest.xfail() manager = grid_utils.run_grid_manager( diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py index d2047e2b38..4735c20334 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py @@ -43,7 +43,7 @@ def test_compute_domain_bounds( grid_savepoint: serialbox.IconGridSavepoint, processor_props: decomposition.ProcessProperties, ) -> None: - if processor_props.single_node() and experiment == definitions.Experiments.EXCLAIM_APE: + if processor_props.is_single_rank() and experiment == definitions.Experiments.EXCLAIM_APE: pytest.mark.xfail( "end index data for single node APE are all 0 - re- serialization should fix that (patch%cells%end_index vs patch%cells%end_idx)" ) From 59fa8a3853d193e9339d764c049b7a36f54bfc23 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 14:02:54 +0100 Subject: [PATCH 159/240] Rename *single_node test to *single_rank --- model/common/tests/common/grid/unit_tests/test_grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index ec8b59d728..f44b75b0ce 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -570,7 +570,7 @@ def test_limited_area_on_grid(grid_descriptor: definitions.GridDescription, expe @pytest.mark.datatest @pytest.mark.parametrize("dim", utils.horizontal_dims()) -def test_decomposition_info_single_node( +def test_decomposition_info_single_rank( dim: gtx.Dimension, experiment: definitions.Experiment, grid_savepoint: serialbox.IconGridSavepoint, From 44038d52860d883ff4e5966eeab49a10ae98dff7 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 14:07:19 +0100 Subject: [PATCH 160/240] Minor formatting change --- model/testing/src/icon4py/model/testing/serialbox.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/model/testing/src/icon4py/model/testing/serialbox.py b/model/testing/src/icon4py/model/testing/serialbox.py index b76b9abe5c..e59c1676f0 100644 --- a/model/testing/src/icon4py/model/testing/serialbox.py +++ b/model/testing/src/icon4py/model/testing/serialbox.py @@ -455,8 +455,7 @@ def _read_field_for_dim(field_name, read_func, dim: gtx.Dimension): ) def owner_mask(self, dim: gtx.Dimension): - mask = self._read_field_for_dim("owner_mask", self._read_bool, dim) - return np.squeeze(mask) + return np.squeeze(self._read_field_for_dim("owner_mask", self._read_bool, dim)) def global_index(self, dim: gtx.Dimension): return self._read_field_for_dim("glb_index", self._read_int32_shift1, dim) From cc1839be01a9480466b225a146dc4bfeb213781e Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 14:28:12 +0100 Subject: [PATCH 161/240] Remove unused variable --- model/common/tests/common/grid/unit_tests/test_grid_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index f44b75b0ce..aea9fa6266 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -66,7 +66,6 @@ ZERO_BASE = icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation() -vertical = v_grid.VerticalGridConfig(num_levels=80) # TODO @magdalena add test cases for hexagon vertices v2e2v From 4b265036f0b3f635303f90707ac289ae26250b01 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 14:31:39 +0100 Subject: [PATCH 162/240] Remove another unused variable --- model/common/tests/common/grid/unit_tests/test_grid_manager.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index aea9fa6266..e425b41bcf 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -65,9 +65,6 @@ MCH_CH_RO4B09_GLOBAL_NUM_CELLS = 83886080 -ZERO_BASE = icon4py.model.common.grid.gridfile.ToZeroBasedIndexTransformation() - - # TODO @magdalena add test cases for hexagon vertices v2e2v # v2e2v: grid,??? From 8c8876b08cde05b9917ce1f5e4edca4c8044f066 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 14:34:20 +0100 Subject: [PATCH 163/240] Add extra assert to test_props --- .../common/decomposition/mpi_tests/test_mpi_decomposition.py | 1 + 1 file changed, 1 insertion(+) diff --git a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py index 47aea8abac..b40b79df8e 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py @@ -57,6 +57,7 @@ def test_props(processor_props: definitions.ProcessProperties) -> None: assert processor_props.comm assert processor_props.comm_size > 1 + assert 0 <= processor_props.rank < processor_props.comm_size @pytest.mark.mpi(min_size=2) From 0c20e614ae52cff53e3f0c22fc5a92d3d6ff4393 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 14:38:41 +0100 Subject: [PATCH 164/240] Rename MissingConnectivity to MissingConnectivityError Also remove duplicate definition in different modules. --- .../common/src/icon4py/model/common/decomposition/halo.py | 2 +- model/common/src/icon4py/model/common/exceptions.py | 2 +- model/common/src/icon4py/model/common/grid/base.py | 8 ++------ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 112d46bc54..c15130b50f 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -145,7 +145,7 @@ def _connectivity(self, offset: gtx.FieldOffset | str) -> data_alloc.NDArray: conn_table = self._connectivities.get(_value(offset)) return conn_table except KeyError as err: - raise exceptions.MissingConnectivity( + raise exceptions.MissingConnectivityError( f"Connectivity for offset {offset} is not available" ) from err diff --git a/model/common/src/icon4py/model/common/exceptions.py b/model/common/src/icon4py/model/common/exceptions.py index 85f9d28bb2..75f79dc7f1 100644 --- a/model/common/src/icon4py/model/common/exceptions.py +++ b/model/common/src/icon4py/model/common/exceptions.py @@ -29,5 +29,5 @@ class IconGridError(RuntimeError): pass -class MissingConnectivity(ValueError): +class MissingConnectivityError(ValueError): pass diff --git a/model/common/src/icon4py/model/common/grid/base.py b/model/common/src/icon4py/model/common/grid/base.py index c96d1b3ff3..ea1f3604a3 100644 --- a/model/common/src/icon4py/model/common/grid/base.py +++ b/model/common/src/icon4py/model/common/grid/base.py @@ -15,7 +15,7 @@ import gt4py.next as gtx from gt4py.next import allocators as gtx_allocators, common as gtx_common -from icon4py.model.common import dimension as dims +from icon4py.model.common import dimension as dims, exceptions from icon4py.model.common.grid import horizontal as h_grid from icon4py.model.common.grid.gridfile import GridFile from icon4py.model.common.utils import data_allocation as data_alloc @@ -24,10 +24,6 @@ _log = logging.getLogger(__name__) -class MissingConnectivity(ValueError): - pass - - class GeometryType(enum.Enum): """Define geometries of the horizontal domain supported by the ICON grid. @@ -159,7 +155,7 @@ def get_connectivity(self, offset: str | gtx.FieldOffset) -> gtx_common.Neighbor if isinstance(offset, gtx.FieldOffset): offset = offset.value if offset not in self.connectivities: - raise MissingConnectivity( + raise exceptions.MissingConnectivityError( f"Missing connectivity for offset {offset} in grid {self.id}." ) connectivity = self.connectivities[offset] From 39bcbfd3ad81680a42e862f68ea8a8f7a5f7f30e Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 14:44:44 +0100 Subject: [PATCH 165/240] Update halo cut example --- model/common/src/icon4py/model/common/decomposition/halo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index c15130b50f..b727c9c6b2 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -286,8 +286,8 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: Cells (c0, c1, c2) are the 1. HALO LEVEL: these are cells that are neighbors of an owned cell Cells (c3, c4. c5) are the 2. HALO LEVEL: cells that "close" the hexagon of a vertex on the cutting line and do not share an edge with an owned cell (that is are not LEVEL 1 cells) this is _not_ the same as the neighboring cells of LEVEL 1 cells, and the definition might be different from ICON. - In the above picture if the cut was along a corner (e0 -> e1-> e8) then cy is part of the 2. LEVEL because it is part of the hexagon on v2, but it is not - in c2e2c(c2e2c(c4)) + In the above picture if the cut was along a corner (e0 -> e1 -> e7) then cz is part of the 2. LEVEL because it is part of the hexagon on v2, but it is not + in c2e2c2e2c(c1). Note that this definition of 1. and 2. line differs from the definition of boundary line counting used in [grid refinement](grid_refinement.py), in terms of "distance" to the cutting line all halo cells have a distance of 1. @@ -308,7 +308,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: For edges a similar pattern is used as for the vertices. - 1. HALO LEVEL: edges that are on owned cells but not owned themselves (these are edges that share 2 vertices with a owned cell). In terms of ownership the same convention is applied as for the vertices: (e0, e1, e2, e3) are in the HALO LEVEL 1 of rank 0, and are owned by rank 1 - - 2. HALO LEVEL: edges that share exactly one vertex with an owned cell. The definition via vertices is important: TODO (halungge): EXAMPLE??? + - 2. HALO LEVEL: edges that share exactly one vertex with an owned cell. The definition via vertices is important: see example above with a cut along e0 -> e1 -> e7. For rank 0 these are the edges (e4, e5, e6, e7, e8, e9, e10) in the example above. - 3. HALO LEVEL: In **ICON4Py ONLY**, edges that "close" the halo cells and share exactly 2 vertices with a HALO LEVEL 2 cell, but none with From a69c3c7da608864562419161161c42f8e33f8b89 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 14:47:51 +0100 Subject: [PATCH 166/240] Reformat halo construction docstring --- .../model/common/decomposition/halo.py | 84 ++++++++++++------- 1 file changed, 55 insertions(+), 29 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index b727c9c6b2..dc8df90229 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -252,11 +252,13 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: Args: face_to_rank: a mapping of cells to a rank - The DecompositionInfo object is constructed for all horizontal dimension starting from the - cell distribution. - Edges and vertices are then handled through their connectivity to the distributed cells. + The DecompositionInfo object is constructed for all horizontal + dimension starting from the cell distribution. Edges and vertices + are then handled through their connectivity to the distributed + cells. - This constructs a halo similar to ICON which consists of **exactly** 2 cell-halo lines + This constructs a halo similar to ICON which consists of + **exactly** 2 cell-halo lines. | /| /| /| | / | / | / | @@ -282,38 +284,62 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: Cells: - The "numbered" cells and edges are relevant for the halo construction from the point of view of rank 0 - Cells (c0, c1, c2) are the 1. HALO LEVEL: these are cells that are neighbors of an owned cell - Cells (c3, c4. c5) are the 2. HALO LEVEL: cells that "close" the hexagon of a vertex on the cutting line and do not share an edge with an owned cell (that is are not LEVEL 1 cells) - this is _not_ the same as the neighboring cells of LEVEL 1 cells, and the definition might be different from ICON. - In the above picture if the cut was along a corner (e0 -> e1 -> e7) then cz is part of the 2. LEVEL because it is part of the hexagon on v2, but it is not - in c2e2c2e2c(c1). - Note that this definition of 1. and 2. line differs from the definition of boundary line counting used in [grid refinement](grid_refinement.py), in terms - of "distance" to the cutting line all halo cells have a distance of 1. + - The "numbered" cells and edges are relevant for the halo + construction from the point of view of rank 0. + - Cells (c0, c1, c2) are the 1. HALO LEVEL: these are cells that + are neighbors of an owned cell. + - Cells (c3, c4. c5) are the 2. HALO LEVEL: cells that "close" the + hexagon of a vertex on the cutting line and do not share an edge + with an owned cell (that is are not LEVEL 1 cells). This is + _not_ the same as the neighboring cells of LEVEL 1 cells, and the + definition might be different from ICON. In the above picture if + the cut was along a corner (e0 -> e1 -> e7) then cz is part of + the 2. LEVEL because it is part of the hexagon on v2, but it is + not in c2e2c2e2c(c1). + + Note that this definition of 1. and 2. line differs from the + definition of boundary line counting used in [grid + refinement](grid_refinement.py), in terms of "distance" to the + cutting line all halo cells have a distance of 1. Vertices: - - 1. HALO LEVEL: are vertices on the cutting line that are not owned, or in a different wording: all vertices of owned cells that are not - owned. - In ICON every element in an array needs **exactly one owner** (otherwise there would be duplicates and double-counting). - For elements on the cutting line (vertices and edges) there is no clear indication which rank should own it, - ICON uses the rank with the higher rank value (see (_update_owner_mask_by_max_rank_convention)) - In the example above (v0, v1, v2, v3) are in the 1. HALO LEVEL of rank 0 and owned by rank 1. - As a consequence, there are ranks that have no 1. HALO LEVEL vertices. - - 2. HALO LEVEL: are vertices that are on HALO LEVEL cells, but not on owned. For rank 0 these are (v4, v5, v6, v7) + + - 1. HALO LEVEL: are vertices on the cutting line that are not + owned, or in a different wording: all vertices of owned cells + that are not owned. + - In ICON every element in an array needs **exactly one owner** + (otherwise there would be duplicates and double-counting). For + elements on the cutting line (vertices and edges) there is no + clear indication which rank should own it, ICON uses the rank + with the higher rank value (see + (_update_owner_mask_by_max_rank_convention)) In the example above + (v0, v1, v2, v3) are in the 1. HALO LEVEL of rank 0 and owned by + rank 1. As a consequence, there are ranks that have no 1. HALO + LEVEL vertices. + - 2. HALO LEVEL: are vertices that are on HALO LEVEL cells, but not + on owned. For rank 0 these are (v4, v5, v6, v7). Edges: - For edges a similar pattern is used as for the vertices. - - 1. HALO LEVEL: edges that are on owned cells but not owned themselves (these are edges that share 2 vertices with a owned cell). - In terms of ownership the same convention is applied as for the vertices: (e0, e1, e2, e3) are in the HALO LEVEL 1 of rank 0, and are owned by rank 1 - - 2. HALO LEVEL: edges that share exactly one vertex with an owned cell. The definition via vertices is important: see example above with a cut along e0 -> e1 -> e7. - For rank 0 these are the edges (e4, e5, e6, e7, e8, e9, e10) in the example above. - - 3. HALO LEVEL: - In **ICON4Py ONLY**, edges that "close" the halo cells and share exactly 2 vertices with a HALO LEVEL 2 cell, but none with - an owned cell. These edges are **not** included in the halo in ICON. These are (e11, e12, e13) for rank 0 in the example above. - This is the HALO LINE which makes the C2E connectivity complete (= without skip value) for a distributed setup. + + - For edges a similar pattern is used as for the vertices. + - 1. HALO LEVEL: edges that are on owned cells but not owned + themselves (these are edges that share 2 vertices with a owned + cell). In terms of ownership the same convention is applied as + for the vertices: (e0, e1, e2, e3) are in the HALO LEVEL 1 of + rank 0, and are owned by rank 1. + - 2. HALO LEVEL: edges that share exactly one vertex with an owned + cell. The definition via vertices is important: see example above + with a cut along e0 -> e1 -> e7. For rank 0 these are the edges + (e4, e5, e6, e7, e8, e9, e10) in the example above. + - 3. HALO LEVEL: In **ICON4Py ONLY**, edges that "close" the halo + cells and share exactly 2 vertices with a HALO LEVEL 2 cell, but + none with an owned cell. These edges are **not** included in the + halo in ICON. These are (e11, e12, e13) for rank 0 in the example + above. This is the HALO LINE which makes the C2E connectivity + complete (= without skip value) for a distributed setup. """ self._validate_mapping(face_to_rank) From df3bb06db2ff4c2894627cb12aafe07cde9c2aeb Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 14:54:41 +0100 Subject: [PATCH 167/240] Remove unnecessary numpy.unique from halo construction --- model/common/src/icon4py/model/common/decomposition/halo.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index dc8df90229..3eed5318a1 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -167,9 +167,7 @@ def next_halo_line( cells_so_far = self._xp.hstack((depot, cells)) if depot is not None else cells - next_halo_cells = self._xp.setdiff1d( - self._xp.unique(cell_neighbors), cells_so_far, assume_unique=True - ) + next_halo_cells = self._xp.setdiff1d(cell_neighbors, cells_so_far, assume_unique=True) return next_halo_cells def _find_neighbors( From 05e992caab3ff80d258e413b7d2331a2d46a7413 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 14:58:31 +0100 Subject: [PATCH 168/240] Minor renamings in grid_manager.py --- .../src/icon4py/model/common/grid/grid_manager.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 19340cad27..4e5393cc0a 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -402,7 +402,7 @@ def _construct_decomposed_grid( limited_area = refinement.is_limited_area_grid(cell_refinement, array_ns=xp) cell_to_cell_neighbors = self._get_index_field(gridfile.ConnectivityName.C2E2C, array_ns=xp) - neighbor_tables = { + global_neighbor_tables = { dims.C2E2C: cell_to_cell_neighbors, dims.C2E: self._get_index_field(gridfile.ConnectivityName.C2E, array_ns=xp), dims.E2C: self._get_index_field(gridfile.ConnectivityName.E2C, array_ns=xp), @@ -417,20 +417,17 @@ def _construct_decomposed_grid( # HALO CONSTRUCTION # TODO(halungge): reduce the set of neighbor tables used in the halo construction # TODO(halungge): figure out where to do the host to device copies (xp.asarray...) - neighbor_tables_for_halo_construction = neighbor_tables halo_constructor = halo.get_halo_constructor( run_properties=run_properties, full_grid_size=global_size, - connectivities=neighbor_tables_for_halo_construction, + connectivities=global_neighbor_tables, allocator=allocator, ) self._decomposition_info = halo_constructor(cells_to_rank_mapping) distributed_size = self._decomposition_info.get_horizontal_size() - neighbor_tables = self._get_local_connectivities( - neighbor_tables_for_halo_construction, array_ns=xp - ) + neighbor_tables = self._get_local_connectivities(global_neighbor_tables, array_ns=xp) # COMPUTE remaining derived connectivities neighbor_tables.update(_get_derived_connectivities(neighbor_tables, array_ns=xp)) @@ -466,7 +463,7 @@ def _construct_decomposed_grid( def _get_local_connectivities( self, - neighbor_tables_for_halo_construction: dict[gtx.FieldOffset, data_alloc.NDArray], + neighbor_tables_global: dict[gtx.FieldOffset, data_alloc.NDArray], array_ns, ) -> dict[gtx.FieldOffset, data_alloc.NDArray]: global_to_local = functools.partial(halo.global_to_local, array_ns=array_ns) @@ -476,10 +473,10 @@ def _get_local_connectivities( self._decomposition_info.global_index(k.source), v[self._decomposition_info.global_index(k.target[0])], ) - for k, v in neighbor_tables_for_halo_construction.items() + for k, v in neighbor_tables_global.items() } else: - return neighbor_tables_for_halo_construction + return neighbor_tables_global def _construct_global_params( self, From 0c6942adfdb36e9ddfcff21cca4c7b2e4b74392e Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 15:06:11 +0100 Subject: [PATCH 169/240] Simplify some returns in halo.py --- .../src/icon4py/model/common/decomposition/halo.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 3eed5318a1..78bcead237 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -142,8 +142,7 @@ def _assert_all_neighbor_tables(self) -> None: def _connectivity(self, offset: gtx.FieldOffset | str) -> data_alloc.NDArray: try: - conn_table = self._connectivities.get(_value(offset)) - return conn_table + return self._connectivities.get(_value(offset)) except KeyError as err: raise exceptions.MissingConnectivityError( f"Connectivity for offset {offset} is not available" @@ -167,8 +166,7 @@ def next_halo_line( cells_so_far = self._xp.hstack((depot, cells)) if depot is not None else cells - next_halo_cells = self._xp.setdiff1d(cell_neighbors, cells_so_far, assume_unique=True) - return next_halo_cells + return self._xp.setdiff1d(cell_neighbors, cells_so_far, assume_unique=True) def _find_neighbors( self, source_indices: data_alloc.NDArray, connectivity: data_alloc.NDArray @@ -176,8 +174,7 @@ def _find_neighbors( """Get a flattened list of all (unique) neighbors to a given global index list""" neighbors = connectivity[source_indices, :] shp = neighbors.shape - unique_neighbors = self._xp.unique(neighbors.reshape(shp[0] * shp[1])) - return unique_neighbors + return self._xp.unique(neighbors.reshape(shp[0] * shp[1])) def _find_cell_neighbors(self, cells: data_alloc.NDArray) -> data_alloc.NDArray: """Find all neighboring cells of a list of cells.""" From 34a8eb2fd27f79edfb7f4eec5b6ddc72a3b20861 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 15:07:10 +0100 Subject: [PATCH 170/240] Simplify _find_neighbors --- model/common/src/icon4py/model/common/decomposition/halo.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 78bcead237..0effa5cec0 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -172,9 +172,7 @@ def _find_neighbors( self, source_indices: data_alloc.NDArray, connectivity: data_alloc.NDArray ) -> data_alloc.NDArray: """Get a flattened list of all (unique) neighbors to a given global index list""" - neighbors = connectivity[source_indices, :] - shp = neighbors.shape - return self._xp.unique(neighbors.reshape(shp[0] * shp[1])) + return self._xp.unique(connectivity[source_indices, :].flatten()) def _find_cell_neighbors(self, cells: data_alloc.NDArray) -> data_alloc.NDArray: """Find all neighboring cells of a list of cells.""" From 5c11cb41ed3d99939511af63b8014ce23e123963 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 15:37:39 +0100 Subject: [PATCH 171/240] Remove some trivial helper functions in halo.py --- .../model/common/decomposition/halo.py | 44 +++++-------------- 1 file changed, 10 insertions(+), 34 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 0effa5cec0..5991f1988a 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -84,33 +84,9 @@ def __init__( self._connectivities = {_value(k): v for k, v in connectivities.items()} self._assert_all_neighbor_tables() - @property - def face_face_connectivity(self) -> data_alloc.NDArray: - return self._connectivity(dims.C2E2C) - - @property - def edge_face_connectivity(self) -> data_alloc.NDArray: - return self._connectivity(dims.E2C) - - @property - def face_edge_connectivity(self) -> data_alloc.NDArray: - return self._connectivity(dims.C2E) - - @property - def node_edge_connectivity(self) -> data_alloc.NDArray: - return self._connectivity(dims.V2E) - - @property - def node_face_connectivity(self) -> data_alloc.NDArray: - return self._connectivity(dims.V2C) - - @property - def face_node_connectivity(self) -> data_alloc.NDArray: - return self._connectivity(dims.C2V) - def _validate_mapping(self, face_to_rank_mapping: data_alloc.NDArray) -> None: # validate the distribution mapping: - num_cells = self.face_face_connectivity.shape[0] + num_cells = self._connectivity(dims.C2E2C).shape[0] expected_shape = (num_cells,) if not face_to_rank_mapping.shape == expected_shape: raise exceptions.ValidationError( @@ -169,30 +145,30 @@ def next_halo_line( return self._xp.setdiff1d(cell_neighbors, cells_so_far, assume_unique=True) def _find_neighbors( - self, source_indices: data_alloc.NDArray, connectivity: data_alloc.NDArray + self, source_indices: data_alloc.NDArray, offset: gtx.FieldOffset | str ) -> data_alloc.NDArray: """Get a flattened list of all (unique) neighbors to a given global index list""" - return self._xp.unique(connectivity[source_indices, :].flatten()) + return self._xp.unique(self._connectivity(offset)[source_indices, :].flatten()) def _find_cell_neighbors(self, cells: data_alloc.NDArray) -> data_alloc.NDArray: """Find all neighboring cells of a list of cells.""" - return self._find_neighbors(cells, connectivity=self.face_face_connectivity) + return self._find_neighbors(cells, dims.C2E2C2E2C) def find_edge_neighbors_for_cells(self, cell_line: data_alloc.NDArray) -> data_alloc.NDArray: - return self._find_neighbors(cell_line, connectivity=self.face_edge_connectivity) + return self._find_neighbors(cell_line, dims.C2E) def find_edge_neighbors_for_vertices( self, vertex_line: data_alloc.NDArray ) -> data_alloc.NDArray: - return self._find_neighbors(vertex_line, connectivity=self.node_edge_connectivity) + return self._find_neighbors(vertex_line, dims.V2E) def find_vertex_neighbors_for_cells(self, cell_line: data_alloc.NDArray) -> data_alloc.NDArray: - return self._find_neighbors(cell_line, connectivity=self.face_node_connectivity) + return self._find_neighbors(cell_line, dims.C2V) def find_cell_neighbors_for_vertices( self, vertex_line: data_alloc.NDArray ) -> data_alloc.NDArray: - return self._find_neighbors(vertex_line, connectivity=self.node_face_connectivity) + return self._find_neighbors(vertex_line, dims.V2C) def owned_cells(self, face_to_rank: data_alloc.NDArray) -> data_alloc.NDArray: """Returns the full-grid indices of the cells owned by this rank""" @@ -393,7 +369,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: vertex_owner_mask, all_vertices, vertex_on_cutting_line, - self.node_face_connectivity, + self._connectivity(dims.V2C), ) vertex_second_level = self._xp.setdiff1d(vertex_on_halo_cells, vertex_on_owned_cells) vertex_halo_levels = self._xp.full( @@ -420,7 +396,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: edge_owner_mask, all_edges, edges_on_cutting_line, - self.edge_face_connectivity, + self._connectivity(dims.E2C), ) edge_halo_levels = self._xp.full( From 3a67b94d86a7952e6ace9368a88f0685dfcae8d4 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 15:41:09 +0100 Subject: [PATCH 172/240] Clean up and fix typo in docstring in decomposition utils --- .../tests/common/decomposition/utils.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/model/common/tests/common/decomposition/utils.py b/model/common/tests/common/decomposition/utils.py index f523caeee7..6a45731873 100644 --- a/model/common/tests/common/decomposition/utils.py +++ b/model/common/tests/common/decomposition/utils.py @@ -16,20 +16,24 @@ """ -TESTDATA using the [SimpleGrid](../../../src/icon4py/model/common/grid/simple.py) -The distribution maps all of the 18 cells of the simple grid to ranks 0..3 +TESTDATA using the +[SimpleGrid](../../../src/icon4py/model/common/grid/simple.py) The distribution +maps all of the 18 cells of the simple grid to ranks 0..3. -the dictionaries contain the mapping from rank to global (in the simple grid) index of the dimension: -_CELL_OWN: rank -> owned cells, essentially the inversion of the SIMPLE_DISTRIBUTION -_EDGE_OWN: rank -> owned edges -_VERTEX_OWN: rank -> owned vertices +The dictionaries contain the mapping from rank to global (in the simple grid) +index of the dimension: +- _CELL_OWN: rank -> owned cells, essentially the inversion of the + SIMPLE_DISTRIBUTION +- _EDGE_OWN: rank -> owned edges + - _VERTEX_OWN: rank -> owned vertices -the decision as to whether a "secondary" dimension (edge, vertices) is owned by a rank are made according to the -rules and conventions described in (../../../src/icon4py/model/common/decomposition/halo.py) +The decision as to whether a "secondary" dimension (edge, vertices) is owned by +a rank are made according to the rules and conventions described in +(../../../src/icon4py/model/common/decomposition/halo.py). _CELL_FIRST_HALO_LINE: -_CELL_SECON_HALO_LINE: +_CELL_SECOND_HALO_LINE: _EDGE_FIRST_HALO_LINE: _EDGE_SECOND_HALO_LINE: _VERTEX_FIRST_HALO_LINE: From 0fc37b8e598114e86e6233f0671849d7dc50065b Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 15:58:39 +0100 Subject: [PATCH 173/240] Add fixture for global grids (not limited area grids) --- model/common/tests/common/grid/fixtures.py | 1 + .../mpi_tests/test_parallel_grid_manager.py | 57 ++++++++++--------- .../model/testing/fixtures/datatest.py | 17 ++++++ 3 files changed, 47 insertions(+), 28 deletions(-) diff --git a/model/common/tests/common/grid/fixtures.py b/model/common/tests/common/grid/fixtures.py index bf1ce2b54f..8682291e0a 100644 --- a/model/common/tests/common/grid/fixtures.py +++ b/model/common/tests/common/grid/fixtures.py @@ -13,6 +13,7 @@ download_ser_data, experiment, flat_height, + global_grid_descriptor, grid_savepoint, htop_moist_proc, icon_grid, diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 40341ba686..82feaf7d89 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -34,7 +34,7 @@ from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import definitions as test_defs, grid_utils, test_utils -from ..fixtures import backend, processor_props +from ..fixtures import backend, global_grid_descriptor, processor_props from . import utils @@ -50,8 +50,11 @@ @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi(min_size=2) -def test_grid_manager_validate_decomposer(processor_props: decomp_defs.ProcessProperties) -> None: - file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) +def test_grid_manager_validate_decomposer( + processor_props: decomp_defs.ProcessProperties, + global_grid_descriptor: test_defs.GridDescription, +) -> None: + file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) manager = gm.GridManager(file, utils.NUM_LEVELS, gridfile.ToZeroBasedIndexTransformation()) with pytest.raises(exceptions.InvalidConfigError) as e: manager( @@ -75,11 +78,13 @@ def _get_neighbor_tables(grid: base.Grid) -> dict: @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_fields_distribute_and_gather( - processor_props: decomp_defs.ProcessProperties, caplog: Any + processor_props: decomp_defs.ProcessProperties, + global_grid_descriptor: test_defs.GridDescription, + caplog: Any, ) -> None: caplog.set_level(logging.INFO) print(f"myrank - {processor_props.rank}: running with processor_props = {processor_props}") - file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) + file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid global_cell_area = single_node.geometry_fields[gridfile.GeometryName.CELL_AREA] @@ -206,13 +211,12 @@ def assert_gathered_field_against_global( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -@pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) def test_halo_neighbor_access_c2e( processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, - grid: test_defs.GridDescription, + global_grid_descriptor: test_defs.GridDescription, ) -> None: - file = grid_utils.resolve_full_grid_file_name(grid) + file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid @@ -298,13 +302,12 @@ def test_halo_neighbor_access_c2e( @pytest.mark.embedded_remap_error @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -@pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) def test_halo_neighbor_access_e2c2v( processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, - grid: test_defs.GridDescription, + global_grid_descriptor: test_defs.GridDescription, ) -> None: - file = grid_utils.resolve_full_grid_file_name(grid) + file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( @@ -391,13 +394,12 @@ def test_halo_neighbor_access_e2c2v( @pytest.mark.embedded_remap_error @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -@pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) def test_halo_neighbor_access_e2c( processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, - grid: test_defs.GridDescription, + global_grid_descriptor: test_defs.GridDescription, ) -> None: - file = grid_utils.resolve_full_grid_file_name(grid) + file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( @@ -482,14 +484,13 @@ def test_halo_neighbor_access_e2c( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -@pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) def test_halo_neighbor_access_e2v( processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, - grid: test_defs.GridDescription, + global_grid_descriptor: test_defs.GridDescription, ) -> None: print(f"running on {processor_props.comm}") - file = grid_utils.resolve_full_grid_file_name(grid) + file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid single_node_geometry = geometry.GridGeometry( @@ -560,13 +561,12 @@ def test_halo_neighbor_access_e2v( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -@pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) def test_halo_neighbor_access_v2e( processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, - grid: test_defs.GridDescription, + global_grid_descriptor: test_defs.GridDescription, ) -> None: - file = grid_utils.resolve_full_grid_file_name(grid) + file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) print(f"running on {processor_props.comm}") single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid @@ -659,13 +659,12 @@ def test_halo_neighbor_access_v2e( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -@pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) def test_halo_neighbor_access_c2e2c( processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, - grid: test_defs.GridDescription, + global_grid_descriptor: test_defs.GridDescription, ) -> None: - file = grid_utils.resolve_full_grid_file_name(grid) + file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) center_weight = 0.3 xp = data_alloc.import_array_ns(allocator=backend) start_zone = h_grid.cell_domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2) @@ -744,9 +743,11 @@ def test_halo_neighbor_access_c2e2c( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_halo_neighbor_access_v2c( - processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend + processor_props: decomp_defs.ProcessProperties, + global_grid_descriptor: test_defs.GridDescription, + backend: gtx_typing.Backend, ) -> None: - file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) + file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) print(f"running on {processor_props.comm}") single_node = utils.run_grid_manager_for_singlenode(file) single_node_grid = single_node.grid @@ -882,11 +883,11 @@ def test_halo_neighbor_access_v2c( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -@pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) def test_validate_skip_values_in_distributed_connectivities( - processor_props: decomp_defs.ProcessProperties, grid: test_defs.GridDescription + processor_props: decomp_defs.ProcessProperties, + global_grid_descriptor: test_defs.GridDescription, ) -> None: - file = grid_utils.resolve_full_grid_file_name(grid) + file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, diff --git a/model/testing/src/icon4py/model/testing/fixtures/datatest.py b/model/testing/src/icon4py/model/testing/fixtures/datatest.py index 0727c962ed..d0d023a21e 100644 --- a/model/testing/src/icon4py/model/testing/fixtures/datatest.py +++ b/model/testing/src/icon4py/model/testing/fixtures/datatest.py @@ -81,6 +81,23 @@ def cpu_allocator() -> gtx_typing.FieldBufferAllocationUtil: return model_backends.get_allocator(None) +@pytest.fixture( + params=[ + definitions.Grids.R02B04_GLOBAL, + definitions.Grids.TORUS_50000x5000, + ], + ids=lambda r: r.name, +) +def global_grid_descriptor(request: pytest.FixtureRequest) -> definitions.GridDescription: + """ + Return a global grid descriptor. + + "Global" in this context means "not limited area", i.e. a full icosahedral + grid or a torus grid, both with full connectivity and no boundary layers. + """ + return request.param + + @pytest.fixture( params=[ definitions.Experiments.MCH_CH_R04B09, From f895bb3daa9d0325fd874f97180893f98e3c51a6 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 16:11:47 +0100 Subject: [PATCH 174/240] Minor bugfix for LAM/distributed configuration --- model/common/src/icon4py/model/common/grid/icon.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/icon.py b/model/common/src/icon4py/model/common/grid/icon.py index 02c64b2e2b..3480958a6b 100644 --- a/model/common/src/icon4py/model/common/grid/icon.py +++ b/model/common/src/icon4py/model/common/grid/icon.py @@ -208,8 +208,10 @@ def icon_grid( data_alloc.import_array_ns(allocator).asarray(table), skip_value=-1 if _has_skip_values(offset, limited_area_or_distributed) else None, allocator=allocator, - replace_skip_values=_should_replace_skip_values( - offset, config.keep_skip_values, config.limited_area + replace_skip_values=( + _should_replace_skip_values( + offset, config.keep_skip_values, limited_area_or_distributed + ), ), ) for offset, table in neighbor_tables.items() From 412c5546c0c5b27bd2ee3492486f62d06d873cb8 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 16:14:55 +0100 Subject: [PATCH 175/240] Rename SimpleMetisDecomposer to MetisDecomposer --- .../icon4py/model/common/decomposition/halo.py | 2 +- .../mpi_tests/test_parallel_halo.py | 2 +- .../mpi_tests/test_parallel_grid_manager.py | 18 +++++++++--------- .../grid/mpi_tests/test_parallel_icon.py | 4 ++-- .../grid/unit_tests/test_grid_manager.py | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 5991f1988a..b25fe12157 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -440,7 +440,7 @@ def __call__( ... -class SimpleMetisDecomposer(Decomposer): +class MetisDecomposer(Decomposer): """ A simple decomposer using METIS for partitioning a grid topology. diff --git a/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py index 16e62d76ad..82b4b0c6de 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py @@ -90,7 +90,7 @@ def test_element_ownership_is_unique( def decompose(grid: base_grid.Grid, processor_props): - partitioner = halo.SimpleMetisDecomposer() + partitioner = halo.MetisDecomposer() labels = partitioner( grid.connectivities[dims.C2E2C].asnumpy(), n_part=processor_props.comm_size ) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 82feaf7d89..b696eac696 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -94,7 +94,7 @@ def test_fields_distribute_and_gather( multinode = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.SimpleMetisDecomposer(), + decomposer=halo.MetisDecomposer(), ) decomposition_info = multinode.decomposition_info @@ -252,7 +252,7 @@ def test_halo_neighbor_access_c2e( multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.SimpleMetisDecomposer(), + decomposer=halo.MetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid extra_geometry_fields = multinode_grid_manager.geometry_fields @@ -326,7 +326,7 @@ def test_halo_neighbor_access_e2c2v( multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.SimpleMetisDecomposer(), + decomposer=halo.MetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid extra_geometry_fields = multinode_grid_manager.geometry_fields @@ -418,7 +418,7 @@ def test_halo_neighbor_access_e2c( multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.SimpleMetisDecomposer(), + decomposer=halo.MetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid extra_geometry_fields = multinode_grid_manager.geometry_fields @@ -509,7 +509,7 @@ def test_halo_neighbor_access_e2v( multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.SimpleMetisDecomposer(), + decomposer=halo.MetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid decomposition_info = multinode_grid_manager.decomposition_info @@ -610,7 +610,7 @@ def test_halo_neighbor_access_v2e( multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.SimpleMetisDecomposer(), + decomposer=halo.MetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid decomposition_info = multinode_grid_manager.decomposition_info @@ -694,7 +694,7 @@ def test_halo_neighbor_access_c2e2c( multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.SimpleMetisDecomposer(), + decomposer=halo.MetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid decomposition_info = multinode_grid_manager.decomposition_info @@ -794,7 +794,7 @@ def test_halo_neighbor_access_v2c( multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.SimpleMetisDecomposer(), + decomposer=halo.MetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid decomposition_info = multinode_grid_manager.decomposition_info @@ -891,7 +891,7 @@ def test_validate_skip_values_in_distributed_connectivities( multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.SimpleMetisDecomposer(), + decomposer=halo.MetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid for k, c in distributed_grid.connectivities.items(): diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py index 33d9310e65..3cd2b75ba2 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py @@ -22,7 +22,7 @@ definitions as decomp_defs, definitions as decomposition, ) -from icon4py.model.common.decomposition.halo import SimpleMetisDecomposer +from icon4py.model.common.decomposition.halo import MetisDecomposer from icon4py.model.common.grid import base as base_grid, gridfile, horizontal as h_grid, icon from icon4py.model.testing import definitions as test_defs, grid_utils, parallel_helpers @@ -194,7 +194,7 @@ def test_skip_values_on_distributed_grid( ) -> None: file = grid_utils.resolve_full_grid_file_name(grid) grid_manager = parallel_utils.run_gridmananger_for_multinode( - file, processor_props, decomposer=SimpleMetisDecomposer() + file, processor_props, decomposer=MetisDecomposer() ) mesh = grid_manager.grid assert not np.any(mesh.get_connectivity(dims.C2V).asnumpy() == gridfile.GridFile.INVALID_INDEX) diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index e425b41bcf..838fff3969 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -595,7 +595,7 @@ def test_local_connectivity( ) -> None: processor_props = decomp_utils.DummyProps(rank=rank) caplog.set_level(logging.INFO) # type: ignore [attr-defined] - partitioner = halo.SimpleMetisDecomposer() + partitioner = halo.MetisDecomposer() allocator = model_backends.get_allocator(backend_like) file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) manager = gm.GridManager(num_levels=10, grid_file=file) From d33c0da9d3822679d5da8794cfed1354dc01650c Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 16:22:22 +0100 Subject: [PATCH 176/240] Minor reformatting and more torus tests in test_gridfile.py --- .../common/grid/unit_tests/test_gridfile.py | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/model/common/tests/common/grid/unit_tests/test_gridfile.py b/model/common/tests/common/grid/unit_tests/test_gridfile.py index 800c1daad3..3448e3d368 100644 --- a/model/common/tests/common/grid/unit_tests/test_gridfile.py +++ b/model/common/tests/common/grid/unit_tests/test_gridfile.py @@ -21,6 +21,7 @@ data_provider, download_ser_data, experiment, + global_grid_descriptor, grid_savepoint, processor_props, ranked_data_path, @@ -78,12 +79,11 @@ def test_grid_file_vertex_cell_edge_dimensions( parser.close() -@pytest.mark.parametrize("grid_descriptor", (definitions.Grids.R02B04_GLOBAL,)) @pytest.mark.parametrize("apply_transformation", (True, False)) def test_int_variable( - grid_descriptor: definitions.GridDescription, apply_transformation: bool + global_grid_descriptor: definitions.GridDescription, apply_transformation: bool ) -> None: - file = gridtest_utils.resolve_full_grid_file_name(grid_descriptor) + file = gridtest_utils.resolve_full_grid_file_name(global_grid_descriptor) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: edge_dim = parser.dimension(gridfile.DynamicDimension.EDGE_NAME) # use a test field that does not contain Pentagons @@ -96,27 +96,22 @@ def test_int_variable( assert max_value == np.max(test_field) -def index_selection() -> Iterable[list[int]]: - return ( - x - for x in [ - [0, 1, 2, 3, 4, 5], - [], - [0, 2, 4, 6, 7, 8, 24, 57], - [1, 2, 12, 13, 23, 24, 2306], - ] - ) +_index_selection: Iterable[list[int]] = [ + [0, 1, 2, 3, 4, 5], + [], + [0, 2, 4, 6, 7, 8, 24, 57], + [1, 2, 12, 13, 23, 24, 2306], +] @pytest.mark.parametrize( "selection", - index_selection(), + _index_selection, ) -@pytest.mark.parametrize("grid_descriptor", (definitions.Grids.R02B04_GLOBAL,)) def test_index_read_for_1d_fields( - grid_descriptor: definitions.GridDescription, selection: list[int] + global_grid_descriptor: definitions.GridDescription, selection: list[int] ) -> None: - file = gridtest_utils.resolve_full_grid_file_name(grid_descriptor) + file = gridtest_utils.resolve_full_grid_file_name(global_grid_descriptor) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: indices_to_read = np.asarray(selection) if len(selection) > 0 else None full_field = parser.variable(gridfile.CoordinateName.CELL_LATITUDE) @@ -128,25 +123,31 @@ def test_index_read_for_1d_fields( @pytest.mark.parametrize( "selection", - index_selection(), + _index_selection, ) -@pytest.mark.parametrize("grid_descriptor", (definitions.Grids.R02B04_GLOBAL,)) @pytest.mark.parametrize( "field", - (gridfile.ConnectivityName.V2E, gridfile.ConnectivityName.V2C, gridfile.ConnectivityName.E2V), + ( + gridfile.ConnectivityName.V2E, + gridfile.ConnectivityName.V2C, + gridfile.ConnectivityName.E2V, + ), ) @pytest.mark.parametrize("apply_offset", (True, False)) def test_index_read_for_2d_connectivity( - grid_descriptor: definitions.GridDescription, + global_grid_descriptor: definitions.GridDescription, selection: list[int], field: gridfile.FieldName, apply_offset: bool, ) -> None: - file = gridtest_utils.resolve_full_grid_file_name(grid_descriptor) + file = gridtest_utils.resolve_full_grid_file_name(global_grid_descriptor) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: indices_to_read = np.asarray(selection) if len(selection) > 0 else None full_field = parser.int_variable(field, transpose=True, apply_transformation=apply_offset) selective_field = parser.int_variable( - field, indices=indices_to_read, transpose=True, apply_transformation=apply_offset + field, + indices=indices_to_read, + transpose=True, + apply_transformation=apply_offset, ) assert np.allclose(full_field[indices_to_read], selective_field) From 0d95056f43ea06bb32bac78e9e050369a70d9c57 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 16:50:56 +0100 Subject: [PATCH 177/240] Fix connectivity --- model/common/src/icon4py/model/common/decomposition/halo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index b25fe12157..7683b109a5 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -152,7 +152,7 @@ def _find_neighbors( def _find_cell_neighbors(self, cells: data_alloc.NDArray) -> data_alloc.NDArray: """Find all neighboring cells of a list of cells.""" - return self._find_neighbors(cells, dims.C2E2C2E2C) + return self._find_neighbors(cells, dims.C2E2C) def find_edge_neighbors_for_cells(self, cell_line: data_alloc.NDArray) -> data_alloc.NDArray: return self._find_neighbors(cell_line, dims.C2E) From 9ebc74771c5a10275b5cedc5456d0c4f4d648677 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 16:51:12 +0100 Subject: [PATCH 178/240] Remove debugging --- .../common/grid/mpi_tests/test_parallel_grid_manager.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index b696eac696..be6a29cb46 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -873,12 +873,6 @@ def test_halo_neighbor_access_v2c( global_reference_field=reference.asnumpy(), local_field=output.asnumpy(), ) - nn = np.where(single_node_grid.get_connectivity(dims.V2C).asnumpy() == 3855) - if processor_props.rank == 0: - print(nn) - print(single_node_grid.get_connectivity(dims.V2C).asnumpy()[nn[0]]) - - print(f"rank={processor_props.rank}/{processor_props.comm_size}: ") @pytest.mark.mpi From aaaa51446aaeaac50b880f3bc1dedca18024dd09 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 17:21:35 +0100 Subject: [PATCH 179/240] Add simple test to check that metis decomposition is roughly balanced --- .../common/grid/unit_tests/test_grid_manager.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index 838fff3969..328e7c4e75 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -53,6 +53,7 @@ data_provider, download_ser_data, experiment, + global_grid_descriptor, grid_savepoint, processor_props, ranked_data_path, @@ -644,3 +645,19 @@ def test_local_connectivity( assert np.count_nonzero( (connectivity[level_index] == gridfile.GridFile.INVALID_INDEX) > 0 ), f"missing invalid index in {dim} - offset {field_offset}" + + +@pytest.mark.parametrize("ranks", (2, 3, 4)) +def test_decomposition_size( + ranks: int, + global_grid_descriptor: test_defs.GridDescription, +) -> None: + decomposer = halo.MetisDecomposer() + file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) + with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: + partitions = decomposer(parser.int_variable(gridfile.ConnectivityName.C2E2C), ranks) + sizes = [np.count_nonzero(partitions == r) for r in range(ranks)] + # Verify that sizes are close to each other. This is not a hard + # requirement, but simply a sanity check to make sure that partitions + # are relatively balanced. + assert max(sizes) - min(sizes) <= 2 From 2ed874eb1865a658c8f821acc992826a9a638aab Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 17:30:09 +0100 Subject: [PATCH 180/240] Expand test_local_connectivity parametrization --- .../grid/unit_tests/test_grid_manager.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index 328e7c4e75..869548e2d3 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -583,10 +583,26 @@ def test_decomposition_info_single_rank( assert np.all(data_alloc.as_numpy(result.halo_levels(dim)) == expected.halo_levels(dim)) -@pytest.mark.parametrize("rank", (0, 1, 2, 3)) +@pytest.mark.parametrize("rank", (0, 1, 2, 3), ids=lambda rank: f"rank{rank}") @pytest.mark.parametrize( "field_offset", - [dims.C2V, dims.E2V, dims.V2C, dims.E2C, dims.C2E, dims.V2E, dims.C2E2C, dims.V2E2V], + [ + dims.C2V, + dims.E2V, + dims.V2C, + dims.E2C, + dims.C2E, + dims.V2E, + dims.C2E2C, + dims.V2E2V, + dims.C2E2CO, + dims.C2E2C2E, + dims.C2E2C2E2C, + dims.E2C2V, + dims.E2C2E, + dims.E2C2EO, + ], + ids=lambda offset: offset.value, ) def test_local_connectivity( rank: int, From 19fc9d5eabff02730a6c827fad9685ef21060cd4 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 3 Feb 2026 17:45:21 +0100 Subject: [PATCH 181/240] Add note about partitioning being deterministic --- model/common/src/icon4py/model/common/decomposition/halo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 7683b109a5..dc4d2351c8 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -468,6 +468,8 @@ def __call__( import pymetis # type: ignore [import-untyped] + # The partitioning is done on all ranks, and this assumes that the + # partitioning is deterministic. _, partition_index = pymetis.part_graph(nparts=num_partitions, adjacency=adjacency_matrix) return data_alloc.array_ns_from_array(adjacency_matrix).array(partition_index) From 294956b7689c29c60e33d108eae6413a9de3cdfc Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 4 Feb 2026 15:01:12 +0100 Subject: [PATCH 182/240] Rename face to cell in halo.py --- .../model/common/decomposition/halo.py | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index dc4d2351c8..f665a2e793 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -29,9 +29,9 @@ def _value(k: gtx.FieldOffset | str) -> str: @runtime_checkable class HaloConstructor(Protocol): - """Callable that takes a mapping from faces (aka cells) to ranks""" + """Callable that takes a mapping from cells to ranks""" - def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: ... + def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: ... class NoHalos(HaloConstructor): @@ -43,7 +43,7 @@ def __init__( self._size = horizontal_size self._allocator = allocator - def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: + def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: xp = data_alloc.import_array_ns(self._allocator) create_arrays = functools.partial(_create_dummy_decomposition_arrays, array_ns=xp) decomposition_info = defs.DecompositionInfo() @@ -84,18 +84,18 @@ def __init__( self._connectivities = {_value(k): v for k, v in connectivities.items()} self._assert_all_neighbor_tables() - def _validate_mapping(self, face_to_rank_mapping: data_alloc.NDArray) -> None: + def _validate_mapping(self, cell_to_rank_mapping: data_alloc.NDArray) -> None: # validate the distribution mapping: num_cells = self._connectivity(dims.C2E2C).shape[0] expected_shape = (num_cells,) - if not face_to_rank_mapping.shape == expected_shape: + if not cell_to_rank_mapping.shape == expected_shape: raise exceptions.ValidationError( "rank_mapping", - f"should have shape {expected_shape} but is {face_to_rank_mapping.shape}", + f"should have shape {expected_shape} but is {cell_to_rank_mapping.shape}", ) # the decomposition should match the communicator size - if self._xp.max(face_to_rank_mapping) > self._props.comm_size - 1: + if self._xp.max(cell_to_rank_mapping) > self._props.comm_size - 1: raise exceptions.ValidationError( "rank_mapping", f"The distribution assumes more nodes than the current run is scheduled on {self._props} ", @@ -170,14 +170,14 @@ def find_cell_neighbors_for_vertices( ) -> data_alloc.NDArray: return self._find_neighbors(vertex_line, dims.V2C) - def owned_cells(self, face_to_rank: data_alloc.NDArray) -> data_alloc.NDArray: + def owned_cells(self, cell_to_rank: data_alloc.NDArray) -> data_alloc.NDArray: """Returns the full-grid indices of the cells owned by this rank""" - owned_cells = face_to_rank == self._props.rank + owned_cells = cell_to_rank == self._props.rank return self._xp.asarray(owned_cells).nonzero()[0] def _update_owner_mask_by_max_rank_convention( self, - face_to_rank: data_alloc.NDArray, + cell_to_rank: data_alloc.NDArray, owner_mask: data_alloc.NDArray, all_indices: data_alloc.NDArray, indices_on_cutting_line: data_alloc.NDArray, @@ -194,13 +194,13 @@ def _update_owner_mask_by_max_rank_convention( owner_mask: owner mask for the dimension all_indices: (global) indices of the dimension indices_on_cutting_line: global indices of the elements on the cutting line - target_connectivity: connectivity matrix mapping the dimension d to faces + target_connectivity: connectivity matrix mapping the dimension d to cell Returns: updated owner mask """ for index in indices_on_cutting_line: local_index = self._xp.nonzero(all_indices == index)[0][0] - owning_ranks = face_to_rank[target_connectivity[index]] + owning_ranks = cell_to_rank[target_connectivity[index]] assert ( self._xp.unique(owning_ranks).size > 1 ), f"rank {self._props.rank}: all neighboring cells are owned by the same rank" @@ -214,12 +214,12 @@ def _update_owner_mask_by_max_rank_convention( owner_mask[local_index] = True return owner_mask - def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: + def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: """ Constructs the DecompositionInfo for the current rank. Args: - face_to_rank: a mapping of cells to a rank + cell_to_rank: a mapping of cells to a rank The DecompositionInfo object is constructed for all horizontal dimension starting from the cell distribution. Edges and vertices @@ -311,10 +311,10 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: complete (= without skip value) for a distributed setup. """ - self._validate_mapping(face_to_rank) + self._validate_mapping(cell_to_rank) #: cells - owned_cells = self.owned_cells(face_to_rank) # global indices of owned cells + owned_cells = self.owned_cells(cell_to_rank) # global indices of owned cells first_halo_cells = self.next_halo_line(owned_cells) #: vertices vertex_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) @@ -365,7 +365,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: decomp_info.set_dimension(dims.CellDim, all_cells, cell_owner_mask, cell_halo_levels) vertex_owner_mask = self._xp.isin(all_vertices, vertex_on_owned_cells) vertex_owner_mask = self._update_owner_mask_by_max_rank_convention( - face_to_rank, + cell_to_rank, vertex_owner_mask, all_vertices, vertex_on_cutting_line, @@ -392,7 +392,7 @@ def __call__(self, face_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: edge_owner_mask = self._xp.isin(all_edges, edges_on_owned_cells) edge_owner_mask = self._update_owner_mask_by_max_rank_convention( - face_to_rank, + cell_to_rank, edge_owner_mask, all_edges, edges_on_cutting_line, @@ -434,7 +434,7 @@ def __call__( Call the decomposition. Args: - adjacency_matrix: face-to-face connectivity matrix on the global (undecomposed) grid. In the Icon4py context this C2E2C + adjacency_matrix: cell-to-cell connectivity matrix on the global (undecomposed) grid. In the Icon4py context this C2E2C n_part: number of nodes """ ... From 95e5ef1c7e7f0fbe66315d6019bcabfa15446385 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 4 Feb 2026 16:05:09 +0100 Subject: [PATCH 183/240] Move decomposer to a separate file --- .../model/common/decomposition/decomposer.py | 73 +++++++++++++++++++ .../model/common/decomposition/halo.py | 60 --------------- .../icon4py/model/common/grid/grid_manager.py | 14 ++-- .../mpi_tests/test_parallel_grid_manager.py | 26 ++++--- .../grid/mpi_tests/test_parallel_icon.py | 2 +- .../tests/common/grid/mpi_tests/utils.py | 6 +- 6 files changed, 101 insertions(+), 80 deletions(-) create mode 100644 model/common/src/icon4py/model/common/decomposition/decomposer.py diff --git a/model/common/src/icon4py/model/common/decomposition/decomposer.py b/model/common/src/icon4py/model/common/decomposition/decomposer.py new file mode 100644 index 0000000000..e2973b14a3 --- /dev/null +++ b/model/common/src/icon4py/model/common/decomposition/decomposer.py @@ -0,0 +1,73 @@ +# ICON4Py - ICON inspired code in Python and GT4Py +# +# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss +# All rights reserved. +# +# Please, refer to the LICENSE file in the root directory. +# SPDX-License-Identifier: BSD-3-Clause + +from typing import Protocol, runtime_checkable + +import gt4py.next as gtx + +from icon4py.model.common.utils import data_allocation as data_alloc + + +@runtime_checkable +class Decomposer(Protocol): + def __call__( + self, adjacency_matrix: data_alloc.NDArray, num_partitions: int + ) -> data_alloc.NDArray: + """ + Call the decomposition. + + Args: + adjacency_matrix: cell-to-cell connectivity matrix on the global (undecomposed) grid. In the Icon4py context this C2E2C + n_part: number of nodes + """ + ... + + +class MetisDecomposer(Decomposer): + """ + A simple decomposer using METIS for partitioning a grid topology. + + We use the simple pythonic interface to pymetis: just passing the adjacency matrix, which for ICON is + the full grid C2E2C neigbhor table. + if more control is needed (for example by using weights we need to switch to the C like interface) + https://documen.tician.de/pymetis/functionality.html + """ + + def __call__( + self, adjacency_matrix: data_alloc.NDArray, num_partitions: int + ) -> data_alloc.NDArray: + """ + Generate partition labels for this grid topology using METIS: + https://github.com/KarypisLab/METIS + + This method utilizes the pymetis Python bindings: + https://github.com/inducer/pymetis + + Args: + n_part: int, number of partitions to create + adjacency_matrix: nd array: neighbor table describing of the main dimension object to be distributed: for example cell -> cell neighbors + Returns: data_alloc.NDArray: array with partition label (int, rank number) for each cell + """ + + import pymetis # type: ignore [import-untyped] + + # The partitioning is done on all ranks, and this assumes that the + # partitioning is deterministic. + _, partition_index = pymetis.part_graph(nparts=num_partitions, adjacency=adjacency_matrix) + return data_alloc.array_ns_from_array(adjacency_matrix).array(partition_index) + + +class SingleNodeDecomposer(Decomposer): + def __call__( + self, adjacency_matrix: data_alloc.NDArray, num_partitions: int = 1 + ) -> data_alloc.NDArray: + """Dummy decomposer for single node: assigns all cells to rank = 0""" + return data_alloc.array_ns_from_array(adjacency_matrix).zeros( + adjacency_matrix.shape[0], + dtype=gtx.int32, # type: ignore [attr-defined] + ) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index f665a2e793..6bddd9bfaf 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -425,66 +425,6 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: return decomp_info -@runtime_checkable -class Decomposer(Protocol): - def __call__( - self, adjacency_matrix: data_alloc.NDArray, num_partitions: int - ) -> data_alloc.NDArray: - """ - Call the decomposition. - - Args: - adjacency_matrix: cell-to-cell connectivity matrix on the global (undecomposed) grid. In the Icon4py context this C2E2C - n_part: number of nodes - """ - ... - - -class MetisDecomposer(Decomposer): - """ - A simple decomposer using METIS for partitioning a grid topology. - - We use the simple pythonic interface to pymetis: just passing the adjacency matrix, which for ICON is - the full grid C2E2C neigbhor table. - if more control is needed (for example by using weights we need to switch to the C like interface) - https://documen.tician.de/pymetis/functionality.html - """ - - def __call__( - self, adjacency_matrix: data_alloc.NDArray, num_partitions: int - ) -> data_alloc.NDArray: - """ - Generate partition labels for this grid topology using METIS: - https://github.com/KarypisLab/METIS - - This method utilizes the pymetis Python bindings: - https://github.com/inducer/pymetis - - Args: - n_part: int, number of partitions to create - adjacency_matrix: nd array: neighbor table describing of the main dimension object to be distributed: for example cell -> cell neighbors - Returns: data_alloc.NDArray: array with partition label (int, rank number) for each cell - """ - - import pymetis # type: ignore [import-untyped] - - # The partitioning is done on all ranks, and this assumes that the - # partitioning is deterministic. - _, partition_index = pymetis.part_graph(nparts=num_partitions, adjacency=adjacency_matrix) - return data_alloc.array_ns_from_array(adjacency_matrix).array(partition_index) - - -class SingleNodeDecomposer(Decomposer): - def __call__( - self, adjacency_matrix: data_alloc.NDArray, num_partitions: int = 1 - ) -> data_alloc.NDArray: - """Dummy decomposer for single node: assigns all cells to rank = 0""" - return data_alloc.array_ns_from_array(adjacency_matrix).zeros( - adjacency_matrix.shape[0], - dtype=gtx.int32, # type: ignore [attr-defined] - ) - - def get_halo_constructor( run_properties: defs.ProcessProperties, full_grid_size: base.HorizontalGridSize, diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 4e5393cc0a..0fcaf62eee 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -16,14 +16,18 @@ import numpy as np from icon4py.model.common import dimension as dims, type_alias as ta -from icon4py.model.common.decomposition import definitions as decomposition, halo +from icon4py.model.common.decomposition import ( + decomposer as decomp, + definitions as decomposition, + halo, +) from icon4py.model.common.exceptions import InvalidConfigError from icon4py.model.common.grid import base, grid_refinement as refinement, gridfile, icon from icon4py.model.common.utils import data_allocation as data_alloc _log = logging.getLogger(__name__) -_single_node_decomposer = halo.SingleNodeDecomposer() +_single_node_decomposer = decomp.SingleNodeDecomposer() _single_process_props = decomposition.SingleNodeProcessProperties() _fortan_to_python_transformer = gridfile.ToZeroBasedIndexTransformation() @@ -95,11 +99,11 @@ def __call__( self, allocator: gtx_typing.FieldBufferAllocationUtil | None, keep_skip_values: bool, - decomposer: halo.Decomposer = _single_node_decomposer, + decomposer: decomp.Decomposer = _single_node_decomposer, run_properties=_single_process_props, ): if not run_properties.is_single_rank() and isinstance( - decomposer, halo.SingleNodeDecomposer + decomposer, decomp.SingleNodeDecomposer ): raise InvalidConfigError("Need a Decomposer for multi node run") @@ -383,7 +387,7 @@ def _construct_decomposed_grid( allocator: gtx_typing.FieldBufferAllocationUtil | None, with_skip_values: bool, geometry_type: base.GeometryType, - decomposer: halo.Decomposer, + decomposer: decomp.Decomposer, run_properties: decomposition.ProcessProperties, ) -> None: """Construct the grid topology from the icon grid file. diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index be6a29cb46..68d05d72a2 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -16,7 +16,11 @@ from gt4py.next import common as gtx_common, typing as gtx_typing from icon4py.model.common import dimension as dims, exceptions -from icon4py.model.common.decomposition import definitions as decomp_defs, halo, mpi_decomposition +from icon4py.model.common.decomposition import ( + decomposer as decomp, + definitions as decomp_defs, + mpi_decomposition, +) from icon4py.model.common.grid import ( base, geometry, @@ -61,7 +65,7 @@ def test_grid_manager_validate_decomposer( keep_skip_values=True, allocator=None, run_properties=processor_props, - decomposer=halo.SingleNodeDecomposer(), + decomposer=decomp.SingleNodeDecomposer(), ) assert "Need a Decomposer for multi" in e.value.args[0] @@ -94,7 +98,7 @@ def test_fields_distribute_and_gather( multinode = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.MetisDecomposer(), + decomposer=decomp.MetisDecomposer(), ) decomposition_info = multinode.decomposition_info @@ -252,7 +256,7 @@ def test_halo_neighbor_access_c2e( multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.MetisDecomposer(), + decomposer=decomp.MetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid extra_geometry_fields = multinode_grid_manager.geometry_fields @@ -326,7 +330,7 @@ def test_halo_neighbor_access_e2c2v( multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.MetisDecomposer(), + decomposer=decomp.MetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid extra_geometry_fields = multinode_grid_manager.geometry_fields @@ -418,7 +422,7 @@ def test_halo_neighbor_access_e2c( multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.MetisDecomposer(), + decomposer=decomp.MetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid extra_geometry_fields = multinode_grid_manager.geometry_fields @@ -509,7 +513,7 @@ def test_halo_neighbor_access_e2v( multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.MetisDecomposer(), + decomposer=decomp.MetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid decomposition_info = multinode_grid_manager.decomposition_info @@ -610,7 +614,7 @@ def test_halo_neighbor_access_v2e( multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.MetisDecomposer(), + decomposer=decomp.MetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid decomposition_info = multinode_grid_manager.decomposition_info @@ -694,7 +698,7 @@ def test_halo_neighbor_access_c2e2c( multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.MetisDecomposer(), + decomposer=decomp.MetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid decomposition_info = multinode_grid_manager.decomposition_info @@ -794,7 +798,7 @@ def test_halo_neighbor_access_v2c( multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.MetisDecomposer(), + decomposer=decomp.MetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid decomposition_info = multinode_grid_manager.decomposition_info @@ -885,7 +889,7 @@ def test_validate_skip_values_in_distributed_connectivities( multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, - decomposer=halo.MetisDecomposer(), + decomposer=decomp.MetisDecomposer(), ) distributed_grid = multinode_grid_manager.grid for k, c in distributed_grid.connectivities.items(): diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py index 3cd2b75ba2..a89811ae11 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py @@ -22,7 +22,7 @@ definitions as decomp_defs, definitions as decomposition, ) -from icon4py.model.common.decomposition.halo import MetisDecomposer +from icon4py.model.common.decomposition.decomposer import MetisDecomposer from icon4py.model.common.grid import base as base_grid, gridfile, horizontal as h_grid, icon from icon4py.model.testing import definitions as test_defs, grid_utils, parallel_helpers diff --git a/model/common/tests/common/grid/mpi_tests/utils.py b/model/common/tests/common/grid/mpi_tests/utils.py index a20ec444df..6414a61d04 100644 --- a/model/common/tests/common/grid/mpi_tests/utils.py +++ b/model/common/tests/common/grid/mpi_tests/utils.py @@ -8,7 +8,7 @@ import pathlib -from icon4py.model.common.decomposition import definitions as decomp_defs, halo +from icon4py.model.common.decomposition import decomposer as decomp, definitions as decomp_defs from icon4py.model.common.grid import grid_manager as gm @@ -17,7 +17,7 @@ def run_grid_manager_for_singlenode(file: pathlib.Path) -> gm.GridManager: manager( keep_skip_values=True, run_properties=decomp_defs.SingleNodeProcessProperties(), - decomposer=halo.SingleNodeDecomposer(), + decomposer=decomp.SingleNodeDecomposer(), allocator=None, ) return manager @@ -31,7 +31,7 @@ def _grid_manager(file: pathlib.Path, num_levels: int) -> gm.GridManager: def run_gridmananger_for_multinode( file: pathlib.Path, run_properties: decomp_defs.ProcessProperties, - decomposer: halo.Decomposer, + decomposer: decomp.Decomposer, ) -> gm.GridManager: manager = _grid_manager(file, num_levels=NUM_LEVELS) manager( From 3926eaa8463a484042aed2e327c9be32b2ffd574 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 4 Feb 2026 16:09:59 +0100 Subject: [PATCH 184/240] Revert num_levels/vertical config change --- .../src/icon4py/model/common/grid/grid_manager.py | 14 ++++++++++---- .../grid/mpi_tests/test_parallel_grid_manager.py | 7 ++++++- model/common/tests/common/grid/mpi_tests/utils.py | 8 +++++--- .../common/grid/unit_tests/test_grid_manager.py | 6 +++--- .../model/standalone_driver/driver_utils.py | 5 ++++- .../src/icon4py/model/testing/grid_utils.py | 5 +++-- 6 files changed, 31 insertions(+), 14 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 0fcaf62eee..cec34301c8 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -22,7 +22,13 @@ halo, ) from icon4py.model.common.exceptions import InvalidConfigError -from icon4py.model.common.grid import base, grid_refinement as refinement, gridfile, icon +from icon4py.model.common.grid import ( + base, + grid_refinement as refinement, + gridfile, + icon, + vertical as v_grid, +) from icon4py.model.common.utils import data_allocation as data_alloc @@ -58,13 +64,13 @@ class GridManager: def __init__( self, grid_file: pathlib.Path | str, - num_levels: int, + config: v_grid.VerticalGridConfig, # TODO(msimberg): remove to separate vertical and horizontal grid transformation: gridfile.IndexTransformation = _fortan_to_python_transformer, global_reductions: decomposition.Reductions = decomposition.single_node_reductions, ): self._transformation = transformation self._file_name = str(grid_file) - self._num_levels = num_levels + self._vertical_config = config # Output self._grid: icon.IconGrid | None = None self._decomposition_info: decomposition.DecompositionInfo | None = None @@ -448,7 +454,7 @@ def _construct_decomposed_grid( grid_config = base.GridConfig( horizontal_size=distributed_size, - vertical_size=self._num_levels, + vertical_size=self._vertical_config.num_levels, limited_area=limited_area, keep_skip_values=with_skip_values, ) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 68d05d72a2..8f1989afdd 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -30,6 +30,7 @@ gridfile, horizontal as h_grid, icon, + vertical as v_grid, ) from icon4py.model.common.interpolation import interpolation_fields from icon4py.model.common.interpolation.stencils.compute_cell_2_vertex_interpolation import ( @@ -59,7 +60,11 @@ def test_grid_manager_validate_decomposer( global_grid_descriptor: test_defs.GridDescription, ) -> None: file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) - manager = gm.GridManager(file, utils.NUM_LEVELS, gridfile.ToZeroBasedIndexTransformation()) + manager = gm.GridManager( + grid_file=file, + config=v_grid.VerticalGridConfig(num_levels=utils.NUM_LEVELS), + transformation=gridfile.ToZeroBasedIndexTransformation(), + ) with pytest.raises(exceptions.InvalidConfigError) as e: manager( keep_skip_values=True, diff --git a/model/common/tests/common/grid/mpi_tests/utils.py b/model/common/tests/common/grid/mpi_tests/utils.py index 6414a61d04..fedfafd5d3 100644 --- a/model/common/tests/common/grid/mpi_tests/utils.py +++ b/model/common/tests/common/grid/mpi_tests/utils.py @@ -9,7 +9,7 @@ import pathlib from icon4py.model.common.decomposition import decomposer as decomp, definitions as decomp_defs -from icon4py.model.common.grid import grid_manager as gm +from icon4py.model.common.grid import grid_manager as gm, vertical as v_grid def run_grid_manager_for_singlenode(file: pathlib.Path) -> gm.GridManager: @@ -24,7 +24,9 @@ def run_grid_manager_for_singlenode(file: pathlib.Path) -> gm.GridManager: def _grid_manager(file: pathlib.Path, num_levels: int) -> gm.GridManager: - manager = gm.GridManager(str(file), num_levels=num_levels) + manager = gm.GridManager( + grid_file=str(file), config=v_grid.VerticalGridConfig(num_levels=num_levels) + ) return manager @@ -33,7 +35,7 @@ def run_gridmananger_for_multinode( run_properties: decomp_defs.ProcessProperties, decomposer: decomp.Decomposer, ) -> gm.GridManager: - manager = _grid_manager(file, num_levels=NUM_LEVELS) + manager = _grid_manager(file, NUM_LEVELS) manager( keep_skip_values=True, allocator=None, run_properties=run_properties, decomposer=decomposer ) diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index 869548e2d3..5b44162c68 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -351,8 +351,8 @@ def test_gridmanager_given_file_not_found_then_abort( fname = "./unknown_grid.nc" with pytest.raises(FileNotFoundError) as error: manager = gm.GridManager( - fname, - num_levels=80, + grid_file=fname, + config=v_grid.VerticalGridConfig(num_levels=80), transformation=icon4py.model.common.grid.gridfile.NoTransformation(), ) manager(allocator=cpu_allocator, keep_skip_values=True) @@ -615,7 +615,7 @@ def test_local_connectivity( partitioner = halo.MetisDecomposer() allocator = model_backends.get_allocator(backend_like) file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) - manager = gm.GridManager(num_levels=10, grid_file=file) + manager = gm.GridManager(config=v_grid.VerticalGridConfig(num_levels=10), grid_file=file) manager( decomposer=partitioner, allocator=allocator, diff --git a/model/standalone_driver/src/icon4py/model/standalone_driver/driver_utils.py b/model/standalone_driver/src/icon4py/model/standalone_driver/driver_utils.py index ae02baa36e..cd1db437b1 100644 --- a/model/standalone_driver/src/icon4py/model/standalone_driver/driver_utils.py +++ b/model/standalone_driver/src/icon4py/model/standalone_driver/driver_utils.py @@ -64,7 +64,10 @@ def create_grid_manager( global_reductions: decomposition_defs.Reductions = decomposition_defs.single_node_reductions, ) -> gm.GridManager: grid_manager = gm.GridManager( - gm.ToZeroBasedIndexTransformation(), grid_file_path, vertical_grid_config, global_reductions + grid_file=grid_file_path, + config=vertical_grid_config, + transformation=gm.ToZeroBasedIndexTransformation(), + global_reductions=global_reductions, ) grid_manager(allocator=allocator, keep_skip_values=True) diff --git a/model/testing/src/icon4py/model/testing/grid_utils.py b/model/testing/src/icon4py/model/testing/grid_utils.py index 95e5f9c709..19de3c250d 100644 --- a/model/testing/src/icon4py/model/testing/grid_utils.py +++ b/model/testing/src/icon4py/model/testing/grid_utils.py @@ -15,6 +15,7 @@ geometry_attributes as geometry_attrs, grid_manager as gm, gridfile, + vertical as v_grid, ) from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import config, data_handling, definitions, locking @@ -64,8 +65,8 @@ def get_grid_manager( backend: the gt4py Backend we are running on """ manager = gm.GridManager( - filename, - num_levels=num_levels, + grid_file=filename, + config=v_grid.VerticalGridConfig(num_levels=num_levels), transformation=gridfile.ToZeroBasedIndexTransformation(), ) manager(allocator=allocator, keep_skip_values=keep_skip_values) From 80685291383602705e7a584961f97d62beabe9d2 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 4 Feb 2026 16:54:15 +0100 Subject: [PATCH 185/240] Remove helper next_halo_line --- .../model/common/decomposition/halo.py | 27 ++++--------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 6bddd9bfaf..9dd0746099 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -124,26 +124,6 @@ def _connectivity(self, offset: gtx.FieldOffset | str) -> data_alloc.NDArray: f"Connectivity for offset {offset} is not available" ) from err - def next_halo_line( - self, cells: data_alloc.NDArray, depot: data_alloc.NDArray | None = None - ) -> data_alloc.NDArray: - """Returns the full-grid indices of the next halo line. - - If a depot is given the function only return indices that are not in the depot - - Args: - cells: 1d array, full-grid indices of cells we want to find the neighbors of - depot: full-grid indices that have already been collected - Returns: - next_halo_cells: full-grid indices of the next halo line - """ - assert cells.ndim == 1, "input should be 1d array" - cell_neighbors = self._find_cell_neighbors(cells) - - cells_so_far = self._xp.hstack((depot, cells)) if depot is not None else cells - - return self._xp.setdiff1d(cell_neighbors, cells_so_far, assume_unique=True) - def _find_neighbors( self, source_indices: data_alloc.NDArray, offset: gtx.FieldOffset | str ) -> data_alloc.NDArray: @@ -315,12 +295,15 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: #: cells owned_cells = self.owned_cells(cell_to_rank) # global indices of owned cells - first_halo_cells = self.next_halo_line(owned_cells) + first_halo_cells = self._find_cell_neighbors(owned_cells) #: vertices vertex_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) vertex_on_halo_cells = self.find_vertex_neighbors_for_cells( self._xp.hstack( - (first_halo_cells, (self.next_halo_line(first_halo_cells, owned_cells))) + ( + first_halo_cells, + (self._find_cell_neighbors(self._xp.union1d(first_halo_cells, owned_cells))), + ) ) ) vertex_on_cutting_line = self._xp.intersect1d(vertex_on_owned_cells, vertex_on_halo_cells) From 5e7b984a900cded0cbf8fbca4d3bdc0c1f0559b3 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 4 Feb 2026 17:10:35 +0100 Subject: [PATCH 186/240] Simplify owned_cells --- model/common/src/icon4py/model/common/decomposition/halo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 9dd0746099..61ad51c71b 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -152,8 +152,8 @@ def find_cell_neighbors_for_vertices( def owned_cells(self, cell_to_rank: data_alloc.NDArray) -> data_alloc.NDArray: """Returns the full-grid indices of the cells owned by this rank""" - owned_cells = cell_to_rank == self._props.rank - return self._xp.asarray(owned_cells).nonzero()[0] + assert cell_to_rank.ndim == 1 + return self._xp.where(cell_to_rank == self._props.rank)[0] def _update_owner_mask_by_max_rank_convention( self, From 32fc3dcec956c7e415a7d6552ae737c82de1f8fb Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 4 Feb 2026 17:10:56 +0100 Subject: [PATCH 187/240] Remove unnecessary xfail --- .../common/decomposition/unit_tests/test_definitions.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index 7cd0fd87fb..e4a398621e 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -43,13 +43,7 @@ def get_neighbor_tables_for_simple_grid() -> dict[str, data_alloc.NDArray]: @pytest.mark.parametrize("dim", grid_utils.main_horizontal_dims()) -def test_decomposition_info_single_node_empty_halo( - dim: gtx.Dimension, - processor_props: definitions.ProcessProperties, -) -> None: - if not processor_props.is_single_rank(): - pytest.xfail() - +def test_decomposition_info_single_node_empty_halo(dim: gtx.Dimension) -> None: manager = grid_utils.run_grid_manager( test_defs.Grids.MCH_CH_R04B09_DSL, keep_skip_values=True, backend=None ) From cc642c7cdea6247fe1153611989506fd551cf129 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 4 Feb 2026 17:14:08 +0100 Subject: [PATCH 188/240] Remove owned_cells helper --- .../common/src/icon4py/model/common/decomposition/halo.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 61ad51c71b..28f99dc859 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -150,11 +150,6 @@ def find_cell_neighbors_for_vertices( ) -> data_alloc.NDArray: return self._find_neighbors(vertex_line, dims.V2C) - def owned_cells(self, cell_to_rank: data_alloc.NDArray) -> data_alloc.NDArray: - """Returns the full-grid indices of the cells owned by this rank""" - assert cell_to_rank.ndim == 1 - return self._xp.where(cell_to_rank == self._props.rank)[0] - def _update_owner_mask_by_max_rank_convention( self, cell_to_rank: data_alloc.NDArray, @@ -294,7 +289,8 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: self._validate_mapping(cell_to_rank) #: cells - owned_cells = self.owned_cells(cell_to_rank) # global indices of owned cells + # global indices of owned cells + owned_cells = self._xp.where(cell_to_rank == self._props.rank)[0] first_halo_cells = self._find_cell_neighbors(owned_cells) #: vertices vertex_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) From d275bf80f7dfc6c23ca2be0aa567c75379f5f2b1 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 4 Feb 2026 17:17:15 +0100 Subject: [PATCH 189/240] Add check to disallow limited area grids with distributed runs --- .../icon4py/model/common/grid/grid_manager.py | 3 +++ .../mpi_tests/test_parallel_grid_manager.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index cec34301c8..3f8e7dffa1 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -411,6 +411,9 @@ def _construct_decomposed_grid( global_params = self._construct_global_params(allocator, global_size, geometry_type) limited_area = refinement.is_limited_area_grid(cell_refinement, array_ns=xp) + if limited_area and not run_properties.is_single_rank(): + raise NotImplementedError("Limited-area grids are not supported in distributed runs") + cell_to_cell_neighbors = self._get_index_field(gridfile.ConnectivityName.C2E2C, array_ns=xp) global_neighbor_tables = { dims.C2E2C: cell_to_cell_neighbors, diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 8f1989afdd..4ca0220efe 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -908,3 +908,20 @@ def test_validate_skip_values_in_distributed_connectivities( assert ( c in icon.CONNECTIVITIES_ON_BOUNDARIES or icon.CONNECTIVITIES_ON_PENTAGONS ), f"rank={processor_props.rank} / {processor_props.comm_size}: {k} has skip found in table" + + +@pytest.mark.mpi +@pytest.mark.parametrize("processor_props", [True], indirect=True) +@pytest.mark.parametrize("grid", [test_defs.Grids.MCH_CH_R04B09_DSL]) +def test_limited_area_raises( + processor_props: decomp_defs.ProcessProperties, + grid: test_defs.GridDescription, +) -> None: + with pytest.raises( + NotImplementedError, match="Limited-area grids are not supported in distributed runs" + ): + _ = utils.run_gridmananger_for_multinode( + file=grid_utils.resolve_full_grid_file_name(grid), + run_properties=processor_props, + decomposer=decomp.MetisDecomposer(), + ) From 0706f6eb01b5b41d54c10f983bd428e184a5fe0f Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Thu, 5 Feb 2026 10:05:34 +0100 Subject: [PATCH 190/240] Revert "Remove helper next_halo_line" This reverts commit 80685291383602705e7a584961f97d62beabe9d2. --- .../model/common/decomposition/halo.py | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 28f99dc859..1478d42461 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -124,6 +124,26 @@ def _connectivity(self, offset: gtx.FieldOffset | str) -> data_alloc.NDArray: f"Connectivity for offset {offset} is not available" ) from err + def next_halo_line( + self, cells: data_alloc.NDArray, depot: data_alloc.NDArray | None = None + ) -> data_alloc.NDArray: + """Returns the full-grid indices of the next halo line. + + If a depot is given the function only return indices that are not in the depot + + Args: + cells: 1d array, full-grid indices of cells we want to find the neighbors of + depot: full-grid indices that have already been collected + Returns: + next_halo_cells: full-grid indices of the next halo line + """ + assert cells.ndim == 1, "input should be 1d array" + cell_neighbors = self._find_cell_neighbors(cells) + + cells_so_far = self._xp.hstack((depot, cells)) if depot is not None else cells + + return self._xp.setdiff1d(cell_neighbors, cells_so_far, assume_unique=True) + def _find_neighbors( self, source_indices: data_alloc.NDArray, offset: gtx.FieldOffset | str ) -> data_alloc.NDArray: @@ -291,15 +311,12 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: #: cells # global indices of owned cells owned_cells = self._xp.where(cell_to_rank == self._props.rank)[0] - first_halo_cells = self._find_cell_neighbors(owned_cells) + first_halo_cells = self.next_halo_line(owned_cells) #: vertices vertex_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) vertex_on_halo_cells = self.find_vertex_neighbors_for_cells( self._xp.hstack( - ( - first_halo_cells, - (self._find_cell_neighbors(self._xp.union1d(first_halo_cells, owned_cells))), - ) + (first_halo_cells, (self.next_halo_line(first_halo_cells, owned_cells))) ) ) vertex_on_cutting_line = self._xp.intersect1d(vertex_on_owned_cells, vertex_on_halo_cells) From d2919af20050b35c01bbaa29e8d3918a7d6bbd10 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Thu, 5 Feb 2026 10:10:27 +0100 Subject: [PATCH 191/240] Simplify, but don't remove, next_halo_line helper --- .../icon4py/model/common/decomposition/halo.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 1478d42461..1bcee8fc0b 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -124,25 +124,17 @@ def _connectivity(self, offset: gtx.FieldOffset | str) -> data_alloc.NDArray: f"Connectivity for offset {offset} is not available" ) from err - def next_halo_line( - self, cells: data_alloc.NDArray, depot: data_alloc.NDArray | None = None - ) -> data_alloc.NDArray: + def next_halo_line(self, cells: data_alloc.NDArray) -> data_alloc.NDArray: """Returns the full-grid indices of the next halo line. - If a depot is given the function only return indices that are not in the depot - Args: cells: 1d array, full-grid indices of cells we want to find the neighbors of - depot: full-grid indices that have already been collected Returns: next_halo_cells: full-grid indices of the next halo line """ assert cells.ndim == 1, "input should be 1d array" cell_neighbors = self._find_cell_neighbors(cells) - - cells_so_far = self._xp.hstack((depot, cells)) if depot is not None else cells - - return self._xp.setdiff1d(cell_neighbors, cells_so_far, assume_unique=True) + return self._xp.setdiff1d(cell_neighbors, cells, assume_unique=True) def _find_neighbors( self, source_indices: data_alloc.NDArray, offset: gtx.FieldOffset | str @@ -316,7 +308,10 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: vertex_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) vertex_on_halo_cells = self.find_vertex_neighbors_for_cells( self._xp.hstack( - (first_halo_cells, (self.next_halo_line(first_halo_cells, owned_cells))) + ( + first_halo_cells, + (self.next_halo_line(self._xp.union1d(first_halo_cells, owned_cells))), + ) ) ) vertex_on_cutting_line = self._xp.intersect1d(vertex_on_owned_cells, vertex_on_halo_cells) From 46371057b5c952822b14e2db4cb36eb510fc9f11 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 6 Feb 2026 15:05:38 +0100 Subject: [PATCH 192/240] Refactor test_parallel_grid_manager.py --- .../mpi_tests/test_parallel_grid_manager.py | 890 +++++------------- 1 file changed, 224 insertions(+), 666 deletions(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 4ca0220efe..32f110527f 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -32,10 +32,15 @@ icon, vertical as v_grid, ) -from icon4py.model.common.interpolation import interpolation_fields +from icon4py.model.common.interpolation import ( + interpolation_attributes, + interpolation_factory, + interpolation_fields, +) from icon4py.model.common.interpolation.stencils.compute_cell_2_vertex_interpolation import ( _compute_cell_2_vertex_interpolation, ) +from icon4py.model.common.metrics import metrics_attributes, metrics_factory from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import definitions as test_defs, grid_utils, test_utils @@ -84,72 +89,10 @@ def _get_neighbor_tables(grid: base.Grid) -> dict: } -@pytest.mark.mpi -@pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_fields_distribute_and_gather( - processor_props: decomp_defs.ProcessProperties, - global_grid_descriptor: test_defs.GridDescription, - caplog: Any, -) -> None: - caplog.set_level(logging.INFO) - print(f"myrank - {processor_props.rank}: running with processor_props = {processor_props}") - file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) - single_node = utils.run_grid_manager_for_singlenode(file) - single_node_grid = single_node.grid - global_cell_area = single_node.geometry_fields[gridfile.GeometryName.CELL_AREA] - global_edge_lat = single_node.coordinates[dims.EdgeDim]["lat"] - global_vertex_lon = single_node.coordinates[dims.VertexDim]["lon"] - - multinode = utils.run_gridmananger_for_multinode( - file=file, - run_properties=processor_props, - decomposer=decomp.MetisDecomposer(), - ) - decomposition_info = multinode.decomposition_info - - local_cell_area = multinode.geometry_fields[gridfile.GeometryName.CELL_AREA] - local_edge_lat = multinode.coordinates[dims.EdgeDim]["lat"] - local_vertex_lon = multinode.coordinates[dims.VertexDim]["lon"] - print( - f"rank = {processor_props.rank} has size(cell_area): {local_cell_area.ndarray.shape}, " - f"has size(edge_length): {local_edge_lat.ndarray.shape}, has size(vertex_length): {local_vertex_lon.ndarray.shape}" - ) - global_num_cells = single_node_grid.config.num_cells - - # the local number of cells must be at most the global number of cells (analytically computed) - assert ( - local_cell_area.asnumpy().shape[0] <= global_num_cells - ), "local field is larger than global field" - # global read: read the same (global fields) - - assert_gathered_field_against_global( - decomposition_info, - processor_props, - dims.CellDim, - global_cell_area.asnumpy(), - local_cell_area.asnumpy(), - ) - - assert_gathered_field_against_global( - decomposition_info, - processor_props, - dims.EdgeDim, - global_edge_lat.asnumpy(), - local_edge_lat.asnumpy(), - ) - assert_gathered_field_against_global( - decomposition_info, - processor_props, - dims.VertexDim, - global_vertex_lon.asnumpy(), - local_vertex_lon.asnumpy(), - ) - - def gather_field(field: np.ndarray, props: decomp_defs.ProcessProperties) -> tuple: constant_dims = tuple(field.shape[1:]) print(f"gather_field on rank={props.rank} - gathering field of local shape {field.shape}") - constant_length = functools.reduce(operator.mul, constant_dims) if len(constant_dims) > 0 else 1 + constant_length = functools.reduce(operator.mul, constant_dims, 1) local_sizes = np.array(props.comm.gather(field.size, root=0)) if props.rank == 0: recv_buffer = np.empty(np.sum(local_sizes), dtype=field.dtype) @@ -172,7 +115,7 @@ def gather_field(field: np.ndarray, props: decomp_defs.ProcessProperties) -> tup return local_first_dim, gathered_field -def assert_gathered_field_against_global( +def check_local_global_field( decomposition_info: decomp_defs.DecompositionInfo, processor_props: decomp_defs.ProcessProperties, # F811 # fixture dim: gtx.Dimension, @@ -188,6 +131,26 @@ def assert_gathered_field_against_global( 0 ] ) + + # Compare full local field, including halos, against global reference field. + # TODO(msimberg): Is halo always expected to be populated? + global_indices_local_field = decomposition_info.global_index( + dim, + decomp_defs.DecompositionInfo.EntryType.OWNED, # ALL if checking halos + ) + local_indices_local_field = decomposition_info.local_index( + dim, + decomp_defs.DecompositionInfo.EntryType.OWNED, # ALL if checking halos + ) + local_field_from_global_field = global_reference_field[global_indices_local_field] + local_field_from_local_field = local_field[local_indices_local_field] + np.testing.assert_allclose( + local_field_from_global_field, local_field_from_local_field, atol=0.0, verbose=True + ) + + # Compare owned local field, excluding halos, against global reference + # field, by gathering owned entries to the first rank. This ensures that in + # total we have the full global field distributed on all ranks. owned_entries = local_field[ decomposition_info.local_index(dim, decomp_defs.DecompositionInfo.EntryType.OWNED) ] @@ -197,7 +160,7 @@ def assert_gathered_field_against_global( decomposition_info.global_index(dim, decomp_defs.DecompositionInfo.EntryType.OWNED), processor_props, ) - rtol = 1e-12 + if processor_props.rank == 0: print(f"rank = {processor_props.rank}: asserting gathered fields: ") @@ -215,354 +178,85 @@ def assert_gathered_field_against_global( print( f" rank = {processor_props.rank}: SHAPES: global reference field {global_reference_field.shape}, gathered = {gathered_field.shape}" ) - np.testing.assert_allclose(sorted_, global_reference_field, rtol=rtol, verbose=True) + + # We expect an exact match, since the starting point is the same (grid + # file) and we are doing the exact same computations in single rank and + # multi rank mode. + np.testing.assert_allclose(sorted_, global_reference_field, atol=0.0, verbose=True) @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_halo_neighbor_access_c2e( +@pytest.mark.parametrize( + "attrs_name, dim", + [ + # TODO(msimberg): Get dim out of field? + (geometry_attributes.CELL_AREA, dims.CellDim), + (geometry_attributes.EDGE_LENGTH, dims.EdgeDim), + (geometry_attributes.VERTEX_LAT, dims.VertexDim), + (geometry_attributes.EDGE_NORMAL_VERTEX_U, dims.EdgeDim), + (geometry_attributes.EDGE_NORMAL_VERTEX_V, dims.EdgeDim), + (geometry_attributes.EDGE_NORMAL_CELL_U, dims.EdgeDim), + (geometry_attributes.EDGE_NORMAL_CELL_V, dims.EdgeDim), + (geometry_attributes.EDGE_TANGENT_X, dims.EdgeDim), + (geometry_attributes.EDGE_TANGENT_Y, dims.EdgeDim), + ], +) +def test_geometry_fields_compare_single_multi_rank( processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, + # TODO(msimberg): Maybe use regular grid fixture and skip local area grids? global_grid_descriptor: test_defs.GridDescription, + attrs_name: str, + dim: gtx.Dimension, ) -> None: + # TODO(msimberg): Add fixture for "always single rank" grid manager file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") - single_node = utils.run_grid_manager_for_singlenode(file) - single_node_grid = single_node.grid + single_node_grid_manager = utils.run_grid_manager_for_singlenode(file) + # TODO(msimberg): Add fixture for "always single rank" geometry single_node_geometry = geometry.GridGeometry( backend=backend, - grid=single_node_grid, - coordinates=single_node.coordinates, - decomposition_info=single_node.decomposition_info, - extra_fields=single_node.geometry_fields, + grid=single_node_grid_manager.grid, + coordinates=single_node_grid_manager.coordinates, + decomposition_info=single_node_grid_manager.decomposition_info, + extra_fields=single_node_grid_manager.geometry_fields, metadata=geometry_attributes.attrs, ) - - print( - f"rank = {processor_props.rank} : single node grid has size {single_node.decomposition_info.get_horizontal_size()!r}" - ) - reference = data_alloc.zero_field(single_node_grid, dims.CellDim, dims.C2EDim) - single_node_edge_length = single_node_geometry.get(geometry_attributes.EDGE_LENGTH) - single_node_cell_area = single_node_geometry.get(geometry_attributes.CELL_AREA) - single_node_edge_orientation = single_node_geometry.get( - geometry_attributes.CELL_NORMAL_ORIENTATION - ) - # has to be computed in gt4py-embedded - interpolation_fields.compute_geofac_div.with_backend(None)( - primal_edge_length=single_node_edge_length, - area=single_node_cell_area, - edge_orientation=single_node_edge_orientation, - out=reference, - offset_provider={"C2E": single_node_grid.get_connectivity("C2E")}, - ) print( - f"rank = {processor_props.rank} : single node computed field reference has size {reference.asnumpy().shape}" - ) - multinode_grid_manager = utils.run_gridmananger_for_multinode( - file=file, - run_properties=processor_props, - decomposer=decomp.MetisDecomposer(), + f"rank = {processor_props.rank} : single node grid has size {single_node_grid_manager.decomposition_info.get_horizontal_size()!r}" ) - distributed_grid = multinode_grid_manager.grid - extra_geometry_fields = multinode_grid_manager.geometry_fields - decomposition_info = multinode_grid_manager.decomposition_info - - print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") - print( - f"rank = {processor_props.rank}: halo size for 'CellDim' " - f"(1 : {decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " - f"(2: {decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" - ) - distributed_coordinates = multinode_grid_manager.coordinates - distributed_geometry = geometry.GridGeometry( - backend=backend, - grid=distributed_grid, - coordinates=distributed_coordinates, - decomposition_info=decomposition_info, - extra_fields=extra_geometry_fields, - metadata=geometry_attributes.attrs, - ) - - edge_length = distributed_geometry.get(geometry_attributes.EDGE_LENGTH) - cell_area = distributed_geometry.get(geometry_attributes.CELL_AREA) - edge_orientation = distributed_geometry.get(geometry_attributes.CELL_NORMAL_ORIENTATION) - - geofac_div = data_alloc.zero_field(distributed_grid, dims.CellDim, dims.C2EDim) - # has to be computed in gt4py-embedded - interpolation_fields.compute_geofac_div.with_backend(None)( - primal_edge_length=edge_length, - area=cell_area, - edge_orientation=edge_orientation, - out=geofac_div, - offset_provider={"C2E": distributed_grid.get_connectivity("C2E")}, - ) - - assert_gathered_field_against_global( - decomposition_info, - processor_props, - dims.CellDim, - global_reference_field=reference.asnumpy(), - local_field=geofac_div.asnumpy(), - ) - - print(f"rank = {processor_props.rank} - DONE") - -@pytest.mark.embedded_remap_error -@pytest.mark.mpi -@pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_halo_neighbor_access_e2c2v( - processor_props: decomp_defs.ProcessProperties, - backend: gtx_typing.Backend | None, - global_grid_descriptor: test_defs.GridDescription, -) -> None: - file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) - single_node = utils.run_grid_manager_for_singlenode(file) - single_node_grid = single_node.grid - single_node_geometry = geometry.GridGeometry( - backend=backend, - grid=single_node_grid, - coordinates=single_node.coordinates, - decomposition_info=single_node.decomposition_info, - extra_fields=single_node.geometry_fields, - metadata=geometry_attributes.attrs, - ) - print( - f"rank = {processor_props.rank} : single node grid has size {single_node.decomposition_info.get_horizontal_size()!r}" - ) - reference_u = single_node_geometry.get(geometry_attributes.EDGE_NORMAL_VERTEX_U).asnumpy() - reference_v = single_node_geometry.get(geometry_attributes.EDGE_NORMAL_VERTEX_V).asnumpy() - multinode_grid_manager = utils.run_gridmananger_for_multinode( + # TODO(msimberg): Use regular grid manager fixture (should anyway be multi rank by default) + multi_node_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, decomposer=decomp.MetisDecomposer(), ) - distributed_grid = multinode_grid_manager.grid - extra_geometry_fields = multinode_grid_manager.geometry_fields - decomposition_info = multinode_grid_manager.decomposition_info - print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'EdgeDim' " - f"(1 : {decomposition_info.get_halo_size(dims.EdgeDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " - f"(2: {decomposition_info.get_halo_size(dims.EdgeDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" + f"rank = {processor_props.rank} : {multi_node_grid_manager.decomposition_info.get_horizontal_size()!r}" ) - distributed_coordinates = multinode_grid_manager.coordinates - distributed_geometry = geometry.GridGeometry( - backend=backend, - grid=distributed_grid, - coordinates=distributed_coordinates, - decomposition_info=decomposition_info, - extra_fields=extra_geometry_fields, - metadata=geometry_attributes.attrs, - ) - vertex_lat = distributed_geometry.get(geometry_attributes.VERTEX_LAT) - vertex_lon = distributed_geometry.get(geometry_attributes.VERTEX_LON) - x = distributed_geometry.get(geometry_attributes.EDGE_NORMAL_X) - y = distributed_geometry.get(geometry_attributes.EDGE_NORMAL_Y) - z = distributed_geometry.get(geometry_attributes.EDGE_NORMAL_Z) - - u0 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) - v0 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) - u1 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) - v1 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) - u2 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) - v2 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) - u3 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) - v3 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) - horizontal_end = distributed_grid.end_index(h_grid.edge_domain(h_grid.Zone.HALO)) - - geometry_stencils.zonal_and_meridional_component_of_edge_field_at_vertex.with_backend(backend)( - vertex_lat, - vertex_lon, - x, - y, - z, - domain={dims.EdgeDim: (0, horizontal_end)}, - out=(u0, v0, u1, v1, u2, v2, u3, v3), - offset_provider={"E2C2V": distributed_grid.get_connectivity(dims.E2C2V)}, - ) - u_component = np.vstack((u0.asnumpy(), u1.asnumpy(), u2.asnumpy(), u3.asnumpy())).T - v_component = np.vstack((v0.asnumpy(), v1.asnumpy(), v2.asnumpy(), v3.asnumpy())).T - - assert_gathered_field_against_global( - decomposition_info, - processor_props, - dims.EdgeDim, - global_reference_field=reference_u, - local_field=u_component, - ) - assert_gathered_field_against_global( - decomposition_info, - processor_props, - dims.EdgeDim, - global_reference_field=reference_v, - local_field=v_component, - ) - - -@pytest.mark.embedded_remap_error -@pytest.mark.mpi -@pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_halo_neighbor_access_e2c( - processor_props: decomp_defs.ProcessProperties, - backend: gtx_typing.Backend | None, - global_grid_descriptor: test_defs.GridDescription, -) -> None: - file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) - single_node = utils.run_grid_manager_for_singlenode(file) - single_node_grid = single_node.grid - single_node_geometry = geometry.GridGeometry( - backend=backend, - grid=single_node_grid, - coordinates=single_node.coordinates, - decomposition_info=single_node.decomposition_info, - extra_fields=single_node.geometry_fields, - metadata=geometry_attributes.attrs, - ) - print( - f"rank = {processor_props.rank} : single node grid has size {single_node.decomposition_info.get_horizontal_size()!r}" - ) - reference_u = single_node_geometry.get(geometry_attributes.EDGE_NORMAL_CELL_U).asnumpy() - reference_v = single_node_geometry.get(geometry_attributes.EDGE_NORMAL_CELL_V).asnumpy() - multinode_grid_manager = utils.run_gridmananger_for_multinode( - file=file, - run_properties=processor_props, - decomposer=decomp.MetisDecomposer(), - ) - distributed_grid = multinode_grid_manager.grid - extra_geometry_fields = multinode_grid_manager.geometry_fields - decomposition_info = multinode_grid_manager.decomposition_info - - print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( - f"rank = {processor_props.rank}: halo size for 'EdgeDim' " - f"(1 : {decomposition_info.get_halo_size(dims.EdgeDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " - f"(2: {decomposition_info.get_halo_size(dims.EdgeDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" + f"rank = {processor_props.rank}: halo size for 'CellDim' " + f"(1: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " + f"(2: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) - distributed_coordinates = multinode_grid_manager.coordinates - distributed_geometry = geometry.GridGeometry( + # TODO(msimberg): Use regular geometry fixture + multi_node_geometry = geometry.GridGeometry( backend=backend, - grid=distributed_grid, - coordinates=distributed_coordinates, - decomposition_info=decomposition_info, - extra_fields=extra_geometry_fields, + grid=multi_node_grid_manager.grid, + coordinates=multi_node_grid_manager.coordinates, + decomposition_info=multi_node_grid_manager.decomposition_info, + extra_fields=multi_node_grid_manager.geometry_fields, metadata=geometry_attributes.attrs, ) - cell_lat = distributed_geometry.get(geometry_attributes.CELL_LAT) - cell_lon = distributed_geometry.get(geometry_attributes.CELL_LON) - x = distributed_geometry.get(geometry_attributes.EDGE_NORMAL_X) - y = distributed_geometry.get(geometry_attributes.EDGE_NORMAL_Y) - z = distributed_geometry.get(geometry_attributes.EDGE_NORMAL_Z) - - u0 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) - v0 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) - u1 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) - v1 = data_alloc.random_field(distributed_grid, dims.EdgeDim, allocator=backend) - horizontal_end = distributed_grid.end_index(h_grid.edge_domain(h_grid.Zone.HALO)) - - geometry_stencils.zonal_and_meridional_component_of_edge_field_at_cell_center.with_backend( - backend - )( - cell_lat, - cell_lon, - x, - y, - z, - out=(u0, v0, u1, v1), - domain={dims.EdgeDim: (0, horizontal_end)}, - offset_provider={"E2C": distributed_grid.get_connectivity(dims.E2C)}, - ) - u_component = np.vstack((u0.asnumpy(), u1.asnumpy())).T - v_component = np.vstack((v0.asnumpy(), v1.asnumpy())).T - assert_gathered_field_against_global( - decomposition_info, - processor_props, - dims.EdgeDim, - global_reference_field=reference_u, - local_field=u_component, - ) - assert_gathered_field_against_global( - decomposition_info, - processor_props, - dims.EdgeDim, - global_reference_field=reference_v, - local_field=v_component, - ) - - -@pytest.mark.mpi -@pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_halo_neighbor_access_e2v( - processor_props: decomp_defs.ProcessProperties, - backend: gtx_typing.Backend | None, - global_grid_descriptor: test_defs.GridDescription, -) -> None: - print(f"running on {processor_props.comm}") - file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) - single_node = utils.run_grid_manager_for_singlenode(file) - single_node_grid = single_node.grid - single_node_geometry = geometry.GridGeometry( - backend=backend, - grid=single_node_grid, - coordinates=single_node.coordinates, - decomposition_info=single_node.decomposition_info, - extra_fields=single_node.geometry_fields, - metadata=geometry_attributes.attrs, - ) - reference_tangent_x = single_node_geometry.get(geometry_attributes.EDGE_TANGENT_X).asnumpy() - reference_tangent_y = single_node_geometry.get(geometry_attributes.EDGE_TANGENT_Y).asnumpy() - print( - f"rank = {processor_props.rank} : single node computed field reference has size {reference_tangent_x.shape}" - ) - multinode_grid_manager = utils.run_gridmananger_for_multinode( - file=file, - run_properties=processor_props, - decomposer=decomp.MetisDecomposer(), - ) - distributed_grid = multinode_grid_manager.grid - decomposition_info = multinode_grid_manager.decomposition_info - - print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") - print( - f"rank = {processor_props.rank}: halo size for 'EdgeDim' " - f"(1 : {decomposition_info.get_halo_size(dims.EdgeDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " - f"(2: {decomposition_info.get_halo_size(dims.EdgeDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" - ) - distributed_coordinates = multinode_grid_manager.coordinates - vertex_lat = distributed_coordinates[dims.VertexDim]["lat"] - vertex_lon = distributed_coordinates[dims.VertexDim]["lon"] - tangent_orientation = multinode_grid_manager.geometry_fields.get( - gridfile.GeometryName.TANGENT_ORIENTATION - ) - - tangent_x = data_alloc.zero_field(distributed_grid, dims.EdgeDim) - tangent_y = data_alloc.zero_field(distributed_grid, dims.EdgeDim) - tangent_z = data_alloc.zero_field(distributed_grid, dims.EdgeDim) - - geometry_stencils.cartesian_coordinates_of_edge_tangent.with_backend(backend)( - vertex_lat=vertex_lat, - vertex_lon=vertex_lon, - edge_orientation=tangent_orientation, - domain={dims.EdgeDim: (0, distributed_grid.num_edges)}, - offset_provider=distributed_grid.connectivities, - out=(tangent_x, tangent_y, tangent_z), - ) - - # only the computation of the tangent uses neighbor access - assert_gathered_field_against_global( - decomposition_info, - processor_props, - dims.EdgeDim, - global_reference_field=reference_tangent_x, - local_field=tangent_x.asnumpy(), - ) - assert_gathered_field_against_global( - decomposition_info, - processor_props, - dims.EdgeDim, - global_reference_field=reference_tangent_y, - local_field=tangent_y.asnumpy(), + check_local_global_field( + decomposition_info=multi_node_grid_manager.decomposition_info, + processor_props=processor_props, + dim=dim, + global_reference_field=single_node_geometry.get(attrs_name).asnumpy(), + local_field=multi_node_geometry.get(attrs_name).asnumpy(), ) print(f"rank = {processor_props.rank} - DONE") @@ -570,318 +264,182 @@ def test_halo_neighbor_access_e2v( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_halo_neighbor_access_v2e( +@pytest.mark.parametrize( + "attrs_name, dim", + [ + (interpolation_attributes.GEOFAC_DIV, dims.CellDim), + (interpolation_attributes.GEOFAC_ROT, dims.VertexDim), + (interpolation_attributes.C_BLN_AVG, dims.CellDim), + ], +) +def test_interpolation_fields_compare_single_multi_rank( processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, + # TODO(msimberg): Maybe use regular grid fixture and skip local area grids? global_grid_descriptor: test_defs.GridDescription, + attrs_name: str, + dim: gtx.Dimension, ) -> None: + # TODO(msimberg): Add fixture for "always single rank" grid manager file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) - print(f"running on {processor_props.comm}") - single_node = utils.run_grid_manager_for_singlenode(file) - single_node_grid = single_node.grid + print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") + single_node_grid_manager = utils.run_grid_manager_for_singlenode(file) + # TODO(msimberg): Add fixture for "always single rank" geometry single_node_geometry = geometry.GridGeometry( backend=backend, - grid=single_node_grid, - coordinates=single_node.coordinates, - decomposition_info=single_node.decomposition_info, - extra_fields=single_node.geometry_fields, + grid=single_node_grid_manager.grid, + coordinates=single_node_grid_manager.coordinates, + decomposition_info=single_node_grid_manager.decomposition_info, + extra_fields=single_node_grid_manager.geometry_fields, metadata=geometry_attributes.attrs, ) - single_node_dual_edge_length = single_node_geometry.get(geometry_attributes.DUAL_EDGE_LENGTH) - single_node_edge_orientation = single_node_geometry.get( - geometry_attributes.VERTEX_EDGE_ORIENTATION - ) - single_node_dual_area = single_node_geometry.get(geometry_attributes.DUAL_AREA) - single_node_owner_mask = gtx.as_field( - (dims.VertexDim,), - data=single_node.decomposition_info.owner_mask(dims.VertexDim), # type: ignore [arg-type] - dtype=bool, - ) - reference = data_alloc.zero_field(single_node_grid, dims.VertexDim, dims.V2EDim) - lateral_boundary_start = single_node_grid.start_index( - h_grid.vertex_domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2) - ) - horizontal_end = single_node_grid.start_index(h_grid.vertex_domain(h_grid.Zone.END)) - - interpolation_fields.compute_geofac_rot.with_backend(None)( - single_node_dual_edge_length, - single_node_edge_orientation, - single_node_dual_area, - single_node_owner_mask, - out=reference, - domain={dims.VertexDim: (lateral_boundary_start, horizontal_end)}, - offset_provider={"V2E": single_node_grid.get_connectivity(dims.V2E)}, + # TODO(msimberg): Add fixture for "always single rank" interpolation factory + single_node_interpolation = interpolation_factory.InterpolationFieldsFactory( + grid=single_node_grid_manager.grid, + decomposition_info=single_node_grid_manager.decomposition_info, + geometry_source=single_node_geometry, + backend=backend, + metadata=interpolation_attributes.attrs, + exchange=decomp_defs.SingleNodeExchange(), ) - print( - f"rank = {processor_props.rank} : single node computed field reference has size {reference.asnumpy().shape}" + f"rank = {processor_props.rank} : single node grid has size {single_node_grid_manager.decomposition_info.get_horizontal_size()!r}" ) - multinode_grid_manager = utils.run_gridmananger_for_multinode( + + # TODO(msimberg): Use regular grid manager fixture (should anyway be multi rank by default) + multi_node_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, decomposer=decomp.MetisDecomposer(), ) - distributed_grid = multinode_grid_manager.grid - decomposition_info = multinode_grid_manager.decomposition_info - - print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") - print( - f"rank = {processor_props.rank}: halo size for 'EdgeDim' " - f"(1 : {decomposition_info.get_halo_size(dims.EdgeDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)})," - f" (2: {decomposition_info.get_halo_size(dims.EdgeDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" - ) - distributed_coordinates = multinode_grid_manager.coordinates - extra_geometry_fields = multinode_grid_manager.geometry_fields - distributed_geometry = geometry.GridGeometry( - backend=backend, - grid=distributed_grid, - coordinates=distributed_coordinates, - decomposition_info=decomposition_info, - extra_fields=extra_geometry_fields, - metadata=geometry_attributes.attrs, - ) - - dual_edge_length = distributed_geometry.get(geometry_attributes.DUAL_EDGE_LENGTH) - edge_orientation = distributed_geometry.get(geometry_attributes.VERTEX_EDGE_ORIENTATION) - dual_area = distributed_geometry.get(geometry_attributes.DUAL_AREA) - geofac_rot = data_alloc.zero_field(distributed_grid, dims.VertexDim, dims.V2EDim) - onwner_mask = gtx.as_field((dims.VertexDim,), decomposition_info.owner_mask(dims.VertexDim)) # type: ignore [arg-type] - interpolation_fields.compute_geofac_rot.with_backend(None)( - dual_edge_length=dual_edge_length, - edge_orientation=edge_orientation, - dual_area=dual_area, - owner_mask=onwner_mask, - out=geofac_rot, - offset_provider={"V2E": distributed_grid.get_connectivity(dims.V2E)}, - ) - - assert_gathered_field_against_global( - decomposition_info, - processor_props, - dims.VertexDim, - global_reference_field=reference.asnumpy(), - local_field=geofac_rot.asnumpy(), - ) - - print(f"rank = {processor_props.rank}/{processor_props.comm_size} - DONE") - - -@pytest.mark.mpi -@pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_halo_neighbor_access_c2e2c( - processor_props: decomp_defs.ProcessProperties, - backend: gtx_typing.Backend | None, - global_grid_descriptor: test_defs.GridDescription, -) -> None: - file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) - center_weight = 0.3 - xp = data_alloc.import_array_ns(allocator=backend) - start_zone = h_grid.cell_domain(h_grid.Zone.LATERAL_BOUNDARY_LEVEL_2) - print(f"running on {processor_props.comm}") - single_node = utils.run_grid_manager_for_singlenode(file) - single_node_grid = single_node.grid - single_node_geometry = geometry.GridGeometry( - backend=backend, - grid=single_node_grid, - coordinates=single_node.coordinates, - decomposition_info=single_node.decomposition_info, - extra_fields=single_node.geometry_fields, - metadata=geometry_attributes.attrs, - ) - reference = interpolation_fields._compute_c_bln_avg( - single_node_grid.get_connectivity(dims.C2E2C).ndarray, - single_node_geometry.get(geometry_attributes.CELL_LAT).ndarray, - single_node_geometry.get(geometry_attributes.CELL_LON).ndarray, - center_weight, - horizontal_start=single_node_grid.start_index(start_zone), - array_ns=xp, - ) - print( - f"rank = {processor_props.rank} : single node computed field reference has size {reference.shape}" - ) - multinode_grid_manager = utils.run_gridmananger_for_multinode( - file=file, - run_properties=processor_props, - decomposer=decomp.MetisDecomposer(), + f"rank = {processor_props.rank} : {multi_node_grid_manager.decomposition_info.get_horizontal_size()!r}" ) - distributed_grid = multinode_grid_manager.grid - decomposition_info = multinode_grid_manager.decomposition_info - - print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") print( f"rank = {processor_props.rank}: halo size for 'CellDim' " - f"(1 : {decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " - f"(2: {decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" + f"(1: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " + f"(2: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) - distributed_coordinates = multinode_grid_manager.coordinates - extra_geometry_fields = multinode_grid_manager.geometry_fields - distributed_geometry = geometry.GridGeometry( + # TODO(msimberg): Use regular geometry fixture + multi_node_geometry = geometry.GridGeometry( backend=backend, - grid=distributed_grid, - coordinates=distributed_coordinates, - decomposition_info=decomposition_info, - extra_fields=extra_geometry_fields, + grid=multi_node_grid_manager.grid, + coordinates=multi_node_grid_manager.coordinates, + decomposition_info=multi_node_grid_manager.decomposition_info, + extra_fields=multi_node_grid_manager.geometry_fields, metadata=geometry_attributes.attrs, ) - - c_bln_avg = interpolation_fields._compute_c_bln_avg( - distributed_grid.get_connectivity(dims.C2E2C).ndarray, - distributed_geometry.get(geometry_attributes.CELL_LAT).ndarray, - distributed_geometry.get(geometry_attributes.CELL_LON).ndarray, - center_weight, - horizontal_start=distributed_grid.start_index(start_zone), - array_ns=xp, - ) - - print( - f"rank = {processor_props.rank}/{processor_props.comm_size} - computed field has shape =({c_bln_avg.shape})" - ) - - assert_gathered_field_against_global( - decomposition_info, - processor_props, - dims.CellDim, - global_reference_field=reference, - local_field=c_bln_avg, - ) - - print(f"rank = {processor_props.rank}/{processor_props.comm_size} - DONE") - - -@pytest.mark.mpi -@pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_halo_neighbor_access_v2c( - processor_props: decomp_defs.ProcessProperties, - global_grid_descriptor: test_defs.GridDescription, - backend: gtx_typing.Backend, -) -> None: - file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) - print(f"running on {processor_props.comm}") - single_node = utils.run_grid_manager_for_singlenode(file) - single_node_grid = single_node.grid - - buffer = np.repeat( - single_node.coordinates[dims.CellDim]["lat"].asnumpy()[:, None], - repeats=utils.NUM_LEVELS, - axis=1, - ) - full_cell_k_field = gtx.as_field( - (dims.CellDim, dims.KDim), - data=buffer, # type: ignore [arg-type] - dtype=float, - allocator=backend, - ) - print( - f"rank = {processor_props.rank} / {processor_props.comm_size}: single node input field has size {full_cell_k_field.asnumpy().shape}" - ) - buffer = single_node.coordinates[dims.VertexDim]["lat"].asnumpy() + 1.0 - full_coef = gtx.as_field( - (dims.VertexDim, dims.V2CDim), - data=np.repeat((buffer / np.max(buffer))[:, None], 6, axis=1), # type: ignore [arg-type] - dtype=float, - allocator=backend, - ) - - reference = data_alloc.zero_field( - single_node_grid, - dims.VertexDim, - dims.KDim, - dtype=full_cell_k_field.dtype, - allocator=backend, - ) - _compute_cell_2_vertex_interpolation( - full_cell_k_field, - full_coef, - out=reference, - offset_provider={"V2C": single_node_grid.get_connectivity(dims.V2C)}, - ) - - print( - f"rank = {processor_props.rank}/ {processor_props.comm_size} : single node computed field reference has size {reference.asnumpy().shape}" - ) - multinode_grid_manager = utils.run_gridmananger_for_multinode( - file=file, - run_properties=processor_props, - decomposer=decomp.MetisDecomposer(), + # TODO(msimberg): Use regular interpolation factory fixture + multi_node_interpolation = interpolation_factory.InterpolationFieldsFactory( + grid=multi_node_grid_manager.grid, + decomposition_info=multi_node_grid_manager.decomposition_info, + geometry_source=multi_node_geometry, + backend=backend, + metadata=interpolation_attributes.attrs, + exchange=mpi_decomposition.GHexMultiNodeExchange( + processor_props, multi_node_grid_manager.decomposition_info + ), ) - distributed_grid = multinode_grid_manager.grid - decomposition_info = multinode_grid_manager.decomposition_info - print(f"rank = {processor_props.rank} : {decomposition_info.get_horizontal_size()!r}") - print( - f"rank = {processor_props.rank}: halo size for 'CellDim' " - f"(1 : {decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " - f"(2: {decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" - ) - print( - f"rank = {processor_props.rank}: halo size for 'VertexDim' " - f"(1 : {decomposition_info.get_halo_size(dims.VertexDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " - f"(2: {decomposition_info.get_halo_size(dims.VertexDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" - ) - my_global_cells = decomposition_info.global_index(dims.CellDim) - distributed_cell_k_buffer = ( - full_cell_k_field.asnumpy()[my_global_cells, :] - .ravel(order="K") - .reshape(distributed_grid.num_cells, utils.NUM_LEVELS) - ) - # validate the input data distribution - assert_gathered_field_against_global( - decomposition_info, - processor_props, - dims.CellDim, - global_reference_field=full_cell_k_field.asnumpy(), - local_field=distributed_cell_k_buffer, - ) - print( - f"rank={processor_props.rank}/{processor_props.comm_size}: input field shape = ([{distributed_cell_k_buffer.shape})" - ) + field_ref = single_node_interpolation.get(attrs_name).asnumpy() + field = multi_node_interpolation.get(attrs_name).asnumpy() - distributed_cell_k_field = gtx.as_field( - (dims.CellDim, dims.KDim), - data=distributed_cell_k_buffer, # type: ignore [arg-type] - dtype=distributed_cell_k_buffer.dtype, - allocator=backend, + check_local_global_field( + decomposition_info=multi_node_grid_manager.decomposition_info, + processor_props=processor_props, + dim=dim, + global_reference_field=field_ref, + local_field=field, ) - my_global_vertices = decomposition_info.global_index(dims.VertexDim) + print(f"rank = {processor_props.rank} - DONE") - distributed_coef_buffer = ( - full_coef.asnumpy()[my_global_vertices, :] - .ravel(order="K") - .reshape((distributed_grid.num_vertices, 6)) - ) - distributed_coef = gtx.as_field( - (dims.VertexDim, dims.V2CDim), - data=distributed_coef_buffer, # type: ignore [arg-type] - allocator=backend, - ) - assert_gathered_field_against_global( - decomposition_info, - processor_props, - dims.VertexDim, - global_reference_field=full_coef.asnumpy(), - local_field=distributed_coef_buffer, - ) - print( - f"rank={processor_props.rank}/{processor_props.comm_size}: coefficient shape = ([{distributed_coef_buffer.shape})" - ) - output = data_alloc.zero_field(distributed_grid, dims.VertexDim, dims.KDim, allocator=backend) - _compute_cell_2_vertex_interpolation( - distributed_cell_k_field, - distributed_coef, - out=output, - offset_provider={"V2C": distributed_grid.get_connectivity(dims.V2C)}, - ) - - assert_gathered_field_against_global( - decomposition_info, - processor_props, - dim=dims.VertexDim, - global_reference_field=reference.asnumpy(), - local_field=output.asnumpy(), - ) +# @pytest.mark.mpi +# @pytest.mark.parametrize("processor_props", [True], indirect=True) +# @pytest.mark.parametrize("attrs_name, dim", [(metrics_attributes.DDXT_Z_HALF_E, dims.EdgeDim)]) +# def test_metrics_fields_compare_single_multi_rank( +# processor_props: decomp_defs.ProcessProperties, +# backend: gtx_typing.Backend | None, +# # TODO(msimberg): Maybe use regular grid fixture and skip local area grids? +# global_grid_descriptor: test_defs.GridDescription, +# attrs_name: str, +# dim: gtx.Dimension, +# ) -> None: +# # TODO(msimberg): Add fixture for "always single rank" grid manager +# file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) +# print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") +# single_node_grid_manager = utils.run_grid_manager_for_singlenode(file) +# # TODO(msimberg): Add fixture for "always single rank" geometry +# single_node_geometry = geometry.GridGeometry( +# backend=backend, +# grid=single_node_grid_manager.grid, +# coordinates=single_node_grid_manager.coordinates, +# decomposition_info=single_node_grid_manager.decomposition_info, +# extra_fields=single_node_grid_manager.geometry_fields, +# metadata=geometry_attributes.attrs, +# ) +# # TODO(msimberg): Add fixture for "always single rank" interpolation factory +# single_node_interpolation = interpolation_factory.InterpolationFieldsFactory( +# grid=single_node_grid_manager.grid, +# decomposition_info=single_node_grid_manager.decomposition_info, +# geometry_source=single_node_geometry, +# backend=backend, +# metadata=interpolation_attributes.attrs, +# exchange=decomp_defs.SingleNodeExchange(), +# ) +# # TODO metrics factory +# print( +# f"rank = {processor_props.rank} : single node grid has size {single_node_grid_manager.decomposition_info.get_horizontal_size()!r}" +# ) + +# # TODO(msimberg): Use regular grid manager fixture (should anyway be multi rank by default) +# multi_node_grid_manager = utils.run_gridmananger_for_multinode( +# file=file, +# run_properties=processor_props, +# decomposer=decomp.MetisDecomposer(), +# ) +# print( +# f"rank = {processor_props.rank} : {multi_node_grid_manager.decomposition_info.get_horizontal_size()!r}" +# ) +# print( +# f"rank = {processor_props.rank}: halo size for 'CellDim' " +# f"(1: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " +# f"(2: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" +# ) +# # TODO(msimberg): Use regular geometry fixture +# multi_node_geometry = geometry.GridGeometry( +# backend=backend, +# grid=multi_node_grid_manager.grid, +# coordinates=multi_node_grid_manager.coordinates, +# decomposition_info=multi_node_grid_manager.decomposition_info, +# extra_fields=multi_node_grid_manager.geometry_fields, +# metadata=geometry_attributes.attrs, +# ) +# # TODO(msimberg): Use regular interpolation factory fixture +# multi_node_interpolation = interpolation_factory.InterpolationFieldsFactory( +# grid=multi_node_grid_manager.grid, +# decomposition_info=multi_node_grid_manager.decomposition_info, +# geometry_source=multi_node_geometry, +# backend=backend, +# metadata=interpolation_attributes.attrs, +# exchange=mpi_decomposition.GHexMultiNodeExchange( +# processor_props, multi_node_grid_manager.decomposition_info +# ), +# ) + +# field_ref = single_node_interpolation.get(attrs_name).asnumpy() +# field = multi_node_interpolation.get(attrs_name).asnumpy() + +# check_local_global_field( +# decomposition_info=multi_node_grid_manager.decomposition_info, +# processor_props=processor_props, +# dim=dim, +# global_reference_field=field_ref, +# local_field=field, +# ) + +# print(f"rank = {processor_props.rank} - DONE") @pytest.mark.mpi From dcf72b581b130e8b0a10a5e2dad0e90f902580d3 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 6 Feb 2026 15:25:04 +0100 Subject: [PATCH 193/240] Minor cleanup and todo in test_parallel_halo.py --- .../decomposition/mpi_tests/test_parallel_halo.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py index 82b4b0c6de..18a01a5f73 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py @@ -24,6 +24,7 @@ from icon4py.model.common.decomposition import mpi_decomposition + # TODO(msimberg): Does every test/module need to do this? mpi_decomposition.init_mpi() except ImportError: pytest.skip("Skipping parallel on single node installation", allow_module_level=True) @@ -69,12 +70,12 @@ def test_element_ownership_is_unique( my_size = owned.shape[0] local_sizes = np.array(comm.gather(my_size, root=0)) buffer_size = 27 - send_buf = -1 * np.ones(buffer_size, dtype=int) + send_buf = np.full(buffer_size, -1, dtype=int) send_buf[:my_size] = owned print(f"rank {processor_props.rank} send_buf: {send_buf}") if processor_props.rank == 0: print(f"local_sizes: {local_sizes}") - recv_buffer = -1 * np.ones((4, buffer_size), dtype=int) + recv_buffer = np.full((4, buffer_size), -1, dtype=int) print(f"{recv_buffer.shape}") else: recv_buffer = None @@ -87,11 +88,3 @@ def test_element_ownership_is_unique( assert values.size == len(np.unique(values)) # check the buffer has all global indices assert np.all(np.sort(values) == global_indices(dim)) - - -def decompose(grid: base_grid.Grid, processor_props): - partitioner = halo.MetisDecomposer() - labels = partitioner( - grid.connectivities[dims.C2E2C].asnumpy(), n_part=processor_props.comm_size - ) - return labels From f7eb9bea827b99dc0336882a627ee04efa5440da Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 6 Feb 2026 15:25:26 +0100 Subject: [PATCH 194/240] Cleanup and todo --- .../common/grid/mpi_tests/test_parallel_geometry.py | 3 +-- .../common/grid/mpi_tests/test_parallel_icon.py | 1 + model/common/tests/common/grid/mpi_tests/utils.py | 13 ++++++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_geometry.py b/model/common/tests/common/grid/mpi_tests/test_parallel_geometry.py index 34f223121f..e6a10a588c 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_geometry.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_geometry.py @@ -78,9 +78,8 @@ def test_distributed_geometry_attrs( parallel_helpers.check_comm_size(processor_props) parallel_helpers.log_process_properties(processor_props) parallel_helpers.log_local_field_size(decomposition_info) - grid_geometry = geometry_from_savepoint field_ref = grid_savepoint.__getattribute__(grid_name)().asnumpy() - field = grid_geometry.get(attrs_name).asnumpy() + field = geometry_from_savepoint.get(attrs_name).asnumpy() assert test_utils.dallclose(field, field_ref, atol=1e-12) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py index 0bee89418f..30b6d36c62 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py @@ -184,6 +184,7 @@ def test_start_index_end_index_halo_zones_on_distributed_lam_grid( assert end_index == expected, f"expected start index {1}, but was {start_index}" +# TODO(msimberg): Torus? Above as well. @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi @pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) diff --git a/model/common/tests/common/grid/mpi_tests/utils.py b/model/common/tests/common/grid/mpi_tests/utils.py index fedfafd5d3..1883d0eaae 100644 --- a/model/common/tests/common/grid/mpi_tests/utils.py +++ b/model/common/tests/common/grid/mpi_tests/utils.py @@ -12,6 +12,12 @@ from icon4py.model.common.grid import grid_manager as gm, vertical as v_grid +def _grid_manager(file: pathlib.Path, num_levels: int) -> gm.GridManager: + return gm.GridManager( + grid_file=str(file), config=v_grid.VerticalGridConfig(num_levels=num_levels) + ) + + def run_grid_manager_for_singlenode(file: pathlib.Path) -> gm.GridManager: manager = _grid_manager(file, NUM_LEVELS) manager( @@ -23,13 +29,6 @@ def run_grid_manager_for_singlenode(file: pathlib.Path) -> gm.GridManager: return manager -def _grid_manager(file: pathlib.Path, num_levels: int) -> gm.GridManager: - manager = gm.GridManager( - grid_file=str(file), config=v_grid.VerticalGridConfig(num_levels=num_levels) - ) - return manager - - def run_gridmananger_for_multinode( file: pathlib.Path, run_properties: decomp_defs.ProcessProperties, From 48e9a21edeeaaa7f3be40a2850105e1dd378da8d Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 6 Feb 2026 15:44:32 +0100 Subject: [PATCH 195/240] Clean up todos --- .../grid/mpi_tests/test_parallel_grid_manager.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 32f110527f..45ead92075 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -210,11 +210,12 @@ def test_geometry_fields_compare_single_multi_rank( attrs_name: str, dim: gtx.Dimension, ) -> None: - # TODO(msimberg): Add fixture for "always single rank" grid manager + # TODO(msimberg): Add fixtures for single/mult-rank + # grid/geometry/interpolation/metrics factoriesAdd fixtures for + # single/mult-rank grid/geometry/interpolation/metrics factories. file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") single_node_grid_manager = utils.run_grid_manager_for_singlenode(file) - # TODO(msimberg): Add fixture for "always single rank" geometry single_node_geometry = geometry.GridGeometry( backend=backend, grid=single_node_grid_manager.grid, @@ -227,7 +228,6 @@ def test_geometry_fields_compare_single_multi_rank( f"rank = {processor_props.rank} : single node grid has size {single_node_grid_manager.decomposition_info.get_horizontal_size()!r}" ) - # TODO(msimberg): Use regular grid manager fixture (should anyway be multi rank by default) multi_node_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, @@ -241,7 +241,6 @@ def test_geometry_fields_compare_single_multi_rank( f"(1: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " f"(2: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) - # TODO(msimberg): Use regular geometry fixture multi_node_geometry = geometry.GridGeometry( backend=backend, grid=multi_node_grid_manager.grid, @@ -275,16 +274,13 @@ def test_geometry_fields_compare_single_multi_rank( def test_interpolation_fields_compare_single_multi_rank( processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, - # TODO(msimberg): Maybe use regular grid fixture and skip local area grids? global_grid_descriptor: test_defs.GridDescription, attrs_name: str, dim: gtx.Dimension, ) -> None: - # TODO(msimberg): Add fixture for "always single rank" grid manager file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") single_node_grid_manager = utils.run_grid_manager_for_singlenode(file) - # TODO(msimberg): Add fixture for "always single rank" geometry single_node_geometry = geometry.GridGeometry( backend=backend, grid=single_node_grid_manager.grid, @@ -293,7 +289,6 @@ def test_interpolation_fields_compare_single_multi_rank( extra_fields=single_node_grid_manager.geometry_fields, metadata=geometry_attributes.attrs, ) - # TODO(msimberg): Add fixture for "always single rank" interpolation factory single_node_interpolation = interpolation_factory.InterpolationFieldsFactory( grid=single_node_grid_manager.grid, decomposition_info=single_node_grid_manager.decomposition_info, @@ -306,7 +301,6 @@ def test_interpolation_fields_compare_single_multi_rank( f"rank = {processor_props.rank} : single node grid has size {single_node_grid_manager.decomposition_info.get_horizontal_size()!r}" ) - # TODO(msimberg): Use regular grid manager fixture (should anyway be multi rank by default) multi_node_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, @@ -320,7 +314,6 @@ def test_interpolation_fields_compare_single_multi_rank( f"(1: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " f"(2: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) - # TODO(msimberg): Use regular geometry fixture multi_node_geometry = geometry.GridGeometry( backend=backend, grid=multi_node_grid_manager.grid, @@ -329,7 +322,6 @@ def test_interpolation_fields_compare_single_multi_rank( extra_fields=multi_node_grid_manager.geometry_fields, metadata=geometry_attributes.attrs, ) - # TODO(msimberg): Use regular interpolation factory fixture multi_node_interpolation = interpolation_factory.InterpolationFieldsFactory( grid=multi_node_grid_manager.grid, decomposition_info=multi_node_grid_manager.decomposition_info, From f29e554fc68d4a872f078de5886d90c2a28df440 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 6 Feb 2026 16:30:23 +0100 Subject: [PATCH 196/240] Remove global_grid_descriptor fixture, try to add metrics test --- model/common/tests/common/grid/fixtures.py | 1 - .../mpi_tests/test_parallel_grid_manager.py | 290 ++++++++++++------ .../grid/unit_tests/test_grid_manager.py | 5 +- .../common/grid/unit_tests/test_gridfile.py | 17 +- .../model/testing/fixtures/datatest.py | 17 - 5 files changed, 202 insertions(+), 128 deletions(-) diff --git a/model/common/tests/common/grid/fixtures.py b/model/common/tests/common/grid/fixtures.py index 773bbed224..debad78fc5 100644 --- a/model/common/tests/common/grid/fixtures.py +++ b/model/common/tests/common/grid/fixtures.py @@ -13,7 +13,6 @@ download_ser_data, experiment, flat_height, - global_grid_descriptor, grid_savepoint, htop_moist_proc, icon_grid, diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 45ead92075..c1fca8fa5f 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -15,7 +15,7 @@ from gt4py import next as gtx from gt4py.next import common as gtx_common, typing as gtx_typing -from icon4py.model.common import dimension as dims, exceptions +from icon4py.model.common import dimension as dims, exceptions, model_backends from icon4py.model.common.decomposition import ( decomposer as decomp, definitions as decomp_defs, @@ -44,7 +44,7 @@ from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import definitions as test_defs, grid_utils, test_utils -from ..fixtures import backend, global_grid_descriptor, processor_props +from ..fixtures import backend, experiment, processor_props, topography_savepoint from . import utils @@ -62,9 +62,12 @@ @pytest.mark.mpi(min_size=2) def test_grid_manager_validate_decomposer( processor_props: decomp_defs.ProcessProperties, - global_grid_descriptor: test_defs.GridDescription, + experiment: test_defs.Experiment, ) -> None: - file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) + if experiment == test_defs.Experiments.MCH_CH_R04B09: + pytest.xfail("Limited-area grids not yet supported") + + file = grid_utils.resolve_full_grid_file_name(experiment.grid) manager = gm.GridManager( grid_file=file, config=v_grid.VerticalGridConfig(num_levels=utils.NUM_LEVELS), @@ -206,14 +209,17 @@ def test_geometry_fields_compare_single_multi_rank( processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, # TODO(msimberg): Maybe use regular grid fixture and skip local area grids? - global_grid_descriptor: test_defs.GridDescription, + experiment: test_defs.Experiment, attrs_name: str, dim: gtx.Dimension, ) -> None: - # TODO(msimberg): Add fixtures for single/mult-rank + if experiment == test_defs.Experiments.MCH_CH_R04B09: + pytest.xfail("Limited-area grids not yet supported") + + # TODO(msimberg): Add fixtures for single/multi-rank # grid/geometry/interpolation/metrics factoriesAdd fixtures for # single/mult-rank grid/geometry/interpolation/metrics factories. - file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) + file = grid_utils.resolve_full_grid_file_name(experiment.grid) print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") single_node_grid_manager = utils.run_grid_manager_for_singlenode(file) single_node_geometry = geometry.GridGeometry( @@ -274,11 +280,14 @@ def test_geometry_fields_compare_single_multi_rank( def test_interpolation_fields_compare_single_multi_rank( processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, - global_grid_descriptor: test_defs.GridDescription, + experiment: test_defs.Experiment, attrs_name: str, dim: gtx.Dimension, ) -> None: - file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) + if experiment == test_defs.Experiments.MCH_CH_R04B09: + pytest.xfail("Limited-area grids not yet supported") + + file = grid_utils.resolve_full_grid_file_name(experiment.grid) print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") single_node_grid_manager = utils.run_grid_manager_for_singlenode(file) single_node_geometry = geometry.GridGeometry( @@ -347,100 +356,189 @@ def test_interpolation_fields_compare_single_multi_rank( print(f"rank = {processor_props.rank} - DONE") -# @pytest.mark.mpi -# @pytest.mark.parametrize("processor_props", [True], indirect=True) -# @pytest.mark.parametrize("attrs_name, dim", [(metrics_attributes.DDXT_Z_HALF_E, dims.EdgeDim)]) -# def test_metrics_fields_compare_single_multi_rank( -# processor_props: decomp_defs.ProcessProperties, -# backend: gtx_typing.Backend | None, -# # TODO(msimberg): Maybe use regular grid fixture and skip local area grids? -# global_grid_descriptor: test_defs.GridDescription, -# attrs_name: str, -# dim: gtx.Dimension, -# ) -> None: -# # TODO(msimberg): Add fixture for "always single rank" grid manager -# file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) -# print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") -# single_node_grid_manager = utils.run_grid_manager_for_singlenode(file) -# # TODO(msimberg): Add fixture for "always single rank" geometry -# single_node_geometry = geometry.GridGeometry( -# backend=backend, -# grid=single_node_grid_manager.grid, -# coordinates=single_node_grid_manager.coordinates, -# decomposition_info=single_node_grid_manager.decomposition_info, -# extra_fields=single_node_grid_manager.geometry_fields, -# metadata=geometry_attributes.attrs, -# ) -# # TODO(msimberg): Add fixture for "always single rank" interpolation factory -# single_node_interpolation = interpolation_factory.InterpolationFieldsFactory( -# grid=single_node_grid_manager.grid, -# decomposition_info=single_node_grid_manager.decomposition_info, -# geometry_source=single_node_geometry, -# backend=backend, -# metadata=interpolation_attributes.attrs, -# exchange=decomp_defs.SingleNodeExchange(), -# ) -# # TODO metrics factory -# print( -# f"rank = {processor_props.rank} : single node grid has size {single_node_grid_manager.decomposition_info.get_horizontal_size()!r}" -# ) - -# # TODO(msimberg): Use regular grid manager fixture (should anyway be multi rank by default) -# multi_node_grid_manager = utils.run_gridmananger_for_multinode( -# file=file, -# run_properties=processor_props, -# decomposer=decomp.MetisDecomposer(), -# ) -# print( -# f"rank = {processor_props.rank} : {multi_node_grid_manager.decomposition_info.get_horizontal_size()!r}" -# ) -# print( -# f"rank = {processor_props.rank}: halo size for 'CellDim' " -# f"(1: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " -# f"(2: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" -# ) -# # TODO(msimberg): Use regular geometry fixture -# multi_node_geometry = geometry.GridGeometry( -# backend=backend, -# grid=multi_node_grid_manager.grid, -# coordinates=multi_node_grid_manager.coordinates, -# decomposition_info=multi_node_grid_manager.decomposition_info, -# extra_fields=multi_node_grid_manager.geometry_fields, -# metadata=geometry_attributes.attrs, -# ) -# # TODO(msimberg): Use regular interpolation factory fixture -# multi_node_interpolation = interpolation_factory.InterpolationFieldsFactory( -# grid=multi_node_grid_manager.grid, -# decomposition_info=multi_node_grid_manager.decomposition_info, -# geometry_source=multi_node_geometry, -# backend=backend, -# metadata=interpolation_attributes.attrs, -# exchange=mpi_decomposition.GHexMultiNodeExchange( -# processor_props, multi_node_grid_manager.decomposition_info -# ), -# ) - -# field_ref = single_node_interpolation.get(attrs_name).asnumpy() -# field = multi_node_interpolation.get(attrs_name).asnumpy() - -# check_local_global_field( -# decomposition_info=multi_node_grid_manager.decomposition_info, -# processor_props=processor_props, -# dim=dim, -# global_reference_field=field_ref, -# local_field=field, -# ) - -# print(f"rank = {processor_props.rank} - DONE") +@pytest.mark.mpi +@pytest.mark.parametrize("processor_props", [True], indirect=True) +@pytest.mark.parametrize("attrs_name, dim", [(metrics_attributes.DDXT_Z_HALF_E, dims.EdgeDim)]) +def test_metrics_fields_compare_single_multi_rank( + processor_props: decomp_defs.ProcessProperties, + backend: gtx_typing.Backend | None, + experiment: test_defs.Experiment, + attrs_name: str, + dim: gtx.Dimension, +) -> None: + # TODO(msimberg): Currently segfaults. Are topography and vertical fields + # set up correctly? + pytest.xfail("Segfault") + + if experiment == test_defs.Experiments.MCH_CH_R04B09: + pytest.xfail("Limited-area grids not yet supported") + + file = grid_utils.resolve_full_grid_file_name(experiment.grid) + + ( + lowest_layer_thickness, + model_top_height, + stretch_factor, + damping_height, + rayleigh_coeff, + exner_expol, + vwind_offctr, + rayleigh_type, + thslp_zdiffu, + thhgtd_zdiffu, + ) = test_defs.construct_metrics_config(experiment) + vertical_config = v_grid.VerticalGridConfig( + experiment.num_levels, + lowest_layer_thickness=lowest_layer_thickness, + model_top_height=model_top_height, + stretch_factor=stretch_factor, + rayleigh_damping_height=damping_height, + ) + # TODO(msimberg): Dummy vct_a? Taken from test_io.py. + xp = data_alloc.import_array_ns(backend) + allocator = model_backends.get_allocator(backend) + vertical_grid = v_grid.VerticalGrid( + config=vertical_config, + vct_a=gtx.as_field( + (dims.KDim,), + xp.linspace(12000.0, 0.0, experiment.num_levels + 1), + allocator=allocator, + ), + vct_b=gtx.as_field( + (dims.KDim,), + xp.linspace(12000.0, 0.0, experiment.num_levels + 1), + allocator=allocator, + ), + ) + + print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") + single_node_grid_manager = utils.run_grid_manager_for_singlenode(file) + single_node_geometry = geometry.GridGeometry( + backend=backend, + grid=single_node_grid_manager.grid, + coordinates=single_node_grid_manager.coordinates, + decomposition_info=single_node_grid_manager.decomposition_info, + extra_fields=single_node_grid_manager.geometry_fields, + metadata=geometry_attributes.attrs, + ) + single_node_interpolation = interpolation_factory.InterpolationFieldsFactory( + grid=single_node_grid_manager.grid, + decomposition_info=single_node_grid_manager.decomposition_info, + geometry_source=single_node_geometry, + backend=backend, + metadata=interpolation_attributes.attrs, + exchange=decomp_defs.SingleNodeExchange(), + ) + single_node_metrics = metrics_factory.MetricsFieldsFactory( + grid=single_node_geometry.grid, + vertical_grid=vertical_grid, + decomposition_info=single_node_grid_manager.decomposition_info, + geometry_source=single_node_geometry, + # TODO(msimberg): Valid dummy topography? + topography=( + gtx.as_field( + (dims.CellDim,), + xp.zeros(single_node_geometry.grid.num_cells), + allocator=allocator, + ) + ), + interpolation_source=single_node_interpolation, + backend=backend, + metadata=metrics_attributes.attrs, + rayleigh_type=rayleigh_type, + rayleigh_coeff=rayleigh_coeff, + exner_expol=exner_expol, + vwind_offctr=vwind_offctr, + thslp_zdiffu=thslp_zdiffu, + thhgtd_zdiffu=thhgtd_zdiffu, + exchange=decomp_defs.SingleNodeExchange(), + ) + print( + f"rank = {processor_props.rank} : single node grid has size {single_node_grid_manager.decomposition_info.get_horizontal_size()!r}" + ) + + multi_node_grid_manager = utils.run_gridmananger_for_multinode( + file=file, + run_properties=processor_props, + decomposer=decomp.MetisDecomposer(), + ) + print( + f"rank = {processor_props.rank} : {multi_node_grid_manager.decomposition_info.get_horizontal_size()!r}" + ) + print( + f"rank = {processor_props.rank}: halo size for 'CellDim' " + f"(1: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " + f"(2: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" + ) + multi_node_geometry = geometry.GridGeometry( + backend=backend, + grid=multi_node_grid_manager.grid, + coordinates=multi_node_grid_manager.coordinates, + decomposition_info=multi_node_grid_manager.decomposition_info, + extra_fields=multi_node_grid_manager.geometry_fields, + metadata=geometry_attributes.attrs, + ) + multi_node_interpolation = interpolation_factory.InterpolationFieldsFactory( + grid=multi_node_grid_manager.grid, + decomposition_info=multi_node_grid_manager.decomposition_info, + geometry_source=multi_node_geometry, + backend=backend, + metadata=interpolation_attributes.attrs, + exchange=mpi_decomposition.GHexMultiNodeExchange( + processor_props, multi_node_grid_manager.decomposition_info + ), + ) + multi_node_metrics = metrics_factory.MetricsFieldsFactory( + grid=multi_node_geometry.grid, + vertical_grid=vertical_grid, + decomposition_info=multi_node_grid_manager.decomposition_info, + geometry_source=multi_node_geometry, + # TODO(msimberg): Valid dummy topography? + topography=( + gtx.as_field( + (dims.CellDim,), + xp.zeros(multi_node_geometry.grid.num_cells), + allocator=allocator, + ) + ), + interpolation_source=multi_node_interpolation, + backend=backend, + metadata=metrics_attributes.attrs, + rayleigh_type=rayleigh_type, + rayleigh_coeff=rayleigh_coeff, + exner_expol=exner_expol, + vwind_offctr=vwind_offctr, + thslp_zdiffu=thslp_zdiffu, + thhgtd_zdiffu=thhgtd_zdiffu, + exchange=mpi_decomposition.GHexMultiNodeExchange( + processor_props, multi_node_grid_manager.decomposition_info + ), + ) + + field_ref = single_node_metrics.get(attrs_name).asnumpy() + field = multi_node_metrics.get(attrs_name).asnumpy() + + check_local_global_field( + decomposition_info=multi_node_grid_manager.decomposition_info, + processor_props=processor_props, + dim=dim, + global_reference_field=field_ref, + local_field=field, + ) + + print(f"rank = {processor_props.rank} - DONE") @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) def test_validate_skip_values_in_distributed_connectivities( processor_props: decomp_defs.ProcessProperties, - global_grid_descriptor: test_defs.GridDescription, + experiment: test_defs.Experiment, ) -> None: - file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) + if experiment == test_defs.Experiments.MCH_CH_R04B09: + pytest.xfail("Limited-area grids not yet supported") + + file = grid_utils.resolve_full_grid_file_name(experiment.grid) multinode_grid_manager = utils.run_gridmananger_for_multinode( file=file, run_properties=processor_props, diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index e223024f32..c601d09240 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -54,7 +54,6 @@ data_provider, download_ser_data, experiment, - global_grid_descriptor, grid_savepoint, processor_props, ) @@ -666,10 +665,10 @@ def test_local_connectivity( @pytest.mark.parametrize("ranks", (2, 3, 4)) def test_decomposition_size( ranks: int, - global_grid_descriptor: test_defs.GridDescription, + experiment: test_defs.Experiment, ) -> None: decomposer = halo.MetisDecomposer() - file = grid_utils.resolve_full_grid_file_name(global_grid_descriptor) + file = grid_utils.resolve_full_grid_file_name(experiment.grid) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: partitions = decomposer(parser.int_variable(gridfile.ConnectivityName.C2E2C), ranks) sizes = [np.count_nonzero(partitions == r) for r in range(ranks)] diff --git a/model/common/tests/common/grid/unit_tests/test_gridfile.py b/model/common/tests/common/grid/unit_tests/test_gridfile.py index 9f84f172c4..9eef59eb0f 100644 --- a/model/common/tests/common/grid/unit_tests/test_gridfile.py +++ b/model/common/tests/common/grid/unit_tests/test_gridfile.py @@ -21,7 +21,6 @@ data_provider, download_ser_data, experiment, - global_grid_descriptor, grid_savepoint, processor_props, ) @@ -79,10 +78,8 @@ def test_grid_file_vertex_cell_edge_dimensions( @pytest.mark.parametrize("apply_transformation", (True, False)) -def test_int_variable( - global_grid_descriptor: definitions.GridDescription, apply_transformation: bool -) -> None: - file = gridtest_utils.resolve_full_grid_file_name(global_grid_descriptor) +def test_int_variable(experiment: definitions.Experiment, apply_transformation: bool) -> None: + file = gridtest_utils.resolve_full_grid_file_name(experiment.grid) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: edge_dim = parser.dimension(gridfile.DynamicDimension.EDGE_NAME) # use a test field that does not contain Pentagons @@ -107,10 +104,8 @@ def test_int_variable( "selection", _index_selection, ) -def test_index_read_for_1d_fields( - global_grid_descriptor: definitions.GridDescription, selection: list[int] -) -> None: - file = gridtest_utils.resolve_full_grid_file_name(global_grid_descriptor) +def test_index_read_for_1d_fields(experiment: definitions.Experiment, selection: list[int]) -> None: + file = gridtest_utils.resolve_full_grid_file_name(experiment.grid) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: indices_to_read = np.asarray(selection) if len(selection) > 0 else None full_field = parser.variable(gridfile.CoordinateName.CELL_LATITUDE) @@ -134,12 +129,12 @@ def test_index_read_for_1d_fields( ) @pytest.mark.parametrize("apply_offset", (True, False)) def test_index_read_for_2d_connectivity( - global_grid_descriptor: definitions.GridDescription, + experiment: definitions.Experiment, selection: list[int], field: gridfile.FieldName, apply_offset: bool, ) -> None: - file = gridtest_utils.resolve_full_grid_file_name(global_grid_descriptor) + file = gridtest_utils.resolve_full_grid_file_name(experiment.grid) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: indices_to_read = np.asarray(selection) if len(selection) > 0 else None full_field = parser.int_variable(field, transpose=True, apply_transformation=apply_offset) diff --git a/model/testing/src/icon4py/model/testing/fixtures/datatest.py b/model/testing/src/icon4py/model/testing/fixtures/datatest.py index 72f36cd703..5d59060a33 100644 --- a/model/testing/src/icon4py/model/testing/fixtures/datatest.py +++ b/model/testing/src/icon4py/model/testing/fixtures/datatest.py @@ -79,23 +79,6 @@ def cpu_allocator() -> gtx_typing.FieldBufferAllocationUtil: return model_backends.get_allocator(None) -@pytest.fixture( - params=[ - definitions.Grids.R02B04_GLOBAL, - definitions.Grids.TORUS_50000x5000, - ], - ids=lambda r: r.name, -) -def global_grid_descriptor(request: pytest.FixtureRequest) -> definitions.GridDescription: - """ - Return a global grid descriptor. - - "Global" in this context means "not limited area", i.e. a full icosahedral - grid or a torus grid, both with full connectivity and no boundary layers. - """ - return request.param - - @pytest.fixture( params=[ definitions.Experiments.MCH_CH_R04B09, From e1d05f56c0d85316f448e6ca5a0cdf63c8f47441 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 6 Feb 2026 16:58:18 +0100 Subject: [PATCH 197/240] Fix decomposer import --- .../common/tests/common/grid/unit_tests/test_grid_manager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index c601d09240..d500cf5bc1 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -19,6 +19,7 @@ import icon4py.model.common.grid.gridfile from icon4py.model.common import dimension as dims, model_backends from icon4py.model.common.decomposition import ( + decomposer as decomp, definitions as decomp_defs, definitions as decomposition, halo, @@ -611,7 +612,7 @@ def test_local_connectivity( ) -> None: processor_props = decomp_utils.DummyProps(rank=rank) caplog.set_level(logging.INFO) # type: ignore [attr-defined] - partitioner = halo.MetisDecomposer() + partitioner = decomp.MetisDecomposer() allocator = model_backends.get_allocator(backend_like) file = grid_utils.resolve_full_grid_file_name(test_defs.Grids.R02B04_GLOBAL) manager = gm.GridManager(config=v_grid.VerticalGridConfig(num_levels=10), grid_file=file) @@ -667,7 +668,7 @@ def test_decomposition_size( ranks: int, experiment: test_defs.Experiment, ) -> None: - decomposer = halo.MetisDecomposer() + decomposer = decomp.MetisDecomposer() file = grid_utils.resolve_full_grid_file_name(experiment.grid) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: partitions = decomposer(parser.int_variable(gridfile.ConnectivityName.C2E2C), ranks) From f52a88394be1dac25bc5a6a1afac0d3b90bf461d Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 10:40:46 +0100 Subject: [PATCH 198/240] Update comments in halo.py --- model/common/src/icon4py/model/common/decomposition/halo.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 1bcee8fc0b..712fa8c207 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -396,8 +396,8 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: dtype=gtx.int32, # type: ignore [attr-defined] ) edge_halo_levels[edge_owner_mask] = defs.DecompositionFlag.OWNED - # LEVEL_ONE edges are on an owned cell but are not owned: these are all edges on the cutting line that are not owned (by the convention) + # LEVEL_ONE edges are on an owned cell but are not owned: these are all edges on the cutting line that are not owned (by the convention) edge_halo_levels[ self._xp.logical_not(edge_owner_mask) & self._xp.isin(all_edges, edges_on_cutting_line) ] = defs.DecompositionFlag.FIRST_HALO_LEVEL @@ -407,8 +407,7 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: defs.DecompositionFlag.SECOND_HALO_LEVEL ) - # LEVEL_THREE edges - # LEVEL_TWO edges share exactly one vertex with an owned cell, they are on the first halo-line cells, but not on the cutting line + # LEVEL_THREE edges are the "closing" edges of the second halo line, i.e. share no vertex with owned cells edge_halo_levels[self._xp.isin(all_edges, edge_third_level)] = ( defs.DecompositionFlag.THIRD_HALO_LEVEL ) From 774900c1e05b12ba16d737a2269449e5619064e3 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 10:42:30 +0100 Subject: [PATCH 199/240] Small renaming --- .../model/common/decomposition/halo.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 712fa8c207..729bb10af3 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -325,19 +325,19 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: all_cells = self._xp.hstack((owned_cells, first_halo_cells, second_halo_cells)) #: edges - edges_on_owned_cells = self.find_edge_neighbors_for_cells(owned_cells) - edges_on_any_halo_line = self.find_edge_neighbors_for_cells(total_halo_cells) + edge_on_owned_cells = self.find_edge_neighbors_for_cells(owned_cells) + edge_on_any_halo_line = self.find_edge_neighbors_for_cells(total_halo_cells) - edges_on_cutting_line = self._xp.intersect1d(edges_on_owned_cells, edges_on_any_halo_line) + edge_on_cutting_line = self._xp.intersect1d(edge_on_owned_cells, edge_on_any_halo_line) # needs to be defined as vertex neighbor due to "corners" in the cut. edge_second_level = self._xp.setdiff1d( - self.find_edge_neighbors_for_vertices(vertex_on_cutting_line), edges_on_owned_cells + self.find_edge_neighbors_for_vertices(vertex_on_cutting_line), edge_on_owned_cells ) - edge_third_level = self._xp.setdiff1d(edges_on_any_halo_line, edge_second_level) - edge_third_level = self._xp.setdiff1d(edge_third_level, edges_on_cutting_line) + edge_third_level = self._xp.setdiff1d(edge_on_any_halo_line, edge_second_level) + edge_third_level = self._xp.setdiff1d(edge_third_level, edge_on_cutting_line) - all_edges = self._xp.hstack((edges_on_owned_cells, edge_second_level, edge_third_level)) + all_edges = self._xp.hstack((edge_on_owned_cells, edge_second_level, edge_third_level)) #: construct decomposition info decomp_info = defs.DecompositionInfo() cell_owner_mask = self._xp.isin(all_cells, owned_cells) @@ -381,12 +381,12 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: dims.VertexDim, all_vertices, vertex_owner_mask, vertex_halo_levels ) - edge_owner_mask = self._xp.isin(all_edges, edges_on_owned_cells) + edge_owner_mask = self._xp.isin(all_edges, edge_on_owned_cells) edge_owner_mask = self._update_owner_mask_by_max_rank_convention( cell_to_rank, edge_owner_mask, all_edges, - edges_on_cutting_line, + edge_on_cutting_line, self._connectivity(dims.E2C), ) @@ -399,7 +399,7 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: # LEVEL_ONE edges are on an owned cell but are not owned: these are all edges on the cutting line that are not owned (by the convention) edge_halo_levels[ - self._xp.logical_not(edge_owner_mask) & self._xp.isin(all_edges, edges_on_cutting_line) + self._xp.logical_not(edge_owner_mask) & self._xp.isin(all_edges, edge_on_cutting_line) ] = defs.DecompositionFlag.FIRST_HALO_LEVEL # LEVEL_TWO edges share exactly one vertex with an owned cell, they are on the first halo-line cells, but not on the cutting line From 571d0f690dd4d174d5cefce959bab729c109ae41 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 10:46:07 +0100 Subject: [PATCH 200/240] Copy owner mask --- .../common/src/icon4py/model/common/decomposition/halo.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 729bb10af3..e43101af36 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -185,6 +185,7 @@ def _update_owner_mask_by_max_rank_convention( Returns: updated owner mask """ + updated_owner_mask = owner_mask.copy() for index in indices_on_cutting_line: local_index = self._xp.nonzero(all_indices == index)[0][0] owning_ranks = cell_to_rank[target_connectivity[index]] @@ -196,10 +197,10 @@ def _update_owner_mask_by_max_rank_convention( ), f"rank {self._props.rank}: neither of the neighboring cells: {owning_ranks} is owned by me" # assign the index to the rank with the higher rank if max(owning_ranks) > self._props.rank: - owner_mask[local_index] = False + updated_owner_mask[local_index] = False else: - owner_mask[local_index] = True - return owner_mask + updated_owner_mask[local_index] = True + return updated_owner_mask def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: """ From bcedac66257305b88c97489b9331568449257e46 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 11:03:41 +0100 Subject: [PATCH 201/240] Remove UNDEFINED flag from DecompositionFlag --- .../icon4py/model/common/decomposition/definitions.py | 1 - .../src/icon4py/model/common/decomposition/halo.py | 9 +++------ .../common/decomposition/unit_tests/test_definitions.py | 1 - 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 590172d2a6..7cc6f58290 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -425,7 +425,6 @@ def create_single_reduction_exchange(props: SingleNodeProcessProperties) -> Redu class DecompositionFlag(int, Enum): - UNDEFINED = -1 OWNED = 0 """used for locally owned cells, vertices, edges""" diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index e43101af36..9a369da834 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -344,10 +344,9 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: cell_owner_mask = self._xp.isin(all_cells, owned_cells) cell_halo_levels = self._xp.full( all_cells.size, - defs.DecompositionFlag.UNDEFINED.value, + defs.DecompositionFlag.OWNED.value, dtype=gtx.int32, # type: ignore [attr-defined] ) - cell_halo_levels[cell_owner_mask] = defs.DecompositionFlag.OWNED cell_halo_levels[self._xp.isin(all_cells, first_halo_cells)] = ( defs.DecompositionFlag.FIRST_HALO_LEVEL ) @@ -366,10 +365,9 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: vertex_second_level = self._xp.setdiff1d(vertex_on_halo_cells, vertex_on_owned_cells) vertex_halo_levels = self._xp.full( all_vertices.size, - defs.DecompositionFlag.UNDEFINED.value, + defs.DecompositionFlag.OWNED.value, dtype=gtx.int32, # type: ignore [attr-defined] ) - vertex_halo_levels[vertex_owner_mask] = defs.DecompositionFlag.OWNED vertex_halo_levels[ self._xp.logical_not(vertex_owner_mask) & self._xp.isin(all_vertices, vertex_on_cutting_line) @@ -393,10 +391,9 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: edge_halo_levels = self._xp.full( all_edges.shape, - defs.DecompositionFlag.UNDEFINED.value, + defs.DecompositionFlag.OWNED.value, dtype=gtx.int32, # type: ignore [attr-defined] ) - edge_halo_levels[edge_owner_mask] = defs.DecompositionFlag.OWNED # LEVEL_ONE edges are on an owned cell but are not owned: these are all edges on the cutting line that are not owned (by the convention) edge_halo_levels[ diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index 3f8990909e..d99514a3a6 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -82,7 +82,6 @@ def test_decomposition_info_single_node_empty_halo(dim: gtx.Dimension) -> None: (definitions.DecompositionFlag.SECOND_HALO_LEVEL, True), (definitions.DecompositionFlag.THIRD_HALO_LEVEL, True), (definitions.DecompositionFlag.FIRST_HALO_LEVEL, True), - (definitions.DecompositionFlag.UNDEFINED, False), ], ) def test_decomposition_info_is_distributed(flag, expected) -> None: From 8cc732b5802267c08bf560df52d25177b372c00a Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 11:04:42 +0100 Subject: [PATCH 202/240] Small renaming --- model/common/src/icon4py/model/common/grid/grid_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 3f8e7dffa1..a0110bf27a 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -123,7 +123,7 @@ def __call__( self._construct_decomposed_grid( allocator=allocator, - with_skip_values=keep_skip_values, + keep_skip_values=keep_skip_values, geometry_type=geometry_type, decomposer=decomposer, run_properties=run_properties, @@ -391,7 +391,7 @@ def decomposition_info(self) -> decomposition.DecompositionInfo: def _construct_decomposed_grid( self, allocator: gtx_typing.FieldBufferAllocationUtil | None, - with_skip_values: bool, + keep_skip_values: bool, geometry_type: base.GeometryType, decomposer: decomp.Decomposer, run_properties: decomposition.ProcessProperties, @@ -459,7 +459,7 @@ def _construct_decomposed_grid( horizontal_size=distributed_size, vertical_size=self._vertical_config.num_levels, limited_area=limited_area, - keep_skip_values=with_skip_values, + keep_skip_values=keep_skip_values, ) grid = icon.icon_grid( From 3877652e4c7852632216a5da117a50fb6d4fdf7d Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 11:08:17 +0100 Subject: [PATCH 203/240] Add assertion --- model/common/src/icon4py/model/common/decomposition/halo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 9a369da834..333dd31798 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -140,6 +140,7 @@ def _find_neighbors( self, source_indices: data_alloc.NDArray, offset: gtx.FieldOffset | str ) -> data_alloc.NDArray: """Get a flattened list of all (unique) neighbors to a given global index list""" + assert source_indices.ndim == 1 return self._xp.unique(self._connectivity(offset)[source_indices, :].flatten()) def _find_cell_neighbors(self, cells: data_alloc.NDArray) -> data_alloc.NDArray: From c7f5ef8ba6b45a6bc916e0cd5ed6a517b37934c0 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 11:34:47 +0100 Subject: [PATCH 204/240] Small update to docstring --- model/common/src/icon4py/model/common/grid/grid_manager.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index a0110bf27a..c9656bfd65 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -398,8 +398,10 @@ def _construct_decomposed_grid( ) -> None: """Construct the grid topology from the icon grid file. - Reads connectivity fields from the grid file and constructs derived connectivities needed in - Icon4py from them. Adds constructed start/end index information to the grid. + Reads connectivity fields from the grid file and constructs derived + connectivities needed in Icon4py from them. Adds constructed start/end + index information to the grid. The grid will be distributed or not based + on run_properties. """ xp = data_alloc.import_array_ns(allocator) From 57c5d062506fcd05dcda913237da04d00b718322 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 11:46:04 +0100 Subject: [PATCH 205/240] Check singlenodedecomposer num_partitions --- .../src/icon4py/model/common/decomposition/decomposer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/model/common/src/icon4py/model/common/decomposition/decomposer.py b/model/common/src/icon4py/model/common/decomposition/decomposer.py index e2973b14a3..7a3b6d2b19 100644 --- a/model/common/src/icon4py/model/common/decomposition/decomposer.py +++ b/model/common/src/icon4py/model/common/decomposition/decomposer.py @@ -64,9 +64,14 @@ def __call__( class SingleNodeDecomposer(Decomposer): def __call__( - self, adjacency_matrix: data_alloc.NDArray, num_partitions: int = 1 + self, adjacency_matrix: data_alloc.NDArray, num_partitions: int ) -> data_alloc.NDArray: """Dummy decomposer for single node: assigns all cells to rank = 0""" + if num_partitions != 1: + raise ValueError( + f"SingleNodeDecomposer can only be used for num_partitions=1, but got {num_partitions}" + ) + return data_alloc.array_ns_from_array(adjacency_matrix).zeros( adjacency_matrix.shape[0], dtype=gtx.int32, # type: ignore [attr-defined] From 204a5ab39cd275f9a0860f5adb05af587d16712c Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 11:51:32 +0100 Subject: [PATCH 206/240] Remove test_geometry_stencils.py --- .../grid/unit_tests/test_geometry_stencils.py | 55 ------------------- 1 file changed, 55 deletions(-) delete mode 100644 model/common/tests/common/grid/unit_tests/test_geometry_stencils.py diff --git a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py b/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py deleted file mode 100644 index 4c638452f1..0000000000 --- a/model/common/tests/common/grid/unit_tests/test_geometry_stencils.py +++ /dev/null @@ -1,55 +0,0 @@ -# ICON4Py - ICON inspired code in Python and GT4Py -# -# Copyright (c) 2022-2024, ETH Zurich and MeteoSwiss -# All rights reserved. -# -# Please, refer to the LICENSE file in the root directory. -# SPDX-License-Identifier: BSD-3-Clause - -import numpy as np -import pytest -from gt4py.next import typing as gtx_typing - -from icon4py.model.common import constants, dimension as dims -from icon4py.model.common.grid.geometry_stencils import compute_edge_length -from icon4py.model.common.utils import data_allocation as data_alloc -from icon4py.model.testing import definitions, grid_utils, serialbox - -from ..fixtures import ( - backend, - data_provider, - download_ser_data, - experiment, - grid_savepoint, - processor_props, -) - - -@pytest.mark.level("unit") -@pytest.mark.datatest -def test_edge_length( - experiment: definitions.Experiment, - grid_savepoint: serialbox.IconGridSavepoint, - backend: gtx_typing.Backend, -) -> None: - keep = True - grid_file = experiment.grid - gm = grid_utils.get_grid_manager_from_identifier( - grid_file, num_levels=experiment.num_levels, keep_skip_values=keep, allocator=backend - ) - grid = gm.grid - coordinates = gm.coordinates[dims.VertexDim] - lat = coordinates["lat"] - lon = coordinates["lon"] - length = data_alloc.zero_field(grid, dims.EdgeDim) - compute_edge_length.with_backend(backend)( - vertex_lat=lat, - vertex_lon=lon, - radius=constants.EARTH_RADIUS, - length=length, - horizontal_start=0, - horizontal_end=grid.size[dims.EdgeDim], - offset_provider={"E2V": grid.get_connectivity(dims.E2V)}, - ) - - assert np.allclose(length.asnumpy(), grid_savepoint.primal_edge_length().asnumpy()) From 67cd2193d996120ec199e7bfa6f81c7b0c9eb5e3 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 11:56:06 +0100 Subject: [PATCH 207/240] Revert changes in test_geometry.py --- .../tests/common/grid/unit_tests/test_geometry.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/model/common/tests/common/grid/unit_tests/test_geometry.py b/model/common/tests/common/grid/unit_tests/test_geometry.py index ac03571602..6dad00abfe 100644 --- a/model/common/tests/common/grid/unit_tests/test_geometry.py +++ b/model/common/tests/common/grid/unit_tests/test_geometry.py @@ -10,11 +10,10 @@ import functools from typing import TYPE_CHECKING -import gt4py.next as gtx import numpy as np import pytest -from icon4py.model.common import constants, dimension as dims, model_backends +from icon4py.model.common import constants, dimension as dims from icon4py.model.common.grid import ( base, geometry, @@ -22,6 +21,7 @@ horizontal as h_grid, simple, ) +from icon4py.model.common.grid.geometry import as_sparse_field from icon4py.model.common.math import helpers as math_helpers from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import definitions, grid_utils, test_utils @@ -194,7 +194,7 @@ def test_compute_coordinates_of_edge_tangent_and_normal( assert test_utils.dallclose(x_tangent.asnumpy(), x_tangent_ref.asnumpy(), atol=1e-12) assert test_utils.dallclose(y_tangent.asnumpy(), y_tangent_ref.asnumpy(), atol=1e-12) assert test_utils.dallclose(z_tangent.asnumpy(), z_tangent_ref.asnumpy(), atol=1e-12) - assert test_utils.dallclose(x_normal.asnumpy(), x_normal_ref.asnumpy(), atol=1e-13) + assert test_utils.dallclose(x_normal.asnumpy(), x_normal_ref.asnumpy(), atol=1e-13) # 1e-16 assert test_utils.dallclose(z_normal.asnumpy(), z_normal_ref.asnumpy(), atol=1e-13) assert test_utils.dallclose(y_normal.asnumpy(), y_normal_ref.asnumpy(), atol=1e-12) @@ -407,8 +407,8 @@ def test_sparse_fields_creator() -> None: g1 = data_alloc.random_field(grid, dims.EdgeDim) g2 = data_alloc.random_field(grid, dims.EdgeDim) - sparse = geometry.as_sparse_field((dims.EdgeDim, dims.E2CDim), [(f1, f2), (g1, g2)]) - sparse_e2c = functools.partial(geometry.as_sparse_field, (dims.EdgeDim, dims.E2CDim)) + sparse = as_sparse_field((dims.EdgeDim, dims.E2CDim), [(f1, f2), (g1, g2)]) + sparse_e2c = functools.partial(as_sparse_field, (dims.EdgeDim, dims.E2CDim)) sparse2 = sparse_e2c(((f1, f2), (g1, g2))) assert sparse[0].asnumpy().shape == (grid.num_edges, 2) assert test_utils.dallclose(sparse[0].asnumpy(), sparse2[0].asnumpy()) @@ -421,7 +421,7 @@ def test_create_auxiliary_orientation_coordinates( ) -> None: gm = grid_utils.get_grid_manager_from_identifier( experiment.grid, - num_levels=experiment.num_levels, + num_levels=1, keep_skip_values=True, allocator=backend, ) From 9f298c9fe7064fb4c87a5adee66290202f536ce5 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 12:17:42 +0100 Subject: [PATCH 208/240] Small refactoring --- .../test_parallel_grid_refinement.py | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py index 0489c90f78..7a0e2a909b 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py @@ -47,24 +47,23 @@ def test_compute_domain_bounds( "end index data for single node APE are all 0 - re- serialization should fix that (patch%cells%end_index vs patch%cells%end_idx)" ) - else: - ref_grid = grid_savepoint.construct_icon_grid(backend=None, keep_skip_values=True) - decomposition_info = grid_savepoint.construct_decomposition_info() - refin_ctrl = {dim: grid_savepoint.refin_ctrl(dim) for dim in utils.main_horizontal_dims()} - start_indices, end_indices = grid_refinement.compute_domain_bounds( - dim, refin_ctrl, decomposition_info + ref_grid = grid_savepoint.construct_icon_grid(backend=None, keep_skip_values=True) + decomposition_info = grid_savepoint.construct_decomposition_info() + refin_ctrl = {dim: grid_savepoint.refin_ctrl(dim) for dim in utils.main_horizontal_dims()} + start_indices, end_indices = grid_refinement.compute_domain_bounds( + dim, refin_ctrl, decomposition_info + ) + for domain in h_grid.get_domains_for_dim(dim): + ref_start_index = ref_grid.start_index(domain) + ref_end_index = ref_grid.end_index(domain) + computed_start = start_indices[domain] + computed_end = end_indices[domain] + print( + f"rank = {processor_props.rank}/{processor_props.comm_size}: domain={domain} : start = {computed_start} end = {computed_end} " ) - for domain in h_grid.get_domains_for_dim(dim): - ref_start_index = ref_grid.start_index(domain) - ref_end_index = ref_grid.end_index(domain) - computed_start = start_indices[domain] - computed_end = end_indices[domain] - print( - f"rank = {processor_props.rank}/{processor_props.comm_size}: domain={domain} : start = {computed_start} end = {computed_end} " - ) - assert ( - computed_start == ref_start_index - ), f"rank={processor_props.rank}/{processor_props.comm_size} - experiment = {experiment.name}: start_index for {domain} does not match: is {computed_start}, expected {ref_start_index}" - assert ( - computed_end == ref_end_index - ), f"rank={processor_props.rank}/{processor_props.comm_size} - experiment = {experiment.name}: end_index for {domain} does not match: is {computed_end}, expected {ref_end_index}" + assert ( + computed_start == ref_start_index + ), f"rank={processor_props.rank}/{processor_props.comm_size} - experiment = {experiment.name}: start_index for {domain} does not match: is {computed_start}, expected {ref_start_index}" + assert ( + computed_end == ref_end_index + ), f"rank={processor_props.rank}/{processor_props.comm_size} - experiment = {experiment.name}: end_index for {domain} does not match: is {computed_end}, expected {ref_end_index}" From 6ff1bd2c35dc51d371c570cd047971cda2fb8343 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 13:55:30 +0100 Subject: [PATCH 209/240] Revert "Minor bugfix for LAM/distributed configuration" This reverts commit f895bb3daa9d0325fd874f97180893f98e3c51a6. --- model/common/src/icon4py/model/common/grid/icon.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/icon.py b/model/common/src/icon4py/model/common/grid/icon.py index 3480958a6b..02c64b2e2b 100644 --- a/model/common/src/icon4py/model/common/grid/icon.py +++ b/model/common/src/icon4py/model/common/grid/icon.py @@ -208,10 +208,8 @@ def icon_grid( data_alloc.import_array_ns(allocator).asarray(table), skip_value=-1 if _has_skip_values(offset, limited_area_or_distributed) else None, allocator=allocator, - replace_skip_values=( - _should_replace_skip_values( - offset, config.keep_skip_values, limited_area_or_distributed - ), + replace_skip_values=_should_replace_skip_values( + offset, config.keep_skip_values, config.limited_area ), ) for offset, table in neighbor_tables.items() From aecfb5fdffa2a657f74b62e5f537d2d7012a287d Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 14:37:51 +0100 Subject: [PATCH 210/240] xfail lam grid --- model/common/tests/common/grid/unit_tests/test_grid_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index d500cf5bc1..f3dedd9432 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -668,6 +668,9 @@ def test_decomposition_size( ranks: int, experiment: test_defs.Experiment, ) -> None: + if experiment == test_defs.Experiments.MCH_CH_R04B09: + pytest.xfail("Limited-area grids not yet supported") + decomposer = decomp.MetisDecomposer() file = grid_utils.resolve_full_grid_file_name(experiment.grid) with gridfile.GridFile(str(file), gridfile.ToZeroBasedIndexTransformation()) as parser: From 777a39663c923f46ca9e44e15d5bef9dba652560 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 15:27:25 +0100 Subject: [PATCH 211/240] Assert there are no invalid indices in adjacency matrix for metis --- .../src/icon4py/model/common/decomposition/decomposer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/model/common/src/icon4py/model/common/decomposition/decomposer.py b/model/common/src/icon4py/model/common/decomposition/decomposer.py index 7a3b6d2b19..48fbc549cd 100644 --- a/model/common/src/icon4py/model/common/decomposition/decomposer.py +++ b/model/common/src/icon4py/model/common/decomposition/decomposer.py @@ -56,6 +56,10 @@ def __call__( import pymetis # type: ignore [import-untyped] + # Invalid indices are not allowed here. Metis will segfault or fail + # there are any invalid indices in the adjacency matrix. + assert (adjacency_matrix >= 0).all() + # The partitioning is done on all ranks, and this assumes that the # partitioning is deterministic. _, partition_index = pymetis.part_graph(nparts=num_partitions, adjacency=adjacency_matrix) From 560957048eb6a1129f73fcb470c32b359fd67ba0 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 15:42:47 +0100 Subject: [PATCH 212/240] Small refactoring of test_local_connectivity --- .../grid/unit_tests/test_grid_manager.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index f3dedd9432..6b7175dd66 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -645,22 +645,26 @@ def test_local_connectivity( ), f"max value in the connectivity is {np.max(connectivity)} is larger than the local patch size {max_local_index}" # - outer halo entries have SKIP_VALUE neighbors (depends on offsets) neighbor_dim = field_offset.target[1] # type: ignore [misc] + dim = field_offset.target[0] + last_halo_level = ( + decomp_defs.DecompositionFlag.THIRD_HALO_LEVEL + if neighbor_dim == dims.E2CDim + else decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL + ) + level_index = np.where( + data_alloc.as_numpy(decomposition_info.halo_levels(dim)) == last_halo_level.value + ) if ( neighbor_dim in icon.CONNECTIVITIES_ON_BOUNDARIES or neighbor_dim in icon.CONNECTIVITIES_ON_PENTAGONS ): - dim = field_offset.target[0] - last_halo_level = ( - decomp_defs.DecompositionFlag.THIRD_HALO_LEVEL - if neighbor_dim == dims.E2CDim - else decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL - ) - level_index = np.where( - data_alloc.as_numpy(decomposition_info.halo_levels(dim)) == last_halo_level.value - ) assert np.count_nonzero( (connectivity[level_index] == gridfile.GridFile.INVALID_INDEX) > 0 ), f"missing invalid index in {dim} - offset {field_offset}" + else: + assert np.count_nonzero( + (connectivity[level_index] == gridfile.GridFile.INVALID_INDEX) == 0 + ), f"have invalid index in {dim} - offset {field_offset} when none expected" @pytest.mark.parametrize("ranks", (2, 3, 4)) From d55a20d31b6ba8c36bcc123fba07f2f8560ce991 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 15:59:25 +0100 Subject: [PATCH 213/240] Fix test_local_connectivity --- model/common/tests/common/grid/unit_tests/test_grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index 6b7175dd66..fad7dbdce3 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -648,7 +648,7 @@ def test_local_connectivity( dim = field_offset.target[0] last_halo_level = ( decomp_defs.DecompositionFlag.THIRD_HALO_LEVEL - if neighbor_dim == dims.E2CDim + if dim == dims.EdgeDim else decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL ) level_index = np.where( From b814c9ffa9ab343d301d34be517ee0bc7b1fcaba Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 16:58:36 +0100 Subject: [PATCH 214/240] More tests for test_parallel_icon.py --- .../tests/common/grid/mpi_tests/test_parallel_icon.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py index 30b6d36c62..8501da8f90 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py @@ -30,6 +30,7 @@ backend, data_provider, download_ser_data, + experiment, grid_savepoint, icon_grid, processor_props, @@ -187,12 +188,14 @@ def test_start_index_end_index_halo_zones_on_distributed_lam_grid( # TODO(msimberg): Torus? Above as well. @pytest.mark.parametrize("processor_props", [True], indirect=True) @pytest.mark.mpi -@pytest.mark.parametrize("grid", (test_defs.Grids.R02B04_GLOBAL,)) def test_skip_values_on_distributed_grid( processor_props: decomposition.ProcessProperties, - grid: test_defs.GridDescription, + experiment: test_defs.Experiment, ) -> None: - file = grid_utils.resolve_full_grid_file_name(grid) + if experiment == test_defs.Experiments.MCH_CH_R04B09: + pytest.xfail("Limited-area grids not yet supported") + + file = grid_utils.resolve_full_grid_file_name(experiment.grid) grid_manager = parallel_utils.run_gridmananger_for_multinode( file, processor_props, decomposer=MetisDecomposer() ) From 079f3acdc647b7fa0eb0f11a775dea2acb92c5ff Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 17:15:33 +0100 Subject: [PATCH 215/240] Clean up todo --- .../tests/common/grid/mpi_tests/test_parallel_grid_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index c1fca8fa5f..a26f9a51f6 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -217,8 +217,7 @@ def test_geometry_fields_compare_single_multi_rank( pytest.xfail("Limited-area grids not yet supported") # TODO(msimberg): Add fixtures for single/multi-rank - # grid/geometry/interpolation/metrics factoriesAdd fixtures for - # single/mult-rank grid/geometry/interpolation/metrics factories. + # grid/geometry/interpolation/metrics factories. file = grid_utils.resolve_full_grid_file_name(experiment.grid) print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") single_node_grid_manager = utils.run_grid_manager_for_singlenode(file) From 3fd4d3174fa80d11e04aefacb2245281452f658d Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 17:59:58 +0100 Subject: [PATCH 216/240] Add todos --- .../tests/common/grid/mpi_tests/test_parallel_grid_manager.py | 3 +++ model/common/tests/common/grid/mpi_tests/utils.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index a26f9a51f6..96e5927ee3 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -216,6 +216,9 @@ def test_geometry_fields_compare_single_multi_rank( if experiment == test_defs.Experiments.MCH_CH_R04B09: pytest.xfail("Limited-area grids not yet supported") + # TODO(msimberg): Some of these fail with keep_skip_values=True, pass with + # keep_skip_values=False. What should it be? + # TODO(msimberg): Add fixtures for single/multi-rank # grid/geometry/interpolation/metrics factories. file = grid_utils.resolve_full_grid_file_name(experiment.grid) diff --git a/model/common/tests/common/grid/mpi_tests/utils.py b/model/common/tests/common/grid/mpi_tests/utils.py index 1883d0eaae..6fe5eb8160 100644 --- a/model/common/tests/common/grid/mpi_tests/utils.py +++ b/model/common/tests/common/grid/mpi_tests/utils.py @@ -18,6 +18,7 @@ def _grid_manager(file: pathlib.Path, num_levels: int) -> gm.GridManager: ) +# TODO(msimberg): Single rank, not node. def run_grid_manager_for_singlenode(file: pathlib.Path) -> gm.GridManager: manager = _grid_manager(file, NUM_LEVELS) manager( @@ -29,6 +30,7 @@ def run_grid_manager_for_singlenode(file: pathlib.Path) -> gm.GridManager: return manager +# TODO(msimberg): Fix typos. Consistent naming with above function. def run_gridmananger_for_multinode( file: pathlib.Path, run_properties: decomp_defs.ProcessProperties, From a94be5a8e47fd093ad069b037fdf78cbf7106e30 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 9 Feb 2026 18:00:26 +0100 Subject: [PATCH 217/240] Update icon grid skip values again --- model/common/src/icon4py/model/common/grid/icon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/common/src/icon4py/model/common/grid/icon.py b/model/common/src/icon4py/model/common/grid/icon.py index 02c64b2e2b..2c16a826d1 100644 --- a/model/common/src/icon4py/model/common/grid/icon.py +++ b/model/common/src/icon4py/model/common/grid/icon.py @@ -209,7 +209,7 @@ def icon_grid( skip_value=-1 if _has_skip_values(offset, limited_area_or_distributed) else None, allocator=allocator, replace_skip_values=_should_replace_skip_values( - offset, config.keep_skip_values, config.limited_area + offset, config.keep_skip_values, limited_area_or_distributed ), ) for offset, table in neighbor_tables.items() From 027418963503f9d692e2656b37a7fe23b47b59d3 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 10 Feb 2026 09:10:11 +0100 Subject: [PATCH 218/240] Revert "Remove UNDEFINED flag from DecompositionFlag" This reverts commit bcedac66257305b88c97489b9331568449257e46. --- .../icon4py/model/common/decomposition/definitions.py | 1 + .../src/icon4py/model/common/decomposition/halo.py | 9 ++++++--- .../common/decomposition/unit_tests/test_definitions.py | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 7cc6f58290..590172d2a6 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -425,6 +425,7 @@ def create_single_reduction_exchange(props: SingleNodeProcessProperties) -> Redu class DecompositionFlag(int, Enum): + UNDEFINED = -1 OWNED = 0 """used for locally owned cells, vertices, edges""" diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 333dd31798..05060bf476 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -345,9 +345,10 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: cell_owner_mask = self._xp.isin(all_cells, owned_cells) cell_halo_levels = self._xp.full( all_cells.size, - defs.DecompositionFlag.OWNED.value, + defs.DecompositionFlag.UNDEFINED.value, dtype=gtx.int32, # type: ignore [attr-defined] ) + cell_halo_levels[cell_owner_mask] = defs.DecompositionFlag.OWNED cell_halo_levels[self._xp.isin(all_cells, first_halo_cells)] = ( defs.DecompositionFlag.FIRST_HALO_LEVEL ) @@ -366,9 +367,10 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: vertex_second_level = self._xp.setdiff1d(vertex_on_halo_cells, vertex_on_owned_cells) vertex_halo_levels = self._xp.full( all_vertices.size, - defs.DecompositionFlag.OWNED.value, + defs.DecompositionFlag.UNDEFINED.value, dtype=gtx.int32, # type: ignore [attr-defined] ) + vertex_halo_levels[vertex_owner_mask] = defs.DecompositionFlag.OWNED vertex_halo_levels[ self._xp.logical_not(vertex_owner_mask) & self._xp.isin(all_vertices, vertex_on_cutting_line) @@ -392,9 +394,10 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: edge_halo_levels = self._xp.full( all_edges.shape, - defs.DecompositionFlag.OWNED.value, + defs.DecompositionFlag.UNDEFINED.value, dtype=gtx.int32, # type: ignore [attr-defined] ) + edge_halo_levels[edge_owner_mask] = defs.DecompositionFlag.OWNED # LEVEL_ONE edges are on an owned cell but are not owned: these are all edges on the cutting line that are not owned (by the convention) edge_halo_levels[ diff --git a/model/common/tests/common/decomposition/unit_tests/test_definitions.py b/model/common/tests/common/decomposition/unit_tests/test_definitions.py index d99514a3a6..3f8990909e 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_definitions.py +++ b/model/common/tests/common/decomposition/unit_tests/test_definitions.py @@ -82,6 +82,7 @@ def test_decomposition_info_single_node_empty_halo(dim: gtx.Dimension) -> None: (definitions.DecompositionFlag.SECOND_HALO_LEVEL, True), (definitions.DecompositionFlag.THIRD_HALO_LEVEL, True), (definitions.DecompositionFlag.FIRST_HALO_LEVEL, True), + (definitions.DecompositionFlag.UNDEFINED, False), ], ) def test_decomposition_info_is_distributed(flag, expected) -> None: From e97b1c464cf9bc8a46f9e6c25bc01fede31f8e2c Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 10 Feb 2026 09:11:38 +0100 Subject: [PATCH 219/240] Revert "Remove owned_cells helper" This reverts commit cc642c7cdea6247fe1153611989506fd551cf129. --- .../common/src/icon4py/model/common/decomposition/halo.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 05060bf476..4c792676cc 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -163,6 +163,11 @@ def find_cell_neighbors_for_vertices( ) -> data_alloc.NDArray: return self._find_neighbors(vertex_line, dims.V2C) + def owned_cells(self, cell_to_rank: data_alloc.NDArray) -> data_alloc.NDArray: + """Returns the full-grid indices of the cells owned by this rank""" + assert cell_to_rank.ndim == 1 + return self._xp.where(cell_to_rank == self._props.rank)[0] + def _update_owner_mask_by_max_rank_convention( self, cell_to_rank: data_alloc.NDArray, @@ -303,8 +308,7 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: self._validate_mapping(cell_to_rank) #: cells - # global indices of owned cells - owned_cells = self._xp.where(cell_to_rank == self._props.rank)[0] + owned_cells = self.owned_cells(cell_to_rank) first_halo_cells = self.next_halo_line(owned_cells) #: vertices vertex_on_owned_cells = self.find_vertex_neighbors_for_cells(owned_cells) From 0e24b760995cd5dd24625dcbce3338173d6693f5 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 10 Feb 2026 09:13:40 +0100 Subject: [PATCH 220/240] Fix decomposer import --- .../common/tests/common/decomposition/unit_tests/test_halo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model/common/tests/common/decomposition/unit_tests/test_halo.py b/model/common/tests/common/decomposition/unit_tests/test_halo.py index 7f6d0aacc5..61261b5a29 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_halo.py +++ b/model/common/tests/common/decomposition/unit_tests/test_halo.py @@ -11,7 +11,7 @@ from gt4py.next import common as gtx_common from icon4py.model.common import dimension as dims, exceptions, model_backends -from icon4py.model.common.decomposition import definitions, halo +from icon4py.model.common.decomposition import decomposer as decomp, definitions, halo from icon4py.model.common.grid import base as base_grid, simple from ...fixtures import backend_like, processor_props @@ -126,7 +126,7 @@ def test_halo_constructor_decomposition_info_halo_levels(rank, dim, simple_neigh def test_no_halo(): grid_size = base_grid.HorizontalGridSize(num_cells=9, num_edges=14, num_vertices=6) halo_generator = halo.NoHalos(horizontal_size=grid_size, allocator=None) - decomposition = halo.SingleNodeDecomposer() + decomposition = decomp.SingleNodeDecomposer() decomposition_info = halo_generator(decomposition(np.arange(grid_size.num_cells), 1)) # cells np.testing.assert_allclose( From 356695aa1230296b1ece891b47efef2fdba215d0 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 10 Feb 2026 09:37:58 +0100 Subject: [PATCH 221/240] Use low enough index in test for all tested grids --- model/common/tests/common/grid/unit_tests/test_gridfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/common/tests/common/grid/unit_tests/test_gridfile.py b/model/common/tests/common/grid/unit_tests/test_gridfile.py index 9eef59eb0f..7974bb41d0 100644 --- a/model/common/tests/common/grid/unit_tests/test_gridfile.py +++ b/model/common/tests/common/grid/unit_tests/test_gridfile.py @@ -96,7 +96,7 @@ def test_int_variable(experiment: definitions.Experiment, apply_transformation: [0, 1, 2, 3, 4, 5], [], [0, 2, 4, 6, 7, 8, 24, 57], - [1, 2, 12, 13, 23, 24, 2306], + [1, 2, 12, 13, 23, 24, 513], ] From ee8ce4a0e04ef269df80c699e28c7732bd162f61 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 10 Feb 2026 10:54:34 +0100 Subject: [PATCH 222/240] Add rbf fields to test_parallel_grid_manager.py --- .../grid/mpi_tests/test_parallel_grid_manager.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 96e5927ee3..f44abaa513 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -148,7 +148,7 @@ def check_local_global_field( local_field_from_global_field = global_reference_field[global_indices_local_field] local_field_from_local_field = local_field[local_indices_local_field] np.testing.assert_allclose( - local_field_from_global_field, local_field_from_local_field, atol=0.0, verbose=True + local_field_from_global_field, local_field_from_local_field, atol=1e-9, verbose=True ) # Compare owned local field, excluding halos, against global reference @@ -182,10 +182,11 @@ def check_local_global_field( f" rank = {processor_props.rank}: SHAPES: global reference field {global_reference_field.shape}, gathered = {gathered_field.shape}" ) + # TODO(msimberg): Is this true? Not true for RBF itnerpolation... why? # We expect an exact match, since the starting point is the same (grid # file) and we are doing the exact same computations in single rank and # multi rank mode. - np.testing.assert_allclose(sorted_, global_reference_field, atol=0.0, verbose=True) + np.testing.assert_allclose(sorted_, global_reference_field, atol=1e-9, verbose=True) @pytest.mark.mpi @@ -277,6 +278,11 @@ def test_geometry_fields_compare_single_multi_rank( (interpolation_attributes.GEOFAC_DIV, dims.CellDim), (interpolation_attributes.GEOFAC_ROT, dims.VertexDim), (interpolation_attributes.C_BLN_AVG, dims.CellDim), + (interpolation_attributes.RBF_VEC_COEFF_C1, dims.CellDim), + (interpolation_attributes.RBF_VEC_COEFF_C2, dims.CellDim), + (interpolation_attributes.RBF_VEC_COEFF_E, dims.EdgeDim), + (interpolation_attributes.RBF_VEC_COEFF_V1, dims.VertexDim), + (interpolation_attributes.RBF_VEC_COEFF_V2, dims.VertexDim), ], ) def test_interpolation_fields_compare_single_multi_rank( From 395553f678e072a318651eb1c7b00fd61f520531 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 10 Feb 2026 10:55:20 +0100 Subject: [PATCH 223/240] Remove todo --- .../tests/common/grid/mpi_tests/test_parallel_grid_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index f44abaa513..39b4b626fe 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -209,7 +209,6 @@ def check_local_global_field( def test_geometry_fields_compare_single_multi_rank( processor_props: decomp_defs.ProcessProperties, backend: gtx_typing.Backend | None, - # TODO(msimberg): Maybe use regular grid fixture and skip local area grids? experiment: test_defs.Experiment, attrs_name: str, dim: gtx.Dimension, From cd74e0c4267c68c1fae8a749060bc368ad8a5af1 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 11 Feb 2026 09:53:06 +0100 Subject: [PATCH 224/240] xfail some distributed geometry tests with embedded backend --- .../grid/mpi_tests/test_parallel_grid_manager.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 39b4b626fe..fa0fd21cd2 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -216,6 +216,20 @@ def test_geometry_fields_compare_single_multi_rank( if experiment == test_defs.Experiments.MCH_CH_R04B09: pytest.xfail("Limited-area grids not yet supported") + # backend is embedded and in list of edge_normal* attributes, xfail + if ( + test_utils.is_embedded(backend) + and attrs_name + in ( + geometry_attributes.EDGE_NORMAL_VERTEX_U, + geometry_attributes.EDGE_NORMAL_VERTEX_V, + geometry_attributes.EDGE_NORMAL_CELL_U, + geometry_attributes.EDGE_NORMAL_CELL_V, + ) + and experiment == test_defs.Experiments.EXCLAIM_APE + ): + pytest.xfail("IndexOutOfBounds with embedded backend") + # TODO(msimberg): Some of these fail with keep_skip_values=True, pass with # keep_skip_values=False. What should it be? From 672cd9388f65cfc1f304b54c39d159db2c87c455 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 11 Feb 2026 13:40:37 +0100 Subject: [PATCH 225/240] Remove duplicated test --- .../mpi_tests/test_parallel_grid_manager.py | 6 +++-- .../grid/mpi_tests/test_parallel_icon.py | 25 ------------------- 2 files changed, 4 insertions(+), 27 deletions(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index fa0fd21cd2..7ee0e83901 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -574,9 +574,11 @@ def test_validate_skip_values_in_distributed_connectivities( found_skips == (c.skip_value is not None) ), f"rank={processor_props.rank} / {processor_props.comm_size}: {k} - # of skip values found in table = {skip_values_in_table}, skip value is {c.skip_value}" if skip_values_in_table > 0: + dim = gtx.Dimension(k, gtx.DimensionKind.LOCAL) assert ( - c in icon.CONNECTIVITIES_ON_BOUNDARIES or icon.CONNECTIVITIES_ON_PENTAGONS - ), f"rank={processor_props.rank} / {processor_props.comm_size}: {k} has skip found in table" + dim in icon.CONNECTIVITIES_ON_BOUNDARIES + or dim in icon.CONNECTIVITIES_ON_PENTAGONS + ), f"rank={processor_props.rank} / {processor_props.comm_size}: {k} has skip found in table, expected none" @pytest.mark.mpi diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py index 8501da8f90..684abc283f 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py @@ -183,28 +183,3 @@ def test_start_index_end_index_halo_zones_on_distributed_lam_grid( assert start_index == expected, f"expected start index {expected}, but was {start_index}" expected = HALO_IDX[processor_props.comm_size][dim][rank][level] assert end_index == expected, f"expected start index {1}, but was {start_index}" - - -# TODO(msimberg): Torus? Above as well. -@pytest.mark.parametrize("processor_props", [True], indirect=True) -@pytest.mark.mpi -def test_skip_values_on_distributed_grid( - processor_props: decomposition.ProcessProperties, - experiment: test_defs.Experiment, -) -> None: - if experiment == test_defs.Experiments.MCH_CH_R04B09: - pytest.xfail("Limited-area grids not yet supported") - - file = grid_utils.resolve_full_grid_file_name(experiment.grid) - grid_manager = parallel_utils.run_gridmananger_for_multinode( - file, processor_props, decomposer=MetisDecomposer() - ) - mesh = grid_manager.grid - assert not np.any(mesh.get_connectivity(dims.C2V).asnumpy() == gridfile.GridFile.INVALID_INDEX) - assert not np.any(mesh.get_connectivity(dims.E2V).asnumpy() == gridfile.GridFile.INVALID_INDEX) - assert not np.any(mesh.get_connectivity(dims.E2V).asnumpy() == gridfile.GridFile.INVALID_INDEX) - assert not np.any(mesh.get_connectivity(dims.C2E).asnumpy() == gridfile.GridFile.INVALID_INDEX) - assert np.any(mesh.get_connectivity(dims.E2C).asnumpy() == gridfile.GridFile.INVALID_INDEX) - assert np.any(mesh.get_connectivity(dims.C2E2C).asnumpy() == gridfile.GridFile.INVALID_INDEX) - assert np.any(mesh.get_connectivity(dims.V2E).asnumpy() == gridfile.GridFile.INVALID_INDEX) - assert np.any(mesh.get_connectivity(dims.V2C).asnumpy() == gridfile.GridFile.INVALID_INDEX) From a9b24fcc7ecfcb9711ef54f30c7eab4f740190b4 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 11 Feb 2026 13:44:23 +0100 Subject: [PATCH 226/240] Remove unused imports --- .../mpi_tests/test_mpi_decomposition.py | 1 - .../decomposition/mpi_tests/test_parallel_halo.py | 2 +- .../common/decomposition/unit_tests/test_halo.py | 1 - .../grid/mpi_tests/test_parallel_grid_manager.py | 12 +----------- .../common/grid/mpi_tests/test_parallel_icon.py | 9 ++------- .../common/grid/unit_tests/test_grid_manager.py | 7 +------ .../common/grid/unit_tests/test_grid_refinement.py | 3 +-- 7 files changed, 6 insertions(+), 29 deletions(-) diff --git a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py index 8e4dabbf48..b593a9b644 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py @@ -45,7 +45,6 @@ metrics_savepoint, processor_props, ) -from ..utils import dummy_four_ranks _log = logging.getLogger(__name__) diff --git a/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py b/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py index 18a01a5f73..77fc00162e 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_parallel_halo.py @@ -30,7 +30,7 @@ pytest.skip("Skipping parallel on single node installation", allow_module_level=True) from icon4py.model.common.decomposition import halo -from icon4py.model.common.grid import base as base_grid, simple +from icon4py.model.common.grid import simple from .. import utils diff --git a/model/common/tests/common/decomposition/unit_tests/test_halo.py b/model/common/tests/common/decomposition/unit_tests/test_halo.py index 61261b5a29..aa57270d76 100644 --- a/model/common/tests/common/decomposition/unit_tests/test_halo.py +++ b/model/common/tests/common/decomposition/unit_tests/test_halo.py @@ -15,7 +15,6 @@ from icon4py.model.common.grid import base as base_grid, simple from ...fixtures import backend_like, processor_props -from ...grid import utils as grid_utils from .. import utils from ..fixtures import simple_neighbor_tables from ..utils import dummy_four_ranks diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 7ee0e83901..6fa48a803c 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -8,7 +8,6 @@ import functools import logging import operator -from typing import Any import numpy as np import pytest @@ -25,21 +24,12 @@ base, geometry, geometry_attributes, - geometry_stencils, grid_manager as gm, gridfile, - horizontal as h_grid, icon, vertical as v_grid, ) -from icon4py.model.common.interpolation import ( - interpolation_attributes, - interpolation_factory, - interpolation_fields, -) -from icon4py.model.common.interpolation.stencils.compute_cell_2_vertex_interpolation import ( - _compute_cell_2_vertex_interpolation, -) +from icon4py.model.common.interpolation import interpolation_attributes, interpolation_factory from icon4py.model.common.metrics import metrics_attributes, metrics_factory from icon4py.model.common.utils import data_allocation as data_alloc from icon4py.model.testing import definitions as test_defs, grid_utils, test_utils diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py index 684abc283f..66361c319c 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_icon.py @@ -13,17 +13,12 @@ import gt4py.next as gtx import numpy as np import pytest -from gt4py.next.common import is_neighbor_table import icon4py.model.common.dimension as dims import icon4py.model.common.grid.horizontal as h_grid -import icon4py.model.common.grid.icon -from icon4py.model.common.decomposition import ( - definitions as decomp_defs, - definitions as decomposition, -) +from icon4py.model.common.decomposition import definitions as decomposition from icon4py.model.common.decomposition.decomposer import MetisDecomposer -from icon4py.model.common.grid import base as base_grid, gridfile, horizontal as h_grid, icon +from icon4py.model.common.grid import base as base_grid, gridfile, horizontal as h_grid from icon4py.model.testing import definitions as test_defs, grid_utils, parallel_helpers from ...fixtures import ( diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index fad7dbdce3..183bfc5c87 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -18,12 +18,7 @@ import icon4py.model.common.grid.gridfile from icon4py.model.common import dimension as dims, model_backends -from icon4py.model.common.decomposition import ( - decomposer as decomp, - definitions as decomp_defs, - definitions as decomposition, - halo, -) +from icon4py.model.common.decomposition import decomposer as decomp, definitions as decomp_defs from icon4py.model.common.grid import ( base as base_grid, grid_manager as gm, diff --git a/model/common/tests/common/grid/unit_tests/test_grid_refinement.py b/model/common/tests/common/grid/unit_tests/test_grid_refinement.py index 51682cfeb0..28eaf0fc76 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_refinement.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_refinement.py @@ -15,12 +15,11 @@ from icon4py.model.common import dimension as dims, model_backends from icon4py.model.common.grid import grid_refinement as refinement, horizontal as h_grid from icon4py.model.common.utils import data_allocation as data_alloc, device_utils -from icon4py.model.testing import definitions as test_defs, grid_utils, serialbox +from icon4py.model.testing import definitions as test_defs, grid_utils from icon4py.model.testing.fixtures import backend, cpu_allocator from .. import utils from ..fixtures import data_provider, download_ser_data, experiment, grid_savepoint, processor_props -from ..utils import main_horizontal_dims @pytest.mark.parametrize("dim", utils.main_horizontal_dims()) From fe38371cd3baccc9653e17fadb058b434b2863c3 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 11 Feb 2026 14:24:28 +0100 Subject: [PATCH 227/240] Fix pytest.mark.xfail to pytest.xfail --- .../common/grid/mpi_tests/test_parallel_grid_refinement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py index 7a0e2a909b..e26ea72e43 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_refinement.py @@ -43,7 +43,7 @@ def test_compute_domain_bounds( processor_props: decomposition.ProcessProperties, ) -> None: if processor_props.is_single_rank() and experiment == definitions.Experiments.EXCLAIM_APE: - pytest.mark.xfail( + pytest.xfail( "end index data for single node APE are all 0 - re- serialization should fix that (patch%cells%end_index vs patch%cells%end_idx)" ) From f7f3724293f693fbb6c8028d29eb06e8dcf11f46 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 11 Feb 2026 14:31:34 +0100 Subject: [PATCH 228/240] Clean up docstring for Zone --- .../icon4py/model/common/grid/horizontal.py | 88 +++++++++---------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/horizontal.py b/model/common/src/icon4py/model/common/grid/horizontal.py index 6fe9290df0..d9ca9724b6 100644 --- a/model/common/src/icon4py/model/common/grid/horizontal.py +++ b/model/common/src/icon4py/model/common/grid/horizontal.py @@ -115,23 +115,23 @@ class Zone(enum.Enum): ## CellDim - | ICON constant or value |python | ICON4py Name | - | from mo_impl_constants.f90 |index | | - |:------------------------------------- |:----:|:-------------------------- | - | `min_rlcell_int-3`, `min_rlcell` (-8) | 0 | `END` | - | `min_rlcell_int-3` (-7) | 1 | | - | `min_rlcell_int-2`, (-6) | 2 |`HALO_LEVEL_2` | - | `min_rlcell_int-1` (-5) | 3 |`HALO` | - | `min_rlcell_int`(-4) | 4 |`LOCAL` | - | (-3) | 5 | | unused in icon4py (relevant for nesting) - | (-2) | 6 | | unused in icon4py (relevant for nesting) - | (-1) | 7 | | unused in icon4py (relevant for nesting) - | `0` | 8 |`INTERIOR` | - | `1` | 9 |`LATERAL_BOUNDARY` | - | `2` |10 | `LATERAL_BOUNDARY_LEVEL_2` | - | `3` |11 | `LATERAL_BOUNDARY_LEVEL_3` | - | `grf_bdywidth_c` (4) |12 | `LATERAL_BOUNDARY_LEVEL_4` | - | `grf_bdywith_c +1`,max_rlcell (5) |13 | `NUDGING` | + | ICON constant or value | python| ICON4py Name | + | from mo_impl_constants.f90 | index | | + |:--------------------------------------|:------|:---------------------------| + | `min_rlcell_int-3`, `min_rlcell` (-8) | 0 | `END` | + | `min_rlcell_int-3` (-7) | 1 | | + | `min_rlcell_int-2`, (-6) | 2 | `HALO_LEVEL_2` | + | `min_rlcell_int-1` (-5) | 3 | `HALO` | + | `min_rlcell_int`(-4) | 4 | `LOCAL` | + | (-3) | 5 | | unused in icon4py (relevant for nesting) + | (-2) | 6 | | unused in icon4py (relevant for nesting) + | (-1) | 7 | | unused in icon4py (relevant for nesting) + | `0` | 8 | `INTERIOR` | + | `1` | 9 | `LATERAL_BOUNDARY` | + | `2` | 10 | `LATERAL_BOUNDARY_LEVEL_2` | + | `3` | 11 | `LATERAL_BOUNDARY_LEVEL_3` | + | `grf_bdywidth_c` (4) | 12 | `LATERAL_BOUNDARY_LEVEL_4` | + | `grf_bdywith_c +1`,max_rlcell (5) | 13 | `NUDGING` | Lateral boundary and nudging are only relevant for LAM runs, halo lines only for distributed domains. @@ -139,53 +139,53 @@ class Zone(enum.Enum): ## VertexDim - | ICON constant or value | python | ICON4Py Name | - | from mo_impl_constants.f90 | index | | - |:--------------------------------------- |:------|:-------------------------- | + | ICON constant or value | python| ICON4Py Name | + | from mo_impl_constants.f90 | index | | + |:----------------------------------------|:------|:---------------------------| | `min_rlvert` (-7) | 0 | `END` | - | `min_rlvert+1`, `min_rlvert_int-2` (-6) | 1 |`HALO_LEVEL_2` | - | `min_rlvert_int-1` (-5) | 2 |`HALO` | - | `min_rlvert_int` (-4) | 3 |`LOCAL` | + | `min_rlvert+1`, `min_rlvert_int-2` (-6) | 1 | `HALO_LEVEL_2` | + | `min_rlvert_int-1` (-5) | 2 | `HALO` | + | `min_rlvert_int` (-4) | 3 | `LOCAL` | | (-3) | 4 | | unused in icon4py (relevant for nesting) | (-2) | 5 | | unused in icon4py (relevant for nesting) | (-1) | 6 | | unused in icon4py (relevant for nesting) - | `0` | ` 7 |INTERIOR` | - | `1` | 8 |`LATERAL_BOUNDARY` | - | `2` | 9 |`LATERAL_BOUNDARY_LEVEL_2` | - | `3` | 10 |`LATERAL_BOUNDARY_LEVEL_3` | - | `4` | 11 |`LATERAL_BOUNDARY_LEVEL_4` | - | `max_rlvert` (5) | 12 |`NUDGING` | + | `0` | 7 | `INTERIOR` | + | `1` | 8 | `LATERAL_BOUNDARY` | + | `2` | 9 | `LATERAL_BOUNDARY_LEVEL_2` | + | `3` | 10 | `LATERAL_BOUNDARY_LEVEL_3` | + | `4` | 11 | `LATERAL_BOUNDARY_LEVEL_4` | + | `max_rlvert` (5) | 12 | `NUDGING` | For the meaning see above. ## EdgeDim - | ICON constant or value | python | ICON4Py Name | - | from mo_impl_constants.f90 | index | | - |:-------------------------------------- |:-------|:-------------------------- | - | `min_rledge` (-13) | 0 |`END` | - |(-12) | 1 | | - |(-11) | 2 | | - | `min_rledge_int-2` (-10) | 3 |`HALO_LEVEL_2` | - | `min_rledge_int-1` (-9) | 4 |`HALO` | - | `min_rledge_int` (-8) | 5 |`LOCAL` | + | ICON constant or value | python | ICON4Py Name | + | from mo_impl_constants.f90 | index | | + |:---------------------------------------|:-------|:---------------------------| + | `min_rledge` (-13) | 0 | `END` | + | (-12) | 1 | | + | (-11) | 2 | | + | `min_rledge_int-2` (-10) | 3 | `HALO_LEVEL_2` | + | `min_rledge_int-1` (-9) | 4 | `HALO` | + | `min_rledge_int` (-8) | 5 | `LOCAL` | | (-7) | 6 | | unused in icon4py (relevant for nesting) | (-6) | 7 | | unused in icon4py (relevant for nesting) | (-5) | 8 | | unused in icon4py (relevant for nesting) | (-4) | 9 | | unused in icon4py (relevant for nesting) | (-3) | 10 | | unused in icon4py (relevant for nesting) | (-2) | 11 | | unused in icon4py (relevant for nesting) - |(-1) | 12 | | unused in icon4py (relevant for nesting) + | (-1) | 12 | | unused in icon4py (relevant for nesting) | `0` | 13 | `INTERIOR` | | `1` | 14 | `LATERAL_BOUNDARY` | | `2` | 15 | `LATERAL_BOUNDARY_LEVEL_2` | - | `3` | 16 |`LATERAL_BOUNDARY_LEVEL_3` | - | `4` | 17 |`LATERAL_BOUNDARY_LEVEL_4` | - | `5` | 18 |`LATERAL_BOUNDARY_LEVEL_5` | - | `6` | 18 |`LATERAL_BOUNDARY_LEVEL_6` | + | `3` | 16 | `LATERAL_BOUNDARY_LEVEL_3` | + | `4` | 17 | `LATERAL_BOUNDARY_LEVEL_4` | + | `5` | 18 | `LATERAL_BOUNDARY_LEVEL_5` | + | `6` | 19 | `LATERAL_BOUNDARY_LEVEL_6` | | `7` | 20 | `LATERAL_BOUNDARY_LEVEL_7` | - | `8` | 12 | `LATERAL_BOUNDARY_LEVEL_8`| + | `8` | 21 | `LATERAL_BOUNDARY_LEVEL_8` | | `grf_bdywidth_e` (9) | 22 | `NUDGING` | | `grf_bdywidth_e+1`, `max_rledge` (10) | 23 | `NUDGING_LEVEL_2` | From c7d8a74b480f5e2ed0be3e6c954975b51892d1b8 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 11 Feb 2026 14:35:06 +0100 Subject: [PATCH 229/240] Don't use get on dict to ensure exception on missing entries --- model/common/src/icon4py/model/common/decomposition/halo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index 4c792676cc..e8bbe734b9 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -118,7 +118,7 @@ def _assert_all_neighbor_tables(self) -> None: def _connectivity(self, offset: gtx.FieldOffset | str) -> data_alloc.NDArray: try: - return self._connectivities.get(_value(offset)) + return self._connectivities[_value(offset)] except KeyError as err: raise exceptions.MissingConnectivityError( f"Connectivity for offset {offset} is not available" From c8bfbc6c927853408292eb547cac5a46b6faa7ae Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 11 Feb 2026 14:36:50 +0100 Subject: [PATCH 230/240] Fix import for ToZeroBasedIndexTransformation --- .../src/icon4py/model/standalone_driver/driver_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/model/standalone_driver/src/icon4py/model/standalone_driver/driver_utils.py b/model/standalone_driver/src/icon4py/model/standalone_driver/driver_utils.py index cd1db437b1..94275a4e57 100644 --- a/model/standalone_driver/src/icon4py/model/standalone_driver/driver_utils.py +++ b/model/standalone_driver/src/icon4py/model/standalone_driver/driver_utils.py @@ -35,6 +35,7 @@ geometry as grid_geometry, geometry_attributes as geometry_meta, grid_manager as gm, + gridfile, icon as icon_grid, states as grid_states, vertical as v_grid, @@ -66,7 +67,7 @@ def create_grid_manager( grid_manager = gm.GridManager( grid_file=grid_file_path, config=vertical_grid_config, - transformation=gm.ToZeroBasedIndexTransformation(), + transformation=gridfile.ToZeroBasedIndexTransformation(), global_reductions=global_reductions, ) grid_manager(allocator=allocator, keep_skip_values=True) From 86da00423aa574ceb591bc13a24aedfb5a18b027 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 11 Feb 2026 14:53:30 +0100 Subject: [PATCH 231/240] Remove unused import --- model/common/tests/common/grid/unit_tests/test_horizontal.py | 1 - 1 file changed, 1 deletion(-) diff --git a/model/common/tests/common/grid/unit_tests/test_horizontal.py b/model/common/tests/common/grid/unit_tests/test_horizontal.py index 6fa4fbab9d..a859cef62e 100644 --- a/model/common/tests/common/grid/unit_tests/test_horizontal.py +++ b/model/common/tests/common/grid/unit_tests/test_horizontal.py @@ -17,7 +17,6 @@ import icon4py.model.common.grid.horizontal as h_grid from .. import utils -from ..fixtures import * # noqa: F403 if TYPE_CHECKING: From 814814f27b64460f0aa1f60eb1f83d9f72074f73 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 11 Feb 2026 16:44:46 +0100 Subject: [PATCH 232/240] Rename single/multi-node to single/multi-rank --- .../mpi_tests/test_mpi_decomposition.py | 2 +- .../mpi_tests/test_parallel_grid_manager.py | 182 +++++++++--------- .../tests/common/grid/mpi_tests/utils.py | 4 +- 3 files changed, 94 insertions(+), 94 deletions(-) diff --git a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py index b593a9b644..7624d85b47 100644 --- a/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py +++ b/model/common/tests/common/decomposition/mpi_tests/test_mpi_decomposition.py @@ -260,7 +260,7 @@ def test_decomposition_info_matches_gridsize( @pytest.mark.mpi @pytest.mark.parametrize("processor_props", [True], indirect=True) -def test_create_multi_node_runtime_with_mpi( +def test_create_multi_rank_runtime_with_mpi( decomposition_info: definitions.DecompositionInfo, processor_props: definitions.ProcessProperties, ) -> None: diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index 6fa48a803c..de958ecd7e 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -227,47 +227,47 @@ def test_geometry_fields_compare_single_multi_rank( # grid/geometry/interpolation/metrics factories. file = grid_utils.resolve_full_grid_file_name(experiment.grid) print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") - single_node_grid_manager = utils.run_grid_manager_for_singlenode(file) - single_node_geometry = geometry.GridGeometry( + single_rank_grid_manager = utils.run_grid_manager_for_single_rank(file) + single_rank_geometry = geometry.GridGeometry( backend=backend, - grid=single_node_grid_manager.grid, - coordinates=single_node_grid_manager.coordinates, - decomposition_info=single_node_grid_manager.decomposition_info, - extra_fields=single_node_grid_manager.geometry_fields, + grid=single_rank_grid_manager.grid, + coordinates=single_rank_grid_manager.coordinates, + decomposition_info=single_rank_grid_manager.decomposition_info, + extra_fields=single_rank_grid_manager.geometry_fields, metadata=geometry_attributes.attrs, ) print( - f"rank = {processor_props.rank} : single node grid has size {single_node_grid_manager.decomposition_info.get_horizontal_size()!r}" + f"rank = {processor_props.rank} : single node grid has size {single_rank_grid_manager.decomposition_info.get_horizontal_size()!r}" ) - multi_node_grid_manager = utils.run_gridmananger_for_multinode( + multi_rank_grid_manager = utils.run_grid_manager_for_multi_rank( file=file, run_properties=processor_props, decomposer=decomp.MetisDecomposer(), ) print( - f"rank = {processor_props.rank} : {multi_node_grid_manager.decomposition_info.get_horizontal_size()!r}" + f"rank = {processor_props.rank} : {multi_rank_grid_manager.decomposition_info.get_horizontal_size()!r}" ) print( f"rank = {processor_props.rank}: halo size for 'CellDim' " - f"(1: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " - f"(2: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" + f"(1: {multi_rank_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " + f"(2: {multi_rank_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) - multi_node_geometry = geometry.GridGeometry( + multi_rank_geometry = geometry.GridGeometry( backend=backend, - grid=multi_node_grid_manager.grid, - coordinates=multi_node_grid_manager.coordinates, - decomposition_info=multi_node_grid_manager.decomposition_info, - extra_fields=multi_node_grid_manager.geometry_fields, + grid=multi_rank_grid_manager.grid, + coordinates=multi_rank_grid_manager.coordinates, + decomposition_info=multi_rank_grid_manager.decomposition_info, + extra_fields=multi_rank_grid_manager.geometry_fields, metadata=geometry_attributes.attrs, ) check_local_global_field( - decomposition_info=multi_node_grid_manager.decomposition_info, + decomposition_info=multi_rank_grid_manager.decomposition_info, processor_props=processor_props, dim=dim, - global_reference_field=single_node_geometry.get(attrs_name).asnumpy(), - local_field=multi_node_geometry.get(attrs_name).asnumpy(), + global_reference_field=single_rank_geometry.get(attrs_name).asnumpy(), + local_field=multi_rank_geometry.get(attrs_name).asnumpy(), ) print(f"rank = {processor_props.rank} - DONE") @@ -300,64 +300,64 @@ def test_interpolation_fields_compare_single_multi_rank( file = grid_utils.resolve_full_grid_file_name(experiment.grid) print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") - single_node_grid_manager = utils.run_grid_manager_for_singlenode(file) - single_node_geometry = geometry.GridGeometry( + single_rank_grid_manager = utils.run_grid_manager_for_single_rank(file) + single_rank_geometry = geometry.GridGeometry( backend=backend, - grid=single_node_grid_manager.grid, - coordinates=single_node_grid_manager.coordinates, - decomposition_info=single_node_grid_manager.decomposition_info, - extra_fields=single_node_grid_manager.geometry_fields, + grid=single_rank_grid_manager.grid, + coordinates=single_rank_grid_manager.coordinates, + decomposition_info=single_rank_grid_manager.decomposition_info, + extra_fields=single_rank_grid_manager.geometry_fields, metadata=geometry_attributes.attrs, ) - single_node_interpolation = interpolation_factory.InterpolationFieldsFactory( - grid=single_node_grid_manager.grid, - decomposition_info=single_node_grid_manager.decomposition_info, - geometry_source=single_node_geometry, + single_rank_interpolation = interpolation_factory.InterpolationFieldsFactory( + grid=single_rank_grid_manager.grid, + decomposition_info=single_rank_grid_manager.decomposition_info, + geometry_source=single_rank_geometry, backend=backend, metadata=interpolation_attributes.attrs, exchange=decomp_defs.SingleNodeExchange(), ) print( - f"rank = {processor_props.rank} : single node grid has size {single_node_grid_manager.decomposition_info.get_horizontal_size()!r}" + f"rank = {processor_props.rank} : single node grid has size {single_rank_grid_manager.decomposition_info.get_horizontal_size()!r}" ) - multi_node_grid_manager = utils.run_gridmananger_for_multinode( + multi_rank_grid_manager = utils.run_grid_manager_for_multi_rank( file=file, run_properties=processor_props, decomposer=decomp.MetisDecomposer(), ) print( - f"rank = {processor_props.rank} : {multi_node_grid_manager.decomposition_info.get_horizontal_size()!r}" + f"rank = {processor_props.rank} : {multi_rank_grid_manager.decomposition_info.get_horizontal_size()!r}" ) print( f"rank = {processor_props.rank}: halo size for 'CellDim' " - f"(1: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " - f"(2: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" + f"(1: {multi_rank_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " + f"(2: {multi_rank_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) - multi_node_geometry = geometry.GridGeometry( + multi_rank_geometry = geometry.GridGeometry( backend=backend, - grid=multi_node_grid_manager.grid, - coordinates=multi_node_grid_manager.coordinates, - decomposition_info=multi_node_grid_manager.decomposition_info, - extra_fields=multi_node_grid_manager.geometry_fields, + grid=multi_rank_grid_manager.grid, + coordinates=multi_rank_grid_manager.coordinates, + decomposition_info=multi_rank_grid_manager.decomposition_info, + extra_fields=multi_rank_grid_manager.geometry_fields, metadata=geometry_attributes.attrs, ) - multi_node_interpolation = interpolation_factory.InterpolationFieldsFactory( - grid=multi_node_grid_manager.grid, - decomposition_info=multi_node_grid_manager.decomposition_info, - geometry_source=multi_node_geometry, + multi_rank_interpolation = interpolation_factory.InterpolationFieldsFactory( + grid=multi_rank_grid_manager.grid, + decomposition_info=multi_rank_grid_manager.decomposition_info, + geometry_source=multi_rank_geometry, backend=backend, metadata=interpolation_attributes.attrs, exchange=mpi_decomposition.GHexMultiNodeExchange( - processor_props, multi_node_grid_manager.decomposition_info + processor_props, multi_rank_grid_manager.decomposition_info ), ) - field_ref = single_node_interpolation.get(attrs_name).asnumpy() - field = multi_node_interpolation.get(attrs_name).asnumpy() + field_ref = single_rank_interpolation.get(attrs_name).asnumpy() + field = multi_rank_interpolation.get(attrs_name).asnumpy() check_local_global_field( - decomposition_info=multi_node_grid_manager.decomposition_info, + decomposition_info=multi_rank_grid_manager.decomposition_info, processor_props=processor_props, dim=dim, global_reference_field=field_ref, @@ -423,37 +423,37 @@ def test_metrics_fields_compare_single_multi_rank( ) print(f"running on {processor_props.comm} with {processor_props.comm_size} ranks") - single_node_grid_manager = utils.run_grid_manager_for_singlenode(file) - single_node_geometry = geometry.GridGeometry( + single_rank_grid_manager = utils.run_grid_manager_for_single_rank(file) + single_rank_geometry = geometry.GridGeometry( backend=backend, - grid=single_node_grid_manager.grid, - coordinates=single_node_grid_manager.coordinates, - decomposition_info=single_node_grid_manager.decomposition_info, - extra_fields=single_node_grid_manager.geometry_fields, + grid=single_rank_grid_manager.grid, + coordinates=single_rank_grid_manager.coordinates, + decomposition_info=single_rank_grid_manager.decomposition_info, + extra_fields=single_rank_grid_manager.geometry_fields, metadata=geometry_attributes.attrs, ) - single_node_interpolation = interpolation_factory.InterpolationFieldsFactory( - grid=single_node_grid_manager.grid, - decomposition_info=single_node_grid_manager.decomposition_info, - geometry_source=single_node_geometry, + single_rank_interpolation = interpolation_factory.InterpolationFieldsFactory( + grid=single_rank_grid_manager.grid, + decomposition_info=single_rank_grid_manager.decomposition_info, + geometry_source=single_rank_geometry, backend=backend, metadata=interpolation_attributes.attrs, exchange=decomp_defs.SingleNodeExchange(), ) - single_node_metrics = metrics_factory.MetricsFieldsFactory( - grid=single_node_geometry.grid, + single_rank_metrics = metrics_factory.MetricsFieldsFactory( + grid=single_rank_geometry.grid, vertical_grid=vertical_grid, - decomposition_info=single_node_grid_manager.decomposition_info, - geometry_source=single_node_geometry, + decomposition_info=single_rank_grid_manager.decomposition_info, + geometry_source=single_rank_geometry, # TODO(msimberg): Valid dummy topography? topography=( gtx.as_field( (dims.CellDim,), - xp.zeros(single_node_geometry.grid.num_cells), + xp.zeros(single_rank_geometry.grid.num_cells), allocator=allocator, ) ), - interpolation_source=single_node_interpolation, + interpolation_source=single_rank_interpolation, backend=backend, metadata=metrics_attributes.attrs, rayleigh_type=rayleigh_type, @@ -465,54 +465,54 @@ def test_metrics_fields_compare_single_multi_rank( exchange=decomp_defs.SingleNodeExchange(), ) print( - f"rank = {processor_props.rank} : single node grid has size {single_node_grid_manager.decomposition_info.get_horizontal_size()!r}" + f"rank = {processor_props.rank} : single node grid has size {single_rank_grid_manager.decomposition_info.get_horizontal_size()!r}" ) - multi_node_grid_manager = utils.run_gridmananger_for_multinode( + multi_rank_grid_manager = utils.run_grid_manager_for_multi_rank( file=file, run_properties=processor_props, decomposer=decomp.MetisDecomposer(), ) print( - f"rank = {processor_props.rank} : {multi_node_grid_manager.decomposition_info.get_horizontal_size()!r}" + f"rank = {processor_props.rank} : {multi_rank_grid_manager.decomposition_info.get_horizontal_size()!r}" ) print( f"rank = {processor_props.rank}: halo size for 'CellDim' " - f"(1: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " - f"(2: {multi_node_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" + f"(1: {multi_rank_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.FIRST_HALO_LEVEL)}), " + f"(2: {multi_rank_grid_manager.decomposition_info.get_halo_size(dims.CellDim, decomp_defs.DecompositionFlag.SECOND_HALO_LEVEL)})" ) - multi_node_geometry = geometry.GridGeometry( + multi_rank_geometry = geometry.GridGeometry( backend=backend, - grid=multi_node_grid_manager.grid, - coordinates=multi_node_grid_manager.coordinates, - decomposition_info=multi_node_grid_manager.decomposition_info, - extra_fields=multi_node_grid_manager.geometry_fields, + grid=multi_rank_grid_manager.grid, + coordinates=multi_rank_grid_manager.coordinates, + decomposition_info=multi_rank_grid_manager.decomposition_info, + extra_fields=multi_rank_grid_manager.geometry_fields, metadata=geometry_attributes.attrs, ) - multi_node_interpolation = interpolation_factory.InterpolationFieldsFactory( - grid=multi_node_grid_manager.grid, - decomposition_info=multi_node_grid_manager.decomposition_info, - geometry_source=multi_node_geometry, + multi_rank_interpolation = interpolation_factory.InterpolationFieldsFactory( + grid=multi_rank_grid_manager.grid, + decomposition_info=multi_rank_grid_manager.decomposition_info, + geometry_source=multi_rank_geometry, backend=backend, metadata=interpolation_attributes.attrs, exchange=mpi_decomposition.GHexMultiNodeExchange( - processor_props, multi_node_grid_manager.decomposition_info + processor_props, multi_rank_grid_manager.decomposition_info ), ) - multi_node_metrics = metrics_factory.MetricsFieldsFactory( - grid=multi_node_geometry.grid, + multi_rank_metrics = metrics_factory.MetricsFieldsFactory( + grid=multi_rank_geometry.grid, vertical_grid=vertical_grid, - decomposition_info=multi_node_grid_manager.decomposition_info, - geometry_source=multi_node_geometry, + decomposition_info=multi_rank_grid_manager.decomposition_info, + geometry_source=multi_rank_geometry, # TODO(msimberg): Valid dummy topography? topography=( gtx.as_field( (dims.CellDim,), - xp.zeros(multi_node_geometry.grid.num_cells), + xp.zeros(multi_rank_geometry.grid.num_cells), allocator=allocator, ) ), - interpolation_source=multi_node_interpolation, + interpolation_source=multi_rank_interpolation, backend=backend, metadata=metrics_attributes.attrs, rayleigh_type=rayleigh_type, @@ -522,15 +522,15 @@ def test_metrics_fields_compare_single_multi_rank( thslp_zdiffu=thslp_zdiffu, thhgtd_zdiffu=thhgtd_zdiffu, exchange=mpi_decomposition.GHexMultiNodeExchange( - processor_props, multi_node_grid_manager.decomposition_info + processor_props, multi_rank_grid_manager.decomposition_info ), ) - field_ref = single_node_metrics.get(attrs_name).asnumpy() - field = multi_node_metrics.get(attrs_name).asnumpy() + field_ref = single_rank_metrics.get(attrs_name).asnumpy() + field = multi_rank_metrics.get(attrs_name).asnumpy() check_local_global_field( - decomposition_info=multi_node_grid_manager.decomposition_info, + decomposition_info=multi_rank_grid_manager.decomposition_info, processor_props=processor_props, dim=dim, global_reference_field=field_ref, @@ -550,12 +550,12 @@ def test_validate_skip_values_in_distributed_connectivities( pytest.xfail("Limited-area grids not yet supported") file = grid_utils.resolve_full_grid_file_name(experiment.grid) - multinode_grid_manager = utils.run_gridmananger_for_multinode( + multi_rank_grid_manager = utils.run_grid_manager_for_multi_rank( file=file, run_properties=processor_props, decomposer=decomp.MetisDecomposer(), ) - distributed_grid = multinode_grid_manager.grid + distributed_grid = multi_rank_grid_manager.grid for k, c in distributed_grid.connectivities.items(): if gtx_common.is_neighbor_connectivity(c): skip_values_in_table = np.count_nonzero(c.asnumpy() == c.skip_value) @@ -581,7 +581,7 @@ def test_limited_area_raises( with pytest.raises( NotImplementedError, match="Limited-area grids are not supported in distributed runs" ): - _ = utils.run_gridmananger_for_multinode( + _ = utils.run_grid_manager_for_multi_rank( file=grid_utils.resolve_full_grid_file_name(grid), run_properties=processor_props, decomposer=decomp.MetisDecomposer(), diff --git a/model/common/tests/common/grid/mpi_tests/utils.py b/model/common/tests/common/grid/mpi_tests/utils.py index 6fe5eb8160..d774e3083c 100644 --- a/model/common/tests/common/grid/mpi_tests/utils.py +++ b/model/common/tests/common/grid/mpi_tests/utils.py @@ -19,7 +19,7 @@ def _grid_manager(file: pathlib.Path, num_levels: int) -> gm.GridManager: # TODO(msimberg): Single rank, not node. -def run_grid_manager_for_singlenode(file: pathlib.Path) -> gm.GridManager: +def run_grid_manager_for_single_rank(file: pathlib.Path) -> gm.GridManager: manager = _grid_manager(file, NUM_LEVELS) manager( keep_skip_values=True, @@ -31,7 +31,7 @@ def run_grid_manager_for_singlenode(file: pathlib.Path) -> gm.GridManager: # TODO(msimberg): Fix typos. Consistent naming with above function. -def run_gridmananger_for_multinode( +def run_grid_manager_for_multi_rank( file: pathlib.Path, run_properties: decomp_defs.ProcessProperties, decomposer: decomp.Decomposer, From f7dae6edc9f135116cf06ace7ff6549f5dd19e98 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 11 Feb 2026 16:45:12 +0100 Subject: [PATCH 233/240] Add missing word --- .../common/src/icon4py/model/common/decomposition/decomposer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/common/src/icon4py/model/common/decomposition/decomposer.py b/model/common/src/icon4py/model/common/decomposition/decomposer.py index 48fbc549cd..beaf30be14 100644 --- a/model/common/src/icon4py/model/common/decomposition/decomposer.py +++ b/model/common/src/icon4py/model/common/decomposition/decomposer.py @@ -56,7 +56,7 @@ def __call__( import pymetis # type: ignore [import-untyped] - # Invalid indices are not allowed here. Metis will segfault or fail + # Invalid indices are not allowed here. Metis will segfault or fail if # there are any invalid indices in the adjacency matrix. assert (adjacency_matrix >= 0).all() From 7abfc3df23e26cddd0b3fefc0343e969ddacf185 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 11 Feb 2026 16:45:44 +0100 Subject: [PATCH 234/240] Remove todos --- model/common/tests/common/grid/mpi_tests/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/model/common/tests/common/grid/mpi_tests/utils.py b/model/common/tests/common/grid/mpi_tests/utils.py index d774e3083c..483a96b64e 100644 --- a/model/common/tests/common/grid/mpi_tests/utils.py +++ b/model/common/tests/common/grid/mpi_tests/utils.py @@ -18,7 +18,6 @@ def _grid_manager(file: pathlib.Path, num_levels: int) -> gm.GridManager: ) -# TODO(msimberg): Single rank, not node. def run_grid_manager_for_single_rank(file: pathlib.Path) -> gm.GridManager: manager = _grid_manager(file, NUM_LEVELS) manager( @@ -30,7 +29,6 @@ def run_grid_manager_for_single_rank(file: pathlib.Path) -> gm.GridManager: return manager -# TODO(msimberg): Fix typos. Consistent naming with above function. def run_grid_manager_for_multi_rank( file: pathlib.Path, run_properties: decomp_defs.ProcessProperties, From 3973bfbb00f8512e86f2ce89d1f03daafccac1d3 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 11 Feb 2026 16:48:46 +0100 Subject: [PATCH 235/240] Remove todo --- .../common/src/icon4py/model/common/decomposition/definitions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/model/common/src/icon4py/model/common/decomposition/definitions.py b/model/common/src/icon4py/model/common/decomposition/definitions.py index 590172d2a6..df0b98fb52 100644 --- a/model/common/src/icon4py/model/common/decomposition/definitions.py +++ b/model/common/src/icon4py/model/common/decomposition/definitions.py @@ -149,7 +149,6 @@ def get_horizontal_size(self) -> base.HorizontalGridSize: def get_halo_size(self, dim: gtx.Dimension, flag: DecompositionFlag) -> int: level_mask = self.halo_level_mask(dim, flag) - # TODO(msimberg): Just do array_ns? return data_alloc.array_ns_from_array(level_mask).count_nonzero(level_mask) def halo_levels(self, dim: gtx.Dimension) -> data_alloc.NDArray: From 85b091d677d84e4885d7817e8da7d01c8a2e7bbf Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 11 Feb 2026 17:01:03 +0100 Subject: [PATCH 236/240] Remove unused logger --- model/common/src/icon4py/model/common/decomposition/halo.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index e8bbe734b9..e43dc85ada 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -6,7 +6,6 @@ # Please, refer to the LICENSE file in the root directory. # SPDX-License-Identifier: BSD-3-Clause import functools -import logging from types import ModuleType from typing import Protocol, runtime_checkable @@ -20,9 +19,6 @@ from icon4py.model.common.utils import data_allocation as data_alloc -log = logging.getLogger(__name__) - - def _value(k: gtx.FieldOffset | str) -> str: return str(k.value) if isinstance(k, gtx.FieldOffset) else k From 8e853907c9a97cb0f4a4fc2cba1d260de8a0c5c2 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 11 Feb 2026 17:17:55 +0100 Subject: [PATCH 237/240] Small refactor in halo.py --- .../model/common/decomposition/halo.py | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/model/common/src/icon4py/model/common/decomposition/halo.py b/model/common/src/icon4py/model/common/decomposition/halo.py index e43dc85ada..701ab8323f 100644 --- a/model/common/src/icon4py/model/common/decomposition/halo.py +++ b/model/common/src/icon4py/model/common/decomposition/halo.py @@ -19,10 +19,6 @@ from icon4py.model.common.utils import data_allocation as data_alloc -def _value(k: gtx.FieldOffset | str) -> str: - return str(k.value) if isinstance(k, gtx.FieldOffset) else k - - @runtime_checkable class HaloConstructor(Protocol): """Callable that takes a mapping from cells to ranks""" @@ -41,7 +37,7 @@ def __init__( def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: xp = data_alloc.import_array_ns(self._allocator) - create_arrays = functools.partial(_create_dummy_decomposition_arrays, array_ns=xp) + create_arrays = functools.partial(self._create_dummy_decomposition_arrays, array_ns=xp) decomposition_info = defs.DecompositionInfo() decomposition_info.set_dimension(dims.EdgeDim, *create_arrays(self._size.num_edges)) @@ -49,14 +45,14 @@ def __call__(self, cell_to_rank: data_alloc.NDArray) -> defs.DecompositionInfo: decomposition_info.set_dimension(dims.VertexDim, *create_arrays(self._size.num_vertices)) return decomposition_info - -def _create_dummy_decomposition_arrays( - size: int, array_ns: ModuleType = np -) -> tuple[data_alloc.NDArray, data_alloc.NDArray, data_alloc.NDArray]: - indices = array_ns.arange(size, dtype=gtx.int32) # type: ignore [attr-defined] - owner_mask = array_ns.full((size,), True, dtype=bool) - halo_levels = array_ns.full((size,), defs.DecompositionFlag.OWNED.value, dtype=gtx.int32) # type: ignore [attr-defined] - return indices, owner_mask, halo_levels + @staticmethod + def _create_dummy_decomposition_arrays( + size: int, array_ns: ModuleType = np + ) -> tuple[data_alloc.NDArray, data_alloc.NDArray, data_alloc.NDArray]: + indices = array_ns.arange(size, dtype=gtx.int32) # type: ignore [attr-defined] + owner_mask = array_ns.full((size,), True, dtype=bool) + halo_levels = array_ns.full((size,), defs.DecompositionFlag.OWNED.value, dtype=gtx.int32) # type: ignore [attr-defined] + return indices, owner_mask, halo_levels class IconLikeHaloConstructor(HaloConstructor): @@ -77,9 +73,13 @@ def __init__( """ self._xp = data_alloc.import_array_ns(allocator) self._props = run_properties - self._connectivities = {_value(k): v for k, v in connectivities.items()} + self._connectivities = {self._value(k): v for k, v in connectivities.items()} self._assert_all_neighbor_tables() + @staticmethod + def _value(k: gtx.FieldOffset | str) -> str: + return str(k.value) if isinstance(k, gtx.FieldOffset) else k + def _validate_mapping(self, cell_to_rank_mapping: data_alloc.NDArray) -> None: # validate the distribution mapping: num_cells = self._connectivity(dims.C2E2C).shape[0] @@ -114,7 +114,7 @@ def _assert_all_neighbor_tables(self) -> None: def _connectivity(self, offset: gtx.FieldOffset | str) -> data_alloc.NDArray: try: - return self._connectivities[_value(offset)] + return self._connectivities[self._value(offset)] except KeyError as err: raise exceptions.MissingConnectivityError( f"Connectivity for offset {offset} is not available" @@ -178,7 +178,6 @@ def _update_owner_mask_by_max_rank_convention( according to a remark in `mo_decomposition_tools.f90` ICON puts them to the node with the higher rank. - # TODO(halungge): can we add an assert for the target dimension of the connectivity being cells? Args: owner_mask: owner mask for the dimension all_indices: (global) indices of the dimension From 9e3f0fc0b3bdef1ec85b68e64695c1e41d03a66d Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 11 Feb 2026 17:26:49 +0100 Subject: [PATCH 238/240] Revert horizontal_size/horizontal_config rename --- model/common/src/icon4py/model/common/grid/base.py | 8 ++++---- .../common/src/icon4py/model/common/grid/grid_manager.py | 2 +- model/common/src/icon4py/model/common/grid/simple.py | 2 +- model/testing/src/icon4py/model/testing/serialbox.py | 2 +- tools/src/icon4py/tools/py2fgen/wrappers/common.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/base.py b/model/common/src/icon4py/model/common/grid/base.py index ea1f3604a3..e011295b18 100644 --- a/model/common/src/icon4py/model/common/grid/base.py +++ b/model/common/src/icon4py/model/common/grid/base.py @@ -46,7 +46,7 @@ def __repr__(self): @dataclasses.dataclass(frozen=True, kw_only=True) class GridConfig: - horizontal_size: HorizontalGridSize + horizontal_config: HorizontalGridSize # TODO(halungge): Decouple the vertical from horizontal grid. vertical_size: int limited_area: bool = True @@ -61,15 +61,15 @@ def num_levels(self): @property def num_vertices(self): - return self.horizontal_size.num_vertices + return self.horizontal_config.num_vertices @property def num_edges(self): - return self.horizontal_size.num_edges + return self.horizontal_config.num_edges @property def num_cells(self): - return self.horizontal_size.num_cells + return self.horizontal_config.num_cells @dataclasses.dataclass(frozen=True) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index c9656bfd65..67ddbb4d1e 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -458,7 +458,7 @@ def _construct_decomposed_grid( start_index, end_index = icon.get_start_and_end_index(domain_bounds_constructor) grid_config = base.GridConfig( - horizontal_size=distributed_size, + horizontal_config=distributed_size, vertical_size=self._vertical_config.num_levels, limited_area=limited_area, keep_skip_values=keep_skip_values, diff --git a/model/common/src/icon4py/model/common/grid/simple.py b/model/common/src/icon4py/model/common/grid/simple.py index c548815643..408a369459 100644 --- a/model/common/src/icon4py/model/common/grid/simple.py +++ b/model/common/src/icon4py/model/common/grid/simple.py @@ -438,7 +438,7 @@ def simple_end_index(domain: h_grid.Domain) -> gtx.int32: ) vertical_grid_config = VerticalGridConfig(num_levels=num_levels) config = base.GridConfig( - horizontal_size=horizontal_grid_size, + horizontal_config=horizontal_grid_size, vertical_size=vertical_grid_config.num_levels, limited_area=False, ) diff --git a/model/testing/src/icon4py/model/testing/serialbox.py b/model/testing/src/icon4py/model/testing/serialbox.py index 57c49a9fa6..93a4bc21ce 100644 --- a/model/testing/src/icon4py/model/testing/serialbox.py +++ b/model/testing/src/icon4py/model/testing/serialbox.py @@ -484,7 +484,7 @@ def construct_icon_grid( with_repeated_index: bool = True, ) -> icon.IconGrid: config = base.GridConfig( - horizontal_size=base.HorizontalGridSize( + horizontal_config=base.HorizontalGridSize( num_vertices=self.num(dims.VertexDim), num_cells=self.num(dims.CellDim), num_edges=self.num(dims.EdgeDim), diff --git a/tools/src/icon4py/tools/py2fgen/wrappers/common.py b/tools/src/icon4py/tools/py2fgen/wrappers/common.py index b7f702f11b..0e22449fac 100644 --- a/tools/src/icon4py/tools/py2fgen/wrappers/common.py +++ b/tools/src/icon4py/tools/py2fgen/wrappers/common.py @@ -165,7 +165,7 @@ def construct_icon_grid( e2c2e0 = add_origin(xp, e2c2e) config = base.GridConfig( - horizontal_size=base.HorizontalGridSize( + horizontal_config=base.HorizontalGridSize( num_vertices=num_vertices, num_cells=num_cells, num_edges=num_edges, From 2bd04a7694428a5894393872e5234bd59db49930 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 11 Feb 2026 17:31:16 +0100 Subject: [PATCH 239/240] Fix typo --- model/common/src/icon4py/model/common/grid/grid_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index 67ddbb4d1e..fe7de2f964 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -35,7 +35,7 @@ _log = logging.getLogger(__name__) _single_node_decomposer = decomp.SingleNodeDecomposer() _single_process_props = decomposition.SingleNodeProcessProperties() -_fortan_to_python_transformer = gridfile.ToZeroBasedIndexTransformation() +_fortran_to_python_transformer = gridfile.ToZeroBasedIndexTransformation() class IconGridError(RuntimeError): @@ -65,7 +65,7 @@ def __init__( self, grid_file: pathlib.Path | str, config: v_grid.VerticalGridConfig, # TODO(msimberg): remove to separate vertical and horizontal grid - transformation: gridfile.IndexTransformation = _fortan_to_python_transformer, + transformation: gridfile.IndexTransformation = _fortran_to_python_transformer, global_reductions: decomposition.Reductions = decomposition.single_node_reductions, ): self._transformation = transformation From 2bf0d2a59d044ef55e5382af80965204fcb27d78 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 11 Feb 2026 17:45:50 +0100 Subject: [PATCH 240/240] Small renaming related to index offset/transformation --- .../icon4py/model/common/grid/grid_manager.py | 12 ++++++------ .../src/icon4py/model/common/grid/gridfile.py | 18 ++++++++---------- .../mpi_tests/test_parallel_grid_manager.py | 2 +- .../grid/unit_tests/test_grid_manager.py | 2 +- .../common/grid/unit_tests/test_gridfile.py | 2 +- .../model/standalone_driver/driver_utils.py | 2 +- .../src/icon4py/model/testing/grid_utils.py | 2 +- 7 files changed, 19 insertions(+), 21 deletions(-) diff --git a/model/common/src/icon4py/model/common/grid/grid_manager.py b/model/common/src/icon4py/model/common/grid/grid_manager.py index fe7de2f964..ebd4d600b8 100644 --- a/model/common/src/icon4py/model/common/grid/grid_manager.py +++ b/model/common/src/icon4py/model/common/grid/grid_manager.py @@ -65,10 +65,10 @@ def __init__( self, grid_file: pathlib.Path | str, config: v_grid.VerticalGridConfig, # TODO(msimberg): remove to separate vertical and horizontal grid - transformation: gridfile.IndexTransformation = _fortran_to_python_transformer, + offset_transformation: gridfile.IndexTransformation = _fortran_to_python_transformer, global_reductions: decomposition.Reductions = decomposition.single_node_reductions, ): - self._transformation = transformation + self._offset_transformation = offset_transformation self._file_name = str(grid_file) self._vertical_config = config # Output @@ -81,7 +81,7 @@ def __init__( def open(self): """Open the gridfile resource for reading.""" - self._reader = gridfile.GridFile(self._file_name, self._transformation) + self._reader = gridfile.GridFile(self._file_name, self._offset_transformation) self._reader.open() def close(self): @@ -329,7 +329,7 @@ def _read_geometry_fields( self._reader.int_variable( gridfile.GeometryName.EDGE_ORIENTATION_ON_VERTEX, transpose=True, - apply_transformation=False, + apply_offset=False, indices=my_vertex_indices, ), allocator=allocator, @@ -364,7 +364,7 @@ def _read_grid_refinement_fields( name, indices=self._decomposition_info.global_index(dim), transpose=False, - apply_transformation=False, + apply_offset=False, ), allocator=allocator, ) @@ -541,7 +541,7 @@ def _get_index_field( ): return array_ns.asarray( self._reader.int_variable( - field, indices=indices, transpose=transpose, apply_transformation=apply_offset + field, indices=indices, transpose=transpose, apply_offset=apply_offset ) ) diff --git a/model/common/src/icon4py/model/common/grid/gridfile.py b/model/common/src/icon4py/model/common/grid/gridfile.py index 4ca5900c3f..c51cdcc651 100644 --- a/model/common/src/icon4py/model/common/grid/gridfile.py +++ b/model/common/src/icon4py/model/common/grid/gridfile.py @@ -32,7 +32,7 @@ def __init__(self, *args, **kwargs): class IndexTransformation(Protocol): - """Return a transformation field to be applied to index fields""" + """Return an offset field to be applied to index fields""" def __call__( self, @@ -309,9 +309,9 @@ class GridFile: INVALID_INDEX = -1 - def __init__(self, file_name: str, transformation: IndexTransformation): + def __init__(self, file_name: str, offset_transformation: IndexTransformation): self._filename = file_name - self._offset = transformation + self._offset_transformation = offset_transformation self._dataset = None def dimension(self, name: DimensionName) -> int: @@ -335,7 +335,7 @@ def int_variable( name: FieldName, indices: data_alloc.NDArray | None = None, transpose: bool = True, - apply_transformation: bool = True, + apply_offset: bool = True, ) -> np.ndarray: """Read a integer field from the grid file. @@ -345,18 +345,16 @@ def int_variable( name: name of the field to read indices: list of indices to read transpose: flag to indicate whether the file should be transposed (for 2d fields) - apply_transformation: flag to indicate whether the transformation should be applied + apply_offset: flag to indicate whether the offset should be applied to the indices, defaults to True Returns: NDArray: field data """ - _log.debug( - f"reading {name}: transposing = {transpose} apply_transformation={apply_transformation}" - ) + _log.debug(f"reading {name}: transposing = {transpose} apply_offset={apply_offset}") variable = self.variable(name, indices, transpose=transpose, dtype=gtx.int32) - if apply_transformation: - return variable + self._offset(variable) + if apply_offset: + return variable + self._offset_transformation(variable) return variable def variable( diff --git a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py index de958ecd7e..7a255bb388 100644 --- a/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py +++ b/model/common/tests/common/grid/mpi_tests/test_parallel_grid_manager.py @@ -61,7 +61,7 @@ def test_grid_manager_validate_decomposer( manager = gm.GridManager( grid_file=file, config=v_grid.VerticalGridConfig(num_levels=utils.NUM_LEVELS), - transformation=gridfile.ToZeroBasedIndexTransformation(), + offset_transformation=gridfile.ToZeroBasedIndexTransformation(), ) with pytest.raises(exceptions.InvalidConfigError) as e: manager( diff --git a/model/common/tests/common/grid/unit_tests/test_grid_manager.py b/model/common/tests/common/grid/unit_tests/test_grid_manager.py index 183bfc5c87..24e915c41a 100644 --- a/model/common/tests/common/grid/unit_tests/test_grid_manager.py +++ b/model/common/tests/common/grid/unit_tests/test_grid_manager.py @@ -348,7 +348,7 @@ def test_gridmanager_given_file_not_found_then_abort( manager = gm.GridManager( grid_file=fname, config=v_grid.VerticalGridConfig(num_levels=80), - transformation=icon4py.model.common.grid.gridfile.NoTransformation(), + offset_transformation=icon4py.model.common.grid.gridfile.NoTransformation(), ) manager(allocator=cpu_allocator, keep_skip_values=True) assert error.value == 1 diff --git a/model/common/tests/common/grid/unit_tests/test_gridfile.py b/model/common/tests/common/grid/unit_tests/test_gridfile.py index 7974bb41d0..7fd8decba2 100644 --- a/model/common/tests/common/grid/unit_tests/test_gridfile.py +++ b/model/common/tests/common/grid/unit_tests/test_gridfile.py @@ -34,7 +34,7 @@ def test_grid_file_dimension() -> None: grid_descriptor = definitions.Grids.R02B04_GLOBAL global_grid_file = str(gridtest_utils.resolve_full_grid_file_name(grid_descriptor)) - parser = gridfile.GridFile(global_grid_file, transformation=gridfile.NoTransformation()) + parser = gridfile.GridFile(global_grid_file, offset_transformation=gridfile.NoTransformation()) try: parser.open() assert ( diff --git a/model/standalone_driver/src/icon4py/model/standalone_driver/driver_utils.py b/model/standalone_driver/src/icon4py/model/standalone_driver/driver_utils.py index 94275a4e57..cc6be3b747 100644 --- a/model/standalone_driver/src/icon4py/model/standalone_driver/driver_utils.py +++ b/model/standalone_driver/src/icon4py/model/standalone_driver/driver_utils.py @@ -67,7 +67,7 @@ def create_grid_manager( grid_manager = gm.GridManager( grid_file=grid_file_path, config=vertical_grid_config, - transformation=gridfile.ToZeroBasedIndexTransformation(), + offset_transformation=gridfile.ToZeroBasedIndexTransformation(), global_reductions=global_reductions, ) grid_manager(allocator=allocator, keep_skip_values=True) diff --git a/model/testing/src/icon4py/model/testing/grid_utils.py b/model/testing/src/icon4py/model/testing/grid_utils.py index 166b1b1081..8f9f6a358e 100644 --- a/model/testing/src/icon4py/model/testing/grid_utils.py +++ b/model/testing/src/icon4py/model/testing/grid_utils.py @@ -67,7 +67,7 @@ def get_grid_manager( manager = gm.GridManager( grid_file=filename, config=v_grid.VerticalGridConfig(num_levels=num_levels), - transformation=gridfile.ToZeroBasedIndexTransformation(), + offset_transformation=gridfile.ToZeroBasedIndexTransformation(), ) manager(allocator=allocator, keep_skip_values=keep_skip_values) return manager