Skip to content

Commit

Permalink
Discover periodic neighbours
Browse files Browse the repository at this point in the history
  • Loading branch information
@jwallwork23
jwallwork23 committed Oct 25, 2024
1 parent afaa5ae commit 3f23975
Show file tree
Hide file tree
Showing 2 changed files with 117 additions and 4 deletions.
101 changes: 99 additions & 2 deletions Partitioner.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/*!
* @file Partitioner.cpp
* @author Athena Elafrou <ae488@cam.ac.uk>
* @date 12 Sep 2024
* @date 25 Oct 2024
*/

#include "Partitioner.hpp"
Expand Down Expand Up @@ -42,6 +42,23 @@ void Partitioner::get_neighbours(
}
}

void Partitioner::get_neighbours_periodic(
std::vector<std::vector<int>>& ids_p, std::vector<std::vector<int>>& halo_sizes_p) const
{
// NOTE: Neighbours are looped in the order left-right-bottom-top
for (int d = 0; d < NDIMS; d++) {
if ((d == 0 && _p0) || (d == 1 && _p1)) {
for (int i = 0; i < 2; i++) {
int idx = 2 * d + i;
for (auto it = _neighbours_p[idx].begin(); it != _neighbours_p[idx].end(); ++it) {
ids_p[idx].push_back(it->first);
halo_sizes_p[idx].push_back(it->second);
}
}
}
}
}

void Partitioner::save_mask(const std::string& filename) const
{
// Use C API for parallel I/O
Expand Down Expand Up @@ -113,6 +130,16 @@ void Partitioner::save_metadata(const std::string& filename) const
CHECK_MPI(MPI_Exscan(&num_neighbours[idx], &offsets[idx], 1, MPI_INT, MPI_SUM, _comm));
}

// Prepare periodic neighbour data
std::vector<std::vector<int>> ids_p(NNBRS), halos_p(NNBRS);
get_neighbours_periodic(ids_p, halos_p);
std::vector<int> num_neighbours_p(NNBRS), dims_p(NNBRS, 0), offsets_p(NNBRS, 0);
for (int idx = 0; idx < NNBRS; idx++) {
num_neighbours_p[idx] = (int)ids_p[idx].size();
CHECK_MPI(MPI_Allreduce(&num_neighbours_p[idx], &dims_p[idx], 1, MPI_INT, MPI_SUM, _comm));
CHECK_MPI(MPI_Exscan(&num_neighbours_p[idx], &offsets_p[idx], 1, MPI_INT, MPI_SUM, _comm));
}

// Define dimensions in netCDF file
int dimid;
std::vector<int> dimids(NNBRS);
Expand All @@ -121,6 +148,13 @@ void Partitioner::save_metadata(const std::string& filename) const
NC_CHECK(nc_def_dim(nc_id, dir_chars[idx].c_str(), dims[idx], &dimids[idx]));
}

// Define periodic dimensions in netCDF file
std::vector<int> dimids_p(NNBRS);
for (int idx = 0; idx < NNBRS; idx++) {
NC_CHECK(
nc_def_dim(nc_id, (dir_chars[idx] + "_periodic").c_str(), dims_p[idx], &dimids_p[idx]));
}

// Define groups in netCDF file
int bbox_gid, connectivity_gid;
NC_CHECK(nc_def_grp(nc_id, "bounding_boxes", &bbox_gid));
Expand Down Expand Up @@ -148,6 +182,19 @@ void Partitioner::save_metadata(const std::string& filename) const
NC_CHECK(nc_def_var(connectivity_gid, (dir_names[idx] + "_neighbour_halos").c_str(), NC_INT,
1, &dimids[idx], &halos_vid[idx]));
}
int num_vid_p[NNBRS];
int ids_vid_p[NNBRS];
int halos_vid_p[NNBRS];
for (int idx = 0; idx < NNBRS; idx++) {
// Periodic members of connectivity group
NC_CHECK(nc_def_var(connectivity_gid, (dir_names[idx] + "_neighbours_periodic").c_str(),
NC_INT, 1, &dimid, &num_vid_p[idx]));
NC_CHECK(nc_def_var(connectivity_gid, (dir_names[idx] + "_neighbour_ids_periodic").c_str(),
NC_INT, 1, &dimids_p[idx], &ids_vid_p[idx]));
NC_CHECK(
nc_def_var(connectivity_gid, (dir_names[idx] + "_neighbour_halos_periodic").c_str(),
NC_INT, 1, &dimids_p[idx], &halos_vid_p[idx]));
}

// Write metadata to file
NC_CHECK(nc_enddef(nc_id));
Expand All @@ -161,16 +208,30 @@ void Partitioner::save_metadata(const std::string& filename) const
NC_CHECK(nc_put_var1_int(bbox_gid, cnt_vid[idx], &start, &_local_ext_new[idx]));
}
for (int idx = 0; idx < NNBRS; idx++) {
// Numbers of neighbours
size_t start = _rank;
NC_CHECK(nc_var_par_access(connectivity_gid, num_vid[idx], NC_COLLECTIVE));
NC_CHECK(nc_put_var1_int(connectivity_gid, num_vid[idx], &start, &num_neighbours[idx]));
// Numbers of neighbours for periodic dimensions
NC_CHECK(nc_var_par_access(connectivity_gid, num_vid_p[idx], NC_COLLECTIVE));
NC_CHECK(nc_put_var1_int(connectivity_gid, num_vid_p[idx], &start, &num_neighbours_p[idx]));
// IDs and halos
start = offsets[idx];
size_t count = num_neighbours[idx];
NC_CHECK(nc_var_par_access(connectivity_gid, ids_vid[idx], NC_COLLECTIVE));
NC_CHECK(nc_put_vara_int(connectivity_gid, ids_vid[idx], &start, &count, ids[idx].data()));
NC_CHECK(nc_var_par_access(connectivity_gid, halos_vid[idx], NC_COLLECTIVE));
NC_CHECK(
nc_put_vara_int(connectivity_gid, halos_vid[idx], &start, &count, halos[idx].data()));
// IDs and halos for periodic dimensions
start = offsets_p[idx];
count = num_neighbours_p[idx];
NC_CHECK(nc_var_par_access(connectivity_gid, ids_vid_p[idx], NC_COLLECTIVE));
NC_CHECK(
nc_put_vara_int(connectivity_gid, ids_vid_p[idx], &start, &count, ids_p[idx].data()));
NC_CHECK(nc_var_par_access(connectivity_gid, halos_vid_p[idx], NC_COLLECTIVE));
NC_CHECK(nc_put_vara_int(
connectivity_gid, halos_vid_p[idx], &start, &count, halos_p[idx].data()));
}
NC_CHECK(nc_close(nc_id));
}
Expand Down Expand Up @@ -266,7 +327,7 @@ void Partitioner::discover_neighbours()
// i.e., the left edge of domain[_rank] has to match the right edge of domain[p]
if (domains[_rank].p1.x == domains[p].p2.x) {
// next compute overlap (if overlap is zero then they are not neighbours)
// domain overlap is equivalent to the amount of data transfered in halo exchnage.
// domain overlap is equivalent to the amount of data transfered in halo exchange.
int halo_size = domainOverlap(domains[_rank], domains[p], 'y');
if (halo_size) {
_neighbours[0].insert(std::pair<int, int>(p, halo_size));
Expand Down Expand Up @@ -296,6 +357,42 @@ void Partitioner::discover_neighbours()
_neighbours[3].insert(std::pair<int, int>(p, halo_size));
}
}

if (_p0) {
// Find my left periodic neighbours
if ((domains[_rank].p1.x == 0) && (domains[p].p2.x == _global_ext[0])) {
int halo_size = domainOverlap(domains[_rank], domains[p], 'y');
if (halo_size) {
_neighbours_p[0].insert(std::pair<int, int>(p, halo_size));
}
}

// Find my right periodic neighbours
if ((domains[_rank].p2.x == _global_ext[0]) && (domains[p].p1.x == 0)) {
int halo_size = domainOverlap(domains[_rank], domains[p], 'y');
if (halo_size) {
_neighbours_p[1].insert(std::pair<int, int>(p, halo_size));
}
}
}

if (_p1) {
// Find my bottom periodic neighbours
if ((domains[_rank].p1.y == 0) && (domains[p].p2.y == _global_ext[1])) {
int halo_size = domainOverlap(domains[_rank], domains[p], 'x');
if (halo_size) {
_neighbours_p[2].insert(std::pair<int, int>(p, halo_size));
}
}

// Find my top periodic neighbours
if ((domains[_rank].p2.y == _global_ext[1]) && (domains[p].p1.y == 0)) {
int halo_size = domainOverlap(domains[_rank], domains[p], 'x');
if (halo_size) {
_neighbours_p[3].insert(std::pair<int, int>(p, halo_size));
}
}
}
}
}
}
20 changes: 18 additions & 2 deletions Partitioner.hpp
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/*!
* @file Partitioner.hpp
* @author Athena Elafrou <ae488@cam.ac.uk>
* @date 11 Sep 2024
* @date 25 Oct 2024
*/

#pragma once
Expand Down Expand Up @@ -65,6 +65,17 @@ class LIB_EXPORT Partitioner {
void get_neighbours(
std::vector<std::vector<int>>& ids, std::vector<std::vector<int>>& halo_sizes) const;

/*!
* @brief Returns vectors containing the MPI ranks and halo sizes of the periodic neighbours
* for this process after partitioning. The neighbours are ordered left, right, bottom, top.
*
* @param ids MPI ranks of the neighbours for each direction
* @param halo_sizes Halo sizes of the neighbours for each direction
*/
// TODO: Consider merging with the method above and providing a boolean flag to specify periodic
void get_neighbours_periodic(
std::vector<std::vector<int>>& ids, std::vector<std::vector<int>>& halo_sizes) const;

/*!
* @brief Saves the partition IDs of the latest 2D domain decomposition in a
* NetCDF file.
Expand Down Expand Up @@ -103,7 +114,7 @@ class LIB_EXPORT Partitioner {
// created in the heap to ensure it's dtor is executed before MPI_Finalize()
Partitioner(MPI_Comm comm);

// Discover the processe's neighbours and halo sizes after partitioning
// Discover the neighbours and halo sizes of the process after partitioning
void discover_neighbours();

protected:
Expand All @@ -112,6 +123,8 @@ class LIB_EXPORT Partitioner {
int _total_num_procs = -1; // Total number of processes in communicator
static const int NDIMS = 2; // Number of dimensions
static const int NNBRS = 2 * NDIMS; // Number of neighbours (two per dimension)
bool _p0 = false; // Periodic boundary in the 1st dimension
bool _p1 = false; // Periodic boundary in the 2nd dimension

// Letters used for each dimension
std::vector<std::string> dim_chars = { "x", "y" };
Expand Down Expand Up @@ -149,6 +162,9 @@ class LIB_EXPORT Partitioner {
// Vector of maps of neighbours to their halo sizes after partitioning
std::vector<std::map<int, int>> _neighbours = std::vector<std::map<int, int>>(NNBRS);

// Vector of maps of periodic neighbours to their halo sizes after partitioning
std::vector<std::map<int, int>> _neighbours_p = std::vector<std::map<int, int>>(NNBRS);

public:
struct LIB_EXPORT Factory {
/*!
Expand Down

0 comments on commit 3f23975

Please sign in to comment.