Skip to content

Commit

Permalink
feat: remove blocking
Browse files Browse the repository at this point in the history 
Blocking was added back in commit 3161a06 but it does not have a use
case in the nextsimdg code. It adds unnecessary complexity to the
`decomp` tool and so we have decided to remove it. For discussion see
issue [#50](#50)
  • Loading branch information
@TomMelt
TomMelt committed Oct 5, 2024
1 parent 61bd1df commit 3c56d8c
Show file tree
Hide file tree
Showing 11 changed files with 115 additions and 436 deletions.
149 changes: 42 additions & 107 deletions Grid.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,35 +26,18 @@ static void find_factors(int n, int& factor_a, int& factor_b)

Grid* Grid::create(MPI_Comm comm, const std::string& filename, bool ignore_mask)
{
return new Grid(comm, filename, "x", "y", "mask", 1, 1, ignore_mask);
}

Grid* Grid::create(
MPI_Comm comm, const std::string& filename, int blk_dim0, int blk_dim1, bool ignore_mask)
{
return new Grid(comm, filename, "x", "y", "mask", blk_dim0, blk_dim1, ignore_mask);
return new Grid(comm, filename, "x", "y", "mask", ignore_mask);
}

Grid* Grid::create(MPI_Comm comm, const std::string& filename, const std::string dim0_name,
const std::string dim1_name, const std::string mask_name, bool ignore_mask)
{
return new Grid(comm, filename, dim0_name, dim1_name, mask_name, 1, 1, ignore_mask);
}

Grid* Grid::create(MPI_Comm comm, const std::string& filename, const std::string dim0_name,
const std::string dim1_name, const std::string mask_name, int blk_dim0, int blk_dim1,
bool ignore_mask)
{
return new Grid(
comm, filename, dim0_name, dim1_name, mask_name, blk_dim0, blk_dim1, ignore_mask);
return new Grid(comm, filename, dim0_name, dim1_name, mask_name, ignore_mask);
}

Grid::Grid(MPI_Comm comm, const std::string& filename, const std::string& dim0_name,
const std::string& dim1_name, const std::string& mask_name, int blk_dim0, int blk_dim1,
bool ignore_mask)
const std::string& dim1_name, const std::string& mask_name, bool ignore_mask)
: _comm(comm)
, _blk_factor_0(blk_dim0)
, _blk_factor_1(blk_dim1)
{
// Use C API for parallel I/O
int nc_id, nc_o_mode;
Expand Down Expand Up @@ -84,8 +67,6 @@ Grid::Grid(MPI_Comm comm, const std::string& filename, const std::string& dim0_n
NC_CHECK(nc_inq_dimlen(data_nc_id, dim0_nc_id, &tmp_1));
_global_ext_0 = static_cast<int>(tmp_0);
_global_ext_1 = static_cast<int>(tmp_1);
_global_ext_blk_0 = ceil(static_cast<float>(_global_ext_0) / _blk_factor_0);
_global_ext_blk_1 = ceil(static_cast<float>(_global_ext_1) / _blk_factor_1);

// Initially we partition assuming there is no land mask
// Figure out my subset of objects
Expand All @@ -96,29 +77,21 @@ Grid::Grid(MPI_Comm comm, const std::string& filename, const std::string& dim0_n
_num_procs_1 = 1;
find_factors(_num_procs, _num_procs_0, _num_procs_1);

_local_ext_blk_0 = ceil(static_cast<float>(_global_ext_blk_0) / _num_procs_0);
_local_ext_blk_1 = ceil(static_cast<float>(_global_ext_blk_1) / _num_procs_1);
_global_blk_0 = (_rank / _num_procs_1) * _local_ext_blk_0;
_global_blk_1 = (_rank % _num_procs_1) * _local_ext_blk_1;
if ((_rank / _num_procs_1) == _num_procs_0 - 1) {
_local_ext_blk_0 = _global_ext_blk_0 - (_rank / _num_procs_1) * _local_ext_blk_0;
}
if ((_rank % _num_procs_1) == _num_procs_1 - 1) {
_local_ext_blk_1 = _global_ext_blk_1 - (_rank % _num_procs_1) * _local_ext_blk_1;
}
_local_ext_0 = ceil((float)_global_ext_0 / (_num_procs_0));
_local_ext_1 = ceil((float)_global_ext_1 / (_num_procs_1));
_global_0 = (_rank / _num_procs_1) * _local_ext_0;
_global_1 = (_rank % _num_procs_1) * _local_ext_1;

_local_ext_0 = _local_ext_blk_0 * _blk_factor_0;
_local_ext_1 = _local_ext_blk_1 * _blk_factor_1;
// _local_ext_0 = _local_ext_blk_0 * _blk_factor_0;
// _local_ext_1 = _local_ext_blk_1 * _blk_factor_1;
if ((_rank / _num_procs_1) == _num_procs_0 - 1) {
_local_ext_0 = _global_ext_0 - _global_blk_0 * _blk_factor_0;
_local_ext_0 = _global_ext_0 - (_rank / _num_procs_1) * _local_ext_0;
}
if ((_rank % _num_procs_1) == _num_procs_1 - 1) {
_local_ext_1 = _global_ext_1 - _global_blk_1 * _blk_factor_1;
_local_ext_1 = _global_ext_1 - (_rank % _num_procs_1) * _local_ext_1;
}
_global_0 = _global_blk_0 * _blk_factor_0;
_global_1 = _global_blk_1 * _blk_factor_1;

_num_objects = _local_ext_0 * _local_ext_1;
_num_blks = _local_ext_blk_0 * _local_ext_blk_1;

// Retrieve the land mask, if available and enabled
int mask_nc_id;
Expand Down Expand Up @@ -170,101 +143,63 @@ Grid::Grid(MPI_Comm comm, const std::string& filename, const std::string& dim0_n
}

// Apply land mask
if (_blk_factor_0 == 1 && _blk_factor_1 == 1) {
for (int i = 0; i < _num_objects; i++) {
// The convention is that sea data points will have a positive value
// and land points a zero value
if (_land_mask[i] > 0) {
int local_0 = i / _local_ext_1;
int local_1 = i % _local_ext_1;
int global_0 = local_0 + _global_0;
int global_1 = local_1 + _global_1;
_object_id.push_back(global_0 * _global_ext_1 + global_1);
_sparse_to_dense.push_back(i);
_num_nonzero_objects++;
}
}
_num_nonzero_blks = _num_nonzero_objects;
} else {
// Compute blocked land mask. A block is considered land, when all its
// grid points are land, otherwise it is considered to be sea. The
// convention is that sea data points will have a positive value and
// land points a zero value.
_land_mask_blk.resize(_num_blks, 0);
for (int i = 0; i < _num_objects; i++) {
int local_0 = (i / _local_ext_1) / _blk_factor_1;
int local_1 = (i % _local_ext_1) / _blk_factor_1;
if (_land_mask[i] > 0) {
_land_mask_blk[local_0 * _local_ext_blk_1 + local_1] = 1;
_num_nonzero_objects++;
}
}
for (int i = 0; i < _num_blks; i++) {
// The convention is that sea data points will have a positive value
// and land points a zero value
if (_land_mask_blk[i] > 0) {
int local_0 = i / _local_ext_blk_1;
int local_1 = i % _local_ext_blk_1;
int global_0 = local_0 + _global_blk_0;
int global_1 = local_1 + _global_blk_1;
_object_id.push_back(global_0 * _global_ext_blk_1 + global_1);
_sparse_to_dense.push_back(i);
_num_nonzero_blks++;
}
for (int i = 0; i < _num_objects; i++) {
// The convention is that sea data points will have a positive value
// and land points a zero value
if (_land_mask[i] > 0) {
int local_0 = i / _local_ext_1;
int local_1 = i % _local_ext_1;
int global_0 = local_0 + _global_0;
int global_1 = local_1 + _global_1;
_object_id.push_back(global_0 * _global_ext_1 + global_1);
_sparse_to_dense.push_back(i);
_num_nonzero_objects++;
}
}
} else {
_num_nonzero_objects = _num_objects;
_num_nonzero_blks = _num_blks;
for (int i = 0; i < _num_blks; i++) {
int local_0 = i / _local_ext_blk_1;
int local_1 = i % _local_ext_blk_1;
int global_0 = local_0 + _global_blk_0;
int global_1 = local_1 + _global_blk_1;
_object_id.push_back(global_0 * _global_ext_blk_1 + global_1);
for (int i = 0; i < _num_objects; i++) {
int local_0 = i / _local_ext_1;
int local_1 = i % _local_ext_1;
int global_0 = local_0 + _global_0;
int global_1 = local_1 + _global_1;
_object_id.push_back(global_0 * _global_ext_1 + global_1);
}
}

NC_CHECK(nc_close(nc_id));
}

int Grid::get_global_ext_0() const { return _global_ext_blk_0; }
int Grid::get_num_objects() const { return _num_objects; }

int Grid::get_global_ext_1() const { return _global_ext_blk_1; }
int Grid::get_num_nonzero_objects() const { return _num_nonzero_objects; }

int Grid::get_global_ext_orig_0() const { return _global_ext_0; }
int Grid::get_global_ext_0() const { return _global_ext_0; }

int Grid::get_global_ext_orig_1() const { return _global_ext_1; }
int Grid::get_global_ext_1() const { return _global_ext_1; }

int Grid::get_blk_factor_0() const { return _blk_factor_0; }
int Grid::get_local_ext_0() const { return _local_ext_0; }

int Grid::get_blk_factor_1() const { return _blk_factor_1; }
int Grid::get_local_ext_1() const { return _local_ext_1; }

int Grid::get_num_objects() const { return _num_blks; }
int Grid::get_global_0() const { return _global_0; }

int Grid::get_num_nonzero_objects() const { return _num_nonzero_blks; }
int Grid::get_global_1() const { return _global_1; }

int Grid::get_num_procs_0() const { return _num_procs_0; }

int Grid::get_num_procs_1() const { return _num_procs_1; }

const int* Grid::get_land_mask() const
{
if (_blk_factor_0 == 1 && _blk_factor_1 == 1) {
return _land_mask.data();
} else {
return _land_mask_blk.data();
}
}
const int* Grid::get_land_mask() const { return _land_mask.data(); }

const int* Grid::get_sparse_to_dense() const { return _sparse_to_dense.data(); }

const int* Grid::get_nonzero_object_ids() const { return _object_id.data(); }

void Grid::get_bounding_box(int& global_0, int& global_1, int& local_ext_0, int& local_ext_1) const
{
global_0 = _global_blk_0;
global_1 = _global_blk_1;
local_ext_0 = _local_ext_blk_0;
local_ext_1 = _local_ext_blk_1;
global_0 = _global_0;
global_1 = _global_1;
local_ext_0 = _local_ext_0;
local_ext_1 = _local_ext_1;
}
Loading

0 comments on commit 3c56d8c

Please sign in to comment.