Skip to content

Commit d32edb1

Browse files
author
Antonio Rago
committed
bug fixing for the MPI blocking
1 parent 463e01c commit d32edb1

File tree

5 files changed

+51
-49
lines changed

5 files changed

+51
-49
lines changed

‎Include/Core/global.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,10 @@ GLB_VAR(int, NP_Y, = 1); /* number of processes in direction Y */
5050
GLB_VAR(int, NP_Z, = 1); /* number of processes in direction Z */
5151

5252
/* global MPI arrangment attributes */
53-
GLB_VAR(int, MPI_NBLK_T, = 1); /* MPI arrangment size of the sub-block in direction T */
54-
GLB_VAR(int, MPI_NBLK_X, = 1); /* MPI arrangment size of the sub-block in direction X */
55-
GLB_VAR(int, MPI_NBLK_Y, = 1); /* MPI arrangment size of the sub-block in direction Y */
56-
GLB_VAR(int, MPI_NBLK_Z, = 1); /* MPI arrangment size of the sub-block in direction Z */
53+
GLB_VAR(int, MPI_BLK_T, = 1); /* MPI arrangment size of the sub-block in direction T */
54+
GLB_VAR(int, MPI_BLK_X, = 1); /* MPI arrangment size of the sub-block in direction X */
55+
GLB_VAR(int, MPI_BLK_Y, = 1); /* MPI arrangment size of the sub-block in direction Y */
56+
GLB_VAR(int, MPI_BLK_Z, = 1); /* MPI arrangment size of the sub-block in direction Z */
5757

5858
/* global lattice attributes */
5959
GLB_VAR(int, GLB_T, = 0); /* global size of the lattice in direction T */

‎Include/IO/input_par.h

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ typedef struct input_glb {
2525
/* THIS ARE DEFINED GLOBALLY !!! */
2626
/* int GLB_T, GLB_X, GLB_Y, GLB_Z; */
2727
/* int NP_T, NP_X, NP_Y, NP_Z; */
28-
/* int MPI_NBLK_T, MPI_NBLK_X, MPI_NBLK_Y, MPI_NBLK_Z; */
28+
/* int MPI_BLK_T, MPI_BLK_X, MPI_BLK_Y, MPI_BLK_Z; */
2929
/* int N_REP; */
3030

3131
/* for the reading function */
@@ -45,10 +45,10 @@ typedef struct input_glb {
4545
{ "NP_X", "NP_X = %d", INT_T, &NP_X }, \
4646
{ "NP_Y", "NP_Y = %d", INT_T, &NP_Y }, \
4747
{ "NP_Z", "NP_Z = %d", INT_T, &NP_Z }, \
48-
{ "MPI_NBLK_T", "MPI_NBLK_T = %d", INT_T, &MPI_NBLK_T }, \
49-
{ "MPI_NBLK_X", "MPI_NBLK_X = %d", INT_T, &MPI_NBLK_X }, \
50-
{ "MPI_NBLK_Y", "MPI_NBLK_Y = %d", INT_T, &MPI_NBLK_Y }, \
51-
{ "MPI_NBLK_Z", "MPI_NBLK_Z = %d", INT_T, &MPI_NBLK_Z }, \
48+
{ "MPI_BLK_T", "MPI_BLK_T = %d", INT_T, &MPI_BLK_T }, \
49+
{ "MPI_BLK_X", "MPI_BLK_X = %d", INT_T, &MPI_BLK_X }, \
50+
{ "MPI_BLK_Y", "MPI_BLK_Y = %d", INT_T, &MPI_BLK_Y }, \
51+
{ "MPI_BLK_Z", "MPI_BLK_Z = %d", INT_T, &MPI_BLK_Z }, \
5252
{ "N_REP", "N_REP = %d", INT_T, &N_REP }, \
5353
{ NULL, NULL, INT_T, NULL } \
5454
} \

‎LibHR/Geometry/geometry_init.c

Lines changed: 37 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -174,30 +174,30 @@ static void compute_psign() {
174174
}
175175

176176
#ifndef WITH_MPI_CART
177-
#define MPI_coord_to_id(x0, x1, x2, x3, ib0, ib1, ib2, ib3) \
178-
(x1 + (NP_X / MPI_NBLK_X) * (x2 + (NP_Y / MPI_NBLK_Y) * (x3 + (NP_Z / MPI_NBLK_Z) * x0))) + \
179-
(NP_X / MPI_NBLK_X) * (NP_Y / MPI_NBLK_Y) * (NP_Z / MPI_NBLK_Z) * (NP_T / MPI_NBLK_T) * \
180-
(ib1 + (MPI_NBLK_X) * (ib2 + (MPI_NBLK_Y) * (ib3 + (MPI_NBLK_Z) * ib0)))
177+
#define MPI_coord_to_id(x0, x1, x2, x3, ib0, ib1, ib2, ib3 ) \
178+
((x1) + (MPI_BLK_X) * ((x2) + (MPI_BLK_Y) * ((x3) + (MPI_BLK_Z) * (x0)))) + \
179+
(MPI_BLK_X) * (MPI_BLK_Y) * (MPI_BLK_Z) * (MPI_BLK_T) * \
180+
((ib1) + (NP_X/MPI_BLK_X) * ((ib2) + (NP_Y/MPI_BLK_Y) * ((ib3) + (NP_Z/MPI_BLK_Z) * (ib0))))
181181

182-
void MPI_id_to_coord(int MPI_coord_id, int *ixc, int *ibc, int *ix) {
182+
void MPI_id_to_coord (int MPI_coord_id, int *ixc, int *ibc, int *ix) {
183183
int nib, nix;
184-
nib = MPI_coord_id / ((NP_X / MPI_NBLK_X) * (NP_Y / MPI_NBLK_Y) * (NP_Z / MPI_NBLK_Z) * (NP_T / MPI_NBLK_T));
185-
nix = MPI_coord_id % ((NP_X / MPI_NBLK_X) * (NP_Y / MPI_NBLK_Y) * (NP_Z / MPI_NBLK_Z) * (NP_T / MPI_NBLK_T));
186-
187-
ixc[1] = nix % (NP_X / MPI_NBLK_X);
188-
ixc[2] = (nix / (NP_X / MPI_NBLK_X)) % (NP_Y / MPI_NBLK_Y);
189-
ixc[3] = ((nix / (NP_X / MPI_NBLK_X)) / (NP_Y / MPI_NBLK_Y)) % (NP_Z / MPI_NBLK_Z);
190-
ixc[0] = (((nix / (NP_X / MPI_NBLK_X)) / (NP_Y / MPI_NBLK_Y)) / (NP_Z / MPI_NBLK_Z));
191-
192-
ibc[1] = nib % (MPI_NBLK_X);
193-
ibc[2] = (nib / (MPI_NBLK_X)) % (MPI_NBLK_Y);
194-
ibc[3] = ((nib / (MPI_NBLK_X)) / (MPI_NBLK_Y)) % (MPI_NBLK_Z);
195-
ibc[0] = (((nib / (MPI_NBLK_X)) / (MPI_NBLK_Y)) / (MPI_NBLK_Z));
196-
197-
ix[0] = ixc[0] + ibc[0] * (NP_T / MPI_NBLK_T);
198-
ix[1] = ixc[1] + ibc[1] * (NP_X / MPI_NBLK_X);
199-
ix[2] = ixc[2] + ibc[2] * (NP_Y / MPI_NBLK_Y);
200-
ix[3] = ixc[3] + ibc[3] * (NP_Z / MPI_NBLK_Z);
184+
nib = MPI_coord_id / ((MPI_BLK_X) * (MPI_BLK_Y) * (MPI_BLK_Z) * (MPI_BLK_T));
185+
nix = MPI_coord_id % ((MPI_BLK_X) * (MPI_BLK_Y) * (MPI_BLK_Z) * (MPI_BLK_T));
186+
187+
ixc[1] = nix % (MPI_BLK_X);
188+
ixc[2] = (nix / (MPI_BLK_X)) % (MPI_BLK_Y);
189+
ixc[3] = ((nix / (MPI_BLK_X)) / (MPI_BLK_Y)) % (MPI_BLK_Z);
190+
ixc[0] = (((nix / (MPI_BLK_X)) / (MPI_BLK_Y)) / (MPI_BLK_Z));
191+
192+
ibc[1] = nib % (NP_X/MPI_BLK_X);
193+
ibc[2] = (nib / (NP_X/MPI_BLK_X)) % (NP_Y/MPI_BLK_Y);
194+
ibc[3] = ((nib / (NP_X/MPI_BLK_X)) / (NP_Y/MPI_BLK_Y)) % (NP_Z/MPI_BLK_Z);
195+
ibc[0] = (((nib / (NP_X/MPI_BLK_X)) / (NP_Y/MPI_BLK_Y)) / (NP_Z/MPI_BLK_Z));
196+
197+
ix[0] = ixc[0] + ibc[0] * (MPI_BLK_T);
198+
ix[1] = ixc[1] + ibc[1] * (MPI_BLK_X);
199+
ix[2] = ixc[2] + ibc[2] * (MPI_BLK_Y);
200+
ix[3] = ixc[3] + ibc[3] * (MPI_BLK_Z);
201201
}
202202

203203
#endif
@@ -332,14 +332,14 @@ int geometry_init() {
332332
}
333333
#else
334334

335-
error(NP_T % MPI_NBLK_T != 0, 1, "geometry_init " __FILE__,
336-
"MPI_NBLK_T must be an exact divisor of NP_T (NP_T % MPI_NBLK_T == 0)");
337-
error(NP_X % MPI_NBLK_X != 0, 1, "geometry_init " __FILE__,
338-
"MPI_NBLK_X must be an exact divisor of NP_X (NP_X % MPI_NBLK_X == 0)");
339-
error(NP_Y % MPI_NBLK_Y != 0, 1, "geometry_init " __FILE__,
340-
"MPI_NBLK_Y must be an exact divisor of NP_Y (NP_Y % MPI_NBLK_Y == 0)");
341-
error(NP_Z % MPI_NBLK_Z != 0, 1, "geometry_init " __FILE__,
342-
"MPI_NBLK_Z must be an exact divisor of NP_Z (NP_Z % MPI_NBLK_Z == 0)");
335+
error(NP_T % MPI_BLK_T != 0, 1, "geometry_init " __FILE__,
336+
"MPI_BLK_T must be an exact divisor of NP_T (NP_T % MPI_BLK_T == 0)");
337+
error(NP_X % MPI_BLK_X != 0, 1, "geometry_init " __FILE__,
338+
"MPI_BLK_X must be an exact divisor of NP_X (NP_X % MPI_BLK_X == 0)");
339+
error(NP_Y % MPI_BLK_Y != 0, 1, "geometry_init " __FILE__,
340+
"MPI_BLK_Y must be an exact divisor of NP_Y (NP_Y % MPI_BLK_Y == 0)");
341+
error(NP_Z % MPI_BLK_Z != 0, 1, "geometry_init " __FILE__,
342+
"MPI_BLK_Z must be an exact divisor of NP_Z (NP_Z % MPI_BLK_Z == 0)");
343343

344344
cart_comm = GLB_COMM;
345345
MPI_Comm_size(cart_comm, &CART_SIZE);
@@ -387,8 +387,8 @@ int geometry_init() {
387387
lprintf("GEOMETRY_INIT", 0, "Extended local size is %dx%dx%dx%d\n", T_EXT, X_EXT, Y_EXT, Z_EXT);
388388
lprintf("GEOMETRY_INIT", 0, "The lattice borders are (%d,%d,%d,%d)\n", T_BORDER, X_BORDER, Y_BORDER, Z_BORDER);
389389
lprintf("GEOMETRY_INIT", 0, "Size of the bulk subblocking (%d,%d,%d,%d)\n", PB_T, PB_X, PB_Y, PB_Z);
390-
if (MPI_NBLK_T + MPI_NBLK_X + MPI_NBLK_Y + MPI_NBLK_Z < 40) {
391-
lprintf("GEOMETRY_INIT", 0, "MPI Blocking arrangment (%d,%d,%d,%d)\n", MPI_NBLK_T, MPI_NBLK_X, MPI_NBLK_Y, MPI_NBLK_Z);
390+
if (MPI_BLK_T * MPI_BLK_X * MPI_BLK_Y + MPI_BLK_Z > 1) {
391+
lprintf("GEOMETRY_INIT", 0, "MPI Blocking size (%d,%d,%d,%d)\n", MPI_BLK_T, MPI_BLK_X, MPI_BLK_Y, MPI_BLK_Z);
392392
}
393393

394394
lprintf("GEOMETRY_INIT", 0, "Process sign is %d\n", PSIGN);
@@ -423,7 +423,7 @@ int proc_up(int id, int dir) {
423423

424424
return outid;
425425
#else
426-
int MPIBLS[4] = { (NP_T / MPI_NBLK_T), (NP_X / MPI_NBLK_X), (NP_Y / MPI_NBLK_Y), (NP_Z / MPI_NBLK_Z) };
426+
int MPIBLS[4] = { (MPI_BLK_T), (MPI_BLK_X), (MPI_BLK_Y), (MPI_BLK_Z) };
427427
int NPROC[4] = { NP_T, NP_X, NP_Y, NP_Z };
428428

429429
int ixc[4], ibc[4], ix[4], newpoint;
@@ -450,7 +450,7 @@ int proc_dn(int id, int dir) {
450450

451451
return outid;
452452
#else
453-
int MPIBLS[4] = { (NP_T / MPI_NBLK_T), (NP_X / MPI_NBLK_X), (NP_Y / MPI_NBLK_Y), (NP_Z / MPI_NBLK_Z) };
453+
int MPIBLS[4] = { (MPI_BLK_T), (MPI_BLK_X), (MPI_BLK_Y), (MPI_BLK_Z) };
454454
int NPROC[4] = { NP_T, NP_X, NP_Y, NP_Z };
455455
int ixc[4], ibc[4], ix[4], newpoint;
456456
MPI_id_to_coord(id, ixc, ibc, ix);
@@ -472,9 +472,9 @@ int proc_id(int coords[4]) {
472472
MPI_Cart_rank(cart_comm, coords, &outid);
473473
return outid;
474474
#else
475-
return MPI_coord_to_id(coords[0] % (NP_T / MPI_NBLK_T), coords[1] % (NP_X / MPI_NBLK_X), coords[2] % (NP_Y / MPI_NBLK_Y),
476-
coords[3] % (NP_Z / MPI_NBLK_Z), coords[0] / (NP_T / MPI_NBLK_T), coords[1] / (NP_X / MPI_NBLK_X),
477-
coords[2] / (NP_Y / MPI_NBLK_Y), coords[3] / (NP_Z / MPI_NBLK_Z));
475+
return MPI_coord_to_id(coords[0] % (MPI_BLK_T), coords[1] % (MPI_BLK_X), coords[2] % (MPI_BLK_Y),
476+
coords[3] % (MPI_BLK_Z), coords[0] / (MPI_BLK_T), coords[1] / (MPI_BLK_X),
477+
coords[2] / (MPI_BLK_Y), coords[3] / (MPI_BLK_Z));
478478

479479
#endif
480480
#else

‎LibHR/Geometry/new_geom.c

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,10 @@
88
#include <string.h>
99

1010
// this will include the 4th L2 corner in the geometry
11-
// it's not needed, was here for testing purposed
12-
// #define _INCLUDE_UP_UP_L2
13-
11+
// it is needed only if CSW is used
12+
#if (defined(WITH_EXPCLOVER)) || (defined(WITH_CLOVER))
13+
#define _INCLUDE_UP_UP_L2
14+
#endif
1415
char *const LOGTAG = "GEOMETRY DEFINE";
1516

1617
// ██ ███ ██ ██████ ███████ ██ ██ ██ ███ ██ ██████

‎Make/LUMI-G.env

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ module load LUMI partition/G
22
module load PrgEnv-cray
33
module load craype-accel-amd-gfx90a
44
module load rocm
5+
module load buildtools
56
export LC_CTYPE=en_US.UTF-8
67
export LC_ALL=en_US.UTF-8
78
export PATH=$PATH:`pwd`/Make/

0 commit comments

Comments
 (0)