From 8b7ff158b55aad01e5f7375a75b872017ebbbcc1 Mon Sep 17 00:00:00 2001 From: wkliao Date: Tue, 19 Mar 2024 13:04:08 -0500 Subject: [PATCH 01/20] Name strlen of dim/var/attr is expected < NC_MAX_INT --- examples/C/global_attributes.c | 2 +- src/dispatchers/attribute.c | 8 ++++---- src/dispatchers/dimension.c | 4 ++-- src/dispatchers/variable.c | 4 ++-- src/drivers/ncmpio/ncmpio_hash_func.c | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/C/global_attributes.c b/examples/C/global_attributes.c index aa4724c09..d2b920a38 100644 --- a/examples/C/global_attributes.c +++ b/examples/C/global_attributes.c @@ -123,7 +123,7 @@ int main(int argc, char** argv) sprintf(str_att, "Mon Aug 13 21:27:48 2018"); /* make sure the time string are consistent among all processes */ - MPI_Bcast(str_att, strlen(str_att), MPI_CHAR, 0, MPI_COMM_WORLD); + MPI_Bcast(str_att, (int)strlen(str_att), MPI_CHAR, 0, MPI_COMM_WORLD); err = ncmpi_put_att_text(ncid, NC_GLOBAL, "history", strlen(str_att), &str_att[0]); ERR diff --git a/src/dispatchers/attribute.c b/src/dispatchers/attribute.c index 6fe113725..d406c92e5 100644 --- a/src/dispatchers/attribute.c +++ b/src/dispatchers/attribute.c @@ -187,7 +187,7 @@ ncmpi_copy_att(int ncid_in, /* check if name is consistent among all processes */ assert(name != NULL); - root_name_len = strlen(name) + 1; + root_name_len = (int)strlen(name) + 1; TRACE_COMM(MPI_Bcast)(&root_name_len, 1, MPI_INT, 0, pncp_out->comm); if (mpireturn != MPI_SUCCESS) return ncmpii_error_mpi2nc(mpireturn, "MPI_Bcast root_name_len"); @@ -300,7 +300,7 @@ ncmpi_rename_att(int ncid, /* check if name is consistent among all processes */ assert(name != NULL); - root_name_len = strlen(name) + 1; + root_name_len = (int)strlen(name) + 1; TRACE_COMM(MPI_Bcast)(&root_name_len, 1, MPI_INT, 0, pncp->comm); if (mpireturn != MPI_SUCCESS) return ncmpii_error_mpi2nc(mpireturn, "MPI_Bcast root_name_len"); @@ -318,7 +318,7 @@ ncmpi_rename_att(int ncid, /* check if newname is consistent among all processes */ assert(newname != NULL); - root_name_len = strlen(newname) + 1; + root_name_len = (int)strlen(newname) + 1; TRACE_COMM(MPI_Bcast)(&root_name_len, 1, MPI_INT, 0, pncp->comm); if (mpireturn != MPI_SUCCESS) return ncmpii_error_mpi2nc(mpireturn, "MPI_Bcast root_name_len"); @@ -410,7 +410,7 @@ ncmpi_del_att(int ncid, /* check if name is consistent among all processes */ assert(name != NULL); - root_name_len = strlen(name) + 1; + root_name_len = (int)strlen(name) + 1; TRACE_COMM(MPI_Bcast)(&root_name_len, 1, MPI_INT, 0, pncp->comm); if (mpireturn != MPI_SUCCESS) return ncmpii_error_mpi2nc(mpireturn, "MPI_Bcast root_name_len"); diff --git a/src/dispatchers/dimension.c b/src/dispatchers/dimension.c index d1a912926..5b009ee4b 100644 --- a/src/dispatchers/dimension.c +++ b/src/dispatchers/dimension.c @@ -117,7 +117,7 @@ ncmpi_def_dim(int ncid, /* IN: file ID */ /* check if name is consistent among all processes */ assert(name != NULL); - root_name_len = strlen(name) + 1; + root_name_len = (int)strlen(name) + 1; TRACE_COMM(MPI_Bcast)(&root_name_len, 1, MPI_INT, 0, pncp->comm); if (mpireturn != MPI_SUCCESS) return ncmpii_error_mpi2nc(mpireturn, "MPI_Bcast root_name_len"); @@ -297,7 +297,7 @@ ncmpi_rename_dim(int ncid, /* IN: file ID */ /* check if name is consistent among all processes */ assert(newname != NULL); - root_name_len = strlen(newname) + 1; + root_name_len = (int)strlen(newname) + 1; TRACE_COMM(MPI_Bcast)(&root_name_len, 1, MPI_INT, 0, pncp->comm); if (mpireturn != MPI_SUCCESS) return ncmpii_error_mpi2nc(mpireturn, "MPI_Bcast root_name_len"); diff --git a/src/dispatchers/variable.c b/src/dispatchers/variable.c index f72aaaf86..8fd48c534 100644 --- a/src/dispatchers/variable.c +++ b/src/dispatchers/variable.c @@ -141,7 +141,7 @@ ncmpi_def_var(int ncid, /* IN: file ID */ /* check if name is consistent among all processes */ assert(name != NULL); - root_name_len = strlen(name) + 1; + root_name_len = (int)strlen(name) + 1; TRACE_COMM(MPI_Bcast)(&root_name_len, 1, MPI_INT, 0, pncp->comm); if (mpireturn != MPI_SUCCESS) return ncmpii_error_mpi2nc(mpireturn, "MPI_Bcast root_name_len"); @@ -666,7 +666,7 @@ ncmpi_rename_var(int ncid, /* IN: file ID */ /* check if newname is consistent among all processes */ assert(newname != NULL); - root_name_len = strlen(newname) + 1; + root_name_len = (int)strlen(newname) + 1; TRACE_COMM(MPI_Bcast)(&root_name_len, 1, MPI_INT, 0, pncp->comm); if (mpireturn != MPI_SUCCESS) return ncmpii_error_mpi2nc(mpireturn, "MPI_Bcast root_name_len"); diff --git a/src/drivers/ncmpio/ncmpio_hash_func.c b/src/drivers/ncmpio/ncmpio_hash_func.c index eabea7797..b59003512 100644 --- a/src/drivers/ncmpio/ncmpio_hash_func.c +++ b/src/drivers/ncmpio/ncmpio_hash_func.c @@ -116,7 +116,7 @@ int ncmpio_Pearson_hash(const char *str_name, int hash_size) } else { size_t len=strlen(str_name); - unsigned int i, hash=len; + unsigned int i, hash=(int)len; for (i=0; i Date: Tue, 19 Mar 2024 13:06:24 -0500 Subject: [PATCH 02/20] File name strlen is expected < NC_MAX_INT --- src/drivers/common/utils.c | 2 +- src/drivers/ncbbio/ncbbio_log.c | 4 ++-- test/common/testutils.c | 2 +- test/testcases/modes.c | 2 +- test/testcases/noclobber.c | 2 +- test/testcases/tst_symlink.c | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/drivers/common/utils.c b/src/drivers/common/utils.c index b0933f80f..e60b6e30a 100644 --- a/src/drivers/common/utils.c +++ b/src/drivers/common/utils.c @@ -79,7 +79,7 @@ char* ncmpii_remove_file_system_type_prefix(const char *filename) /* check if prefix is one of recognized file system types */ int i=0; while (fstypes[i] != NULL) { - int prefix_len = strlen(fstypes[i]); + size_t prefix_len = strlen(fstypes[i]); if (!strncmp(filename, fstypes[i], prefix_len)) { /* found */ ret_filename += prefix_len + 1; break; diff --git a/src/drivers/ncbbio/ncbbio_log.c b/src/drivers/ncbbio/ncbbio_log.c index e6f60fe02..5b22b8339 100644 --- a/src/drivers/ncbbio/ncbbio_log.c +++ b/src/drivers/ncbbio/ncbbio_log.c @@ -28,7 +28,7 @@ int ncbbio_log_create(NC_bb* ncbbp, __attribute__((unused)) MPI_Info info) { - int i, rank, np, err, flag, masterrank, procname_len; + int rank, np, err, flag, masterrank, procname_len; char logbase[NC_LOG_MAX_PATH], basename[NC_LOG_MAX_PATH]; char procname[MPI_MAX_PROCESSOR_NAME]; char *abspath, *fname, *path, *fdir = NULL; @@ -86,7 +86,7 @@ int ncbbio_log_create(NC_bb* ncbbp, logbasep = ncmpii_remove_file_system_type_prefix(ncbbp->logbase); } else { - i = strlen(path); + size_t i = strlen(path); fdir = (char*)NCI_Malloc((i + 1) * sizeof(char)); strncpy(fdir, path, i + 1); /* Search for first '\' from the back */ diff --git a/test/common/testutils.c b/test/common/testutils.c index b8e90593b..3e37aa84b 100644 --- a/test/common/testutils.c +++ b/test/common/testutils.c @@ -273,7 +273,7 @@ char* remove_file_system_type_prefix(const char *filename) /* check if prefix is one of recognized file system types */ int i=0; while (fstypes[i] != NULL) { - int prefix_len = strlen(fstypes[i]); + size_t prefix_len = strlen(fstypes[i]); if (!strncmp(filename, fstypes[i], prefix_len)) { /* found */ ret_filename += prefix_len + 1; break; diff --git a/test/testcases/modes.c b/test/testcases/modes.c index 60e2bfdef..2fbaf3bcc 100644 --- a/test/testcases/modes.c +++ b/test/testcases/modes.c @@ -194,7 +194,7 @@ int main(int argc, char** argv) } if (argc == 2) filename = strdup(argv[1]); else filename = strdup("testfile.nc"); - len = strlen(filename) + 1; + len = (int)strlen(filename) + 1; MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(filename, len, MPI_CHAR, 0, MPI_COMM_WORLD); diff --git a/test/testcases/noclobber.c b/test/testcases/noclobber.c index d4a949b0e..38d366f07 100644 --- a/test/testcases/noclobber.c +++ b/test/testcases/noclobber.c @@ -55,7 +55,7 @@ int main(int argc, char **argv) { } if (argc == 2) filename = strdup(argv[1]); else filename = strdup("testfile.nc"); - len = strlen(filename) + 1; + len = (int)strlen(filename) + 1; MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(filename, len, MPI_CHAR, 0, MPI_COMM_WORLD); diff --git a/test/testcases/tst_symlink.c b/test/testcases/tst_symlink.c index cf9d5fbc5..0ab154921 100644 --- a/test/testcases/tst_symlink.c +++ b/test/testcases/tst_symlink.c @@ -47,7 +47,7 @@ int main(int argc, char **argv) { } if (argc == 2) filename = strdup(argv[1]); else filename = strdup("testfile.nc"); - len = strlen(filename) + 1; + len = (int)strlen(filename) + 1; MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(filename, len, MPI_CHAR, 0, MPI_COMM_WORLD); From 0344d072ea2c22216ef3436d08dfcf206f8580af Mon Sep 17 00:00:00 2001 From: wkliao Date: Tue, 19 Mar 2024 13:12:15 -0500 Subject: [PATCH 03/20] use size_t for C functions returning integer of type size_t --- src/drivers/ncmpio/ncmpio_wait.c | 3 ++- src/utils/ncmpidump/ncmpidump.c | 3 ++- src/utils/ncmpilogdump/ncmpilogdump.m4 | 12 ++++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/drivers/ncmpio/ncmpio_wait.c b/src/drivers/ncmpio/ncmpio_wait.c index 587c2f83b..1e7884ab0 100644 --- a/src/drivers/ncmpio/ncmpio_wait.c +++ b/src/drivers/ncmpio/ncmpio_wait.c @@ -127,7 +127,8 @@ ncmpio_cancel(void *ncdp, int *req_ids, /* [num_req]: IN/OUT */ int *statuses) /* [num_req] can be NULL (ignore status) */ { - int i, j, status=NC_NOERR; + int status=NC_NOERR; + size_t i, j; NC *ncp=(NC*)ncdp; if (num_req == 0) return NC_NOERR; diff --git a/src/utils/ncmpidump/ncmpidump.c b/src/utils/ncmpidump/ncmpidump.c index e8ce61114..99f042f26 100644 --- a/src/utils/ncmpidump/ncmpidump.c +++ b/src/utils/ncmpidump/ncmpidump.c @@ -743,7 +743,8 @@ enum FILE_KIND check_file_signature(char *path) char *cdf_signature="CDF"; char *hdf5_signature="\211HDF\r\n\032\n"; char signature[8]; - int fd, rlen; + int fd; + size_t rlen; if ((fd = open(path, O_RDONLY, 0700)) == -1) { fprintf(stderr,"%s error at opening file %s (%s)\n",progname,path,strerror(errno)); diff --git a/src/utils/ncmpilogdump/ncmpilogdump.m4 b/src/utils/ncmpilogdump/ncmpilogdump.m4 index 185df3063..78e293131 100644 --- a/src/utils/ncmpilogdump/ncmpilogdump.m4 +++ b/src/utils/ncmpilogdump/ncmpilogdump.m4 @@ -36,8 +36,8 @@ define(`PRINTTYPE',dnl #include int main(int argc, char *argv[]) { - int i, j, k, fd, ret, err = NC_NOERR; - size_t offset; + int i, j, k, fd, err = NC_NOERR; + size_t offset, rlen; FILE *fmeta=NULL, *fdata=NULL; struct stat metastat; struct stat datastat; @@ -71,8 +71,8 @@ int main(int argc, char *argv[]) { Meta = (char*)malloc(metastat.st_size); /* Read the metadata */ - ret = fread(Meta, metastat.st_size, 1, fmeta); - if (ret != 1) { + rlen = fread(Meta, metastat.st_size, 1, fmeta); + if (rlen != 1) { err = NC_EBADLOG; goto fn_exit; } @@ -96,8 +96,8 @@ int main(int argc, char *argv[]) { /* Allocate buffer */ Data = (char*)malloc(datastat.st_size); /* Read the data */ - ret = fread(Data, datastat.st_size, 1, fdata); - if (ret != 1) { + rlen = fread(Data, datastat.st_size, 1, fdata); + if (rlen != 1) { err = NC_EBADLOG; goto fn_exit; } From 777d607642fe62899fea5db3e3dd4c7010378201 Mon Sep 17 00:00:00 2001 From: wkliao Date: Tue, 19 Mar 2024 13:15:37 -0500 Subject: [PATCH 04/20] proper type cast, as these test programs test small problem size --- examples/C/block_cyclic.c | 4 ++-- test/nc_test/test_iput.m4 | 4 ++-- test/nc_test/test_read.m4 | 2 +- test/nc_test/test_write.m4 | 2 +- test/nc_test/util.c | 2 +- test/nonblocking/mcoll_perf.c | 9 ++++++--- test/testcases/flexible.c | 8 ++++---- test/testcases/test_vard.c | 22 +++++++++++----------- test/testcases/test_vard_multiple.c | 8 ++++---- test/testcases/varn_int.c | 2 +- test/testcases/vectors.c | 3 ++- 11 files changed, 35 insertions(+), 31 deletions(-) diff --git a/examples/C/block_cyclic.c b/examples/C/block_cyclic.c index 65eab0423..dbd8625cb 100644 --- a/examples/C/block_cyclic.c +++ b/examples/C/block_cyclic.c @@ -223,7 +223,7 @@ int main(int argc, char** argv) { i, start[0],start[1], count[0],count[1]); if (i % block_len == block_len-1) { - int stride = MIN(myNX-1-i, block_len); + MPI_Offset stride = MIN(myNX-1-i, block_len); block_start += block_len * nprocs; start[1] = block_start + stride * rank; } @@ -297,7 +297,7 @@ int main(int argc, char** argv) { ERR if (i % block_len == block_len-1) { - int stride = MIN(myNX-1-i, block_len); + MPI_Offset stride = MIN(myNX-1-i, block_len); block_start += block_len * nprocs; start[1] = block_start + stride * rank; } diff --git a/test/nc_test/test_iput.m4 b/test/nc_test/test_iput.m4 index 2fc5b759a..d9f0da58c 100644 --- a/test/nc_test/test_iput.m4 +++ b/test/nc_test/test_iput.m4 @@ -102,9 +102,9 @@ hash2nc(const nc_type var_type, int var_rank, MPI_Offset *index) } static int -dbls2ncs(int nels, int var_type, double *inBuf, void *outBuf) +dbls2ncs(size_t nels, int var_type, double *inBuf, void *outBuf) { - int i; + size_t i; char *p = (char*)outBuf; for (i=0; i Date: Tue, 19 Mar 2024 13:17:30 -0500 Subject: [PATCH 05/20] proper type cast and remove tab characters --- test/subfile/test_subfile.c | 96 ++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/test/subfile/test_subfile.c b/test/subfile/test_subfile.c index ecb786a2b..dbbb9cf6b 100644 --- a/test/subfile/test_subfile.c +++ b/test/subfile/test_subfile.c @@ -61,28 +61,28 @@ int main(int argc, char **argv) /* process 0 takes the file name as a command-line argument and broadcasts it to other processes */ if (rank == 0) { - while ((opt = getopt(argc, argv, "f:s:p:n:l:r")) != EOF) { - switch (opt) { - case 'f': fbasename = optarg; - break; - case 's': num_sf = (int)strtol(optarg,NULL,10); - break; - case 'r': do_read = 1; - break; + while ((opt = getopt(argc, argv, "f:s:p:n:l:r")) != EOF) { + switch (opt) { + case 'f': fbasename = optarg; + break; + case 's': num_sf = (int)strtol(optarg,NULL,10); + break; + case 'r': do_read = 1; + break; case 'p': par_dim_id = (int)strtol(optarg,NULL,10); break; case 'n': nvars = (int)strtol(optarg,NULL,10); break; case 'l': length = (int)strtol(optarg,NULL,10); break; - default: - break; - } - } - if (fbasename == NULL) { - fprintf(stderr, "\n*# Usage: test_subfile -f pathname -s num_sf -p par_dim_id \n\n"); - nerrs++; - } + default: + break; + } + } + if (fbasename == NULL) { + fprintf(stderr, "\n*# Usage: test_subfile -f pathname -s num_sf -p par_dim_id \n\n"); + nerrs++; + } } MPI_Bcast(&nerrs, 1, MPI_INT, 0, MPI_COMM_WORLD); if (nerrs > 0) { @@ -91,12 +91,12 @@ int main(int argc, char **argv) } if (rank == 0) { - len = (fbasename == NULL) ? 0 : strlen(fbasename); - MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); + len = (fbasename == NULL) ? 0 : (int)strlen(fbasename); + MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); } else { - MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); - fbasename = (char *) malloc(len+1); + MPI_Bcast(&len, 1, MPI_INT, 0, MPI_COMM_WORLD); + fbasename = (char *) malloc(len+1); } MPI_Bcast(fbasename, len+1, MPI_CHAR, 0, MPI_COMM_WORLD); MPI_Bcast(&num_sf, 1, MPI_INT, 0, MPI_COMM_WORLD); @@ -141,16 +141,16 @@ int main(int argc, char **argv) } for (i=0; i Date: Tue, 19 Mar 2024 13:23:06 -0500 Subject: [PATCH 06/20] put_att consistency check: use MPI large count APIs if available Throw out NC_EINTOVERFLOW if not able to use MPI large count APIs --- src/dispatchers/attr_getput.m4 | 35 ++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/src/dispatchers/attr_getput.m4 b/src/dispatchers/attr_getput.m4 index f4138bd70..4cebf37b5 100644 --- a/src/dispatchers/attr_getput.m4 +++ b/src/dispatchers/attr_getput.m4 @@ -145,7 +145,7 @@ check_consistency_put(MPI_Comm comm, MPI_Comm_rank(comm, &rank); /* check if attribute name is consistent among all processes */ - root_name_len = strlen(name) + 1; + root_name_len = (int)strlen(name) + 1; TRACE_COMM(MPI_Bcast)(&root_name_len, 1, MPI_INT, 0, comm); if (mpireturn != MPI_SUCCESS) return ncmpii_error_mpi2nc(mpireturn, "MPI_Bcast root_name_len"); @@ -179,27 +179,54 @@ check_consistency_put(MPI_Comm comm, /* check if nelems is consistent across all processes */ root_nelems = nelems; + +#if !defined(HAVE_MPI_TYPE_SIZE_C) && !defined(HAVE_MPI_TYPE_SIZE_X) + if (root_nelems > NC_MAX_INT) { + DEBUG_ASSIGN_ERROR(err, NC_EINTOVERFLOW) + root_nelems = 0; + } +#endif + TRACE_COMM(MPI_Bcast)(&root_nelems, 1, MPI_OFFSET, 0, comm); if (mpireturn != MPI_SUCCESS) return ncmpii_error_mpi2nc(mpireturn, "MPI_Bcast"); if (err == NC_NOERR && root_nelems != nelems) DEBUG_ASSIGN_ERROR(err, NC_EMULTIDEFINE_ATTR_LEN) + /* must continue to participate collective calls below, even if error */ + /* check if buf contents is consistent across all processes */ if (root_nelems > 0) { /* non-scalar attribute */ /* note xsz is aligned, thus must use the exact size of buf */ - int itype_size, rank, buf_size; + int rank, itype_size; + size_t buf_size; void *root_buf; MPI_Comm_rank(comm, &rank); + + /* for attributes, itype is nc_type, so its size is small. Thus, no + * need to check against NC_MAX_INT. + */ MPI_Type_size(itype, &itype_size); - buf_size = (int)root_nelems * itype_size; + buf_size = root_nelems * itype_size; + if (rank > 0) root_buf = (void*) NCI_Malloc(buf_size); else root_buf = (void*)buf; - TRACE_COMM(MPI_Bcast)(root_buf, root_nelems, itype, 0, comm); + if (root_nelems > NC_MAX_INT) { +#ifdef HAVE_MPI_BCAST_C + TRACE_COMM(MPI_Bcast_c)(root_buf, root_nelems, itype, 0, comm); +#else + /* Note root_nelems has been bcast-ed, same value among all ranks */ + DEBUG_ASSIGN_ERROR(err, NC_EMULTIDEFINE_ATTR_LEN) +#endif + } + else + TRACE_COMM(MPI_Bcast)(root_buf, (int)root_nelems, itype, 0, comm); + if (mpireturn != MPI_SUCCESS) return ncmpii_error_mpi2nc(mpireturn, "MPI_Bcast"); + if (err == NC_NOERR && (root_nelems != nelems || memcmp(root_buf, buf, buf_size))) DEBUG_ASSIGN_ERROR(err, NC_EMULTIDEFINE_ATTR_VAL) From 125d40da5d4259a977aaad62e0059f35b7cb88e4 Mon Sep 17 00:00:00 2001 From: wkliao Date: Tue, 19 Mar 2024 13:26:28 -0500 Subject: [PATCH 07/20] check against X_INT_MAX --- src/drivers/common/ncx.m4 | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/drivers/common/ncx.m4 b/src/drivers/common/ncx.m4 index dbcdf120b..f60463513 100644 --- a/src/drivers/common/ncx.m4 +++ b/src/drivers/common/ncx.m4 @@ -2120,9 +2120,14 @@ APIPrefix`x_put_off_t'(void **xpp, const off_t *lp, size_t sizeof_off_t) #ifdef WORDS_BIGENDIAN memcpy(*xpp, lp, 4); #else - int xtmp, itmp = *lp; - xtmp = SWAP4(itmp); - memcpy(*xpp, &xtmp, 4); + if (*lp > X_INT_MAX) + DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) + else { + int xtmp, itmp; + itmp = (int)*lp; + xtmp = SWAP4(itmp); + memcpy(*xpp, &xtmp, 4); + } #endif } else { #ifdef WORDS_BIGENDIAN From bfa3b8f850e69b6b01c14abed8de28eb15cfe2b7 Mon Sep 17 00:00:00 2001 From: wkliao Date: Tue, 19 Mar 2024 13:29:00 -0500 Subject: [PATCH 08/20] use size_t for memory buffer operations --- src/drivers/ncbbio/ncbbio_driver.h | 8 ++++---- src/drivers/ncbbio/ncbbio_nonblocking.c | 8 +++++--- src/drivers/ncbbio/ncbbio_sharedfile.c | 8 ++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/drivers/ncbbio/ncbbio_driver.h b/src/drivers/ncbbio/ncbbio_driver.h index a6a7250cf..a9912bd05 100644 --- a/src/drivers/ncbbio/ncbbio_driver.h +++ b/src/drivers/ncbbio/ncbbio_driver.h @@ -70,7 +70,7 @@ typedef struct NC_bb_metadataheader { MPI_Offset max_ndims; MPI_Offset num_entries; MPI_Offset entry_begin; - int basenamelen; + size_t basenamelen; char basename[1]; /* The hack to keep basename inside the structure */ } NC_bb_metadataheader; @@ -97,7 +97,7 @@ typedef struct NC_bb_metadataptr { typedef struct NC_bb_metadataidx { NC_bb_metadataptr *entries; int nused; - int nalloc; + size_t nalloc; } NC_bb_metadataidx; /* Buffer structure */ @@ -134,8 +134,8 @@ typedef struct NC_bb_put_req { typedef struct NC_bb_put_list { NC_bb_put_req *reqs; // Array of request object int *ids; // Array of request ids - int nalloc; // Size of the pool - int nused; // Number of ids issued + size_t nalloc; // Size of the pool + size_t nused; // Number of ids issued } NC_bb_put_list; /* Shared file object */ diff --git a/src/drivers/ncbbio/ncbbio_nonblocking.c b/src/drivers/ncbbio/ncbbio_nonblocking.c index 18cc667b2..317090512 100644 --- a/src/drivers/ncbbio/ncbbio_nonblocking.c +++ b/src/drivers/ncbbio/ncbbio_nonblocking.c @@ -88,8 +88,7 @@ int ncbbio_put_list_init(NC_bb *ncbbp) { */ static int ncbbio_put_list_resize(NC_bb *ncbbp) { - int i; - ssize_t nsize; + size_t i, nsize; void *ptr; NC_bb_put_list *lp = &(ncbbp->putlist); @@ -109,8 +108,11 @@ static int ncbbio_put_list_resize(NC_bb *ncbbp) /* Initialize values of ids and reqs * Assign increasing unique id */ + if (nsize > NC_MAX_INT) + DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) + for (i=lp->nalloc; iids[i] = i; // Unique ids + lp->ids[i] = (int)i; // Unique ids lp->reqs[i].valid = 0; // Not in use } diff --git a/src/drivers/ncbbio/ncbbio_sharedfile.c b/src/drivers/ncbbio/ncbbio_sharedfile.c index 26ac7ba34..883b8e06d 100644 --- a/src/drivers/ncbbio/ncbbio_sharedfile.c +++ b/src/drivers/ncbbio/ncbbio_sharedfile.c @@ -144,8 +144,8 @@ int ncbbio_sharedfile_pwrite(NC_bb_sharedfile *f, size_t count, off_t offset) { - int i, err; - int sblock, eblock; // start and end block + int err; + size_t i, sblock, eblock; // start and end block off_t offstart, offend, wsize = 0; // Start and end offset to write for current block in physical file ssize_t ioret; @@ -317,8 +317,8 @@ int ncbbio_sharedfile_pread(NC_bb_sharedfile *f, size_t count, off_t offset) { - int i, err; - int sblock, eblock; // start and end block + int err; + size_t i, sblock, eblock; // start and end block off_t offstart, offend, rsize = 0; // Start and end offset to write for current block in physical file ssize_t ioret; From 63cc3be55015dda058eb512794fe0d891fa1a86f Mon Sep 17 00:00:00 2001 From: wkliao Date: Tue, 19 Mar 2024 13:35:21 -0500 Subject: [PATCH 09/20] type cast, check NC_EINTOVERFLOW, and fix MPI error checking --- src/drivers/ncmpio/ncmpio_subfile.c | 58 ++++++++++++++++------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/src/drivers/ncmpio/ncmpio_subfile.c b/src/drivers/ncmpio/ncmpio_subfile.c index eef114dbd..6e86468f8 100644 --- a/src/drivers/ncmpio/ncmpio_subfile.c +++ b/src/drivers/ncmpio/ncmpio_subfile.c @@ -28,14 +28,15 @@ enum {ONE, BALANCED}; static int delegate_scheme = BALANCED; /* default: any proc can be delegate proc */ static int is_partitioned = 0; -#define check_err(fn_name_) \ +#define check_mpi_err(fn_name) \ if (mpireturn != MPI_SUCCESS) { \ - errs++; \ - int _len; \ + int err, err_len; \ char err_str_[MPI_MAX_ERROR_STRING]; \ - MPI_Error_string(mpireturn, err_str_, &_len); \ - fprintf(stderr, #fn_name_ " failed at line %d, mpireturn=%d: %s\n", \ - __LINE__, mpireturn, err_str_); \ + MPI_Error_string(mpireturn, err_str_, &err_len); \ + fprintf(stderr, "Error: %s line %d calling MPI func=%s: %s\n", \ + __func__,__LINE__, fn_name, err_str_); \ + err = ncmpii_error_mpi2nc(mpireturn, fn_name); \ + DEBUG_RETURN_ERROR(err) \ } \ #if 0 @@ -328,15 +329,14 @@ int ncmpio_subfile_partition(NC *ncp) status = ncmpio_put_att(ncp, i, "_PnetCDF_SubFiling.ndims_org", NC_INT, 1, &vpp[i]->ndims, MPI_INT); if (status != NC_NOERR) DEBUG_RETURN_ERROR(status) - int sf_range[ncp->num_subfiles][var_ndims][3]; + long long sf_range[ncp->num_subfiles][var_ndims][3]; /* j: each dimension */ /* subfile: create a var with partitioned dim sizes */ for(j=0; jdimids[j]]->name; dim_sz0 = dpp[vpp[i]->dimids[j]]->size; /* init both to org dim sz */ @@ -354,9 +354,9 @@ int ncmpio_subfile_partition(NC *ncp) { double xx = x*(double)color; double yy = x*(double)(color+1); - min = (int)xx+(color==0||(xx-(int)xx==0.0)?0:1); - max = (int)yy-(yy-(int)yy==0.0?1:0); - if ((MPI_Offset)max >= dim_sz0) max = dim_sz0-1; + min = (MPI_Offset)xx+(color==0||(xx-(MPI_Offset)xx==0.0)?0:1); + max = (MPI_Offset)yy-(yy-(MPI_Offset)yy==0.0?1:0); + if (max >= dim_sz0) max = dim_sz0-1; dim_sz = max-min+1; } @@ -374,12 +374,12 @@ int ncmpio_subfile_partition(NC *ncp) double xx, yy; sprintf(key[jj][j], "_PnetCDF_SubFiling.range(%s).subfile.%d", dim_name, jj); /* dim name*/ xx = x*(double)jj; - min = (int)xx+(jj==0||(xx-(int)xx==0.0)?0:1); + min = (MPI_Offset)xx+(jj==0||(xx-(MPI_Offset)xx==0.0)?0:1); yy = x*(double)(jj+1); - max = (int)yy-(yy-(int)yy==0.0?1:0); - if ((MPI_Offset)max >= dim_sz0) max = dim_sz0-1; + max = (MPI_Offset)yy-(yy-(MPI_Offset)yy==0.0?1:0); + if (max >= dim_sz0) max = dim_sz0-1; #ifdef SUBFILE_DEBUG - if (myrank == 0) printf("subfile(%d): min=%d, max=%d\n", jj, min, max); + if (myrank == 0) printf("subfile(%d): min=%lld, max=%lld\n", jj, min, max); #endif if (j == par_dim_id) { /* partitioning dims? */ sf_range[jj][j][0] = min; @@ -421,7 +421,7 @@ int ncmpio_subfile_partition(NC *ncp) for (jj=0; jj < ncp->num_subfiles; jj++) for (k=0; k < var_ndims; k++) { status = ncmpio_put_att(ncp, i, key[jj][k], NC_INT, - 2, sf_range[jj][k], MPI_INT); + 2, sf_range[jj][k], MPI_LONG_LONG); if (status != NC_NOERR) DEBUG_RETURN_ERROR(status) } @@ -467,7 +467,7 @@ ncmpio_subfile_getput_vars(NC *ncp, MPI_Datatype buftype, int reqMode) { - int mpireturn, errs=0, status; + int mpireturn, status; NC_subfile_access *my_req, *others_req; int i, j, k, myrank, nprocs; int *count_my_req_per_proc, *count_others_req_per_proc; @@ -578,7 +578,7 @@ ncmpio_subfile_getput_vars(NC *ncp, int jx = j%ndims_org; NC_dim *dimp = ncp->ncp_sf->dims.value[ncp->ncp_sf->vars.value[varid_sf]->dimids[jx]]; int sf_range[2]; - int ii, jj, kk, stride_count; + MPI_Offset ii, jj, kk, stride_count; char key[256], *org_dim_name; org_dim_name = strtok(dimp->name, "."); @@ -622,12 +622,12 @@ ncmpio_subfile_getput_vars(NC *ncp, jj++; else { #ifdef SUBFILE_DEBUG - printf("rank(%d): var(%s): i=%d, j=%d, ii=%d, jj=%d, kk=%d, jx=%d\n", myrank, varp->name, i, j, ii, jj, kk, jx); + printf("rank(%d): var(%s): i=%d, j=%d, ii=%lld, jj=%lld, kk=%lld, jx=%d\n", myrank, varp->name, i, j, ii, jj, kk, jx); #endif if (kk == 0) { my_req[aproc].start[jx] = ii; #ifdef SUBFILE_DEBUG - printf("rank(%d): var(%s): my_req[%d].start[%d]=%d\n", + printf("rank(%d): var(%s): my_req[%d].start[%d]=%lld\n", myrank, varp->name, aproc, jx, ii); #endif } @@ -896,7 +896,7 @@ ncmpio_subfile_getput_vars(NC *ncp, #ifdef TAU_SSON TAU_PHASE_STOP(t52); #endif - check_err(MPI_Waitall); + check_mpi_err("MPI_Waitall"); NCI_Free(statuses); NCI_Free(requests); @@ -933,7 +933,7 @@ ncmpio_subfile_getput_vars(NC *ncp, #ifdef TAU_SSON TAU_PHASE_STOP(t53); #endif - check_err(MPI_Waitall); + check_mpi_err("MPI_Waitall"); #ifdef SUBFILE_DEBUG /* DEBUG: print out others_req.{start,count} */ @@ -986,8 +986,11 @@ ncmpio_subfile_getput_vars(NC *ncp, #ifdef SUBFILE_DEBUG printf("rank(%d): recv from rank %d: buf_count_others[%d]=%d\n", myrank, i, i, buf_count_others[i]); #endif + if (buf_count_others[i] > NC_MAX_INT) + DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) + xbuf[i] = (void*)NCI_Calloc((size_t)buf_count_others[i], (size_t)el_size); - TRACE_COMM(MPI_Irecv)(xbuf[i], buf_count_others[i], (!buftype_is_contig?ptype:buftype), i, i+myrank, ncp->comm, &requests[j++]); + TRACE_COMM(MPI_Irecv)(xbuf[i], (int)buf_count_others[i], (!buftype_is_contig?ptype:buftype), i, i+myrank, ncp->comm, &requests[j++]); } } @@ -1020,7 +1023,10 @@ ncmpio_subfile_getput_vars(NC *ncp, printf("rank(%d): send to rank %d: buf_offset[%d]=%d, buf_count_my[%d]=%d\n", myrank, i, i, buf_offset[i], i, buf_count_my[i]); #endif - TRACE_COMM(MPI_Isend)((char*)cbuf+buf_offset[i], buf_count_my[i], (!buftype_is_contig?ptype:buftype), i, i+myrank, ncp->comm, &requests[j++]); + if (buf_count_my[i] > NC_MAX_INT) + DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) + + TRACE_COMM(MPI_Isend)((char*)cbuf+buf_offset[i], (int)buf_count_my[i], (!buftype_is_contig?ptype:buftype), i, i+myrank, ncp->comm, &requests[j++]); } /* end if() */ } /* end for() */ @@ -1032,7 +1038,7 @@ ncmpio_subfile_getput_vars(NC *ncp, #ifdef TAU_SSON TAU_PHASE_STOP(t54); #endif - check_err(MPI_Waitall); + check_mpi_err("MPI_Waitall"); #ifdef TAU_SSON TAU_PHASE_CREATE_STATIC(t55, "SSON --- getput_vars igetput", "", TAU_USER); From 4209056e9a66465421f7ce9f1b44518923638b04 Mon Sep 17 00:00:00 2001 From: wkliao Date: Tue, 19 Mar 2024 13:36:34 -0500 Subject: [PATCH 10/20] Hints nc_header_read_chunk_size and nc_num_subfiles are of type int This also means theire values both are < NC_MAX_INT --- src/drivers/ncmpio/ncmpio_util.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/drivers/ncmpio/ncmpio_util.c b/src/drivers/ncmpio/ncmpio_util.c index b7d944ade..34d8bdb27 100644 --- a/src/drivers/ncmpio/ncmpio_util.c +++ b/src/drivers/ncmpio/ncmpio_util.c @@ -95,9 +95,9 @@ void ncmpio_set_pnetcdf_hints(NC *ncp, MPI_Info_get(user_info, "nc_header_read_chunk_size", MPI_MAX_INFO_VAL-1, value, &flag); if (flag) { - long int chunk; + int chunk; errno = 0; /* errno must set to zero before calling strtoll */ - chunk = (int) strtol(value, NULL, 10); + chunk = atoi(value); if (errno != 0) ncp->chunk = 0; else if (ncp->chunk < 0) ncp->chunk = 0; @@ -164,7 +164,7 @@ void ncmpio_set_pnetcdf_hints(NC *ncp, value, &flag); if (flag) { errno = 0; - ncp->num_subfiles = strtoll(value, NULL, 10); + ncp->num_subfiles = atoi(value); if (errno != 0) ncp->num_subfiles = 0; else if (ncp->num_subfiles < 0) ncp->num_subfiles = 0; sprintf(value, "%d", ncp->num_subfiles); From 87e646b49d746b8d03eec8a43e298b440a976923 Mon Sep 17 00:00:00 2001 From: wkliao Date: Tue, 19 Mar 2024 13:40:25 -0500 Subject: [PATCH 11/20] req_size has been checked against NC_MAX_INT --- src/drivers/ncmpio/ncmpio_file_io.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/drivers/ncmpio/ncmpio_file_io.c b/src/drivers/ncmpio/ncmpio_file_io.c index e3912c50c..977c73e81 100644 --- a/src/drivers/ncmpio/ncmpio_file_io.c +++ b/src/drivers/ncmpio/ncmpio_file_io.c @@ -64,7 +64,7 @@ ncmpio_read_write(NC *ncp, } /* request size in bytes, may be > NC_MAX_INT */ - req_size = (MPI_Offset)btype_size * buf_count; + req_size = buf_count * btype_size; /* explicitly initialize mpistatus object to 0. For zero-length read, * MPI_Get_count may report incorrect result for some MPICH version, @@ -109,7 +109,7 @@ ncmpio_read_write(NC *ncp, } else { xbuf_type = MPI_BYTE; - xlen = req_size; + xlen = (int)req_size; } xbuf = NCI_Malloc((size_t)req_size); } @@ -217,7 +217,7 @@ ncmpio_read_write(NC *ncp, } else { int pos=0; - xlen = req_size; + xlen = (int)req_size; xbuf = NCI_Malloc(xlen); MPI_Pack(buf, (int)buf_count, buf_type, xbuf, xlen, &pos, MPI_COMM_SELF); From 39bc2e5cfb3f0e5a45a37c52be409ec6fb829451 Mon Sep 17 00:00:00 2001 From: wkliao Date: Tue, 19 Mar 2024 13:41:17 -0500 Subject: [PATCH 12/20] use 'chunk' instead of 'size', better describing its purpose Hint `nc_header_read_chunk_size` is limited to `NC_MAX_INT`. --- src/drivers/ncmpio/ncmpio_NC.h | 4 +- src/drivers/ncmpio/ncmpio_header_get.c | 51 +++++++++++++------------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/drivers/ncmpio/ncmpio_NC.h b/src/drivers/ncmpio/ncmpio_NC.h index 67bbb4a22..6f4d311ff 100644 --- a/src/drivers/ncmpio/ncmpio_NC.h +++ b/src/drivers/ncmpio/ncmpio_NC.h @@ -382,7 +382,7 @@ struct NC { MPI_Comm comm_sf; /* subfile MPI communicator */ #endif int striping_unit; /* stripe size of the file */ - int chunk; /* chunk size for reading header */ + int chunk; /* chunk size for reading header, one chunk at a time */ MPI_Offset h_align; /* file alignment for header size */ MPI_Offset v_align; /* alignment of the beginning of fixed-size variables */ MPI_Offset r_align; /* file alignment for record variable section */ @@ -471,7 +471,7 @@ typedef struct bufferinfo { MPI_File collective_fh; MPI_Offset get_size; /* amount of file read n bytes so far */ MPI_Offset offset; /* current read/write offset in the file */ - size_t size; /* allocated size of the buffer */ + int chunk; /* chunk size for reading the header */ int version; /* 1, 2, and 5 for CDF-1, 2, and 5 respectively */ int safe_mode;/* 0: disabled, 1: enabled */ int rw_mode; /* 0: independent, 1: collective */ diff --git a/src/drivers/ncmpio/ncmpio_header_get.c b/src/drivers/ncmpio/ncmpio_header_get.c index c96ce801e..a09e89282 100644 --- a/src/drivers/ncmpio/ncmpio_header_get.c +++ b/src/drivers/ncmpio/ncmpio_header_get.c @@ -316,7 +316,7 @@ hdr_len_NC_vararray(const NC_vararray *ncap, /*----< hdr_fetch() >--------------------------------------------------------*/ /* Fetch the next header chunk. The chunk buffer, pointed by gbp->base, is of - * size 'gbp->size' bytes. Be careful not to overwrite leftover (yet to be + * size 'gbp->chunk' bytes. Be careful not to overwrite leftover (yet to be * used) data in the buffer before fetching a new chunk. */ static int @@ -329,20 +329,21 @@ hdr_fetch(bufferinfo *gbp) { MPI_Comm_rank(gbp->comm, &rank); if (rank == 0) { char *readBuf; - size_t slack, readLen; + int readLen; + size_t slack; /* any leftover data in the buffer */ - slack = gbp->size - (gbp->pos - gbp->base); - if (slack == gbp->size) slack = 0; + slack = gbp->chunk - (gbp->pos - gbp->base); + if (slack == gbp->chunk) slack = 0; - /* When gbp->size == (gbp->pos - gbp->base), all data in the buffer has + /* When gbp->chunk == (gbp->pos - gbp->base), all data in the buffer has * been consumed. If not, then read additional header of size - * (gbp->size - slack) into a contiguous buffer, pointed by gbp->base + + * (gbp->chunk - slack) into a contiguous buffer, pointed by gbp->base + * slack. */ readBuf = gbp->base; - readLen = gbp->size; + readLen = gbp->chunk; if (slack > 0) { /* move slack to beginning of the buffer, gbp->base */ memmove(gbp->base, gbp->pos, slack); readBuf += slack; @@ -377,7 +378,7 @@ hdr_fetch(bufferinfo *gbp) { /* If actual read amount is shorter than readLen, then we zero-out * the remaining buffer. This is because the MPI_Bcast below - * broadcasts a buffer of a fixed size, gbp->size. Without zeroing + * broadcasts a buffer of a fixed size, gbp->chunk. Without zeroing * out, valgrind will complain about the uninitialized values. */ if (get_size < readLen) @@ -399,7 +400,7 @@ hdr_fetch(bufferinfo *gbp) { } /* broadcast root's read (full or partial header) to other processes */ - TRACE_COMM(MPI_Bcast)(gbp->base, gbp->size, MPI_BYTE, 0, gbp->comm); + TRACE_COMM(MPI_Bcast)(gbp->base, gbp->chunk, MPI_BYTE, 0, gbp->comm); gbp->pos = gbp->base; @@ -516,8 +517,8 @@ hdr_get_NC_name(bufferinfo *gbp, char **namep, size_t *name_len) * NON_NEG = | // CDF-1 and CDF-2 * // CDF-5 */ - int err=NC_NOERR, padding, bufremain, strcount; - size_t nchars; + int err=NC_NOERR; + size_t nchars, padding, bufremain, strcount; char *cpos; *namep = NULL; @@ -551,7 +552,7 @@ hdr_get_NC_name(bufferinfo *gbp, char **namep, size_t *name_len) */ padding = _RNDUP(nchars, X_ALIGN) - nchars; - bufremain = gbp->size - (gbp->pos - gbp->base); + bufremain = gbp->chunk - (gbp->pos - gbp->base); cpos = *namep; @@ -572,7 +573,7 @@ hdr_get_NC_name(bufferinfo *gbp, char **namep, size_t *name_len) *namep = NULL; return err; } - bufremain = gbp->size; + bufremain = gbp->chunk; } } @@ -608,7 +609,7 @@ hdr_get_NC_name(bufferinfo *gbp, char **namep, size_t *name_len) #ifdef ENABLE_NULL_BYTE_HEADER_PADDING char pad[X_ALIGN-1]; memset(pad, 0, X_ALIGN-1); - if (memcmp(gbp->pos, pad, (size_t)padding) != 0) { + if (memcmp(gbp->pos, pad, padding) != 0) { #ifdef PNETCDF_DEBUG fprintf(stderr,"Error in file %s func %s line %d: NetCDF header corrupted, non-zero padding found\n",__FILE__,__func__,__LINE__); #endif @@ -788,22 +789,22 @@ hdr_get_NC_attrV(bufferinfo *gbp, NC_attr *attrp) * doubles = [DOUBLE ...] * padding = <0, 1, 2, or 3 bytes to next 4-byte boundary> */ - int err=NC_NOERR, xsz, padding, bufremain; + int err=NC_NOERR, xsz; void *value = attrp->xvalue; - MPI_Offset nbytes; + size_t nbytes, padding, bufremain; ncmpii_xlen_nc_type(attrp->xtype, &xsz); nbytes = attrp->nelems * xsz; padding = attrp->xsz - nbytes; - bufremain = gbp->size - (gbp->pos - gbp->base); - /* gbp->size is the read chunk size, which is of type 4-byte int. + bufremain = gbp->chunk - (gbp->pos - gbp->base); + /* gbp->chunk is the read chunk size, which is of type 4-byte int. * thus bufremain should be less than INT_MAX */ /* get values */ while (nbytes > 0) { if (bufremain > 0) { - int attcount = MIN(nbytes, bufremain); + size_t attcount = MIN(nbytes, bufremain); memcpy(value, gbp->pos, attcount); nbytes -= attcount; gbp->pos += attcount; @@ -813,7 +814,7 @@ hdr_get_NC_attrV(bufferinfo *gbp, NC_attr *attrp) int err; err = hdr_fetch(gbp); if (err != NC_NOERR) return err; - bufremain = gbp->size; + bufremain = gbp->chunk; } } @@ -849,7 +850,7 @@ hdr_get_NC_attrV(bufferinfo *gbp, NC_attr *attrp) #ifdef ENABLE_NULL_BYTE_HEADER_PADDING char pad[X_ALIGN-1]; memset(pad, 0, X_ALIGN-1); - if (memcmp(gbp->pos, pad, (size_t)padding) != 0) { + if (memcmp(gbp->pos, pad, padding) != 0) { #ifdef PNETCDF_DEBUG fprintf(stderr,"Error in file %s func %s line %d: NetCDF header corrupted, non-zero padding found\n",__FILE__,__func__,__LINE__); #endif @@ -1334,13 +1335,13 @@ ncmpio_hdr_get_NC(NC *ncp) getbuf.rw_mode = (fIsSet(ncp->flags, NC_HCOLL)) ? 1 : 0; /* CDF-5's minimum header size is 4 bytes more than CDF-1 and CDF-2's */ - getbuf.size = _RNDUP( MAX(MIN_NC_XSZ+4, ncp->chunk), X_ALIGN ); + getbuf.chunk = _RNDUP( MAX(MIN_NC_XSZ+4, ncp->chunk), X_ALIGN ); - getbuf.base = (char*) NCI_Malloc((size_t)getbuf.size); + getbuf.base = (char*) NCI_Malloc(getbuf.chunk); getbuf.pos = getbuf.base; - getbuf.end = getbuf.base + getbuf.size; + getbuf.end = getbuf.base + getbuf.chunk; - /* Fetch the next header chunk. The chunk is 'gbp->size' bytes big */ + /* Fetch the next header chunk. The chunk is 'gbp->chunk' bytes big */ err = hdr_fetch(&getbuf); if (err != NC_NOERR) return err; From 6451ff53c2716f53b2df855170fc51d78951cd85 Mon Sep 17 00:00:00 2001 From: wkliao Date: Tue, 19 Mar 2024 14:08:43 -0500 Subject: [PATCH 13/20] Update/overwrite file header in multiple runs if large --- src/drivers/ncmpio/ncmpio_header_put.c | 81 ++++++++++++++------------ 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/src/drivers/ncmpio/ncmpio_header_put.c b/src/drivers/ncmpio/ncmpio_header_put.c index f88937262..45c4315a3 100644 --- a/src/drivers/ncmpio/ncmpio_header_put.c +++ b/src/drivers/ncmpio/ncmpio_header_put.c @@ -449,7 +449,6 @@ ncmpio_hdr_put_NC(NC *ncp, void *buf) putbuf.offset = 0; putbuf.pos = buf; putbuf.base = buf; - putbuf.size = ncp->xsz; putbuf.safe_mode = ncp->safe_mode; putbuf.rw_mode = (fIsSet(ncp->flags, NC_HCOLL)) ? 1 : 0; @@ -512,6 +511,7 @@ ncmpio_hdr_put_NC(NC *ncp, void *buf) int ncmpio_write_header(NC *ncp) { int rank, status=NC_NOERR, mpireturn, err; + size_t i, ntimes; MPI_File fh; MPI_Status mpistatus; @@ -531,60 +531,67 @@ int ncmpio_write_header(NC *ncp) * occupied by the file header */ ncp->xsz = ncmpio_hdr_len_NC(ncp); + ntimes = ncp->xsz / NC_MAX_INT; + if (ncp->xsz % NC_MAX_INT) ntimes++; + MPI_Comm_rank(ncp->comm, &rank); if (rank == 0) { /* only root writes to file header */ + MPI_Offset offset; + size_t remain; size_t bufLen = _RNDUP(ncp->xsz, X_ALIGN); void *buf = NCI_Malloc(bufLen); /* header's write buffer */ /* copy header object to write buffer */ status = ncmpio_hdr_put_NC(ncp, buf); - if (ncp->xsz > NC_MAX_INT) { - NCI_Free(buf); - DEBUG_RETURN_ERROR(NC_EINTOVERFLOW) - } + offset = 0; + remain = ncp->xsz; + for (i=0; iflags, NC_HCOLL)) /* collective write */ - TRACE_IO(MPI_File_write_at_all)(fh, 0, buf, (int)ncp->xsz, + if (fIsSet(ncp->flags, NC_HCOLL)) /* collective write */ + TRACE_IO(MPI_File_write_at_all)(fh, offset, buf, writeLen, + MPI_BYTE, &mpistatus); + else + TRACE_IO(MPI_File_write_at)(fh, offset, buf, writeLen, MPI_BYTE, &mpistatus); - else - TRACE_IO(MPI_File_write_at)(fh, 0, buf, (int)ncp->xsz, - MPI_BYTE, &mpistatus); - if (mpireturn != MPI_SUCCESS) { - err = ncmpii_error_mpi2nc(mpireturn, "MPI_File_write_at"); - if (status == NC_NOERR) { - err = (err == NC_EFILE) ? NC_EWRITE : err; - DEBUG_ASSIGN_ERROR(status, err) + if (mpireturn != MPI_SUCCESS) { + err = ncmpii_error_mpi2nc(mpireturn, "MPI_File_write_at"); + if (status == NC_NOERR) { + err = (err == NC_EFILE) ? NC_EWRITE : err; + DEBUG_ASSIGN_ERROR(status, err) + } } - } - else { - /* update the number of bytes written since file open */ -#ifdef HAVE_MPI_GET_COUNT_C - MPI_Count put_size; - MPI_Get_count_c(&mpistatus, MPI_BYTE, &put_size); - ncp->put_size += put_size; -#else - int put_size; - mpireturn = MPI_Get_count(&mpistatus, MPI_BYTE, &put_size); - if (mpireturn != MPI_SUCCESS || put_size == MPI_UNDEFINED) - ncp->put_size += ncp->xsz; - else - ncp->put_size += put_size; -#endif + else { + /* update the number of bytes written since file open. + * Because each MPI write writes no more than NC_MAX_INT, + * calling MPI_Get_count() is sufficient. No need to call + * MPI_Get_count_c() + */ + int put_size; + mpireturn = MPI_Get_count(&mpistatus, MPI_BYTE, &put_size); + if (mpireturn != MPI_SUCCESS || put_size == MPI_UNDEFINED) + ncp->put_size += ncp->xsz; + else + ncp->put_size += writeLen; + } + offset += writeLen; + remain -= writeLen; } NCI_Free(buf); } else if (fIsSet(ncp->flags, NC_HCOLL)) { /* collective write */ /* other processes participate the collective call */ - TRACE_IO(MPI_File_write_at_all)(fh, 0, NULL, 0, MPI_BYTE, &mpistatus); + for (i=0; isafe_mode == 1) { From c476d70faf9db91b68cbfa03a658d0e881735069 Mon Sep 17 00:00:00 2001 From: wkliao Date: Tue, 19 Mar 2024 14:12:55 -0500 Subject: [PATCH 14/20] use MPI_Type_create_hindexed_c for large datatype --- src/drivers/ncmpio/ncmpio_filetype.c | 59 ++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/src/drivers/ncmpio/ncmpio_filetype.c b/src/drivers/ncmpio/ncmpio_filetype.c index 97b3da0c3..8c1ee7025 100644 --- a/src/drivers/ncmpio/ncmpio_filetype.c +++ b/src/drivers/ncmpio/ncmpio_filetype.c @@ -402,12 +402,12 @@ stride_flatten(int isRecVar, /* whether record variable */ const MPI_Offset *start, /* [ndim] starts of subarray */ const MPI_Offset *count, /* [ndim] counts of subarray */ const MPI_Offset *stride, /* [ndim] strides of subarray */ - int *nblocks, /* OUT: number of blocks */ - int *blocklens,/* OUT: length of each block */ + MPI_Offset *nblocks, /* OUT: number of blocks */ + MPI_Offset *blocklens,/* OUT: length of each block */ MPI_Aint *disps) /* OUT: displacement of each block */ { - int i, j, k, seg_len; - MPI_Offset nstride, array_len, off, subarray_len; + int i, j, k; + MPI_Offset seg_len, nstride, array_len, off, subarray_len; /* scalar variables have been handled before this subroutine is called */ assert (ndim > 0); @@ -478,9 +478,9 @@ ncmpio_filetype_create_vars(const NC *ncp, MPI_Datatype *filetype_ptr, /* OUT */ int *is_filetype_contig) /* OUT */ { - int dim, err, nblocks, *blocklens; + int err, dim, isLargeReq; MPI_Aint *disps; - MPI_Offset offset, nelems, *shape; + MPI_Offset i, offset, nblocks, nelems, *shape, *blocklens; MPI_Datatype filetype=MPI_BYTE; if (stride == NULL) @@ -516,9 +516,9 @@ ncmpio_filetype_create_vars(const NC *ncp, if (is_filetype_contig != NULL) *is_filetype_contig = 0; offset = varp->begin; - blocklens = (int*) NCI_Malloc((size_t)nelems * SIZEOF_INT); - disps = (MPI_Aint*) NCI_Malloc((size_t)nelems * SIZEOF_MPI_AINT); - shape = (MPI_Offset*) NCI_Malloc((size_t)varp->ndims * SIZEOF_MPI_OFFSET); + blocklens = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * nelems); + disps = (MPI_Aint*) NCI_Malloc(sizeof(MPI_Aint) * nelems); + shape = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * varp->ndims); for (dim=0; dimndims; dim++) shape[dim] = varp->shape[dim]; /* for record variable, set shape[0] to the record size */ @@ -535,8 +535,45 @@ ncmpio_filetype_create_vars(const NC *ncp, * We cannot use MPI_Type_indexed because displacement for the record * dimension may not be a multiple of varp->xtype */ - err = MPI_Type_create_hindexed(nblocks, blocklens, disps, MPI_BYTE, - &filetype); + isLargeReq = 0; + if (nblocks > NC_MAX_INT) + isLargeReq = 1; + else { + for (i=0; i NC_MAX_INT) { + isLargeReq = 1; + break; + } + } + } + + if (isLargeReq) { +#ifdef HAVE_MPI_TYPE_CREATE_HINDEXED_C + MPI_Count *blocklens_c, *disps_c; + blocklens_c = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * nelems); + disps_c = (MPI_Count*) NCI_Malloc(sizeof(MPI_Count) * nelems); + for (i=0; i Date: Tue, 19 Mar 2024 14:15:34 -0500 Subject: [PATCH 15/20] --large-single-req has been removed in #131 --- src/utils/README.md | 1 - src/utils/pnetcdf-config.in | 1 - 2 files changed, 2 deletions(-) diff --git a/src/utils/README.md b/src/utils/README.md index 6f4f03ece..5e12adca4 100644 --- a/src/utils/README.md +++ b/src/utils/README.md @@ -894,7 +894,6 @@ Available values for OPTION include: --in-place-swap Whether using buffer in-place Endianness byte swap --erange-fill Whether using fill values for NC_ERANGE error --subfiling Whether subfiling is enabled or disabled - --large-single-req Whether to allow single >2GiB MPI-IO requests --null-byte-header-padding Whether to check null-byte padding in header --burst-buffering Whether burst buffer driver is built or not --profiling Whether internal profiling is enabled or not diff --git a/src/utils/pnetcdf-config.in b/src/utils/pnetcdf-config.in index 4fcb4a62c..4892f2721 100644 --- a/src/utils/pnetcdf-config.in +++ b/src/utils/pnetcdf-config.in @@ -117,7 +117,6 @@ Available values for OPTION include: --in-place-swap Whether using buffer in-place Endianness byte swap --erange-fill Whether using fill values for NC_ERANGE error --subfiling Whether subfiling is enabled or disabled - --large-single-req Whether to allow single >2GiB MPI-IO requests --null-byte-header-padding Whether to check null-byte padding in header --burst-buffering Whether burst buffer driver is built or not --profiling Whether internal profiling is enabled or not From d4f1ffd18361f9a9188344a5add373220ee88f44 Mon Sep 17 00:00:00 2001 From: wkliao Date: Tue, 19 Mar 2024 14:16:14 -0500 Subject: [PATCH 16/20] dump memory footprint in verbose mode --- test/largefile/large_dims_vars_attrs.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/largefile/large_dims_vars_attrs.c b/test/largefile/large_dims_vars_attrs.c index 6786e60f4..1cff876f5 100644 --- a/test/largefile/large_dims_vars_attrs.c +++ b/test/largefile/large_dims_vars_attrs.c @@ -30,7 +30,7 @@ int main(int argc, char** argv) { char filename[256], str[32]; - int i, rank, nprocs, err, nerrs=0; + int i, rank, nprocs, err, nerrs=0, verbose=0; int ncid, cmode, *varid, *dimids, intBuf[1]; MPI_Init(&argc, &argv); @@ -143,9 +143,11 @@ int main(int argc, char** argv) } } - err = ncmpi_inq_malloc_max_size(&malloc_size); - printf("%d: PnetCDF internal memory footprint high water mark %.2f MB\n", - rank, (float)malloc_size/1048576); + if (verbose) { + err = ncmpi_inq_malloc_max_size(&malloc_size); + printf("\n%d: PnetCDF internal memory footprint high water mark %.2f MB\n", + rank, (float)malloc_size/1048576); + } MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (rank == 0) { From 62022034a9af85c43ce5f015836eac31e5169203 Mon Sep 17 00:00:00 2001 From: wkliao Date: Tue, 19 Mar 2024 14:39:50 -0500 Subject: [PATCH 17/20] free allocated memory before return NC_EINTOVERFLOW --- src/drivers/ncmpio/ncmpio_filetype.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/drivers/ncmpio/ncmpio_filetype.c b/src/drivers/ncmpio/ncmpio_filetype.c index 8c1ee7025..9ec721467 100644 --- a/src/drivers/ncmpio/ncmpio_filetype.c +++ b/src/drivers/ncmpio/ncmpio_filetype.c @@ -561,7 +561,11 @@ ncmpio_filetype_create_vars(const NC *ncp, NCI_Free(blocklens_c); NCI_Free(disps_c); #else - DEBUG_ASSIGN_ERROR(ret, NC_ESMALL) + NCI_Free(disps); + NCI_Free(blocklens); + *offset_ptr = offset; + *filetype_ptr = MPI_DATATYPE_NULL; + return NC_EINTOVERFLOW; #endif } else { @@ -623,7 +627,7 @@ ncmpio_file_set_view(const NC *ncp, /* check if header size > 2^31 */ if (ncp->begin_var > NC_MAX_INT) { - status = NC_EINTOVERFLOW; + DEBUG_ASSIGN_ERROR(status, NC_EINTOVERFLOW); goto err_out; } #endif @@ -640,7 +644,7 @@ ncmpio_file_set_view(const NC *ncp, #if !defined(HAVE_MPI_LARGE_COUNT) && (SIZEOF_MPI_AINT != SIZEOF_MPI_OFFSET) if (*offset > NC_MAX_INT) { - status = NC_EINTOVERFLOW; + DEBUG_ASSIGN_ERROR(status, NC_EINTOVERFLOW); goto err_out; } #endif From 923e7a7b25ef4f16c610595c0497f022b5f8071f Mon Sep 17 00:00:00 2001 From: wkliao Date: Tue, 19 Mar 2024 16:16:35 -0500 Subject: [PATCH 18/20] wrap error code for debugging --- src/drivers/ncmpio/ncmpio_file_io.c | 2 +- src/drivers/ncmpio/ncmpio_fill.c | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/drivers/ncmpio/ncmpio_file_io.c b/src/drivers/ncmpio/ncmpio_file_io.c index 977c73e81..4f0f2849e 100644 --- a/src/drivers/ncmpio/ncmpio_file_io.c +++ b/src/drivers/ncmpio/ncmpio_file_io.c @@ -51,7 +51,7 @@ ncmpio_read_write(NC *ncp, err = (err == NC_EFILE) ? NC_EREAD : err; } else if (btype_size == MPI_UNDEFINED) - err = NC_EINTOVERFLOW; + DEBUG_ASSIGN_ERROR(err, NC_EINTOVERFLOW) if (err != NC_NOERR) { if (coll_indep == NC_REQ_COLL) { diff --git a/src/drivers/ncmpio/ncmpio_fill.c b/src/drivers/ncmpio/ncmpio_fill.c index 437070dd8..2589e47a4 100644 --- a/src/drivers/ncmpio/ncmpio_fill.c +++ b/src/drivers/ncmpio/ncmpio_fill.c @@ -642,7 +642,9 @@ fillerup_aggregate(NC *ncp, NC *old_ncp) buf_len = 1; } #else - if (status == NC_NOERR) status = NC_EINTOVERFLOW; + if (status == NC_NOERR) + DEBUG_ASSIGN_ERROR(status, NC_EINTOVERFLOW) + buf_len = 0; /* participate collective write with 0-length request */ #endif } From eb9d70fb4fbe3e24baa6bfb124a7dbe8b170390c Mon Sep 17 00:00:00 2001 From: wkliao Date: Tue, 19 Mar 2024 16:18:21 -0500 Subject: [PATCH 19/20] fix: nelems may not be = nblocks --- src/drivers/ncmpio/ncmpio_filetype.c | 154 +++++++++++++++------------ 1 file changed, 83 insertions(+), 71 deletions(-) diff --git a/src/drivers/ncmpio/ncmpio_filetype.c b/src/drivers/ncmpio/ncmpio_filetype.c index 9ec721467..88c009218 100644 --- a/src/drivers/ncmpio/ncmpio_filetype.c +++ b/src/drivers/ncmpio/ncmpio_filetype.c @@ -395,75 +395,96 @@ filetype_create_vara(const NC *ncp, * blocklens[] and disps[] */ static int -stride_flatten(int isRecVar, /* whether record variable */ - int ndim, /* number of dimensions */ - int el_size, /* array element size */ - const MPI_Offset *dimlen, /* [ndim] dimension lengths */ - const MPI_Offset *start, /* [ndim] starts of subarray */ - const MPI_Offset *count, /* [ndim] counts of subarray */ - const MPI_Offset *stride, /* [ndim] strides of subarray */ - MPI_Offset *nblocks, /* OUT: number of blocks */ - MPI_Offset *blocklens,/* OUT: length of each block */ - MPI_Aint *disps) /* OUT: displacement of each block */ +stride_flatten(const NC_var *varp, + MPI_Offset recsize, /* record size */ + const MPI_Offset *start, /* [ndim] starts of subarray */ + const MPI_Offset *count, /* [ndim] counts of subarray */ + const MPI_Offset *stride, /* [ndim] strides of subarray */ + MPI_Offset *nblocks, /* OUT: number of blocks */ + MPI_Offset **blocklens,/* OUT: length of each block */ + MPI_Aint **disps) /* OUT: displacement of each block */ { - int i, j, k; - MPI_Offset seg_len, nstride, array_len, off, subarray_len; + int i, j, k, isRecVar, ndims, el_size; + MPI_Offset *dimlen, seg_len, nstride, array_len, off, subarray_len; + + *blocklens = NULL; + *disps = NULL; + + /* whether record variable */ + isRecVar = IS_RECVAR(varp); + + ndims = varp->ndims; /* scalar variables have been handled before this subroutine is called */ - assert (ndim > 0); + assert (ndims > 0); + + /* array element size */ + el_size = varp->xsz; + + /* dimension lengths */ + dimlen = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * ndims); + + for (i=0; ishape[i]; + /* for record variable, set dimlen[0] to the record size */ + if (isRecVar) dimlen[0] = recsize; /* calculate the number of offset-length pairs */ - *nblocks = (stride[ndim-1] == 1) ? 1 : count[ndim-1]; - for (i=0; i 0) { - /* array_len is global array size from lowest up to ndim */ - array_len *= dimlen[ndim]; + while (ndims > 0) { + /* array_len is global array size from lowest up to ndims */ + array_len *= dimlen[ndims]; - /* off is the global array offset for this dimension, ndim-1 + /* off is the global array offset for this dimension, ndims-1 * For record variable, dimlen[0] is the sum of single record sizes */ - if (ndim == 1 && isRecVar) off = start[0] * dimlen[0]; - else off = start[ndim-1] * array_len; + if (ndims == 1 && isRecVar) off = start[0] * dimlen[0]; + else off = start[ndims-1] * array_len; - /* update all offsets from lowest up to dimension ndim-1 */ - for (j=0; jndims; dim++) nelems *= count[dim]; + /* starting file offset of the variable */ + *offset_ptr = varp->begin; + /* when nelems == 0 or varp is a scalar, i.e. varp->ndims == 0, no need to * create a filetype */ if (varp->ndims == 0 || nelems == 0) { - *offset_ptr = varp->begin; *filetype_ptr = MPI_BYTE; if (is_filetype_contig != NULL) *is_filetype_contig = 1; return NC_NOERR; @@ -514,22 +537,12 @@ ncmpio_filetype_create_vars(const NC *ncp, /* hereinafter fileview is noncontiguous, i.e. filetype != MPI_BYTE */ if (is_filetype_contig != NULL) *is_filetype_contig = 0; - offset = varp->begin; - - blocklens = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * nelems); - disps = (MPI_Aint*) NCI_Malloc(sizeof(MPI_Aint) * nelems); - shape = (MPI_Offset*) NCI_Malloc(sizeof(MPI_Offset) * varp->ndims); - - for (dim=0; dimndims; dim++) shape[dim] = varp->shape[dim]; - /* for record variable, set shape[0] to the record size */ - if (IS_RECVAR(varp)) shape[0] = ncp->recsize; /* flatten stride access into list of offsets and lengths stored in * disps[] and blocklens[], respectively. */ - stride_flatten(IS_RECVAR(varp), varp->ndims, varp->xsz, shape, start, - count, stride, &nblocks, blocklens, disps); - NCI_Free(shape); + stride_flatten(varp, ncp->recsize, start, count, stride, &nblocks, + &blocklens, &disps); /* the flattened list allows one single call to hindexed constructor. * We cannot use MPI_Type_indexed because displacement for the record @@ -539,7 +552,7 @@ ncmpio_filetype_create_vars(const NC *ncp, if (nblocks > NC_MAX_INT) isLargeReq = 1; else { - for (i=0; i NC_MAX_INT) { isLargeReq = 1; break; @@ -556,16 +569,13 @@ ncmpio_filetype_create_vars(const NC *ncp, blocklens_c[i] = blocklens[i]; disps_c[i] = disps[i]; } - err = MPI_Type_create_hindexed_c(nblocks, blocklens_c, disps_c, - MPI_BYTE, &filetype); + mpireturn = MPI_Type_create_hindexed_c(nblocks, blocklens_c, disps_c, + MPI_BYTE, &filetype); NCI_Free(blocklens_c); NCI_Free(disps_c); #else - NCI_Free(disps); - NCI_Free(blocklens); - *offset_ptr = offset; *filetype_ptr = MPI_DATATYPE_NULL; - return NC_EINTOVERFLOW; + DEBUG_ASSIGN_ERROR(err, NC_EINTOVERFLOW) #endif } else { @@ -573,22 +583,24 @@ ncmpio_filetype_create_vars(const NC *ncp, blocklens_i = (int*) NCI_Malloc(sizeof(int) * nelems); for (i=0; i---------------------------------------------*/ From 3551cf8330994373218ccd9251f89299cd9da5e4 Mon Sep 17 00:00:00 2001 From: wkliao Date: Tue, 19 Mar 2024 17:29:59 -0500 Subject: [PATCH 20/20] fix typecast 64-to-32 bit int --- examples/C/ghost_cell.c | 3 ++- examples/C/transpose.c | 5 +++-- examples/C/transpose2D.c | 10 +++++----- examples/C/vard_int.c | 3 ++- examples/C/vard_mvars.c | 16 +++++++--------- examples/tutorial/pnetcdf-permute.c | 2 +- examples/tutorial/pnetcdf-read-from-master.c | 7 +++++-- 7 files changed, 25 insertions(+), 21 deletions(-) diff --git a/examples/C/ghost_cell.c b/examples/C/ghost_cell.c index 794b02188..6136222e8 100644 --- a/examples/C/ghost_cell.c +++ b/examples/C/ghost_cell.c @@ -128,9 +128,10 @@ pnetcdf_check_mem_usage(MPI_Comm comm) static int pnetcdf_io(MPI_Comm comm, char *filename, int cmode, int len) { - int i, j, rank, nprocs, ncid, bufsize, err, nerrs=0; + int i, j, rank, nprocs, ncid, err, nerrs=0; int psizes[2], local_rank[2], dimids[2], varid, nghosts; int *buf, *buf_ptr; + size_t bufsize; MPI_Offset gsizes[2], starts[2], counts[2], imap[2]; MPI_Comm_rank(comm, &rank); diff --git a/examples/C/transpose.c b/examples/C/transpose.c index a23452136..aceb1d137 100644 --- a/examples/C/transpose.c +++ b/examples/C/transpose.c @@ -141,9 +141,10 @@ pnetcdf_io(MPI_Comm comm, char *filename, int cmode, int len) for (j=0; j=0; i--) { start[i] = rank / lower_dims % psizes[i]; lower_dims *= psizes[i]; @@ -160,7 +160,7 @@ pnetcdf_io(MPI_Comm comm, char *filename, int cmode, int len) buf = (int *) malloc(bufsize * sizeof(int)); for (i=0; i