Skip to content

Commit

Permalink
Add command-line option '-i' to use MPI independent I/O
Browse files Browse the repository at this point in the history
  • Loading branch information
@wkliao
wkliao committed May 20, 2024
1 parent d6e35f2 commit b41a82c
Show file tree
Hide file tree
Showing 5 changed files with 164 additions and 46 deletions.
12 changes: 8 additions & 4 deletions benchmarks/FLASH-IO/README.md
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
#
# Copyright (C) 2013, Northwestern University
# See COPYRIGHT notice in top-level directory.
#
## PnetCDF FLASH-IO Benchmark

This software benchmarks the performance of PnetCDF method for the I/O and data
partitioning pattern from the FLASH I/O benchmark, developed at the Flash
Expand Down Expand Up @@ -51,6 +48,10 @@ and the I/O time of which dominates the entire benchmark.
```
(this will use file base name prefix: "/pvfs2/flash_io_test_")

+ Command-line options:
* [-q] quiet mode
* [-i] use MPI independent I/O (default is collective I/O)

* Example output on screen:
```c
number of guards : 4
Expand Down Expand Up @@ -97,3 +98,6 @@ and the I/O time of which dominates the entire benchmark.
```
* Copyright (C) 2013, Northwestern University
* See COPYRIGHT notice in top-level directory.
81 changes: 62 additions & 19 deletions benchmarks/FLASH-IO/checkpoint_ncmpi_parallel.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,11 @@ double precision function checkpoint_wr_ncmpi_par (filenum, simtime)
MPI_Wtime() - time_start
#endif

if (indep_io) then
err = nfmpi_begin_indep_data(ncid)
if (err .NE. NF_NOERR) call check(err, "nfmpi_begin_indep_data")
endif

global_offset = n_to_left(MyPE)

!-----------------------------------------------------------------------------
Expand All @@ -408,8 +413,13 @@ double precision function checkpoint_wr_ncmpi_par (filenum, simtime)
err = nfmpi_iput_vara_int(ncid, varid(1), starts, counts, lrefine, reqs(1))
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_int: lrefine")
else
err = nfmpi_put_vara_int_all(ncid, varid(1), starts, counts, lrefine)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: lrefine")
if (indep_io) then
err = nfmpi_put_vara_int(ncid, varid(1), starts, counts, lrefine)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int: lrefine")
else
err = nfmpi_put_vara_int_all(ncid, varid(1), starts, counts, lrefine)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: lrefine")
endif
endif

#ifdef TIMERS
Expand All @@ -426,8 +436,13 @@ double precision function checkpoint_wr_ncmpi_par (filenum, simtime)
err = nfmpi_iput_vara_int(ncid, varid(2), starts, counts, nodetype, reqs(2))
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int: nodetype")
else
err = nfmpi_put_vara_int_all(ncid, varid(2), starts, counts, nodetype)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: nodetype")
if (indep_io) then
err = nfmpi_put_vara_int(ncid, varid(2), starts, counts, nodetype)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int: nodetype")
else
err = nfmpi_put_vara_int_all(ncid, varid(2), starts, counts, nodetype)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: nodetype")
endif
endif

#ifdef TIMERS
Expand All @@ -448,8 +463,13 @@ double precision function checkpoint_wr_ncmpi_par (filenum, simtime)
err = nfmpi_iput_vara_int(ncid, varid(3), starts, counts, gid, reqs(3))
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_int: gid")
else
err = nfmpi_put_vara_int_all(ncid, varid(3), starts, counts, gid)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: gid")
if (indep_io) then
err = nfmpi_put_vara_int(ncid, varid(3), starts, counts, gid)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int: gid")
else
err = nfmpi_put_vara_int_all(ncid, varid(3), starts, counts, gid)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: gid")
endif
endif

#ifdef TIMERS
Expand All @@ -475,8 +495,13 @@ double precision function checkpoint_wr_ncmpi_par (filenum, simtime)
err = nfmpi_iput_vara_double(ncid, varid(4), starts, counts, coord_buf, reqs(4))
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_double: coord")
else
err = nfmpi_put_vara_double_all(ncid, varid(4), starts, counts, coord_buf)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double_all: coord")
if (indep_io) then
err = nfmpi_put_vara_double(ncid, varid(4), starts, counts, coord_buf)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double: coord")
else
err = nfmpi_put_vara_double_all(ncid, varid(4), starts, counts, coord_buf)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double_all: coord")
endif
endif

#ifdef TIMERS
Expand All @@ -498,8 +523,13 @@ double precision function checkpoint_wr_ncmpi_par (filenum, simtime)
err = nfmpi_iput_vara_double(ncid, varid(5), starts, counts, bs_buf, reqs(5))
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_double: size")
else
err = nfmpi_put_vara_double_all(ncid, varid(5), starts, counts, bs_buf)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double_all: size")
if (indep_io) then
err = nfmpi_put_vara_double(ncid, varid(5), starts, counts, bs_buf)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double: size")
else
err = nfmpi_put_vara_double_all(ncid, varid(5), starts, counts, bs_buf)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double_all: size")
endif
endif

#ifdef TIMERS
Expand All @@ -523,8 +553,13 @@ double precision function checkpoint_wr_ncmpi_par (filenum, simtime)
err = nfmpi_iput_vara_double(ncid, varid(6), starts, counts, bb_buf, reqs(6))
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_double: bnd_box")
else
err = nfmpi_put_vara_double_all(ncid, varid(6), starts, counts, bb_buf)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double_all: bnd_box")
if (indep_io) then
err = nfmpi_put_vara_double(ncid, varid(6), starts, counts, bb_buf)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double: bnd_box")
else
err = nfmpi_put_vara_double_all(ncid, varid(6), starts, counts, bb_buf)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double_all: bnd_box")
endif
endif

#ifdef TIMERS
Expand Down Expand Up @@ -595,23 +630,31 @@ double precision function checkpoint_wr_ncmpi_par (filenum, simtime)
if (err .NE. NF_NOERR) &
call check(err, "nfmpi_iput_vara: unknowns")
else
err = nfmpi_put_vara_double_all(ncid, varid(6+i), starts, counts, unk_buf)
if (err .NE. NF_NOERR) &
call check(err, "nfmpi_put_vara_double_all: unknowns")
if (indep_io) then
err = nfmpi_put_vara_double(ncid, varid(6+i), starts, counts, unk_buf)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double: unknowns")
else
err = nfmpi_put_vara_double_all(ncid, varid(6+i), starts, counts, unk_buf)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double_all: unknowns")
endif
endif
enddo

if (use_nonblocking_io) then
! wait for the nonblocking I/O to complete
err = nfmpi_wait_all(ncid, nvar+6, reqs, stats)
if (err .NE. NF_NOERR) &
call check(err, "nfmpi_wait_all: unknowns")
if (indep_io) then
err = nfmpi_wait(ncid, nvar+6, reqs, stats)
if (err .NE. NF_NOERR) call check(err, "nfmpi_wait: unknowns")
else
err = nfmpi_wait_all(ncid, nvar+6, reqs, stats)
if (err .NE. NF_NOERR) call check(err, "nfmpi_wait_all: unknowns")
endif

! check the status of each nonblocking request
do i=1, nvar+6
write(str,'(I2)') i
if (stats(i) .NE. NF_NOERR) &
call check(stats(i), 'In nfmpi_wait_all req '//trim(str))
call check(stats(i), 'In nfmpi_wait(_all) req '//trim(str))
enddo
call MPI_Type_free(buftype, err)
endif
Expand Down
4 changes: 3 additions & 1 deletion benchmarks/FLASH-IO/common.fh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,10 @@

integer info_used
logical use_nonblocking_io
logical indep_io

double precision chk_t(3), corner_t(3), nocorner_t(3)
common /timers/ chk_t, corner_t, nocorner_t, info_used, use_nonblocking_io
common /timers/ chk_t, corner_t, nocorner_t, info_used, &
use_nonblocking_io, indep_io


24 changes: 22 additions & 2 deletions benchmarks/FLASH-IO/flash_benchmark_io.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ program flash_benchmark_io

MasterPE = 0
verbose = .TRUE.
indep_io = .FALSE.

! root process reads command-line arguments
if (MyPE .EQ. MasterPE) then
Expand All @@ -55,13 +56,28 @@ program flash_benchmark_io
if (basenm(1:2) .EQ. '-q') then
verbose = .FALSE.
basenm = "flash_io_test_"
else if (basenm(1:2) .EQ. '-i') then
indep_io = .TRUE.
basenm = "flash_io_test_"
endif
else if (argc .EQ. 2) then
verbose = .FALSE.
call getarg(2, basenm)
call getarg(1, basenm)
if (basenm(1:2) .EQ. '-q') then
verbose = .FALSE.
call getarg(2, basenm)
else if (basenm(1:2) .EQ. '-i') then
indep_io = .TRUE.
call getarg(2, basenm)
else
isArgvRight = .FALSE.
endif
else if (argc .EQ. 3) then
call getarg(3, basenm)
if (basenm(1:2) .EQ. '-q' .OR. basenm(1:2) .EQ. '-i') then
isArgvRight = .FALSE.
endif
verbose = .FALSE.
indep_io = .TRUE.
else if (argc .GT. 2) then
isArgvRight = .FALSE.
endif
Expand All @@ -76,6 +92,10 @@ program flash_benchmark_io
MPI_COMM_WORLD, ierr)
if (.NOT. isArgvRight) goto 999

! broadcast if independent I/O should be used
call MPI_Bcast(indep_io, 1, MPI_LOGICAL, MasterPE, &
MPI_COMM_WORLD, ierr)

! broadcast file base name prefix
call MPI_Bcast(basenm, 128, MPI_CHARACTER, MasterPE, &
MPI_COMM_WORLD, ierr)
Expand Down
89 changes: 69 additions & 20 deletions benchmarks/FLASH-IO/plotfile_ncmpi_parallel.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,11 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
sunklabels, &
varid)

if (indep_io) then
err = nfmpi_begin_indep_data(ncid)
if (err .NE. NF_NOERR) call check(err, "nfmpi_begin_indep_data")
endif

global_offset = n_to_left(MyPE)
!-----------------------------------------------------------------------------
! store the tree information
Expand All @@ -442,17 +447,27 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
err = nfmpi_iput_vara_int(ncid, varid(1), starts, counts, lrefine, reqs(1))
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_int: lrefine sp")
else
err = nfmpi_put_vara_int_all(ncid, varid(1), starts, counts, lrefine)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: lrefine sp")
if (indep_io) then
err = nfmpi_put_vara_int(ncid, varid(1), starts, counts, lrefine)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int: lrefine sp")
else
err = nfmpi_put_vara_int_all(ncid, varid(1), starts, counts, lrefine)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: lrefine sp")
endif
endif

! store the nodetype
if (use_nonblocking_io) then
err = nfmpi_iput_vara_int(ncid, varid(2), starts, counts, nodetype, reqs(2))
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_int: nodetype sp")
else
err = nfmpi_put_vara_int_all(ncid, varid(2), starts, counts, nodetype)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: nodetype sp")
if (indep_io) then
err = nfmpi_put_vara_int(ncid, varid(2), starts, counts, nodetype)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int: nodetype sp")
else
err = nfmpi_put_vara_int_all(ncid, varid(2), starts, counts, nodetype)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: nodetype sp")
endif
endif

! store the global id
Expand All @@ -464,8 +479,13 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
err = nfmpi_iput_vara_int(ncid, varid(3), starts, counts, gid, reqs(3))
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_int: gid sp")
else
err = nfmpi_put_vara_int_all(ncid, varid(3), starts, counts, gid)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: gid sp")
if (indep_io) then
err = nfmpi_put_vara_int(ncid, varid(3), starts, counts, gid)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int: gid sp")
else
err = nfmpi_put_vara_int_all(ncid, varid(3), starts, counts, gid)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: gid sp")
endif
endif

!-----------------------------------------------------------------------------
Expand All @@ -484,8 +504,13 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
err = nfmpi_iput_vara_real(ncid, varid(4), starts, counts, coord_single, reqs(4))
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_real: coord sp")
else
err = nfmpi_put_vara_real_all(ncid, varid(4), starts, counts, coord_single)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_read_all: coord sp")
if (indep_io) then
err = nfmpi_put_vara_real(ncid, varid(4), starts, counts, coord_single)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_read: coord sp")
else
err = nfmpi_put_vara_real_all(ncid, varid(4), starts, counts, coord_single)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_read_all: coord sp")
endif
endif

! store the block size
Expand All @@ -501,8 +526,13 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
err = nfmpi_iput_vara_real(ncid, varid(5), starts, counts, blk_sz_single, reqs(5))
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_real: size sp")
else
err = nfmpi_put_vara_real_all(ncid, varid(5), starts, counts, blk_sz_single)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real_all: size sp")
if (indep_io) then
err = nfmpi_put_vara_real(ncid, varid(5), starts, counts, blk_sz_single)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real: size sp")
else
err = nfmpi_put_vara_real_all(ncid, varid(5), starts, counts, blk_sz_single)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real_all: size sp")
endif
endif

! store the bounding box
Expand All @@ -522,8 +552,13 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
err = nfmpi_iput_vara_real(ncid, varid(6), starts, counts, bnd_single, reqs(6))
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_real: bnd_box")
else
err = nfmpi_put_vara_real_all(ncid, varid(6), starts, counts, bnd_single)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real_all: bnd_box")
if (indep_io) then
err = nfmpi_put_vara_real(ncid, varid(6), starts, counts, bnd_single)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real: bnd_box")
else
err = nfmpi_put_vara_real_all(ncid, varid(6), starts, counts, bnd_single)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real_all: bnd_box")
endif
endif

if (use_nonblocking_io) then
Expand Down Expand Up @@ -614,8 +649,13 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
err = nfmpi_bput_vara_real(ncid, varid(6+ivar), starts, counts, unkt_crn, reqs(ivar+6))
if (err .NE. NF_NOERR) call check(err, "nfmpi_bput_vara_real: unknowns sp")
else
err = nfmpi_put_vara_real_all(ncid, varid(6+ivar), starts, counts, unkt_crn)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real_all: unknowns sp")
if (indep_io) then
err = nfmpi_put_vara_real(ncid, varid(6+ivar), starts, counts, unkt_crn)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real: unknowns sp")
else
err = nfmpi_put_vara_real_all(ncid, varid(6+ivar), starts, counts, unkt_crn)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real_all: unknowns sp")
endif
endif

else
Expand All @@ -638,8 +678,13 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
err = nfmpi_bput_vara_real(ncid, varid(6+ivar), starts, counts, unkt, reqs(ivar+6))
if (err .NE. NF_NOERR) call check(err, "nfmpi_bput_vara_real: unknowns sp")
else
err = nfmpi_put_vara_real_all(ncid, varid(6+ivar), starts, counts, unkt)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real_all: unknowns sp")
if (indep_io) then
err = nfmpi_put_vara_real(ncid, varid(6+ivar), starts, counts, unkt)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real: unknowns sp")
else
err = nfmpi_put_vara_real_all(ncid, varid(6+ivar), starts, counts, unkt)
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real_all: unknowns sp")
endif
endif
endif

Expand All @@ -648,15 +693,19 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
! wait for all nonblocking requests to complete
if (use_nonblocking_io) then
! wait for the nonblocking I/O to complete
err = nfmpi_wait_all(ncid, num_out+6, reqs, stats)
if (err .NE. NF_NOERR) &
call check(err, "(sp) nfmpi_wait_all: ")
if (indep_io) then
err = nfmpi_wait(ncid, num_out+6, reqs, stats)
if (err .NE. NF_NOERR) call check(err, "(sp) nfmpi_wait: ")
else
err = nfmpi_wait_all(ncid, num_out+6, reqs, stats)
if (err .NE. NF_NOERR) call check(err, "(sp) nfmpi_wait_all: ")
endif

! check the status of each nonblocking request
do i=1, num_out+6
write(str,'(I2)') i
if (stats(i) .NE. NF_NOERR) &
call check(stats(i), '(sp) nfmpi_wait_all req '//trim(str))
call check(stats(i), '(sp) nfmpi_wait(_all) req '//trim(str))
enddo

! detach the temporary buffer
Expand Down

0 comments on commit b41a82c

Please sign in to comment.