Skip to content

Commit cf2fcbc

Browse files
max-Hawkinsmax-Hawkins
committed
Standardize timing ranges
1 parent a228bb5 commit cf2fcbc

File tree

5 files changed

+42
-35
lines changed

5 files changed

+42
-35
lines changed

‎src/common/m_mpi_common.fpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ module m_mpi_common
1616
use m_derived_types !< Definitions of the derived types
1717
1818
use m_global_parameters !< Definitions of the global parameters
19+
20+
use m_nvtx
1921
! ==========================================================================
2022
2123
implicit none
@@ -239,6 +241,7 @@ contains
239241
#ifdef MFC_MPI
240242
#ifdef MFC_SIMULATION
241243
244+
call nvtxStartRange("TSTEP-RUNTIME-INFO-REDUCE-CRIT")
242245
! Reducing local extrema of ICFL, VCFL, CCFL and Rc numbers to their
243246
! global extrema and bookkeeping the results on the rank 0 processor
244247
call MPI_REDUCE(icfl_max_loc, icfl_max_glb, 1, &
@@ -253,6 +256,7 @@ contains
253256
MPI_DOUBLE_PRECISION, MPI_MIN, 0, &
254257
MPI_COMM_WORLD, ierr)
255258
end if
259+
call nvtxEndRange
256260
257261
#endif
258262
#endif

‎src/simulation/m_data_output.fpp

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ module m_data_output
3535

3636
implicit none
3737

38-
private;
38+
private;
3939
public :: s_initialize_data_output_module, &
4040
s_open_run_time_information_file, &
4141
s_open_probe_files, &
@@ -255,7 +255,9 @@ contains
255255
real(kind(0d0)) :: fltr_dtheta !<
256256
!! Modified dtheta accounting for Fourier filtering in azimuthal direction.
257257

258+
call nvtxStartRange("TSTEP-RUNTIME-INFO")
258259
! Computing Stability Criteria at Current Time-step ================
260+
call nvtxStartRange("TSTEP-RUNTIME-INFO-COMPUTE-LOCAL")
259261
!$acc parallel loop collapse(3) gang vector default(present) private(alpha_rho, vel, alpha, Re, fltr_dtheta, Nfq)
260262
do l = 0, p
261263
do k = 0, n
@@ -275,6 +277,7 @@ contains
275277
end do
276278
end do
277279
end do
280+
278281
! END: Computing Stability Criteria at Current Time-step ===========
279282

280283
! Determining local stability criteria extrema at current time-step
@@ -304,7 +307,7 @@ contains
304307
!$acc end kernels
305308
end if
306309
#endif
307-
310+
call nvtxEndRange ! RUNTIME-INFO-COMPUTE-LOCAL
308311
! Determining global stability criteria extrema at current time-step
309312
if (num_procs > 1) then
310313
call s_mpi_reduce_stability_criteria_extrema(icfl_max_loc, &
@@ -362,6 +365,7 @@ contains
362365
end if
363366

364367
call s_mpi_barrier()
368+
call nvtxEndRange ! RUNTIME-INFO
365369

366370
end subroutine s_write_run_time_information
367371

‎src/simulation/m_mpi_proxy.fpp

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -874,6 +874,7 @@ contains
874874
875875
#ifdef MFC_MPI
876876
877+
call nvtxStartRange("RHS-COMM-PACKBUF")
877878
!$acc update device(v_size)
878879
879880
if (qbmm .and. .not. polytropic) then
@@ -1066,6 +1067,7 @@ contains
10661067
#:endif
10671068
end if
10681069
#:endfor
1070+
call nvtxEndRange ! Packbuf
10691071
10701072
! Send/Recv
10711073
#:for rdma_mpi in [False, True]
@@ -1075,28 +1077,34 @@ contains
10751077
#:if rdma_mpi
10761078
!$acc data attach(p_send, p_recv)
10771079
!$acc host_data use_device(p_send, p_recv)
1080+
call nvtxStartRange("RHS-COMM-SENDRECV-RDMA")
10781081
#:else
1082+
call nvtxStartRange("RHS-COMM-DEV2HOST")
10791083
!$acc update host(q_cons_buff_send, ib_buff_send)
1084+
call nvtxEndRange
1085+
call nvtxStartRange("RHS-COMM-SENDRECV-NO-RMDA")
10801086
#:endif
10811087
1082-
call nvtxStartRange("RHS-MPI-SENDRECV")
10831088
call MPI_SENDRECV( &
10841089
p_send, buffer_count, MPI_DOUBLE_PRECISION, dst_proc, send_tag, &
10851090
p_recv, buffer_count, MPI_DOUBLE_PRECISION, src_proc, recv_tag, &
10861091
MPI_COMM_WORLD, MPI_STATUS_IGNORE, ierr)
1087-
call nvtxEndRange
1092+
call nvtxEndRange ! RHS-MPI-SENDRECV-(NO)-RDMA
10881093
10891094
#:if rdma_mpi
10901095
!$acc end host_data
10911096
!$acc end data
10921097
!$acc wait
10931098
#:else
1099+
call nvtxStartRange("RHS-COMM-HOST2DEV")
10941100
!$acc update device(q_cons_buff_recv)
1101+
call nvtxEndRange
10951102
#:endif
10961103
end if
10971104
#:endfor
10981105
10991106
! Unpack Received Buffer
1107+
call nvtxStartRange("RHS-COMM-UNPACKBUF")
11001108
#:for mpi_dir in [1, 2, 3]
11011109
if (mpi_dir == ${mpi_dir}$) then
11021110
#:if mpi_dir == 1
@@ -1265,6 +1273,7 @@ contains
12651273
#:endif
12661274
end if
12671275
#:endfor
1276+
call nvtxEndRange
12681277
12691278
#endif
12701279

‎src/simulation/m_rhs.fpp

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -744,7 +744,7 @@ contains
744744
integer :: i, c, j, k, l, q, ii, id !< Generic loop iterators
745745
integer :: term_index
746746

747-
call nvtxStartRange("Compute_RHS")
747+
call nvtxStartRange("RHS")
748748

749749
! Configuring Coordinate Direction Indexes =========================
750750
ix%beg = -buff_size; iy%beg = 0; iz%beg = 0
@@ -807,9 +807,8 @@ contains
807807
ix, iy, iz)
808808
call nvtxEndRange
809809

810-
call nvtxStartRange("RHS-MPI+BufPack")
810+
call nvtxStartRange("RHS-COMM")
811811
call s_populate_variables_buffers(q_prim_qp%vf, pb, mv)
812-
813812
call nvtxEndRange
814813

815814
if (cfl_dt) then
@@ -822,7 +821,7 @@ contains
822821

823822
if (qbmm) call s_mom_inv(q_cons_qp%vf, q_prim_qp%vf, mom_sp, mom_3d, pb, rhs_pb, mv, rhs_mv, ix, iy, iz, nbub)
824823

825-
call nvtxStartRange("Viscous")
824+
call nvtxStartRange("RHS-VISCOUS")
826825
if (any(Re_size > 0)) call s_get_viscous(qL_rsx_vf, qL_rsy_vf, qL_rsz_vf, &
827826
dqL_prim_dx_n, dqL_prim_dy_n, dqL_prim_dz_n, &
828827
qL_prim, &
@@ -834,7 +833,7 @@ contains
834833
ix, iy, iz)
835834
call nvtxEndRange
836835

837-
call nvtxStartRange("Surface_Tension")
836+
call nvtxStartRange("RHS-SURFACE-TENSION")
838837
if (.not. f_is_default(sigma)) call s_get_capilary(q_prim_qp%vf)
839838
call nvtxEndRange
840839

@@ -923,7 +922,7 @@ contains
923922
end if
924923
ix%end = m; iy%end = n; iz%end = p
925924
! ===============================================================
926-
call nvtxStartRange("RHS_riemann_solver")
925+
call nvtxStartRange("RHS-RIEMANN-SOLVER")
927926

928927
! Computing Riemann Solver Flux and Source Flux =================
929928

@@ -947,7 +946,7 @@ contains
947946
! Additional physics and source terms ==============================
948947

949948
! RHS addition for advection source
950-
call nvtxStartRange("RHS_advection_source")
949+
call nvtxStartRange("RHS-ADVECTION-SOURCE")
951950
call s_compute_advection_source_term(id, &
952951
rhs_vf, &
953952
q_cons_qp, &
@@ -956,14 +955,14 @@ contains
956955
call nvtxEndRange
957956

958957
! RHS additions for hypoelasticity
959-
call nvtxStartRange("RHS_Hypoelasticity")
958+
call nvtxStartRange("RHS-HYPOELASTICITY")
960959
if (hypoelasticity) call s_compute_hypoelastic_rhs(id, &
961960
q_prim_qp%vf, &
962961
rhs_vf)
963962
call nvtxEndRange
964963

965964
! RHS additions for viscosity
966-
call nvtxStartRange("RHS_add_phys")
965+
call nvtxStartRange("RHS-ADD-PHYS")
967966
if (any(Re_size > 0d0) .or. (.not. f_is_default(sigma))) then
968967
call s_compute_additional_physics_rhs(id, &
969968
q_prim_qp%vf, &
@@ -977,13 +976,13 @@ contains
977976
call nvtxEndRange
978977

979978
! RHS additions for sub-grid bubbles
980-
call nvtxStartRange("RHS_bubbles")
979+
call nvtxStartRange("RHS-BUBBLES")
981980
if (bubbles) call s_compute_bubbles_rhs(id, &
982981
q_prim_qp%vf)
983982
call nvtxEndRange
984983

985984
! RHS additions for qbmm bubbles
986-
call nvtxStartRange("RHS_qbmm")
985+
call nvtxStartRange("RHS-QBMM")
987986
if (qbmm) call s_compute_qbmm_rhs(id, &
988987
q_cons_qp%vf, &
989988
q_prim_qp%vf, &
@@ -998,7 +997,7 @@ contains
998997

999998
#:if chemistry
1000999
if (chem_params%advection) then
1001-
call nvtxStartRange("RHS_Chem_Advection")
1000+
call nvtxStartRange("RHS-CHEM-ADVECTION")
10021001

10031002
#:for NORM_DIR, XYZ in [(1, 'x'), (2, 'y'), (3, 'z')]
10041003

@@ -1036,15 +1035,15 @@ contains
10361035

10371036
! Additional Physics and Source Temrs ==================================
10381037
! Additions for acoustic_source
1039-
call nvtxStartRange("RHS_acoustic_src")
1038+
call nvtxStartRange("RHS-ACOUSTIC-SRC")
10401039
if (acoustic_source) call s_acoustic_src_calculations(q_cons_qp%vf(1:sys_size), &
10411040
q_prim_qp%vf(1:sys_size), &
10421041
t_step, &
10431042
rhs_vf)
10441043
call nvtxEndRange
10451044

10461045
! Add bubles source term
1047-
call nvtxStartRange("RHS_bubbles")
1046+
call nvtxStartRange("RHS-BUBBLES-SRC")
10481047
if (bubbles .and. (.not. adap_dt) .and. (.not. qbmm)) call s_compute_bubble_source( &
10491048
q_cons_qp%vf(1:sys_size), &
10501049
q_prim_qp%vf(1:sys_size), &
@@ -1054,7 +1053,7 @@ contains
10541053

10551054
#:if chemistry
10561055
if (chem_params%reactions) then
1057-
call nvtxStartRange("RHS_Chem_Reactions")
1056+
call nvtxStartRange("RHS-CHEM-REACTIONS")
10581057
call s_compute_chemistry_reaction_flux(rhs_vf, q_cons_vf, q_prim_qp%vf)
10591058
call nvtxEndRange
10601059
end if

‎src/simulation/m_time_steppers.fpp

Lines changed: 7 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -329,7 +329,7 @@ contains
329329

330330
! Stage 1 of 1 =====================================================
331331

332-
call nvtxStartRange("Time_Step")
332+
call nvtxStartRange("TSTEP")
333333

334334
call s_compute_rhs(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)
335335

@@ -403,9 +403,7 @@ contains
403403
end do
404404
end if
405405

406-
call nvtxStartRange("body_forces")
407406
if (bodyForces) call s_apply_bodyforces(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, dt)
408-
call nvtxEndRange
409407

410408
if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(1)%vf)
411409

@@ -442,7 +440,7 @@ contains
442440

443441
call cpu_time(start)
444442

445-
call nvtxStartRange("Time_Step")
443+
call nvtxStartRange("TSTEP")
446444

447445
call s_compute_rhs(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)
448446

@@ -508,9 +506,7 @@ contains
508506
end do
509507
end if
510508

511-
call nvtxStartRange("body_forces")
512509
if (bodyForces) call s_apply_bodyforces(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, dt)
513-
call nvtxEndRange
514510

515511
if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf)
516512

@@ -583,9 +579,7 @@ contains
583579
end do
584580
end if
585581

586-
call nvtxStartRange("body_forces")
587582
if (bodyForces) call s_apply_bodyforces(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, 2d0*dt/3d0)
588-
call nvtxEndRange
589583

590584
if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(1)%vf)
591585

@@ -626,7 +620,7 @@ contains
626620

627621
if (.not. adap_dt) then
628622
call cpu_time(start)
629-
call nvtxStartRange("Time_Step")
623+
call nvtxStartRange("TSTEP")
630624
end if
631625

632626
call s_compute_rhs(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)
@@ -693,9 +687,7 @@ contains
693687
end do
694688
end if
695689

696-
call nvtxStartRange("body_forces")
697690
if (bodyForces) call s_apply_bodyforces(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, dt)
698-
call nvtxEndRange
699691

700692
if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf)
701693

@@ -768,9 +760,7 @@ contains
768760
end do
769761
end if
770762

771-
call nvtxStartRange("body_forces")
772763
if (bodyForces) call s_apply_bodyforces(q_cons_ts(2)%vf, q_prim_vf, rhs_vf, dt/4d0)
773-
call nvtxEndRange
774764

775765
if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf)
776766

@@ -842,9 +832,7 @@ contains
842832
end do
843833
end if
844834

845-
call nvtxStartRange("body_forces")
846835
if (bodyForces) call s_apply_bodyforces(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, 2d0*dt/3d0)
847-
call nvtxEndRange
848836

849837
if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(1)%vf)
850838

@@ -886,7 +874,7 @@ contains
886874

887875
call cpu_time(start)
888876

889-
call nvtxStartRange("Time_Step")
877+
call nvtxStartRange("TSTEP")
890878

891879
! Stage 1 of 3 =====================================================
892880
call s_adaptive_dt_bubble(t_step)
@@ -998,6 +986,8 @@ contains
998986

999987
integer :: i, j, k, l
1000988

989+
call nvtxStartRange("TSTEP-BODYFORCES")
990+
1001991
call s_compute_body_forces_rhs(q_prim_vf, q_cons_vf, rhs_vf)
1002992

1003993
!$acc parallel loop collapse(4) gang vector default(present)
@@ -1011,6 +1001,7 @@ contains
10111001
end do
10121002
end do
10131003
end do
1004+
call nvtxEndRange
10141005

10151006
end subroutine s_apply_bodyforces
10161007

0 commit comments

Comments
 (0)