From a21d9acdfb4c8df285d1212eb0f20d82a90004f5 Mon Sep 17 00:00:00 2001 From: Ben Wilfong <48168887+wilfonba@users.noreply.github.com> Date: Tue, 12 Sep 2023 23:40:27 -0400 Subject: [PATCH] Final touches fixed tests not passing loop reordering fix bug in the saving of performance data removed too much code --- src/common/m_derived_types.fpp | 11 - src/post_process/m_checker.f90 | 14 +- src/post_process/m_global_parameters.fpp | 16 +- src/pre_process/m_checker.f90 | 14 +- src/pre_process/m_global_parameters.fpp | 16 +- src/simulation/m_boundary_conditions.fpp | 252 +++++++++++------------ src/simulation/m_checker.fpp | 12 +- src/simulation/m_global_parameters.fpp | 16 +- src/simulation/m_mpi_proxy.fpp | 2 +- src/simulation/m_start_up.fpp | 98 +++++---- src/simulation/m_weno.fpp | 2 +- src/simulation/p_main.fpp | 2 + toolchain/mfc/run/case_dicts.py | 9 - 13 files changed, 216 insertions(+), 248 deletions(-) diff --git a/src/common/m_derived_types.fpp b/src/common/m_derived_types.fpp index 929b804d1..f1bb1a7b9 100644 --- a/src/common/m_derived_types.fpp +++ b/src/common/m_derived_types.fpp @@ -43,17 +43,6 @@ module m_derived_types integer :: end end type int_bounds_info - type bc_bounds_info - integer :: beg - integer :: end - real(kind(0d0)) :: vel1b - real(kind(0d0)) :: vel2b - real(kind(0d0)) :: vel3b - real(kind(0d0)) :: vel1e - real(kind(0d0)) :: vel2e - real(kind(0d0)) :: vel3e - end type bc_bounds_info - !> Derived type adding beginning (beg) and end bounds info as attributes type bounds_info real(kind(0d0)) :: beg diff --git a/src/post_process/m_checker.f90 b/src/post_process/m_checker.f90 index 1d6dcce1d..2c252c8c7 100644 --- a/src/post_process/m_checker.f90 +++ b/src/post_process/m_checker.f90 @@ -106,10 +106,10 @@ subroutine s_check_inputs() 'weno_order. Exiting ...') ! Constraints on the boundary conditions in the x-direction - elseif (bc_x%beg < -17 .or. bc_x%beg > -1 .or. bc_x%beg == -14) then + elseif (bc_x%beg < -16 .or. bc_x%beg > -1 .or. bc_x%beg == -14) then call s_mpi_abort('Unsupported choice for the value of '// & 'bc_x%beg. Exiting ...') - elseif (bc_x%end < -17 .or. bc_x%end > -1 .or. bc_x%beg == -14) then + elseif (bc_x%end < -16 .or. bc_x%end > -1 .or. bc_x%beg == -14) then call s_mpi_abort('Unsupported choice for the value of '// & 'bc_x%end. Exiting ...') elseif ((bc_x%beg == -1 .and. bc_x%end /= -1) & @@ -126,16 +126,16 @@ subroutine s_check_inputs() .or. & (cyl_coord .and. p == 0)) & .and. & - (bc_y%beg < -17 .or. bc_y%beg > -1 .or. bc_y%beg == -14)) & + (bc_y%beg < -16 .or. bc_y%beg > -1 .or. bc_y%beg == -14)) & .or. & (cyl_coord .and. p > 0 & .and. & - (bc_y%beg < -17 .or. bc_y%beg > -1)))) then + (bc_y%beg < -16 .or. bc_y%beg > -1)))) then call s_mpi_abort('Unsupported choice for the value of '// & 'bc_y%beg. Exiting ...') elseif (bc_y%end /= dflt_int & .and. & - (bc_y%end < -17 .or. bc_y%end > -1 .or. bc_y%end == -14)) then + (bc_y%end < -16 .or. bc_y%end > -1 .or. bc_y%end == -14)) then call s_mpi_abort('Unsupported choice for the value of '// & 'bc_y%end. Exiting ...') elseif ((n == 0 .and. bc_y%beg /= dflt_int) & @@ -160,12 +160,12 @@ subroutine s_check_inputs() ! Constraints on the boundary conditions in the z-direction elseif (bc_z%beg /= dflt_int & .and. & - (bc_z%beg < -17 .or. bc_z%beg > -1 .or. bc_z%beg == -14)) then + (bc_z%beg < -16 .or. bc_z%beg > -1 .or. bc_z%beg == -14)) then call s_mpi_abort('Unsupported choice for the value of '// & 'bc_z%beg. Exiting ...') elseif (bc_z%end /= dflt_int & .and. & - (bc_z%end < -17 .or. bc_z%end > -1 .or. bc_z%end == -14)) then + (bc_z%end < -16 .or. bc_z%end > -1 .or. bc_z%end == -14)) then call s_mpi_abort('Unsupported choice for the value of '// & 'bc_z%end. Exiting ...') elseif (any((/bc_x%beg, bc_x%end, bc_y%beg, bc_y%end, bc_z%beg, bc_z%end/) == -13)) then diff --git a/src/post_process/m_global_parameters.fpp b/src/post_process/m_global_parameters.fpp index 87ee8bafc..bc2240643 100644 --- a/src/post_process/m_global_parameters.fpp +++ b/src/post_process/m_global_parameters.fpp @@ -107,7 +107,7 @@ module m_global_parameters !> @name Boundary conditions in the x-, y- and z-coordinate directions !> @{ - type(bc_bounds_info) :: bc_x, bc_y, bc_z + type(int_bounds_info) :: bc_x, bc_y, bc_z !> @} logical :: parallel_io !< Format of the data files @@ -271,17 +271,9 @@ contains alt_soundspeed = .false. hypoelasticity = .false. - #:for DIR in ['x', 'y', 'z'] - #:for PARAM in ['beg', 'end'] - bc_${DIR}$%${PARAM}$ = dflt_int - #:endfor - #:endfor - - #:for DIR in ['x', 'y', 'z'] - #:for PARAM in ['vel1b', 'vel2b', 'vel3b', 'vel1e', 'vel2e', 'vel3e'] - bc_${DIR}$%${PARAM}$ = dflt_real - #:endfor - #:endfor + bc_x%beg = dflt_int; bc_x%end = dflt_int + bc_y%beg = dflt_int; bc_y%end = dflt_int + bc_z%beg = dflt_int; bc_z%end = dflt_int ! Fluids physical parameters do i = 1, num_fluids_max diff --git a/src/pre_process/m_checker.f90 b/src/pre_process/m_checker.f90 index adbc921f3..91ea228bd 100644 --- a/src/pre_process/m_checker.f90 +++ b/src/pre_process/m_checker.f90 @@ -403,10 +403,10 @@ subroutine s_check_inputs() 'weno_order. Exiting ...') ! Constraints on the boundary conditions in the x-direction - elseif (bc_x%beg < -17 .or. bc_x%beg > -1 .or. bc_x%beg == -14) then + elseif (bc_x%beg < -16 .or. bc_x%beg > -1 .or. bc_x%beg == -14) then call s_mpi_abort('Unsupported choice for the value of '// & 'bc_x%beg. Exiting ...') - elseif (bc_x%end < -17 .or. bc_x%end > -1 .or. bc_x%beg == -14) then + elseif (bc_x%end < -16 .or. bc_x%end > -1 .or. bc_x%beg == -14) then call s_mpi_abort('Unsupported choice for the value of '// & 'bc_x%end. Exiting ...') elseif ((bc_x%beg == -1 .and. bc_x%end /= -1) & @@ -429,7 +429,7 @@ subroutine s_check_inputs() 'bc_y%beg. Exiting ...') elseif (bc_y%end /= dflt_int & .and. & - (bc_y%end < -17 .or. bc_y%end > -1 .or. bc_y%end == -14)) then + (bc_y%end < -16 .or. bc_y%end > -1 .or. bc_y%end == -14)) then call s_mpi_abort('Unsupported choice for the value of '// & 'bc_y%end. Exiting ...') elseif ((n > 0 .and. bc_y%beg == dflt_int)) then @@ -475,12 +475,12 @@ subroutine s_check_inputs() ! Constraints on the boundary conditions in the y-direction if (bc_y%beg /= dflt_int & .and. & - (bc_y%beg < -17 .or. bc_y%beg > -1 .or. bc_y%beg == -14)) then + (bc_y%beg < -16 .or. bc_y%beg > -1 .or. bc_y%beg == -14)) then call s_mpi_abort('Unsupported choice for the value of '// & 'bc_y%beg. Exiting ...') elseif (bc_y%end /= dflt_int & .and. & - (bc_y%end < -17 .or. bc_y%end > -1 .or. bc_y%end == -14)) then + (bc_y%end < -16 .or. bc_y%end > -1 .or. bc_y%end == -14)) then call s_mpi_abort('Unsupported choice for the value of '// & 'bc_y%end. Exiting ...') elseif ((n == 0 .and. bc_y%beg /= dflt_int) & @@ -505,12 +505,12 @@ subroutine s_check_inputs() ! Constraints on the boundary conditions in the z-direction elseif (bc_z%beg /= dflt_int & .and. & - (bc_z%beg < -17 .or. bc_z%beg > -1 .or. bc_z%beg == -14)) then + (bc_z%beg < -16 .or. bc_z%beg > -1 .or. bc_z%beg == -14)) then call s_mpi_abort('Unsupported choice for the value of '// & 'bc_z%beg. Exiting ...') elseif (bc_z%end /= dflt_int & .and. & - (bc_z%end < -17 .or. bc_z%end > -1 .or. bc_z%end == -14)) then + (bc_z%end < -16 .or. bc_z%end > -1 .or. bc_z%end == -14)) then call s_mpi_abort('Unsupported choice for the value of '// & 'bc_z%end. Exiting ...') elseif (any((/bc_x%beg, bc_x%end, bc_y%beg, bc_y%end, bc_z%beg, bc_z%end/) == -13)) then diff --git a/src/pre_process/m_global_parameters.fpp b/src/pre_process/m_global_parameters.fpp index 46a30903c..3958b4ffd 100644 --- a/src/pre_process/m_global_parameters.fpp +++ b/src/pre_process/m_global_parameters.fpp @@ -91,7 +91,7 @@ module m_global_parameters integer :: pi_inf_idx !< Index of liquid stiffness func. eqn. type(int_bounds_info) :: stress_idx !< Indexes of elastic shear stress eqns. - type(bc_bounds_info) :: bc_x, bc_y, bc_z !< + type(int_bounds_info) :: bc_x, bc_y, bc_z !< !! Boundary conditions in the x-, y- and z-coordinate directions logical :: parallel_io !< Format of the data files @@ -244,17 +244,9 @@ contains hypoelasticity = .false. - #:for DIR in ['x', 'y', 'z'] - #:for PARAM in ['beg', 'end'] - bc_${DIR}$%${PARAM}$ = dflt_int - #:endfor - #:endfor - - #:for DIR in ['x', 'y', 'z'] - #:for PARAM in ['vel1b', 'vel2b', 'vel3b', 'vel1e', 'vel2e', 'vel3e'] - bc_${DIR}$%${PARAM}$ = dflt_real - #:endfor - #:endfor + bc_x%beg = dflt_int; bc_x%end = dflt_int + bc_y%beg = dflt_int; bc_y%end = dflt_int + bc_z%beg = dflt_int; bc_z%end = dflt_int parallel_io = .false. precision = 2 diff --git a/src/simulation/m_boundary_conditions.fpp b/src/simulation/m_boundary_conditions.fpp index 12eeec390..389ebe1b1 100644 --- a/src/simulation/m_boundary_conditions.fpp +++ b/src/simulation/m_boundary_conditions.fpp @@ -173,12 +173,12 @@ module m_boundary_conditions end do if(qbmm .and. .not. polytropic) then - !$acc parallel loop collapse(4) gang vector default(present) + !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do l = 0, p - do k = 0, n - do j = 1, buff_size - do q = 1, nnode + do q = 1, nnode + do l = 0, p + do k = 0, n + do j = 1, buff_size pb(-j, k, l, q, i) = & pb(0, k, l, q, i) mv(-j, k, l, q, i) = & @@ -207,10 +207,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do l = 0, p - do k = 0, n - do j = 1, buff_size - do q = 1, nnode + do q = 1, nnode + do l = 0, p + do k = 0, n + do j = 1, buff_size pb(m + j, k, l, q, i) = & pb(m, k, l, q, i) mv(m + j, k, l, q, i) = & @@ -244,10 +244,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do k = 0, p - do j = 1, buff_size - do l = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do k = 0, p + do j = 1, buff_size + do l = -buff_size, m + buff_size pb(l, -j, k, q, i) = & pb(l, 0, k, q, i) mv(l, -j, k, q, i) = & @@ -276,10 +276,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do k = 0, p - do j = 1, buff_size - do l = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do k = 0, p + do j = 1, buff_size + do l = -buff_size, m + buff_size pb(l, n + j, k, q, i) = & pb(l, n , k, q, i) mv(l, n + j, k, q, i) = & @@ -313,10 +313,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do j = 1, buff_size - do l = -buff_size, n + buff_size - do k = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do j = 1, buff_size + do l = -buff_size, n + buff_size + do k = -buff_size, m + buff_size pb(k, l, -j, q, i) = & pb(k, l, 0, q, i) mv(k, l, -j, q, i) = & @@ -345,10 +345,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do j = 1, buff_size - do l = -buff_size, n + buff_size - do k = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do j = 1, buff_size + do l = -buff_size, n + buff_size + do k = -buff_size, m + buff_size pb(k, l, p+j, q, i) = & pb(k, l, p, q, i) mv(k, l, p+j, q, i) = & @@ -404,10 +404,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do l = 0, p - do k = 0, n - do j = 1, buff_size - do q = 1, nnode + do q = 1, nnode + do l = 0, p + do k = 0, n + do j = 1, buff_size pb(-j, k, l, q, i) = & pb(j - 1, k, l, q, i) mv(-j, k, l, q, i) = & @@ -448,10 +448,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do l = 0, p - do k = 0, n - do j = 1, buff_size - do q = 1, nnode + do q = 1, nnode + do l = 0, p + do k = 0, n + do j = 1, buff_size pb(m + j, k, l, q, i) = & pb(m - (j - 1), k, l, q, i) mv(m + j, k, l, q, i) = & @@ -495,10 +495,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do k = 0, p - do j = 1, buff_size - do l = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do k = 0, p + do j = 1, buff_size + do l = -buff_size, m + buff_size pb(l, -j, k, q, i) = & pb(l, j - 1, k, q, i) mv(l, -j, k, q, i) = & @@ -537,10 +537,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do k = 0, p - do j = 1, buff_size - do l = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do k = 0, p + do j = 1, buff_size + do l = -buff_size, m + buff_size pb(l, n + j, k, q, i) = & pb(l, n - (j-1), k, q, i) mv(l, n + j, k, q, i) = & @@ -584,10 +584,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do j = 1, buff_size - do l = -buff_size, n + buff_size - do k = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do j = 1, buff_size + do l = -buff_size, n + buff_size + do k = -buff_size, m + buff_size pb(k, l, -j, q, i) = & pb(k, l, j-1, q, i) mv(k, l, -j, q, i) = & @@ -626,10 +626,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do j = 1, buff_size - do l = -buff_size, n + buff_size - do k = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do j = 1, buff_size + do l = -buff_size, n + buff_size + do k = -buff_size, m + buff_size pb(k, l, p + j, q, i) = & pb(k, l, p - (j-1), q, i) mv(k, l, p + j, q, i) = & @@ -675,10 +675,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do l = 0, p - do k = 0, n - do j = 1, buff_size - do q = 1, nnode + do q = 1, nnode + do l = 0, p + do k = 0, n + do j = 1, buff_size pb(-j, k, l, q, i) = & pb(m - (j - 1), k, l, q, i) mv(-j, k, l, q, i) = & @@ -707,10 +707,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do l = 0, p - do k = 0, n - do j = 1, buff_size - do q = 1, nnode + do q = 1, nnode + do l = 0, p + do k = 0, n + do j = 1, buff_size pb(m + j, k, l, q, i) = & pb(j - 1, k, l, q, i) mv(m + j, k, l, q, i) = & @@ -744,10 +744,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(4) gang vector default(present) do i = 1, nb - do k = 0, p - do j = 1, buff_size - do l = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do k = 0, p + do j = 1, buff_size + do l = -buff_size, m + buff_size pb(l, -j, k, q, i) = & pb(l, n - (j-1), k, q, i) mv(l, -j, k, q, i) = & @@ -776,10 +776,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do k = 0, p - do j = 1, buff_size - do l = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do k = 0, p + do j = 1, buff_size + do l = -buff_size, m + buff_size pb(l, n + j, k, q, i) = & pb(l, (j-1), k, q, i) mv(l, n + j, k, q, i) = & @@ -813,10 +813,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do j = 1, buff_size - do l = -buff_size, n + buff_size - do k = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do j = 1, buff_size + do l = -buff_size, n + buff_size + do k = -buff_size, m + buff_size pb(k, l, -j, q, i) = & pb(k, l, p - (j-1), q, i) mv(k, l, -j, q, i) = & @@ -845,10 +845,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do j = 1, buff_size - do l = -buff_size, n + buff_size - do k = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do j = 1, buff_size + do l = -buff_size, n + buff_size + do k = -buff_size, m + buff_size pb(k, l, p + j, q, i) = & pb(k, l, j-1, q, i) mv(k, l, p + j, q, i) = & @@ -922,10 +922,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do k = 0, p - do j = 1, buff_size - do l = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do k = 0, p + do j = 1, buff_size + do l = -buff_size, m + buff_size pb(l, -j, k, q, i) = & pb(l, j-1, k - ((p+1)/2), q, i) mv(l, -j, k, q, i) = & @@ -971,10 +971,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(4) gang vector default(present) do i = 1, nb - do l = 0, p - do k = 0, n - do j = 1, buff_size - do q = 1, nnode + do q = 1, nnode + do l = 0, p + do k = 0, n + do j = 1, buff_size pb(-j, k, l, q, i) = & pb(0, k, l, q, i) mv(-j, k, l, q, i) = & @@ -1008,10 +1008,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do l = 0, p - do k = 0, n - do j = 1, buff_size - do q = 1, nnode + do q = 1, nnode + do l = 0, p + do k = 0, n + do j = 1, buff_size pb(m + j, k, l, q, i) = & pb(m, k, l, q, i) mv(m + j, k, l, q, i) = & @@ -1050,10 +1050,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do k = 0, p - do j = 1, buff_size - do l = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do k = 0, p + do j = 1, buff_size + do l = -buff_size, m + buff_size pb(l, -j, k, q, i) = & pb(l, 0, k, q, i) mv(l, -j, k, q, i) = & @@ -1087,10 +1087,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do k = 0, p - do j = 1, buff_size - do l = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do k = 0, p + do j = 1, buff_size + do l = -buff_size, m + buff_size pb(l, n + j, k, q, i) = & pb(l, n , k, q, i) mv(l, n + j, k, q, i) = & @@ -1129,10 +1129,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do j = 1, buff_size - do l = -buff_size, n + buff_size - do k = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do j = 1, buff_size + do l = -buff_size, n + buff_size + do k = -buff_size, m + buff_size pb(k, l, -j, q, i) = & pb(k, l, 0, q, i) mv(k, l, -j, q, i) = & @@ -1166,10 +1166,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do j = 1, buff_size - do l = -buff_size, n + buff_size - do k = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do j = 1, buff_size + do l = -buff_size, n + buff_size + do k = -buff_size, m + buff_size pb(k, l, p+j, q, i) = & pb(k, l, p, q, i) mv(k, l, p+j, q, i) = & @@ -1220,10 +1220,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(4) gang vector default(present) do i = 1, nb - do l = 0, p - do k = 0, n - do j = 1, buff_size - do q = 1, nnode + do q = 1, nnode + do l = 0, p + do k = 0, n + do j = 1, buff_size pb(-j, k, l, q, i) = & pb(0, k, l, q, i) mv(-j, k, l, q, i) = & @@ -1257,10 +1257,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do l = 0, p - do k = 0, n - do j = 1, buff_size - do q = 1, nnode + do q = 1, nnode + do l = 0, p + do k = 0, n + do j = 1, buff_size pb(m + j, k, l, q, i) = & pb(m, k, l, q, i) mv(m + j, k, l, q, i) = & @@ -1299,10 +1299,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do k = 0, p - do j = 1, buff_size - do l = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do k = 0, p + do j = 1, buff_size + do l = -buff_size, m + buff_size pb(l, -j, k, q, i) = & pb(l, 0, k, q, i) mv(l, -j, k, q, i) = & @@ -1336,10 +1336,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do k = 0, p - do j = 1, buff_size - do l = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do k = 0, p + do j = 1, buff_size + do l = -buff_size, m + buff_size pb(l, n + j, k, q, i) = & pb(l, n , k, q, i) mv(l, n + j, k, q, i) = & @@ -1378,10 +1378,10 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do j = 1, buff_size - do l = -buff_size, n + buff_size - do k = -buff_size, m + buff_size - do q = 1, nnode + do q = 1, nnode + do j = 1, buff_size + do l = -buff_size, n + buff_size + do k = -buff_size, m + buff_size pb(k, l, -j, q, i) = & pb(k, l, 0, q, i) mv(k, l, -j, q, i) = & @@ -1415,11 +1415,11 @@ module m_boundary_conditions if(qbmm .and. .not. polytropic) then !$acc parallel loop collapse(5) gang vector default(present) do i = 1, nb - do j = 1, buff_size - do l = -buff_size, n + buff_size - do k = -buff_size, m + buff_size - do q = 1, nnode - pb(k, l, p+j, q, i) = & + do q = 1, nnode + do j = 1, buff_size + do l = -buff_size, n + buff_size + do k = -buff_size, m + buff_size + pb(k, l, p+ j, q, i) = & pb(k, l, p, q, i) mv(k, l, p+j, q, i) = & mv(k, l, p, q, i) diff --git a/src/simulation/m_checker.fpp b/src/simulation/m_checker.fpp index 17423cd2d..8126701d2 100644 --- a/src/simulation/m_checker.fpp +++ b/src/simulation/m_checker.fpp @@ -198,9 +198,9 @@ contains call s_mpi_abort('Unsupported combination of values of '// & 'riemann_solver and avg_state. '// & 'Exiting ...') - elseif (bc_x%beg < -17 .or. bc_x%beg > -1 .or. bc_x%beg == -14) then + elseif (bc_x%beg < -16 .or. bc_x%beg > -1 .or. bc_x%beg == -14) then call s_mpi_abort('Unsupported value of bc_x%beg. Exiting ...') - elseif (bc_x%end < -17 .or. bc_x%end > -1 .or. bc_x%beg == -14) then + elseif (bc_x%end < -16 .or. bc_x%end > -1 .or. bc_x%beg == -14) then call s_mpi_abort('Unsupported value of bc_x%end. Exiting ...') elseif ((bc_x%beg == -1 .and. bc_x%end /= -1) & .or. & @@ -209,7 +209,7 @@ contains 'bc_x%beg and bc_x%end. Exiting ...') elseif (bc_y%beg /= dflt_int & .and. & - (((cyl_coord .neqv. .true.) .and. (bc_y%beg < -17 .or. bc_y%beg > -1 .or. bc_y%beg == -14)) & + (((cyl_coord .neqv. .true.) .and. (bc_y%beg < -16 .or. bc_y%beg > -1 .or. bc_y%beg == -14)) & .or. & (cyl_coord .and. p == 0 .and. bc_y%beg /= -2) & .or. & @@ -217,7 +217,7 @@ contains call s_mpi_abort('Unsupported value of bc_y%beg. Exiting ...') elseif (bc_y%end /= dflt_int & .and. & - (bc_y%end < -17 .or. bc_y%end > -1 .or. bc_y%end == -14)) then + (bc_y%end < -16 .or. bc_y%end > -1 .or. bc_y%end == -14)) then call s_mpi_abort('Unsupported value of bc_y%end. Exiting ...') elseif ((n == 0 .and. bc_y%beg /= dflt_int) & .or. & @@ -236,13 +236,13 @@ contains 'bc_y%beg and bc_y%end. Exiting ...') elseif (bc_z%beg /= dflt_int & .and. & - (bc_z%beg < -17 .or. bc_z%beg > -1 .or. bc_z%beg == -14)) then + (bc_z%beg < -16 .or. bc_z%beg > -1 .or. bc_z%beg == -14)) then call s_mpi_abort('Unsupported value of bc_z%beg. Exiting ...') elseif (any((/bc_x%beg, bc_x%end, bc_y%beg, bc_y%end, bc_z%beg, bc_z%end/) == -13)) then call s_mpi_abort('Unsupported coice of boundary condition -13') elseif (bc_z%end /= dflt_int & .and. & - (bc_z%end < -17 .or. bc_z%end > -1 .or. bc_z%end == -14)) then + (bc_z%end < -16 .or. bc_z%end > -1 .or. bc_z%end == -14)) then call s_mpi_abort('Unsupported value of bc_z%end. Exiting ...') elseif ((p == 0 .and. bc_z%beg /= dflt_int) & .or. & diff --git a/src/simulation/m_global_parameters.fpp b/src/simulation/m_global_parameters.fpp index 5f1158279..a49b78bd2 100644 --- a/src/simulation/m_global_parameters.fpp +++ b/src/simulation/m_global_parameters.fpp @@ -130,7 +130,7 @@ module m_global_parameters !> @name Boundary conditions (BC) in the x-, y- and z-directions, respectively !> @{ - type(bc_bounds_info) :: bc_x, bc_y, bc_z + type(int_bounds_info) :: bc_x, bc_y, bc_z !> @} logical :: parallel_io !< Format of the data files @@ -379,17 +379,9 @@ contains riemann_flat = .true. cu_mpi = .false. - #:for DIR in ['x', 'y', 'z'] - #:for PARAM in ['beg', 'end'] - bc_${DIR}$%${PARAM}$ = dflt_int - #:endfor - #:endfor - - #:for DIR in ['x', 'y', 'z'] - #:for PARAM in ['vel1b', 'vel2b', 'vel3b', 'vel1e', 'vel2e', 'vel3e'] - bc_${DIR}$%${PARAM}$ = dflt_real - #:endfor - #:endfor + bc_x%beg = dflt_int; bc_x%end = dflt_int + bc_y%beg = dflt_int; bc_y%end = dflt_int + bc_z%beg = dflt_int; bc_z%end = dflt_int ! x_domain%beg = dflt_int; x_domain%end = dflt_int; ! y_domain%beg = dflt_int; y_domain%end = dflt_int; diff --git a/src/simulation/m_mpi_proxy.fpp b/src/simulation/m_mpi_proxy.fpp index 828abc612..bfb163cb3 100644 --- a/src/simulation/m_mpi_proxy.fpp +++ b/src/simulation/m_mpi_proxy.fpp @@ -141,7 +141,7 @@ contains #:endfor #:for VAR in [ 'dt','weno_eps','pref','rhoref','R0ref','Web','Ca', & - & 'Re_inv','poly_sigma', 'bc_x%vel1b', 'bc_x%vel2b', 'bc_x%vel3b' ] + & 'Re_inv','poly_sigma' ] call MPI_BCAST(${VAR}$, 1, MPI_DOUBLE_PRECISION, 0, MPI_COMM_WORLD, ierr) #:endfor diff --git a/src/simulation/m_start_up.fpp b/src/simulation/m_start_up.fpp index ee026601a..ca1f29208 100644 --- a/src/simulation/m_start_up.fpp +++ b/src/simulation/m_start_up.fpp @@ -79,7 +79,8 @@ module m_start_up s_initialize_internal_energy_equations, & s_initialize_modules, s_initialize_gpu_vars, & s_initialize_mpi_domain, s_finalize_modules, & - s_perform_time_step, s_save_data + s_perform_time_step, s_save_data, & + s_save_performance_metrics abstract interface ! =================================================== @@ -838,59 +839,67 @@ contains end if ! Time-stepping loop controls + if ((mytime + dt) >= finaltime) dt = finaltime - mytime + t_step = t_step + 1 - if (t_step == t_step_stop) then + end subroutine s_perform_time_step - call s_mpi_barrier() + subroutine s_save_performance_metrics(t_step, time_avg, time_final, io_time_avg, io_time_final, proc_time, io_proc_time, file_exists, start, finish, nt) - if (num_procs > 1) then - call mpi_bcast_time_step_values(proc_time, time_avg) + integer, intent(INOUT) :: t_step + real(kind(0d0)), intent(INOUT) :: time_avg, time_final + real(kind(0d0)), intent(INOUT) :: io_time_avg, io_time_final + real(kind(0d0)), dimension(:), intent(INOUT) :: proc_time + real(kind(0d0)), dimension(:), intent(INOUT) :: io_proc_time + logical, intent(INOUT) :: file_exists + real(kind(0d0)), intent(INOUT) :: start, finish + integer, intent(INOUT) :: nt - call mpi_bcast_time_step_values(io_proc_time, io_time_avg) - end if + call s_mpi_barrier() - if (proc_rank == 0) then - time_final = 0d0 - io_time_final = 0d0 - if (num_procs == 1) then - time_final = time_avg - io_time_final = io_time_avg - print *, "Final Time", time_final - else - time_final = maxval(proc_time) - io_time_final = maxval(io_proc_time) - print *, "Final Time", time_final - end if - inquire (FILE='time_data.dat', EXIST=file_exists) - if (file_exists) then - open (1, file='time_data.dat', position='append', status='old') - write (1, *) num_procs, time_final - close (1) - else - open (1, file='time_data.dat', status='new') - write (1, *) num_procs, time_final - close (1) - end if + if (num_procs > 1) then + call mpi_bcast_time_step_values(proc_time, time_avg) - inquire (FILE='io_time_data.dat', EXIST=file_exists) - if (file_exists) then - open (1, file='io_time_data.dat', position='append', status='old') - write (1, *) num_procs, io_time_final - close (1) - else - open (1, file='io_time_data.dat', status='new') - write (1, *) num_procs, io_time_final - close (1) - end if + call mpi_bcast_time_step_values(io_proc_time, io_time_avg) + end if + + if (proc_rank == 0) then + time_final = 0d0 + io_time_final = 0d0 + if (num_procs == 1) then + time_final = time_avg + io_time_final = io_time_avg + print *, "Final Time", time_final + else + time_final = maxval(proc_time) + io_time_final = maxval(io_proc_time) + print *, "Final Time", time_final + end if + inquire (FILE='time_data.dat', EXIST=file_exists) + if (file_exists) then + open (1, file='time_data.dat', position='append', status='old') + write (1, *) num_procs, time_final + close (1) + else + open (1, file='time_data.dat', status='new') + write (1, *) num_procs, time_final + close (1) + end if + inquire (FILE='io_time_data.dat', EXIST=file_exists) + if (file_exists) then + open (1, file='io_time_data.dat', position='append', status='old') + write (1, *) num_procs, io_time_final + close (1) + else + open (1, file='io_time_data.dat', status='new') + write (1, *) num_procs, io_time_final + close (1) end if - - else - if ((mytime + dt) >= finaltime) dt = finaltime - mytime - t_step = t_step + 1 end if - end subroutine s_perform_time_step + + end subroutine s_save_performance_metrics subroutine s_save_data(t_step, start, finish, io_time_avg, nt) real(kind(0d0)), intent(INOUT) :: start, finish, io_time_avg @@ -930,6 +939,7 @@ contains io_time_avg = (abs(finish - start) + io_time_avg*(nt - 1))/nt end if end if + end subroutine s_save_data subroutine s_initialize_modules() diff --git a/src/simulation/m_weno.fpp b/src/simulation/m_weno.fpp index e661ed090..940a2fac4 100644 --- a/src/simulation/m_weno.fpp +++ b/src/simulation/m_weno.fpp @@ -238,7 +238,7 @@ contains real(kind(0d0)), pointer, dimension(:) :: s_cb => null() !< !! Cell-boundary locations in the s-direction - type(bc_bounds_info) :: bc_s !< Boundary conditions (BC) in the s-direction + type(int_bounds_info) :: bc_s !< Boundary conditions (BC) in the s-direction integer :: i !< Generic loop iterator diff --git a/src/simulation/p_main.fpp b/src/simulation/p_main.fpp index 68af1f9f8..89824207d 100644 --- a/src/simulation/p_main.fpp +++ b/src/simulation/p_main.fpp @@ -57,6 +57,8 @@ program p_main ! Time-stepping Loop ======================================================= do if (t_step == t_step_stop) then + call s_save_performance_metrics(t_step, time_avg, time_final, io_time_avg, & + io_time_final, proc_time, io_proc_time, file_exists, start, finish, nt) exit end if diff --git a/toolchain/mfc/run/case_dicts.py b/toolchain/mfc/run/case_dicts.py index f626b4826..1a753d1b9 100644 --- a/toolchain/mfc/run/case_dicts.py +++ b/toolchain/mfc/run/case_dicts.py @@ -28,9 +28,6 @@ PRE_PROCESS.append(f"bc_{cmp}%beg") PRE_PROCESS.append(f"bc_{cmp}%end") - for var in ["vel1b", "vel2b", "vel3b", "vel1e", "vel2e", "vel3e"]: - PRE_PROCESS.append(f"bc_{cmp}%{var}") - for f_id in range(1, 10+1): PRE_PROCESS.append(f'fluid_rho({f_id})') @@ -90,9 +87,6 @@ SIMULATION.append(f'bc_{cmp}%beg') SIMULATION.append(f'bc_{cmp}%end') - for var in ["vel1b", "vel2b", "vel3b", "vel1e", "vel2e", "vel3e"]: - SIMULATION.append(f"bc_{cmp}%{var}") - for wrt_id in range(1,10+1): for cmp in ["x", "y", "z"]: SIMULATION.append(f'probe_wrt({wrt_id})%{cmp}') @@ -139,9 +133,6 @@ POST_PROCESS.append(f'bc_{cmp}%beg') POST_PROCESS.append(f'bc_{cmp}%end') - for var in ["vel1b", "vel2b", "vel3b", "vel1e", "vel2e", "vel3e"]: - POST_PROCESS.append(f"bc_{cmp}%{var}") - for attribute in ["mom_wrt", "vel_wrt", "flux_wrt", "omega_wrt"]: POST_PROCESS.append(f'{attribute}({cmp_id})')