From bb77a24531a5d079e4393e356be52ec3d3b2d730 Mon Sep 17 00:00:00 2001 From: wilfonba Date: Sun, 8 Dec 2024 16:48:01 -0500 Subject: [PATCH] builds on frontier but doesn't run ): --- src/simulation/m_bubbles_EL.fpp | 108 +++++++++++++++++------- src/simulation/m_bubbles_EL_kernels.fpp | 4 +- src/simulation/m_time_steppers.fpp | 2 +- 3 files changed, 81 insertions(+), 33 deletions(-) diff --git a/src/simulation/m_bubbles_EL.fpp b/src/simulation/m_bubbles_EL.fpp index 9db7145e6..380711cab 100644 --- a/src/simulation/m_bubbles_EL.fpp +++ b/src/simulation/m_bubbles_EL.fpp @@ -471,6 +471,17 @@ contains type(scalar_field), dimension(sys_size), intent(inout) :: rhs_vf integer, intent(in) :: stage + ! manual inline of s_compute_KM + real(kind(0.d0)) :: pliqint, pbubble, deltaP, pinf, termI, aux1 + real(kind(0.d0)) :: aux2, velint, rhol, cson, E, H, qv, gamma, pi_inf + real(kind(0d0)), dimension(num_fluids) :: alpha_rho, alpha + integer, dimension(3) :: cell + real(kind(0.0d0)), dimension(3) :: scoord + + real(kind(0.d0)), dimension(num_dims) :: vel + real(kind(0.d0)), dimension(2) :: Re + ! end inline + real(kind(0.d0)) :: gammaparticle, vaporflux, heatflux integer :: i, j, k, l real(kind(0.d0)) :: preterm1, term2, paux, pint, Romega, term1_fac, Rb @@ -511,9 +522,43 @@ contains ! Radial motion model if (lag_params%bubble_model == 1) then - !$acc parallel loop gang vector default(present) private(k) copyin(stage) + !$acc parallel loop gang vector default(present) private(scoord,alpha,alpha_rho,cell,Re,vel) copyin(stage) do k = 1, nBubs - call s_compute_KM(k, stage, q_prim_vf) + !call s_compute_KM(k, stage, q_prim_vf) + + ! Manual inline of s_compute_KM + pliqint = f_pressureliq_int(k) ! pressure at the bubble wall + + scoord = mtn_s(k, 1:3, 2) + call s_get_cell(scoord, cell) + pinf = f_pressure_inf(k, q_prim_vf, 1, aux1, aux2) ! getting p_inf + + do i = 1, num_fluids + alpha_rho(i) = q_prim_vf(advxb + i - 1)%sf(cell(1), cell(2), cell(3))*q_prim_vf(i)%sf(cell(1), cell(2), cell(3)) + alpha(i) = q_prim_vf(advxb + i - 1)%sf(cell(1), cell(2), cell(3)) + end do + + call s_convert_species_to_mixture_variables_acc(rhol, gamma, pi_inf, qv, alpha, alpha_rho, Re, cell(1), cell(2), cell(3)) + + ! Computing speed of sound + do i = 1, num_dims + vel(i) = q_prim_vf(i + num_fluids)%sf(cell(1), cell(2), cell(3)) + end do + E = gamma*pinf + pi_inf + 0.5d0*rhol*dot_product(vel, vel) + H = (E + pinf)/rhol + cson = sqrt((H - 0.5d0*dot_product(vel, vel))/gamma) + + deltaP = pliqint - pinf + termI = 0.0d0 + velint = intfc_vel(k, 2) - gas_dmvdt(k,stage)/(4.0d0*pi*intfc_rad(k, 2)**2*rhol) + + intfc_dveldt(k, stage) = ((1.0d0 + velint/cson)*deltaP/rhol & + + termI & + + gas_dpdt(k,stage)*intfc_rad(k, 2)/rhol/cson & + - velint**2*3.0d0/2.0d0*(1.0d0 - velint/3.0d0/cson)) & + /(intfc_rad(k, 2)*(1.0d0 - velint/cson)) + ! end of manual inline + intfc_draddt(k, stage) = intfc_vel(k, 2) end do else @@ -674,8 +719,11 @@ contains !! @param step Current time-stage in RKCK stepper, otherwise step = 1 !! @param q_prim_vf Primitive variables subroutine s_compute_KM(nparticles, step, q_prim_vf) - !$acc routine seq - +!#ifdef _CRAYFTN + !!DIR$ INLINEALWAYS s_compute_KM +!#else + !!$acc routine seq +!#endif integer, intent(in) :: nparticles, step type(scalar_field), dimension(sys_size), intent(in) :: q_prim_vf @@ -689,36 +737,36 @@ contains real(kind(0.d0)), dimension(2) :: Re integer :: i - pliqint = f_pressureliq_int(nparticles) ! pressure at the bubble wall + !pliqint = f_pressureliq_int(nparticles) ! pressure at the bubble wall - scoord = mtn_s(nparticles, 1:3, 2) - call s_get_cell(scoord, cell) - pinf = f_pressure_inf(nparticles, q_prim_vf, 1, aux1, aux2) ! getting p_inf + !scoord = mtn_s(nparticles, 1:3, 2) + !call s_get_cell(scoord, cell) + !pinf = f_pressure_inf(nparticles, q_prim_vf, 1, aux1, aux2) ! getting p_inf - do i = 1, num_fluids - alpha_rho(i) = q_prim_vf(advxb + i - 1)%sf(cell(1), cell(2), cell(3))*q_prim_vf(i)%sf(cell(1), cell(2), cell(3)) - alpha(i) = q_prim_vf(advxb + i - 1)%sf(cell(1), cell(2), cell(3)) - end do + !do i = 1, num_fluids + !alpha_rho(i) = q_prim_vf(advxb + i - 1)%sf(cell(1), cell(2), cell(3))*q_prim_vf(i)%sf(cell(1), cell(2), cell(3)) + !alpha(i) = q_prim_vf(advxb + i - 1)%sf(cell(1), cell(2), cell(3)) + !end do - call s_convert_species_to_mixture_variables_acc(rhol, gamma, pi_inf, qv, alpha, alpha_rho, Re, cell(1), cell(2), cell(3)) + !call s_convert_species_to_mixture_variables_acc(rhol, gamma, pi_inf, qv, alpha, alpha_rho, Re, cell(1), cell(2), cell(3)) - ! Computing speed of sound - do i = 1, num_dims - vel(i) = q_prim_vf(i + num_fluids)%sf(cell(1), cell(2), cell(3)) - end do - E = gamma*pinf + pi_inf + 0.5d0*rhol*dot_product(vel, vel) - H = (E + pinf)/rhol - cson = sqrt((H - 0.5d0*dot_product(vel, vel))/gamma) - - deltaP = pliqint - pinf - termI = 0.0d0 - velint = intfc_vel(nparticles, 2) - gas_dmvdt(nparticles, step)/(4.0d0*pi*intfc_rad(nparticles, 2)**2*rhol) - - intfc_dveldt(nparticles, step) = ((1.0d0 + velint/cson)*deltaP/rhol & - + termI & - + gas_dpdt(nparticles, step)*intfc_rad(nparticles, 2)/rhol/cson & - - velint**2*3.0d0/2.0d0*(1.0d0 - velint/3.0d0/cson)) & - /(intfc_rad(nparticles, 2)*(1.0d0 - velint/cson)) + !! Computing speed of sound + !do i = 1, num_dims + !vel(i) = q_prim_vf(i + num_fluids)%sf(cell(1), cell(2), cell(3)) + !end do + !E = gamma*pinf + pi_inf + 0.5d0*rhol*dot_product(vel, vel) + !H = (E + pinf)/rhol + !cson = sqrt((H - 0.5d0*dot_product(vel, vel))/gamma) + + !deltaP = pliqint - pinf + !termI = 0.0d0 + !velint = intfc_vel(nparticles, 2) - gas_dmvdt(nparticles, step)/(4.0d0*pi*intfc_rad(nparticles, 2)**2*rhol) + + !intfc_dveldt(nparticles, step) = ((1.0d0 + velint/cson)*deltaP/rhol & + !+ termI & + !+ gas_dpdt(nparticles, step)*intfc_rad(nparticles, 2)/rhol/cson & + !- velint**2*3.0d0/2.0d0*(1.0d0 - velint/3.0d0/cson)) & + !/(intfc_rad(nparticles, 2)*(1.0d0 - velint/cson)) end subroutine s_compute_KM diff --git a/src/simulation/m_bubbles_EL_kernels.fpp b/src/simulation/m_bubbles_EL_kernels.fpp index 9735739ce..0957e2f9c 100644 --- a/src/simulation/m_bubbles_EL_kernels.fpp +++ b/src/simulation/m_bubbles_EL_kernels.fpp @@ -124,7 +124,7 @@ contains smearGridz = smearGrid if (p == 0) smearGridz = 1 - !$acc parallel loop gang vector default(present) private(l, s_coord, cell, center) copyin(smearGrid, smearGridz) + !$acc parallel loop gang vector default(present) private(nodecoord, l, s_coord, cell, center) copyin(smearGrid, smearGridz) do l = 1, nBubs nodecoord(1:3) = 0 center(1:3) = 0.0d0 @@ -138,7 +138,7 @@ contains strength_vol = volpart strength_vel = 4.0d0*pi*lbk_rad(l, 2)**2*lbk_vel(l, 2) - !$acc loop collapse(3) private(i, j, k, cellaux, nodecoord) + !$acc loop collapse(3) private(cellaux, nodecoord) do i = 1, smearGrid do j = 1, smearGrid do k = 1, smearGridz diff --git a/src/simulation/m_time_steppers.fpp b/src/simulation/m_time_steppers.fpp index c2a047b01..c7430d774 100644 --- a/src/simulation/m_time_steppers.fpp +++ b/src/simulation/m_time_steppers.fpp @@ -266,7 +266,7 @@ contains do j = 1, sys_size @:ALLOCATE(rhs_ts_rkck(i)%vf(j)%sf(0:m, 0:n, 0:p)) end do - @:ACC_SETUP_SFs(rhs_ts_rkck(i)) + @:ACC_SETUP_VFs(rhs_ts_rkck(i)) end do else ! Allocating the cell-average RHS variables