diff --git a/schemes/rrtmgp/rrtmgp_inputs_setup.meta b/schemes/rrtmgp/rrtmgp_inputs_setup.meta index a2b8b4f1..df5f5822 100644 --- a/schemes/rrtmgp/rrtmgp_inputs_setup.meta +++ b/schemes/rrtmgp/rrtmgp_inputs_setup.meta @@ -122,7 +122,7 @@ intent = in [ current_cal_day ] standard_name = fractional_calendar_days_on_end_of_current_timestep - units = 1 + units = days type = real | kind = kind_phys dimensions = () intent = in @@ -232,7 +232,7 @@ standard_name = shortwave_start_and_end_gpoint_for_each_band units = index type = integer - dimensions = (ccpp_constant_two,number_of_bands_for_shortwave_radiation) + dimensions = (constant_dimension_two,number_of_bands_for_shortwave_radiation) intent = out [ irad_always_out ] standard_name = number_of_timesteps_to_force_radiation_calculation_after_initialization diff --git a/schemes/rrtmgp/rrtmgp_lw_gas_optics.F90 b/schemes/rrtmgp/rrtmgp_lw_gas_optics.F90 index 5388b60f..5f258010 100644 --- a/schemes/rrtmgp/rrtmgp_lw_gas_optics.F90 +++ b/schemes/rrtmgp/rrtmgp_lw_gas_optics.F90 @@ -358,6 +358,13 @@ subroutine rrtmgp_lw_gas_optics_run(dolw, iter_num, ncol, rrtmgp_phys_blksz, p_l iCol = ((iter_num - 1) * rrtmgp_phys_blksz) + 1 iCol2= min(iCol + rrtmgp_phys_blksz - 1, ncol) + !$acc data copyin(lw_gas_props%gas_props, p_lay, p_lev, t_lay, & + !$acc tsfg, gas_concs%gas_concs) & + !$acc copy(sources%sources, sources%sources%lay_source, & + !$acc sources%sources%sfc_source, & + !$acc sources%sources%lev_source, & + !$acc sources%sources%sfc_source_jac) + if (include_interface_temp) then errmsg = lw_gas_props%gas_props%gas_optics(& p_lay(iCol:iCol2,:), & ! IN - Pressure @ layer-centers (Pa) @@ -386,6 +393,7 @@ subroutine rrtmgp_lw_gas_optics_run(dolw, iter_num, ncol, rrtmgp_phys_blksz, p_l errflg = 1 end if end if + !$acc end data end subroutine rrtmgp_lw_gas_optics_run diff --git a/schemes/rrtmgp/rrtmgp_lw_gas_optics.meta b/schemes/rrtmgp/rrtmgp_lw_gas_optics.meta index c25d0553..1c90d3bd 100644 --- a/schemes/rrtmgp/rrtmgp_lw_gas_optics.meta +++ b/schemes/rrtmgp/rrtmgp_lw_gas_optics.meta @@ -3,6 +3,8 @@ type = scheme dependencies = ext/rte-kernels/mo_rte_kind.F90 dependencies = objects/ccpp_gas_concentrations.F90 + dependencies = ext/rte-kernels/accel/mo_optical_props_kernels.F90 + dependencies = ext/rrtmgp-kernels/accel/mo_gas_optics_rrtmgp_kernels.F90 [ccpp-arg-table] name = rrtmgp_lw_gas_optics_init diff --git a/schemes/rrtmgp/rrtmgp_lw_rte.F90 b/schemes/rrtmgp/rrtmgp_lw_rte.F90 index f70919db..573c5746 100644 --- a/schemes/rrtmgp/rrtmgp_lw_rte.F90 +++ b/schemes/rrtmgp/rrtmgp_lw_rte.F90 @@ -57,6 +57,19 @@ subroutine rrtmgp_lw_rte_run(doLWrad, doLWclrsky, doGP_lwscat, use_LW_jacobian, if (.not. doLWrad) return + !$acc data copyin(lw_optical_props_clrsky%optical_props,lw_optical_props_clrsky%optical_props%tau, & + !$acc aerlw%optical_props,aerlw%optical_props%tau, & + !$acc lw_optical_props_clouds%optical_props, lw_optical_props_clouds%optical_props%tau, & + !$acc sources%sources,sources%sources%lay_source, & + !$acc sources%sources%sfc_source, & + !$acc sources%sources%lev_source, & + !$acc sources%sources%sfc_source_jac, & + !$acc sfc_emiss_byband) & + !$acc copy(flux_clrsky%fluxes, flux_clrsky%fluxes%flux_net, flux_clrsky%fluxes%flux_up, & + !$acc flux_clrsky%fluxes%flux_dn, flux_allsky%fluxes, flux_allsky%fluxes%flux_net, & + !$acc flux_allsky%fluxes%flux_up, flux_allsky%fluxes%flux_dn, & + !$acc lw_Ds) + ! ################################################################################### ! ! Compute clear-sky fluxes (gaseous+aerosol) (optional) @@ -67,19 +80,18 @@ subroutine rrtmgp_lw_rte_run(doLWrad, doLWclrsky, doGP_lwscat, use_LW_jacobian, call check_error_msg('rrtmgp_lw_rte_increment_aerosol_to_clrsky', errmsg) if (len_trim(errmsg) /= 0) then errflg = 1 - return + ! Can't return from within a top-level acc block end if ! Call RTE solver - if (doLWclrsky) then + if (errflg == 0 .and. doLWclrsky) then if (use_lw_optimal_angles) then errmsg = lw_gas_props%gas_props%compute_optimal_angles(lw_optical_props_clrsky%optical_props,lw_Ds) call check_error_msg('rrtmgp_lw_rte_opt_angle', errmsg) if (len_trim(errmsg) /= 0) then errflg = 1 - return end if - if (nGauss_angles > 1) then + if (nGauss_angles > 1 .and. errflg == 0) then errmsg = rte_lw( & lw_optical_props_clrsky%optical_props, & ! IN - optical-properties sources%sources, & ! IN - source function @@ -87,7 +99,7 @@ subroutine rrtmgp_lw_rte_run(doLWrad, doLWclrsky, doGP_lwscat, use_LW_jacobian, flux_clrsky%fluxes, & ! OUT - Fluxes n_gauss_angles = nGauss_angles, & ! IN - Number of angles in Gaussian quadrature lw_Ds = lw_Ds) ! IN - 1/cos of transport angle per column and g-point - else + else if (errflg == 0) then errmsg = rte_lw( & lw_optical_props_clrsky%optical_props, & ! IN - optical-properties sources%sources, & ! IN - source function @@ -96,14 +108,14 @@ subroutine rrtmgp_lw_rte_run(doLWrad, doLWclrsky, doGP_lwscat, use_LW_jacobian, lw_Ds = lw_Ds) ! IN - 1/cos of transport angle per column and g-point end if else - if (nGauss_angles > 1) then + if (nGauss_angles > 1 .and. errflg == 0) then errmsg = rte_lw( & lw_optical_props_clrsky%optical_props, & ! IN - optical-properties sources%sources, & ! IN - source function sfc_emiss_byband, & ! IN - surface emissivity in each LW band flux_clrsky%fluxes, & ! OUT - Fluxes n_gauss_angles = nGauss_angles) ! IN - Number of angles in Gaussian quadrature - else + else if (errflg == 0) then errmsg = rte_lw( & lw_optical_props_clrsky%optical_props, & ! IN - optical-properties sources%sources, & ! IN - source function @@ -114,7 +126,6 @@ subroutine rrtmgp_lw_rte_run(doLWrad, doLWclrsky, doGP_lwscat, use_LW_jacobian, call check_error_msg('rrtmgp_lw_rte_lw_rte_clrsky', errmsg) if (len_trim(errmsg) /= 0) then errflg = 1 - return end if end if @@ -134,15 +145,14 @@ subroutine rrtmgp_lw_rte_run(doLWrad, doLWclrsky, doGP_lwscat, use_LW_jacobian, ! ################################################################################### ! Include LW cloud-scattering? - if (doGP_lwscat) then + if (doGP_lwscat .and. errflg == 0) then ! Increment errmsg = lw_optical_props_clrsky%optical_props%increment(lw_optical_props_clouds%optical_props) call check_error_msg('rrtmgp_lw_rte_increment_clrsky_to_clouds', errmsg) if (len_trim(errmsg) /= 0) then errflg = 1 - return end if - if (use_LW_jacobian) then + if (use_LW_jacobian .and. errflg == 0) then if (nGauss_angles > 1) then errmsg = rte_lw( & lw_optical_props_clouds%optical_props, & ! IN - optical-properties @@ -159,7 +169,7 @@ subroutine rrtmgp_lw_rte_run(doLWrad, doLWclrsky, doGP_lwscat, use_LW_jacobian, flux_allsky%fluxes, & ! OUT - Fluxes flux_up_Jac = fluxlwUP_jac) ! OUT - surface temperature flux (upward) Jacobian (W m-2 K-1) end if - else + else if (errflg == 0) then if (nGauss_angles > 1) then errmsg = rte_lw( & lw_optical_props_clouds%optical_props, & ! IN - optical-properties @@ -182,10 +192,9 @@ subroutine rrtmgp_lw_rte_run(doLWrad, doLWclrsky, doGP_lwscat, use_LW_jacobian, call check_error_msg('rrtmgp_lw_rte_increment_clouds_to_clrsky', errmsg) if (len_trim(errmsg) /= 0) then errflg = 1 - return end if - if (use_LW_jacobian) then + if (use_LW_jacobian .and. errflg == 0) then if (nGauss_angles > 1) then errmsg = rte_lw( & lw_optical_props_clrsky%optical_props, & ! IN - optical-properties @@ -202,7 +211,7 @@ subroutine rrtmgp_lw_rte_run(doLWrad, doLWclrsky, doGP_lwscat, use_LW_jacobian, flux_allsky%fluxes, & ! OUT - Fluxes flux_up_Jac = fluxlwUP_jac) ! OUT - surface temperature flux (upward) Jacobian (W m-2 K-1) end if - else + else if (errflg == 0) then if (nGauss_angles > 1) then errmsg = rte_lw( & lw_optical_props_clrsky%optical_props, & ! IN - optical-properties @@ -223,6 +232,7 @@ subroutine rrtmgp_lw_rte_run(doLWrad, doLWclrsky, doGP_lwscat, use_LW_jacobian, if (len_trim(errmsg) /= 0) then errflg = 1 end if + !$acc end data end subroutine rrtmgp_lw_rte_run end module rrtmgp_lw_rte diff --git a/schemes/rrtmgp/rrtmgp_lw_rte.meta b/schemes/rrtmgp/rrtmgp_lw_rte.meta index 81c71e77..7baf0978 100644 --- a/schemes/rrtmgp/rrtmgp_lw_rte.meta +++ b/schemes/rrtmgp/rrtmgp_lw_rte.meta @@ -7,6 +7,7 @@ dependencies = ext/rte-frontend/mo_rte_util_array_validation.F90 dependencies = ext/rte-kernels/mo_rte_solver_kernels.F90 dependencies = ext/rte-kernels/mo_fluxes_broadband_kernels.F90 + dependencies = ext/rte-kernels/accel/mo_rte_solver_kernels.F90 dependencies = utils/radiation_tools.F90 dependencies = objects/ccpp_source_functions.F90,objects/ccpp_gas_optics_rrtmgp.F90 dependencies = objects/ccpp_optical_props.F90 diff --git a/schemes/rrtmgp/rrtmgp_pre.meta b/schemes/rrtmgp/rrtmgp_pre.meta index 5689fab2..8cfe0f1e 100644 --- a/schemes/rrtmgp/rrtmgp_pre.meta +++ b/schemes/rrtmgp/rrtmgp_pre.meta @@ -178,7 +178,7 @@ intent = in [ next_cday ] standard_name = fractional_calendar_days_on_end_of_next_timestep - units = 1 + units = days type = real | kind = kind_phys dimensions = () intent = in diff --git a/schemes/rrtmgp/rrtmgp_sw_gas_optics.F90 b/schemes/rrtmgp/rrtmgp_sw_gas_optics.F90 index 8bbccd4e..6191e7b6 100644 --- a/schemes/rrtmgp/rrtmgp_sw_gas_optics.F90 +++ b/schemes/rrtmgp/rrtmgp_sw_gas_optics.F90 @@ -369,6 +369,8 @@ subroutine rrtmgp_sw_gas_optics_run(dosw, iter_num, ncol, rrtmgp_phys_blksz, p_l iCol = ((iter_num - 1) * rrtmgp_phys_blksz) + 1 iCol2= min(iCol + rrtmgp_phys_blksz - 1, ncol) + !$acc data copyin(sw_gas_props%gas_props,p_lay,p_lev,t_lay,gas_concs%gas_concs) & + !$acc copyout(toa_src_sw) errmsg = sw_gas_props%gas_props%gas_optics(& p_lay(iCol:iCol2,:), & ! IN - Pressure @ layer-centers (Pa) p_lev(iCol:iCol2,:), & ! IN - Pressure @ layer-interfaces (Pa) @@ -381,6 +383,7 @@ subroutine rrtmgp_sw_gas_optics_run(dosw, iter_num, ncol, rrtmgp_phys_blksz, p_l if (len_trim(errmsg) /= 0) then errflg = 1 end if + !$acc end data end subroutine rrtmgp_sw_gas_optics_run diff --git a/schemes/rrtmgp/rrtmgp_sw_gas_optics.meta b/schemes/rrtmgp/rrtmgp_sw_gas_optics.meta index b530275c..e81e833e 100644 --- a/schemes/rrtmgp/rrtmgp_sw_gas_optics.meta +++ b/schemes/rrtmgp/rrtmgp_sw_gas_optics.meta @@ -1,6 +1,8 @@ [ccpp-table-properties] name = rrtmgp_sw_gas_optics type = scheme + dependencies = ext/rte-kernels/accel/mo_optical_props_kernels.F90 + dependencies = ext/rrtmgp-kernels/accel/mo_gas_optics_rrtmgp_kernels.F90 [ccpp-arg-table] name = rrtmgp_sw_gas_optics_init diff --git a/schemes/rrtmgp/rrtmgp_sw_rte.F90 b/schemes/rrtmgp/rrtmgp_sw_rte.F90 index 6fd6ecdd..e3cec314 100644 --- a/schemes/rrtmgp/rrtmgp_sw_rte.F90 +++ b/schemes/rrtmgp/rrtmgp_sw_rte.F90 @@ -63,16 +63,24 @@ subroutine rrtmgp_sw_rte_run(doswrad, doswclrsky, doswallsky, nday, iter_num, rr ! Compute clear-sky fluxes (gaseous+aerosol) ! ! ################################################################################### + !$acc data copyin(coszen_day, toa_src_sw, sfc_alb_dir, sfc_alb_dif, & + !$acc sw_optical_props%optical_props, sw_optical_props%optical_props%tau, sw_optical_props%optical_props%ssa, & + !$acc sw_optical_props%optical_props%g, aersw%optical_props%tau, & + !$acc aersw%optical_props, aersw%optical_props%ssa, aersw%optical_props%g, & + !$acc sw_optical_props_clouds%optical_props, sw_optical_props_clouds%optical_props%tau, sw_optical_props_clouds%optical_props%ssa, & + !$acc sw_optical_props_clouds%optical_props%g) & + !$acc copy(flux_clrsky%fluxes, flux_clrsky%fluxes%flux_net,flux_clrsky%fluxes%flux_up,flux_clrsky%fluxes%flux_dn, & + !$acc flux_allsky%fluxes, flux_allsky%fluxes%flux_net,flux_allsky%fluxes%flux_up,flux_allsky%fluxes%flux_dn) ! Increment optics (always) errmsg = aersw%optical_props%increment(sw_optical_props%optical_props) call check_error_msg('rrtmgp_sw_rte_increment_aerosol_to_clrsky', errmsg) if (len_trim(errmsg) /= 0) then errflg = 1 - return + ! Can't return from within a top-level acc block end if ! Optionally compute clear-sky fluxes - if (doswclrsky) then + if (doswclrsky .and. errflg == 0) then errmsg = rte_sw( & sw_optical_props%optical_props, & ! IN - optical-properties coszen_day(iCol:iCol2), & ! IN - Cosine of solar zenith angle @@ -83,7 +91,6 @@ subroutine rrtmgp_sw_rte_run(doswrad, doswclrsky, doswallsky, nday, iter_num, rr call check_error_msg('rrtmgp_sw_rte_rte_sw_clrsky', errmsg) if (len_trim(errmsg) /= 0) then errflg = 1 - return end if end if @@ -93,29 +100,30 @@ subroutine rrtmgp_sw_rte_run(doswrad, doswclrsky, doswallsky, nday, iter_num, rr ! ! ################################################################################### - if (doswallsky) then + if (doswallsky .and. errflg == 0) then ! Increment errmsg = sw_optical_props_clouds%optical_props%increment(sw_optical_props%optical_props) call check_error_msg('rrtmgp_sw_rte_increment_clouds_to_clrsky', errmsg) if (len_trim(errmsg) /= 0) then errflg = 1 - return end if ! Compute fluxes - errmsg = rte_sw( & - sw_optical_props%optical_props, & ! IN - optical-properties - coszen_day(iCol:iCol2), & ! IN - Cosine of solar zenith angle - toa_src_sw, & ! IN - incident solar flux at TOA - sfc_alb_dir, & ! IN - Shortwave surface albedo (direct) - sfc_alb_dif, & ! IN - Shortwave surface albedo (diffuse) - flux_allsky%fluxes) ! OUT - Fluxes, all-sky, 3D (1,nLay,nBand) - call check_error_msg('rrtmgp_sw_rte_rte_sw_allsky', errmsg) - if (len_trim(errmsg) /= 0) then - errflg = 1 - return + if (errflg == 0) then + errmsg = rte_sw( & + sw_optical_props%optical_props, & ! IN - optical-properties + coszen_day(iCol:iCol2), & ! IN - Cosine of solar zenith angle + toa_src_sw, & ! IN - incident solar flux at TOA + sfc_alb_dir, & ! IN - Shortwave surface albedo (direct) + sfc_alb_dif, & ! IN - Shortwave surface albedo (diffuse) + flux_allsky%fluxes) ! OUT - Fluxes, all-sky, 3D (1,nLay,nBand) + call check_error_msg('rrtmgp_sw_rte_rte_sw_allsky', errmsg) + if (len_trim(errmsg) /= 0) then + errflg = 1 + end if end if end if + !$acc end data end subroutine rrtmgp_sw_rte_run end module rrtmgp_sw_rte diff --git a/schemes/rrtmgp/rrtmgp_sw_rte.meta b/schemes/rrtmgp/rrtmgp_sw_rte.meta index 2abcef17..e85c3d59 100644 --- a/schemes/rrtmgp/rrtmgp_sw_rte.meta +++ b/schemes/rrtmgp/rrtmgp_sw_rte.meta @@ -7,10 +7,11 @@ dependencies = ext/rte-frontend/mo_rte_util_array_validation.F90 dependencies = ext/rte-kernels/mo_rte_solver_kernels.F90 dependencies = ext/rte-kernels/mo_fluxes_broadband_kernels.F90 - dependencies = ./utils/radiation_tools.F90 - dependencies = ./objects/ccpp_source_functions.F90,./objects/ccpp_gas_optics_rrtmgp.F90 - dependencies = ./objects/ccpp_optical_props.F90 - dependencies = ./objects/ccpp_fluxes.F90,./objects/ccpp_fluxes_byband.F90 + dependencies = ext/rte-kernels/accel/mo_rte_solver_kernels.F90 + dependencies = utils/radiation_tools.F90 + dependencies = objects/ccpp_source_functions.F90,objects/ccpp_gas_optics_rrtmgp.F90 + dependencies = objects/ccpp_optical_props.F90 + dependencies = objects/ccpp_fluxes.F90,objects/ccpp_fluxes_byband.F90 [ccpp-arg-table] name = rrtmgp_sw_rte_run diff --git a/schemes/rrtmgp/rrtmgp_sw_solar_var.F90 b/schemes/rrtmgp/rrtmgp_sw_solar_var.F90 index 7126a006..0ac8fd8a 100644 --- a/schemes/rrtmgp/rrtmgp_sw_solar_var.F90 +++ b/schemes/rrtmgp/rrtmgp_sw_solar_var.F90 @@ -18,7 +18,7 @@ module rrtmgp_sw_solar_var !> \section arg_table_rrtmgp_sw_solar_var_run Argument Table !! \htmlinclude rrtmgp_sw_solar_var_run.html !! - subroutine rrtmgp_sw_solar_var_run(toa_flux, ccpp_constant_two, band2gpt_sw, nswbands, sol_irrad, wave_end, nbins, sol_tsi, & + subroutine rrtmgp_sw_solar_var_run(toa_flux, constant_dim_two, band2gpt_sw, nswbands, sol_irrad, wave_end, nbins, sol_tsi, & nday, dosw, do_spectral_scaling, sfac, eccf, errmsg, errflg) use rrtmgp_sw_solar_var_setup, only: irrad, radbinmax, radbinmin use ccpp_kinds, only : kind_phys @@ -30,7 +30,7 @@ subroutine rrtmgp_sw_solar_var_run(toa_flux, ccpp_constant_two, band2gpt_sw, nsw real(kind_phys), intent(in) :: wave_end(:) ! wavelength endpoints integer, intent(in) :: nday ! number of daytime points integer, intent(in) :: nbins ! number of bins - integer, intent(in) :: ccpp_constant_two ! dimension for band2gpt_sw + integer, intent(in) :: constant_dim_two ! dimension for band2gpt_sw integer, intent(in) :: band2gpt_sw(:,:) ! array for converting shortwave band limits to g-points integer, intent(in) :: nswbands ! number of shortwave bands logical, intent(in) :: do_spectral_scaling ! flag to do spectral scaling diff --git a/schemes/rrtmgp/rrtmgp_sw_solar_var.meta b/schemes/rrtmgp/rrtmgp_sw_solar_var.meta index b1d22a44..92f0f58b 100644 --- a/schemes/rrtmgp/rrtmgp_sw_solar_var.meta +++ b/schemes/rrtmgp/rrtmgp_sw_solar_var.meta @@ -12,8 +12,8 @@ type = real | kind = kind_phys dimensions = (daytime_columns_dimension,number_of_shortwave_g_point_intervals) intent = inout -[ ccpp_constant_two ] - standard_name = ccpp_constant_two +[ constant_dim_two ] + standard_name = constant_dimension_two units = count type = integer dimensions = () @@ -22,7 +22,7 @@ standard_name = shortwave_start_and_end_gpoint_for_each_band units = index type = integer - dimensions = (ccpp_constant_two,number_of_bands_for_shortwave_radiation) + dimensions = (constant_dimension_two,number_of_bands_for_shortwave_radiation) intent = in [ nswbands ] standard_name = number_of_bands_for_shortwave_radiation diff --git a/schemes/tropopause_find/tropopause_find.meta b/schemes/tropopause_find/tropopause_find.meta index a34f9df0..693d766b 100644 --- a/schemes/tropopause_find/tropopause_find.meta +++ b/schemes/tropopause_find/tropopause_find.meta @@ -107,7 +107,7 @@ intent = in [ calday ] standard_name = fractional_calendar_days_on_end_of_current_timestep - units = 1 + units = days type = real | kind = kind_phys dimensions = () intent = in