diff --git a/.gitmodules b/.gitmodules index f8e36f3893..a14a9f2090 100644 --- a/.gitmodules +++ b/.gitmodules @@ -29,7 +29,7 @@ [submodule "atmos_phys"] path = src/atmos_phys url = https://github.com/ESCOMP/atmospheric_physics - fxtag = atmos_phys0_20_001 + fxtag = atmos_phys0_21_000 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics @@ -139,7 +139,7 @@ fxDONOTUSEurl = https://github.com/ESCOMP/mizuRoute [submodule "ccs_config"] path = ccs_config url = https://github.com/ESMCI/ccs_config_cesm.git -fxtag = ccs_config_cesm1.0.65 +fxtag = ccs_config_cesm1.0.72 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git diff --git a/ccs_config b/ccs_config index 7fda4a54ec..bd39901220 160000 --- a/ccs_config +++ b/ccs_config @@ -1 +1 @@ -Subproject commit 7fda4a54ec6261d2affb2351ca2b8f288308f0e8 +Subproject commit bd39901220166f8f8d7368cdf5aad8b704df3602 diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index 99c44e4b84..40ed9b210f 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -84,7 +84,7 @@ - + @@ -1297,7 +1297,7 @@ - + diff --git a/doc/ChangeLog b/doc/ChangeLog index 18ea031563..0398b12428 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,3 +1,68 @@ +=============================================================== + +Tag name: cam6_4_148 +Originator(s): peverwhee, sjsprecious +Date: 29 January 2026 +One-line Summary: Move GPU directives to atmospheric physics for RRTMGP +Github PR URL: https://github.com/ESCOMP/CAM/pull/1461 + +Purpose of changes (include the issue number and title text for each relevant GitHub issue): + - Moves acc directives for RRTMGP from src/physics/rrtmgp/radiation.F90 to + atmospheric_physics (src/atmos_phys/schemes/rrtmgp/) + +Describe any changes made to build system: N/A + +Describe any changes made to the namelist: N/A + +List any changes to the defaults for the boundary datasets: N/A + +Describe any substantial timing or memory changes: +- GPU test now runs thanks to Jian Sun's workaround in ccs_config, but GPU +test now takes longer (until we have a better fix from CISL) + +Code reviewed by: cacraigucar + +List all files eliminated: N/A + +List all files added and what they do: N/A + +List all existing files that have been modified, and describe the changes: +M cime_config/testdefs/testlist_cam.xml +- Up the wallclock time for all nvhpc tests to 2:00:00 + +M src/physics/rrtmgp/radiation.F90 +- remove acc directives + +M src/physics/rrtmgp/rrtmgp_inputs_cam.F90 +- clean-up unused code that has been moved to atmos_phys + +M .gitmodules +M src/atmos_phys +M ccs_config +- update atmospheric_physics and ccs_config tags + - atmospheric_physics tag includes new directives in rrtmgp schemes + - ccs_config tag changes nvhpc optimization to allow the GPU test to run AND + updates the list of source code to compile with GPU flags + +If there were any failures reported from running test_driver.sh on any test +platform, and checkin with these failures has been OK'd by the gatekeeper, +then copy the lines from the td.*.status files for the failed tests to the +appropriate machine below. All failed tests must be justified. + +derecho/intel/aux_cam: + ERI_D_Ln18.ne16pg3_ne16pg3_mt232.FHIST_C4.derecho_intel.cam-outfrq3s_eri (Overall: FAIL) details: + - pre-existing failure + - ERI test discovered a bug in CICE: See: https://github.com/ESCOMP/CESM_CICE/issues/34 + +derecho/nvhpc/aux_cam: + ERS_Ln9.ne30pg3_ne30pg3_mt232.FHISTC_LTso.derecho_nvhpc.cam-outfrq9s_gpu_default (Overall: PASS) details: + - TEST PASSES AS OF THIS TAG + - generated baselines (with cam6_4_147 + the ccs_config mod) to confirm that + the mods in this tag (cam6_4_148) do not change answers + +izumi/nag/aux_cam: all BFB + +izumi/gnu/aux_cam: all BFB =============================================================== diff --git a/src/atmos_phys b/src/atmos_phys index 585591adde..b3abf21d92 160000 --- a/src/atmos_phys +++ b/src/atmos_phys @@ -1 +1 @@ -Subproject commit 585591adde436a25915881819d52e23edc96aa41 +Subproject commit b3abf21d92b101a899dec719c436e0cc046f1251 diff --git a/src/physics/rrtmgp/radiation.F90 b/src/physics/rrtmgp/radiation.F90 index d4793223f3..664b279399 100644 --- a/src/physics/rrtmgp/radiation.F90 +++ b/src/physics/rrtmgp/radiation.F90 @@ -413,7 +413,6 @@ end function radiation_do subroutine radiation_init(pbuf2d) use rrtmgp_pre, only: rrtmgp_pre_init use rrtmgp_inputs_setup, only: rrtmgp_inputs_setup_init - use rrtmgp_inputs_cam, only: rrtmgp_inputs_cam_init use rrtmgp_cloud_optics_setup, only: rrtmgp_cloud_optics_setup_init use rrtmgp_sw_solar_var_setup, only: rrtmgp_sw_solar_var_setup_init use solar_irrad_data, only: do_spctrl_scaling, has_spectrum @@ -484,10 +483,6 @@ subroutine radiation_init(pbuf2d) call endrun(sub//': '//errmsg) end if - ! Set up CAM-side RRTMGP inputs - will go away once SW radiation is CCPPized - call rrtmgp_inputs_cam_init(ktopcam, ktoprad, idx_sw_diag, idx_nir_diag, idx_uv_diag, idx_sw_cloudsim, idx_lw_diag, & - idx_lw_cloudsim) - ! Set radconstants module-level index variables that we're setting in CCPP-ized scheme now call radconstants_init(idx_sw_diag, idx_nir_diag, idx_uv_diag, idx_lw_diag) @@ -1264,14 +1259,11 @@ subroutine radiation_tend( & ! Compute the gas optics (stored in atm_optics_sw). ! toa_flux is the reference solar source from RRTMGP data. - !$acc data copyin(kdist_sw%gas_props,pmid_day,pint_day,t_day,gas_concs_sw%gas_concs,atm_optics_sw%optical_props) & - !$acc copyout(toa_flux) call rrtmgp_sw_gas_optics_run(dosw, 1, nday, nday, pmid_day, pint_day, t_day, & gas_concs_sw, atm_optics_sw, kdist_sw, toa_flux, errmsg, errflg) if (errflg /= 0) then call endrun(sub//': '//errmsg) end if - !$acc end data ! Scale the solar source call rrtmgp_sw_solar_var_run(toa_flux, 2, band2gpt_sw, nswbands, sol_irrad, we, nbins, sol_tsi, & @@ -1285,26 +1277,16 @@ subroutine radiation_tend( & ! Set SW aerosol optical properties in the aer_sw object. ! This call made even when no daylight columns because it does some ! diagnostic aerosol output. - call rrtmgp_set_aer_sw( & - icall, state, pbuf, nday, idxday, nnite, idxnite, aer_sw) + call rrtmgp_set_aer_sw(ktopcam, ktoprad, icall, state, pbuf, nday, idxday, nnite, idxnite, aer_sw) if (nday > 0) then ! Increment the gas optics (in atm_optics_sw) by the aerosol optics in aer_sw. - !$acc data copyin(coszrs_day, toa_flux, alb_dir, alb_dif, & - !$acc atm_optics_sw%optical_props, atm_optics_sw%optical_props%tau, atm_optics_sw%optical_props%ssa, & - !$acc atm_optics_sw%optical_props%g, aer_sw%optical_props%tau, & - !$acc aer_sw%optical_props, aer_sw%optical_props%ssa, aer_sw%optical_props%g, & - !$acc cloud_sw%optical_props, cloud_sw%optical_props%tau, cloud_sw%optical_props%ssa, & - !$acc cloud_sw%optical_props%g) & - !$acc copy(fswc%fluxes, fswc%fluxes%flux_net,fswc%fluxes%flux_up,fswc%fluxes%flux_dn, & - !$acc fsw%fluxes, fsw%fluxes%flux_net,fsw%fluxes%flux_up,fsw%fluxes%flux_dn) call rrtmgp_sw_rte_run(dosw, .true., .true., nday, 1, nday, atm_optics_sw, cloud_sw, & aer_sw, coszrs_day, toa_flux, alb_dir, alb_dif, fswc, fsw, errmsg, errflg) if (errflg /= 0) then call endrun(sub//': '//errmsg) end if - !$acc end data end if ! Transform RRTMGP outputs to CAM outputs and compute heating rates. @@ -1364,37 +1346,17 @@ subroutine radiation_tend( & end if ! Compute the gas optics and Planck sources. - !$acc data copyin(kdist_lw%gas_props, pmid_rad, pint_rad, t_rad, & - !$acc t_sfc, gas_concs_lw%gas_concs, atm_optics_lw%optical_props) & - !$acc copy(atm_optics_lw%optical_props%tau, & - !$acc sources_lw%sources, sources_lw%sources%lay_source, & - !$acc sources_lw%sources%sfc_source, & - !$acc sources_lw%sources%lev_source, & - !$acc sources_lw%sources%sfc_source_jac) call rrtmgp_lw_gas_optics_run(dolw, 1, ncol, ncol, pmid_rad, pint_rad, t_rad, & t_sfc, gas_concs_lw, atm_optics_lw, sources_lw, t_rad, .false., kdist_lw, errmsg, & errflg) if (errflg /= 0) then call endrun(sub//': '//errmsg) end if - !$acc end data ! Set LW aerosol optical properties in the aer_lw object. - call rrtmgp_set_aer_lw(icall, state, pbuf, aer_lw) + call rrtmgp_set_aer_lw(ktopcam, ktoprad, icall, state, pbuf, aer_lw) ! Call the main rrtmgp_lw driver - !$acc data copyin(atm_optics_lw%optical_props,atm_optics_lw%optical_props%tau, & - !$acc aer_lw%optical_props,aer_lw%optical_props%tau, & - !$acc cloud_lw%optical_props, cloud_lw%optical_props%tau, & - !$acc sources_lw%sources,sources_lw%sources%lay_source, & - !$acc sources_lw%sources%sfc_source, & - !$acc sources_lw%sources%lev_source, & - !$acc sources_lw%sources%sfc_source_jac, & - !$acc emis_sfc) & - !$acc copy(flwc%fluxes, flwc%fluxes%flux_net, flwc%fluxes%flux_up, & - !$acc flwc%fluxes%flux_dn, flw%fluxes, flw%fluxes%flux_net, & - !$acc flw%fluxes%flux_up, flw%fluxes%flux_dn, & - !$acc lw_ds) call rrtmgp_lw_rte_run(dolw, dolw, .false., .false., .false., & 0, atm_optics_lw, cloud_lw, sources_lw, emis_sfc, & kdist_lw, aer_lw, fluxlwup_jac, lw_ds, flwc, flw, & @@ -1402,7 +1364,6 @@ subroutine radiation_tend( & if (errflg /= 0) then call endrun(sub//': '//errmsg) end if - !$acc end data ! Transform RRTMGP outputs to CAM outputs and compute heating rates. call set_lw_diags() diff --git a/src/physics/rrtmgp/rrtmgp_inputs_cam.F90 b/src/physics/rrtmgp/rrtmgp_inputs_cam.F90 index af1b6fca3e..4b30630384 100644 --- a/src/physics/rrtmgp/rrtmgp_inputs_cam.F90 +++ b/src/physics/rrtmgp/rrtmgp_inputs_cam.F90 @@ -44,7 +44,6 @@ module rrtmgp_inputs_cam save public :: & - rrtmgp_inputs_cam_init, & rrtmgp_get_gas_mmrs, & rrtmgp_set_aer_lw, & rrtmgp_set_aer_sw @@ -53,16 +52,6 @@ module rrtmgp_inputs_cam ! This value is to match the arbitrary small value used in RRTMG to decide ! when a quantity is effectively zero. real(r8), parameter :: tiny = 1.0e-80_r8 -real(r8) :: sw_low_bounds(nswbands) -real(r8) :: sw_high_bounds(nswbands) -integer :: ktopcam -integer :: ktoprad -integer :: idx_sw_diag -integer :: idx_nir_diag -integer :: idx_uv_diag -integer :: idx_sw_cloudsim -integer :: idx_lw_diag -integer :: idx_lw_cloudsim ! Mapping from RRTMG shortwave bands to RRTMGP. Currently needed to continue using ! the SW optics datasets from RRTMG (even thought there is a slight mismatch in the @@ -74,43 +63,6 @@ module rrtmgp_inputs_cam contains !================================================================================================== -!================================================================================================== -subroutine rrtmgp_inputs_cam_init(ktcam, ktrad, idx_sw_diag_in, idx_nir_diag_in, idx_uv_diag_in, & - idx_sw_cloudsim_in, idx_lw_diag_in, idx_lw_cloudsim_in) - - ! Note that this routine must be called after the calls to set_wavenumber_bands which set - ! the sw/lw band boundaries in the radconstants module. - - integer, intent(in) :: ktcam - integer, intent(in) :: ktrad - integer, intent(in) :: idx_sw_diag_in - integer, intent(in) :: idx_nir_diag_in - integer, intent(in) :: idx_uv_diag_in - integer, intent(in) :: idx_sw_cloudsim_in - integer, intent(in) :: idx_lw_diag_in - integer, intent(in) :: idx_lw_cloudsim_in - character(len=512) :: errmsg - integer :: errflg - - ktopcam = ktcam - ktoprad = ktrad - idx_sw_diag = idx_sw_diag_in - idx_nir_diag = idx_nir_diag_in - idx_uv_diag = idx_uv_diag_in - idx_sw_cloudsim = idx_sw_cloudsim_in - idx_lw_diag = idx_lw_diag_in - idx_lw_cloudsim = idx_lw_cloudsim_in - - ! Initialize the module data containing the SW band boundaries. - call get_sw_spectral_boundaries_ccpp(sw_low_bounds, sw_high_bounds, 'cm^-1', errmsg, errflg) - if (errflg /= 0) then - call endrun('rrtmgp_inputs_cam_init: error during get_sw_spectral_boundaries_ccpp - message: '//errmsg) - end if - -end subroutine rrtmgp_inputs_cam_init - -!========================================================================================= - subroutine rrtmgp_get_gas_mmrs(icall, state, pbuf, nlay, gas_mmrs) ! Retrieve mass mixing ratios for radiatively active gases from rad_constituents @@ -137,11 +89,13 @@ end subroutine rrtmgp_get_gas_mmrs !================================================================================================== -subroutine rrtmgp_set_aer_lw(icall, state, pbuf, aer_lw) +subroutine rrtmgp_set_aer_lw(ktopcam, ktoprad, icall, state, pbuf, aer_lw) ! Load LW aerosol optical properties into the RRTMGP object. ! Arguments + integer, intent(in) :: ktopcam + integer, intent(in) :: ktoprad integer, intent(in) :: icall type(physics_state), target, intent(in) :: state type(physics_buffer_desc), pointer :: pbuf(:) @@ -178,11 +132,13 @@ end subroutine rrtmgp_set_aer_lw !================================================================================================== subroutine rrtmgp_set_aer_sw( & - icall, state, pbuf, nday, idxday, nnite, idxnite, aer_sw) + ktopcam, ktoprad, icall, state, pbuf, nday, idxday, nnite, idxnite, aer_sw) ! Load SW aerosol optical properties into the RRTMGP object. ! Arguments + integer, intent(in) :: ktopcam + integer, intent(in) :: ktoprad integer, intent(in) :: icall type(physics_state), target, intent(in) :: state type(physics_buffer_desc), pointer :: pbuf(:)