Merge pull request #1461 from peverwhee/rrtmgp-gpu

peverwhee · web-flow · commit d5bff71c5783 · 2026-01-29T19:43:31.000-07:00
cam6_4_148: Move GPU directives to atmospheric physics for RRTMGP
diff --git a/.gitmodules b/.gitmodules
@@ -29,7 +29,7 @@
 [submodule "atmos_phys"]
 	path = src/atmos_phys
 	url = https://github.com/ESCOMP/atmospheric_physics
-	fxtag = atmos_phys0_20_001
+	fxtag = atmos_phys0_21_000
         fxrequired = AlwaysRequired
         fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics
 
@@ -139,7 +139,7 @@ fxDONOTUSEurl = https://github.com/ESCOMP/mizuRoute
 [submodule "ccs_config"]
 path = ccs_config
 url = https://github.com/ESMCI/ccs_config_cesm.git
-fxtag = ccs_config_cesm1.0.65
+fxtag = ccs_config_cesm1.0.72
 fxrequired = ToplevelRequired
 fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git
 
diff --git a/ccs_config b/ccs_config
@@ -1 +1 @@
-Subproject commit 7fda4a54ec6261d2affb2351ca2b8f288308f0e8
+Subproject commit bd39901220166f8f8d7368cdf5aad8b704df3602
diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml
@@ -84,7 +84,7 @@
     </machines>
     <options>
       <option name="comment">CAM7-LT, enable gpu for PUMAS, CLUBB, and RRTMGP</option>
-      <option name="wallclock">00:45:00</option>
+      <option name="wallclock">02:00:00</option>
     </options>
   </test>
 
@@ -1297,7 +1297,7 @@
       <machine name="derecho" compiler="nvhpc" category="prealpha"/>
     </machines>
     <options>
-      <option name="wallclock">00:45:00</option>
+      <option name="wallclock">02:00:00</option>
     </options>
   </test>
   <test compset="F2000dev" grid="ne30pg3_ne30pg3_mg17" name="ERP_D_Ln9" testmods="cam/outfrq9s" supported="false">
diff --git a/doc/ChangeLog b/doc/ChangeLog
@@ -1,3 +1,68 @@
+===============================================================
+
+Tag name: cam6_4_148
+Originator(s): peverwhee, sjsprecious
+Date: 29 January 2026
+One-line Summary: Move GPU directives to atmospheric physics for RRTMGP
+Github PR URL: https://github.com/ESCOMP/CAM/pull/1461
+
+Purpose of changes (include the issue number and title text for each relevant GitHub issue):
+  - Moves acc directives for RRTMGP from src/physics/rrtmgp/radiation.F90 to
+   atmospheric_physics (src/atmos_phys/schemes/rrtmgp/)
+
+Describe any changes made to build system: N/A
+
+Describe any changes made to the namelist: N/A
+
+List any changes to the defaults for the boundary datasets: N/A
+
+Describe any substantial timing or memory changes:
+- GPU test now runs thanks to Jian Sun's workaround in ccs_config, but GPU
+test now takes longer (until we have a better fix from CISL)
+
+Code reviewed by: cacraigucar
+
+List all files eliminated: N/A
+
+List all files added and what they do: N/A
+
+List all existing files that have been modified, and describe the changes:
+M       cime_config/testdefs/testlist_cam.xml
+- Up the wallclock time for all nvhpc tests to 2:00:00
+
+M       src/physics/rrtmgp/radiation.F90
+- remove acc directives
+
+M       src/physics/rrtmgp/rrtmgp_inputs_cam.F90
+- clean-up unused code that has been moved to atmos_phys
+
+M       .gitmodules
+M       src/atmos_phys
+M       ccs_config
+- update atmospheric_physics and ccs_config tags
+  - atmospheric_physics tag includes new directives in rrtmgp schemes
+  - ccs_config tag changes nvhpc optimization to allow the GPU test to run AND
+  updates the list of source code to compile with GPU flags
+
+If there were any failures reported from running test_driver.sh on any test
+platform, and checkin with these failures has been OK'd by the gatekeeper,
+then copy the lines from the td.*.status files for the failed tests to the
+appropriate machine below.  All failed tests must be justified.
+
+derecho/intel/aux_cam:
+  ERI_D_Ln18.ne16pg3_ne16pg3_mt232.FHIST_C4.derecho_intel.cam-outfrq3s_eri (Overall: FAIL) details:
+   - pre-existing failure
+   - ERI test discovered a bug in CICE:  See: https://github.com/ESCOMP/CESM_CICE/issues/34
+
+derecho/nvhpc/aux_cam:
+  ERS_Ln9.ne30pg3_ne30pg3_mt232.FHISTC_LTso.derecho_nvhpc.cam-outfrq9s_gpu_default (Overall: PASS) details:
+   - TEST PASSES AS OF THIS TAG
+   - generated baselines (with cam6_4_147 + the ccs_config mod) to confirm that
+     the mods in this tag (cam6_4_148) do not change answers
+
+izumi/nag/aux_cam: all BFB
+
+izumi/gnu/aux_cam: all BFB
 
 ===============================================================
 
diff --git a/src/atmos_phys b/src/atmos_phys
@@ -1 +1 @@
-Subproject commit 585591adde436a25915881819d52e23edc96aa41
+Subproject commit b3abf21d92b101a899dec719c436e0cc046f1251
diff --git a/src/physics/rrtmgp/radiation.F90 b/src/physics/rrtmgp/radiation.F90
@@ -413,7 +413,6 @@ end function radiation_do
 subroutine radiation_init(pbuf2d)
    use rrtmgp_pre,                only: rrtmgp_pre_init
    use rrtmgp_inputs_setup,       only: rrtmgp_inputs_setup_init
-   use rrtmgp_inputs_cam,         only: rrtmgp_inputs_cam_init
    use rrtmgp_cloud_optics_setup, only: rrtmgp_cloud_optics_setup_init
    use rrtmgp_sw_solar_var_setup, only: rrtmgp_sw_solar_var_setup_init
    use solar_irrad_data,          only: do_spctrl_scaling, has_spectrum
@@ -484,10 +483,6 @@ subroutine radiation_init(pbuf2d)
       call endrun(sub//': '//errmsg)
    end if
 
-   ! Set up CAM-side RRTMGP inputs - will go away once SW radiation is CCPPized
-   call rrtmgp_inputs_cam_init(ktopcam, ktoprad, idx_sw_diag, idx_nir_diag, idx_uv_diag, idx_sw_cloudsim, idx_lw_diag, &
-           idx_lw_cloudsim)
-
    ! Set radconstants module-level index variables that we're setting in CCPP-ized scheme now
    call radconstants_init(idx_sw_diag, idx_nir_diag, idx_uv_diag, idx_lw_diag)
 
@@ -1264,14 +1259,11 @@ subroutine radiation_tend( &
 
                   ! Compute the gas optics (stored in atm_optics_sw).
                   ! toa_flux is the reference solar source from RRTMGP data.
-                  !$acc data copyin(kdist_sw%gas_props,pmid_day,pint_day,t_day,gas_concs_sw%gas_concs,atm_optics_sw%optical_props) &
-                  !$acc     copyout(toa_flux)
                   call rrtmgp_sw_gas_optics_run(dosw, 1, nday, nday, pmid_day, pint_day, t_day,  &
                                gas_concs_sw, atm_optics_sw, kdist_sw, toa_flux, errmsg, errflg)
                   if (errflg /= 0) then
                      call endrun(sub//': '//errmsg)
                   end if
-                  !$acc end data
 
                   ! Scale the solar source
                   call rrtmgp_sw_solar_var_run(toa_flux, 2, band2gpt_sw, nswbands, sol_irrad, we, nbins, sol_tsi, &
@@ -1285,26 +1277,16 @@ subroutine radiation_tend( &
                ! Set SW aerosol optical properties in the aer_sw object.
                ! This call made even when no daylight columns because it does some
                ! diagnostic aerosol output.
-               call rrtmgp_set_aer_sw( &
-                  icall, state, pbuf, nday, idxday, nnite, idxnite, aer_sw)
+               call rrtmgp_set_aer_sw(ktopcam, ktoprad, icall, state, pbuf, nday, idxday, nnite, idxnite, aer_sw)
                   
                if (nday > 0) then
 
                   ! Increment the gas optics (in atm_optics_sw) by the aerosol optics in aer_sw.
-                  !$acc data copyin(coszrs_day, toa_flux, alb_dir, alb_dif, &
-                  !$acc             atm_optics_sw%optical_props, atm_optics_sw%optical_props%tau, atm_optics_sw%optical_props%ssa, &
-                  !$acc             atm_optics_sw%optical_props%g, aer_sw%optical_props%tau,          &
-                  !$acc             aer_sw%optical_props, aer_sw%optical_props%ssa, aer_sw%optical_props%g,                 &
-                  !$acc             cloud_sw%optical_props, cloud_sw%optical_props%tau, cloud_sw%optical_props%ssa,           &
-                  !$acc             cloud_sw%optical_props%g)                                         &
-                  !$acc        copy(fswc%fluxes, fswc%fluxes%flux_net,fswc%fluxes%flux_up,fswc%fluxes%flux_dn,     &
-                  !$acc             fsw%fluxes, fsw%fluxes%flux_net,fsw%fluxes%flux_up,fsw%fluxes%flux_dn)
                   call rrtmgp_sw_rte_run(dosw, .true., .true., nday, 1, nday, atm_optics_sw, cloud_sw,  &
                                  aer_sw, coszrs_day, toa_flux, alb_dir, alb_dif, fswc, fsw, errmsg, errflg)
                   if (errflg /= 0) then
                      call endrun(sub//': '//errmsg)
                   end if
-                  !$acc end data
                end if
 
                ! Transform RRTMGP outputs to CAM outputs and compute heating rates.
@@ -1364,45 +1346,24 @@ subroutine radiation_tend( &
                end if
 
                ! Compute the gas optics and Planck sources.
-               !$acc data copyin(kdist_lw%gas_props, pmid_rad, pint_rad, t_rad,  &
-               !$acc             t_sfc, gas_concs_lw%gas_concs, atm_optics_lw%optical_props)         &
-               !$acc        copy(atm_optics_lw%optical_props%tau,                &
-               !$acc             sources_lw%sources, sources_lw%sources%lay_source,                  &
-               !$acc             sources_lw%sources%sfc_source,                  &
-               !$acc             sources_lw%sources%lev_source,                  &
-               !$acc             sources_lw%sources%sfc_source_jac)
                call rrtmgp_lw_gas_optics_run(dolw, 1, ncol, ncol, pmid_rad, pint_rad, t_rad,  &
                   t_sfc, gas_concs_lw, atm_optics_lw, sources_lw, t_rad, .false., kdist_lw, errmsg, &
                   errflg)
                if (errflg /= 0) then
                   call endrun(sub//': '//errmsg)
                end if
-               !$acc end data
 
                ! Set LW aerosol optical properties in the aer_lw object.
-               call rrtmgp_set_aer_lw(icall, state, pbuf, aer_lw)
+               call rrtmgp_set_aer_lw(ktopcam, ktoprad, icall, state, pbuf, aer_lw)
 
                ! Call the main rrtmgp_lw driver
-               !$acc data copyin(atm_optics_lw%optical_props,atm_optics_lw%optical_props%tau,   &
-               !$acc             aer_lw%optical_props,aer_lw%optical_props%tau,          &
-               !$acc             cloud_lw%optical_props, cloud_lw%optical_props%tau,        &
-               !$acc             sources_lw%sources,sources_lw%sources%lay_source,     &
-               !$acc             sources_lw%sources%sfc_source,     &
-               !$acc             sources_lw%sources%lev_source,     &
-               !$acc             sources_lw%sources%sfc_source_jac, &
-               !$acc             emis_sfc)                          &
-               !$acc        copy(flwc%fluxes, flwc%fluxes%flux_net, flwc%fluxes%flux_up, &
-               !$acc             flwc%fluxes%flux_dn, flw%fluxes, flw%fluxes%flux_net,  &
-               !$acc             flw%fluxes%flux_up, flw%fluxes%flux_dn,    &
-               !$acc             lw_ds)
                call rrtmgp_lw_rte_run(dolw, dolw, .false., .false., .false., &
                                  0, atm_optics_lw, cloud_lw, sources_lw, emis_sfc, &
                                  kdist_lw, aer_lw, fluxlwup_jac, lw_ds, flwc, flw, &
                                  errmsg, errflg)
                if (errflg /= 0) then
                   call endrun(sub//': '//errmsg)
                end if
-               !$acc end data
                
                ! Transform RRTMGP outputs to CAM outputs and compute heating rates.
                call set_lw_diags()
diff --git a/src/physics/rrtmgp/rrtmgp_inputs_cam.F90 b/src/physics/rrtmgp/rrtmgp_inputs_cam.F90
@@ -44,7 +44,6 @@ module rrtmgp_inputs_cam
 save
 
 public :: &
-   rrtmgp_inputs_cam_init, &
    rrtmgp_get_gas_mmrs, &
    rrtmgp_set_aer_lw,   &
    rrtmgp_set_aer_sw
@@ -53,16 +52,6 @@ module rrtmgp_inputs_cam
 ! This value is to match the arbitrary small value used in RRTMG to decide
 ! when a quantity is effectively zero.
 real(r8), parameter :: tiny = 1.0e-80_r8
-real(r8) :: sw_low_bounds(nswbands)
-real(r8) :: sw_high_bounds(nswbands)
-integer :: ktopcam
-integer :: ktoprad
-integer :: idx_sw_diag
-integer :: idx_nir_diag
-integer :: idx_uv_diag
-integer :: idx_sw_cloudsim
-integer :: idx_lw_diag
-integer :: idx_lw_cloudsim
 
 ! Mapping from RRTMG shortwave bands to RRTMGP.  Currently needed to continue using
 ! the SW optics datasets from RRTMG (even thought there is a slight mismatch in the
@@ -74,43 +63,6 @@ module rrtmgp_inputs_cam
 contains
 !==================================================================================================
 
-!==================================================================================================
-subroutine rrtmgp_inputs_cam_init(ktcam, ktrad, idx_sw_diag_in, idx_nir_diag_in, idx_uv_diag_in, &
-               idx_sw_cloudsim_in, idx_lw_diag_in, idx_lw_cloudsim_in)
-      
-   ! Note that this routine must be called after the calls to set_wavenumber_bands which set
-   ! the sw/lw band boundaries in the radconstants module.
-
-   integer, intent(in) :: ktcam
-   integer, intent(in) :: ktrad
-   integer, intent(in) :: idx_sw_diag_in
-   integer, intent(in) :: idx_nir_diag_in
-   integer, intent(in) :: idx_uv_diag_in
-   integer, intent(in) :: idx_sw_cloudsim_in
-   integer, intent(in) :: idx_lw_diag_in
-   integer, intent(in) :: idx_lw_cloudsim_in
-   character(len=512) :: errmsg
-   integer :: errflg
-
-   ktopcam = ktcam
-   ktoprad = ktrad
-   idx_sw_diag = idx_sw_diag_in
-   idx_nir_diag = idx_nir_diag_in
-   idx_uv_diag = idx_uv_diag_in
-   idx_sw_cloudsim = idx_sw_cloudsim_in
-   idx_lw_diag = idx_lw_diag_in
-   idx_lw_cloudsim = idx_lw_cloudsim_in
-
-   ! Initialize the module data containing the SW band boundaries.
-   call get_sw_spectral_boundaries_ccpp(sw_low_bounds, sw_high_bounds, 'cm^-1', errmsg, errflg)
-   if (errflg /= 0) then
-      call endrun('rrtmgp_inputs_cam_init: error during get_sw_spectral_boundaries_ccpp - message: '//errmsg)
-   end if
-
-end subroutine rrtmgp_inputs_cam_init
-
-!=========================================================================================
-
 subroutine rrtmgp_get_gas_mmrs(icall, state, pbuf, nlay, gas_mmrs)
 
    ! Retrieve mass mixing ratios for radiatively active gases from rad_constituents
@@ -137,11 +89,13 @@ end subroutine rrtmgp_get_gas_mmrs
 
 !==================================================================================================
 
-subroutine rrtmgp_set_aer_lw(icall, state, pbuf, aer_lw)
+subroutine rrtmgp_set_aer_lw(ktopcam, ktoprad, icall, state, pbuf, aer_lw)
 
    ! Load LW aerosol optical properties into the RRTMGP object.
 
    ! Arguments
+   integer,                     intent(in) :: ktopcam
+   integer,                     intent(in) :: ktoprad
    integer,                     intent(in) :: icall
    type(physics_state), target, intent(in) :: state
    type(physics_buffer_desc),   pointer    :: pbuf(:)
@@ -178,11 +132,13 @@ end subroutine rrtmgp_set_aer_lw
 !==================================================================================================
 
 subroutine rrtmgp_set_aer_sw( &
-   icall, state, pbuf, nday, idxday, nnite, idxnite, aer_sw)
+   ktopcam, ktoprad, icall, state, pbuf, nday, idxday, nnite, idxnite, aer_sw)
 
    ! Load SW aerosol optical properties into the RRTMGP object.
 
    ! Arguments
+   integer,                     intent(in) :: ktopcam
+   integer,                     intent(in) :: ktoprad
    integer,                     intent(in) :: icall
    type(physics_state), target, intent(in) :: state
    type(physics_buffer_desc), pointer      :: pbuf(:)
