Wrapper PR for #596, #609, #605: zorl interstitial, ocn -> wat, merra2 threading

physics/GFS_PBL_generic.meta

@@ -1272,26 +1272,26 @@
   intent = in
   optional = F
 [stress_wat]
-  standard_name = surface_wind_stress_over_ocean
-  long_name = surface wind stress over ocean
+  standard_name = surface_wind_stress_over_water
+  long_name = surface wind stress over water
   units = m2 s-2
   dimensions = (horizontal_loop_extent)
   type = real
   kind = kind_phys
   intent = in
   optional = F
 [hflx_wat]
-  standard_name = kinematic_surface_upward_sensible_heat_flux_over_ocean
-  long_name = kinematic surface upward sensible heat flux over ocean
+  standard_name = kinematic_surface_upward_sensible_heat_flux_over_water
+  long_name = kinematic surface upward sensible heat flux over water
   units = K m s-1
   dimensions = (horizontal_loop_extent)
   type = real
   kind = kind_phys
   intent = in
   optional = F
 [evap_wat]
-  standard_name = kinematic_surface_upward_latent_heat_flux_over_ocean
-  long_name = kinematic surface upward latent heat flux over ocean
+  standard_name = kinematic_surface_upward_latent_heat_flux_over_water
+  long_name = kinematic surface upward latent heat flux over water
   units = kg kg-1 m s-1
   dimensions = (horizontal_loop_extent)
   type = real

physics/GFS_cloud_diagnostics.meta

@@ -160,4 +160,4 @@
   dimensions = ()
   type = integer
   intent = out
-  optional = F    
+  optional = F    

physics/GFS_debug.meta

@@ -646,6 +646,51 @@
   type = integer
   intent = in
   optional = F
+[zorl]
+  standard_name = surface_roughness_length
+  long_name = surface roughness length
+  units = cm
+  dimensions = (horizontal_loop_extent)
+  type = real
+  kind = kind_phys
+  intent = in
+  optional = F
+[zorlw]
+  standard_name = surface_roughness_length_over_water
+  long_name = surface roughness length over water
+  units = cm
+  dimensions = (horizontal_loop_extent)
+  type = real
+  kind = kind_phys
+  intent = in
+  optional = F
+[zorll]
+  standard_name = surface_roughness_length_over_land
+  long_name = surface roughness length over land
+  units = cm
+  dimensions = (horizontal_loop_extent)
+  type = real
+  kind = kind_phys
+  intent = in
+  optional = F
+[zorli]
+  standard_name = surface_roughness_length_over_ice
+  long_name = surface roughness length over ice
+  units = cm
+  dimensions = (horizontal_loop_extent)
+  type = real
+  kind = kind_phys
+  intent = in
+  optional = F
+[fice]
+  standard_name = sea_ice_concentration
+  long_name = ice fraction over open water
+  units = frac
+  dimensions = (horizontal_loop_extent)
+  type = real
+  kind = kind_phys
+  intent = in
+  optional = F
 [errmsg]
   standard_name = ccpp_error_message
   long_name = error message for error handling in CCPP

physics/GFS_phys_time_vary.fv3.F90

@@ -305,8 +305,8 @@ subroutine GFS_phys_time_vary_init (
            call setindxh2o (im, xlat_d, jindx1_h, jindx2_h, ddy_h)
          endif
 
-!> - Call setindxaer() to initialize aerosols data
 !$OMP section
+!> - Call setindxaer() to initialize aerosols data
          if (iaerclm) then
            call setindxaer (im, xlat_d, jindx1_aer,          &
                             jindx2_aer, ddy_aer, xlon_d,     &
@@ -317,8 +317,8 @@ subroutine GFS_phys_time_vary_init (
            jamin=min(minval(jindx1_aer), jamin)
            jamax=max(maxval(jindx2_aer), jamax)
          endif
-!$OMP section
 
+!$OMP section
 !> - Call setindxci() to initialize IN and CCN data
          if (iccn == 1) then
            call setindxci (im, xlat_d, jindx1_ci,      &
@@ -376,10 +376,14 @@ subroutine GFS_phys_time_vary_init (
 !$OMP end sections
 
 !$OMP end parallel
+
+         if (errflg/=0) return
+
          if (iaerclm) then
-           call read_aerdataf (iamin, iamax, jamin, jamax, me,master,iflip,            &
-                              idate,errmsg,errflg)
-         endif
+           call read_aerdataf (iamin, iamax, jamin, jamax, me, master, iflip, &
+                               idate, errmsg, errflg)
+           if (errflg/=0) return
+         end if
 
          if (lsm == lsm_noahmp) then
            if (all(tvxy < zero)) then
@@ -431,15 +435,34 @@ subroutine GFS_phys_time_vary_init (
              smoiseq(:,:)  = missing_value
              zsnsoxy(:,:)  = missing_value
 
+             imn           = idate(2)
+
+!$OMP parallel do num_threads(nthrds) default(none)                     &
+!$OMP          shared(im,lsoil,con_t0c,landfrac,tsfcl,tvxy,tgxy,tahxy)  &
+!$OMP          shared(snowd,canicexy,canliqxy,canopy,eahxy,cmxy,chxy)   &
+!$OMP          shared(fwetxy,sneqvoxy,weasd,alboldxy,qsnowxy,wslakexy)  &
+!$OMP          shared(taussxy,albdvis,albdnir,albivis,albinir,emiss)    &
+!$OMP          shared(waxy,wtxy,zwtxy,imn,vtype,xlaixy,xsaixy,lfmassxy) &
+!$OMP          shared(stmassxy,rtmassxy,woodxy,stblcpxy,fastcpxy)       &
+!$OMP          shared(isbarren_table,isice_table,isurban_table)         &
+!$omp          shared(iswater_table,laim_table,sla_table,bexp_table)    &
+!$omp          shared(stc,smc,slc,tg3,snowxy,tsnoxy,snicexy,snliqxy)    &
+!$omp          shared(zsnsoxy,STYPE,SMCMAX_TABLE,SMCWLT_TABLE,zs,dzs)   &
+!$omp          shared(DWSAT_TABLE,DKSAT_TABLE,PSISAT_TABLE,smoiseq)     &
+!$OMP          shared(smcwtdxy,deeprechxy,rechxy,errmsg,errflg)         &
+!$OMP          private(vegtyp,masslai,masssai,snd,dzsno,dzsnso,isnow)   &
+!$OMP          private(soiltyp,bexp,smcmax,smcwlt,dwsat,dksat,psisat,ddz)
              do ix=1,im
                if (landfrac(ix) >= drythresh) then
                  tvxy(ix)     = tsfcl(ix)
                  tgxy(ix)     = tsfcl(ix)
                  tahxy(ix)    = tsfcl(ix)
 
-                 if (snowd(ix) > 0.01_kind_phys .and. tsfcl(ix) > con_t0c ) tvxy(ix)  = con_t0c
-                 if (snowd(ix) > 0.01_kind_phys .and. tsfcl(ix) > con_t0c ) tgxy(ix)  = con_t0c
-                 if (snowd(ix) > 0.01_kind_phys .and. tsfcl(ix) > con_t0c ) tahxy(ix) = con_t0c
+                 if (snowd(ix) > 0.01_kind_phys .and. tsfcl(ix) > con_t0c ) then
+                   tvxy(ix)  = con_t0c
+                   tgxy(ix)  = con_t0c
+                   tahxy(ix) = con_t0c
+                 end if
 
                  canicexy(ix) = 0.0_kind_phys
                  canliqxy(ix) = canopy(ix)
@@ -463,14 +486,12 @@ subroutine GFS_phys_time_vary_init (
                  albinir(ix)  = 0.2_kind_phys
                  emiss(ix)    = 0.95_kind_phys
 
-
                  waxy(ix)     = 4900.0_kind_phys
                  wtxy(ix)     = waxy(ix)
                  zwtxy(ix)    = (25.0_kind_phys + 2.0_kind_phys) - waxy(ix) / 1000.0_kind_phys / 0.2_kind_phys
 
                  vegtyp       = vtype(ix)
                  if (vegtyp == 0) vegtyp = 7
-                 imn          = idate(2)
 
                  if ((vegtyp == isbarren_table) .or. (vegtyp == isice_table) .or. (vegtyp == isurban_table) .or. (vegtyp == iswater_table)) then
 
@@ -552,7 +573,6 @@ subroutine GFS_phys_time_vary_init (
                  else
                    errmsg = 'Error in GFS_phys_time_vary.fv3.F90: Problem with the logic assigning snow layers in Noah MP initialization'
                    errflg = 1
-                   return
                  endif
 
 ! Now we have the snowxy field
@@ -628,6 +648,9 @@ subroutine GFS_phys_time_vary_init (
                endif
 
              enddo ! ix
+!$OMP end parallel do
+
+             if (errflg/=0) return
 
              deallocate(dzsno)
              deallocate(dzsnso)
@@ -748,6 +771,20 @@ subroutine GFS_phys_time_vary_timestep_init (
             return
          end if
 
+!$OMP parallel num_threads(nthrds) default(none)                                         &
+!$OMP          shared(kdt,nsswr,lsswr,clstp,imfdeepcnv,cal_pre,random_clds)              &
+!$OMP          shared(fhswr,fhour,seed0,cnx,cny,nrcm,wrk,rannie,rndval)                  &
+!$OMP          shared(rann,im,isc,jsc,imap,jmap,ntoz,me,idate,jindx1_o3,jindx2_o3)       &
+!$OMP          shared(ozpl,ddy_o3,h2o_phys,jindx1_h,jindx2_h,h2opl,ddy_h,iaerclm,master) &
+!$OMP          shared(levs,prsl,iccn,jindx1_ci,jindx2_ci,ddy_ci,iindx1_ci,iindx2_ci)     &
+!$OMP          shared(ddx_ci,in_nm,ccn_nm,do_ugwp_v1,jindx1_tau,jindx2_tau,ddy_j1tau)    &
+!$OMP          shared(ddy_j2tau,tau_amf)                                                 &
+!$OMP          private(iseed,iskip,i,j,k)
+
+!$OMP sections
+
+!$OMP section
+
          !--- switch for saving convective clouds - cnvc90.f
          !--- aka Ken Campana/Yu-Tai Hou legacy
          if ((mod(kdt,nsswr) == 0) .and. (lsswr)) then
@@ -764,6 +801,8 @@ subroutine GFS_phys_time_vary_timestep_init (
            clstp = 0100
          endif
 
+!$OMP section
+
          !--- random number needed for RAS and old SAS and when cal_pre=.true.
          !    imfdeepcnv < 0 when ras = .true.
          if ( (imfdeepcnv <= 0 .or. cal_pre) .and. random_clds ) then
@@ -789,29 +828,23 @@ subroutine GFS_phys_time_vary_timestep_init (
 
          endif  ! imfdeepcnv, cal_re, random_clds
 
+!$OMP section
 !> - Call ozinterpol() to make ozone interpolation
          if (ntoz > 0) then
            call ozinterpol (me, im, idate, fhour, &
                             jindx1_o3, jindx2_o3, &
                             ozpl, ddy_o3)
          endif
 
+!$OMP section
 !> - Call h2ointerpol() to make stratospheric water vapor data interpolation
          if (h2o_phys) then
            call h2ointerpol (me, im, idate, fhour, &
                              jindx1_h, jindx2_h,   &
                              h2opl, ddy_h)
          endif
 
-!> - Call aerinterpol() to make aerosol interpolation
-         if (iaerclm) then
-           call aerinterpol (me, master, im, idate, fhour, &
-                             jindx1_aer, jindx2_aer,       &
-                             ddy_aer, iindx1_aer,          &
-                             iindx2_aer, ddx_aer,          &
-                             levs, prsl, aer_nm)
-         endif
-
+!$OMP section
 !> - Call ciinterpol() to make IN and CCN data interpolation
          if (iccn == 1) then
            call ciinterpol (me, im, idate, fhour,     &
@@ -821,13 +854,28 @@ subroutine GFS_phys_time_vary_timestep_init (
                             levs, prsl, in_nm, ccn_nm)
          endif
 
+!$OMP section
 !> - Call  cires_indx_ugwp to read monthly-mean GW-tau diagnosed from FV3GFS-runs that resolve GW-activ
          if (do_ugwp_v1) then
            call tau_amf_interp(me, master, im, idate, fhour, &
                                jindx1_tau, jindx2_tau,       &
                                ddy_j1tau, ddy_j2tau, tau_amf)
          endif
 
+!$OMP end sections
+!$OMP end parallel
+
+!> - Call aerinterpol() to make aerosol interpolation
+         if (iaerclm) then
+           ! aerinterpol is using threading inside, don't
+           ! move into OpenMP parallel section above
+           call aerinterpol (me, master, nthrds, im, idate, &
+                              fhour, jindx1_aer, jindx2_aer,&
+                             ddy_aer, iindx1_aer,           &
+                             iindx2_aer, ddx_aer,           &
+                             levs, prsl, aer_nm)
+         endif
+
 !> - Call gcycle() to repopulate specific time-varying surface properties for AMIP/forecast runs
          if (nscyc >  0) then
            if (mod(kdt,nscyc) == 1) THEN

physics/GFS_phys_time_vary.fv3.meta

@@ -1793,8 +1793,8 @@
   intent = inout
   optional = F
 [zorlo]
-  standard_name = surface_roughness_length_over_ocean
-  long_name = surface roughness length over ocean
+  standard_name = surface_roughness_length_over_water
+  long_name = surface roughness length over water
   units = cm
   dimensions = (horizontal_dimension)
   type = real

physics/GFS_rrtmgp_thompsonmp_pre.meta

@@ -448,4 +448,4 @@
   dimensions = ()
   type = integer
   intent = out
-  optional = F  
+  optional = F  

physics/GFS_suite_interstitial.F90

@@ -159,7 +159,7 @@ end subroutine GFS_suite_interstitial_2_finalize
 !> \section arg_table_GFS_suite_interstitial_2_run Argument Table
 !! \htmlinclude GFS_suite_interstitial_2_run.html
 !!
-    subroutine GFS_suite_interstitial_2_run (im, levs, lssav, ldiag3d, lsidea, cplflx, flag_cice, shal_cnv, old_monin, mstrat,    &
+    subroutine GFS_suite_interstitial_2_run (im, levs, lssav, ldiag3d, lsidea, flag_cice, shal_cnv, old_monin, mstrat,            &
       do_shoc, frac_grid, imfshalcnv, dtf, xcosz, adjsfcdsw, adjsfcdlw, cice, pgr, ulwsfc_cice, lwhd, htrsw, htrlw, xmu, ctei_rm, &
       work1, work2, prsi, tgrs, prsl, qgrs_water_vapor, qgrs_cloud_water, cp, hvap, prslk, suntim, adjsfculw, adjsfculw_lnd,      &
       adjsfculw_ice, adjsfculw_wat, dlwsfc, ulwsfc, psmean, dt3dt_lw, dt3dt_sw, dt3dt_pbl, dt3dt_dcnv, dt3dt_scnv, dt3dt_mp,      &
@@ -169,7 +169,7 @@ subroutine GFS_suite_interstitial_2_run (im, levs, lssav, ldiag3d, lsidea, cplfl
 
       ! interface variables
       integer,              intent(in   ) :: im, levs, imfshalcnv
-      logical,              intent(in   ) :: lssav, ldiag3d, lsidea, cplflx, shal_cnv
+      logical,              intent(in   ) :: lssav, ldiag3d, lsidea, shal_cnv
       logical,              intent(in   ) :: old_monin, mstrat, do_shoc, frac_grid, use_LW_jacobian
       real(kind=kind_phys), intent(in   ) :: dtf, cp, hvap
 

physics/GFS_suite_interstitial.meta

@@ -321,14 +321,6 @@
   type = logical
   intent = in
   optional = F
-[cplflx]
-  standard_name = flag_for_flux_coupling
-  long_name = flag controlling cplflx collection (default off)
-  units = flag
-  dimensions = ()
-  type = logical
-  intent = in
-  optional = F
 [flag_cice]
   standard_name = flag_for_cice
   long_name = flag for cice
@@ -628,8 +620,8 @@
   intent = in
   optional = F
 [adjsfculw_wat]
-  standard_name = surface_upwelling_longwave_flux_over_ocean_interstitial
-  long_name = surface upwelling longwave flux at current time over ocean (temporary use as interstitial)
+  standard_name = surface_upwelling_longwave_flux_over_water_interstitial
+  long_name = surface upwelling longwave flux at current time over water (temporary use as interstitial)
   units = W m-2
   dimensions = (horizontal_loop_extent)
   type = real