Skip to content

MYNN sfclay restart reproducibility #822

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
climbfuji merged 6 commits into from
Jan 6, 2022
Merged

MYNN sfclay restart reproducibility #822

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e96cc7f
Remove commented line from physics/GFS_surface_composites.F90
climbfuji Dec 29, 2021
68e366d
Bug fixes for restart reproducibility in MYNNSFC
climbfuji Dec 29, 2021
4b6b6d6
Merge branch 'main' of https://github.com/NCAR/ccpp-physics into mynn…
climbfuji Dec 29, 2021
eb248cc
Merge branch 'main' of https://github.com/NCAR/ccpp-physics into mynn…
climbfuji Jan 3, 2022
b7f7612
Merge branch 'main' of https://github.com/NCAR/ccpp-physics into mynn…
climbfuji Jan 6, 2022
cd92c4c
Put back initialization of ustar_{lnd,ice,wat} in physics/module_sf_m…
climbfuji Jan 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions physics/GFS_surface_composites.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -410,8 +410,7 @@ subroutine GFS_surface_composites_post_run (
cmm, cmm_wat, cmm_lnd, cmm_ice, chh, chh_wat, chh_lnd, chh_ice, gflx, gflx_wat, gflx_lnd, gflx_ice, ep1d, ep1d_wat, &
ep1d_lnd, ep1d_ice, weasd, weasd_lnd, weasd_ice, snowd, snowd_lnd, snowd_ice, tprcp, tprcp_wat, &
tprcp_lnd, tprcp_ice, evap, evap_wat, evap_lnd, evap_ice, hflx, hflx_wat, hflx_lnd, hflx_ice, qss, qss_wat, qss_lnd, &
qss_ice, tsfc, tsfco, tsfcl, tsfc_wat, tisfc, hice, cice, tiice, &
! qss_ice, tsfc, tsfco, tsfcl, tsfc_wat, tisfc, hice, cice, min_seaice, tiice, &
qss_ice, tsfc, tsfco, tsfcl, tsfc_wat, tisfc, hice, cice, tiice, &
sigmaf, zvfun, lheatstrg, h0facu, h0facs, hflxq, hffac, stc, &
grav, prsik1, prslk1, prslki, z1, ztmax_wat, ztmax_lnd, ztmax_ice, huge, errmsg, errflg)

Expand Down
15 changes: 11 additions & 4 deletions physics/module_MYNNSFC_wrapper.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ SUBROUTINE mynnsfc_wrapper_run( &
& CHS2, CQS2, rmol, zol, mol, ch, &
& lh, wstar
!LOCAL
real, dimension(im) :: &
real(kind=kind_phys), dimension(im) :: &
& hfx, znt, psim, psih, &
& chs, ck, cd, mavail, xland, GZ1OZ0, &
& cpm, qgh, qfx, snowh_wat
Expand All @@ -228,13 +228,19 @@ SUBROUTINE mynnsfc_wrapper_run( &
! endif

! prep MYNN-only variables
pattern_spp_pbl(:,:) = 0
dz(:,:) = 0
th(:,:) = 0
qv(:,:) = 0
hfx(:) = 0
qfx(:) = 0
rmol(:) = 0
do k=1,2 !levs
do i=1,im
dz(i,k)=(phii(i,k+1) - phii(i,k))*g_inv
th(i,k)=t3d(i,k)/exner(i,k)
!qc(i,k)=MAX(qgrs(i,k,ntcw),0.0)
qv(i,k)=qvsh(i,k)/(1.0 - qvsh(i,k))
pattern_spp_pbl(i,k)=0.0
enddo
enddo
do i=1,im
Expand Down Expand Up @@ -333,8 +339,9 @@ SUBROUTINE mynnsfc_wrapper_run( &
spp_pbl=spp_pbl,pattern_spp_pbl=pattern_spp_pbl, &
ids=1,ide=im, jds=1,jde=1, kds=1,kde=levs, &
ims=1,ime=im, jms=1,jme=1, kms=1,kme=levs, &
its=1,ite=im, jts=1,jte=1, kts=1,kte=levs )

its=1,ite=im, jts=1,jte=1, kts=1,kte=levs, &
errmsg=errmsg, errflg=errflg )
if (errflg/=0) return

!! POST MYNN SURFACE LAYER (INTERSTITIAL) WORK:
!do i = 1, im
Expand Down
50 changes: 33 additions & 17 deletions physics/module_sf_mynn.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,8 @@ SUBROUTINE SFCLAY_mynn( &
spp_pbl,pattern_spp_pbl, &
ids,ide, jds,jde, kds,kde, &
ims,ime, jms,jme, kms,kme, &
its,ite, jts,jte, kts,kte )
its,ite, jts,jte, kts,kte, &
errmsg, errflg )
!-------------------------------------------------------------------
IMPLICIT NONE
!-------------------------------------------------------------------
Expand Down Expand Up @@ -258,6 +259,8 @@ SUBROUTINE SFCLAY_mynn( &
!-- jte end index for j in tile
!-- kts start index for k in tile
!-- kte end index for k in tile
!-- errmsg CCPP error message
!-- errflg CCPP error code
!=================================================================
! SCALARS
!===================================
Expand Down Expand Up @@ -352,9 +355,13 @@ SUBROUTINE SFCLAY_mynn( &
& QFLX_wat, QFLX_lnd, QFLX_ice, &
& qsfc_wat, qsfc_lnd, qsfc_ice

! CCPP error handling
character(len=*), intent(inout) :: errmsg
integer, intent(inout) :: errflg

!ADDITIONAL OUTPUT
!JOE-begin
REAL, DIMENSION( ims:ime ) :: qstar
REAL, DIMENSION( ims:ime ) :: qstar
!JOE-end
!===================================
! 1D LOCAL ARRAYS
Expand Down Expand Up @@ -401,6 +408,7 @@ SUBROUTINE SFCLAY_mynn( &
else
rstoch1D(i)=0.0
endif
qstar(i)=0.0
ENDDO

IF (itimestep==1 .AND. iter==1) THEN
Expand All @@ -410,9 +418,6 @@ SUBROUTINE SFCLAY_mynn( &
UST_LND(i)=MAX(0.04*SQRT(U1D(i)*U1D(i) + V1D(i)*V1D(i)),0.001)
UST_ICE(i)=MAX(0.04*SQRT(U1D(i)*U1D(i) + V1D(i)*V1D(i)),0.001)
MOL(i)=0.0
Copy link
Copy Markdown
Collaborator

@joeolson42 joeolson42 Dec 29, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm surprised removing this helps. I don't see u* initialized before it's first used.

Copy link
Copy Markdown
Collaborator Author

@climbfuji climbfuji Dec 29, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ust_{wat,ice,lnd} are initialized in GFS_surface_composites_pre_run, see lines 206-242.

Copy link
Copy Markdown
Collaborator

@joeolson42 joeolson42 Dec 29, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see... If this helps with restarts, then OK, but I don't know how uustar is being initialized in a cold start, so I may add this code back into the wrapper eventually and make sure it's not used for restarts. Does that seem reasonable?

Copy link
Copy Markdown
Collaborator Author

@climbfuji climbfuji Dec 29, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

According to FV3/io/FV3GFS_io.F90, it comes from the initial conditions. It's in the INPUT/sfc_data.tile?.nc files, for example:

data:

 uustar =
  0.171213456605573, 0.245438690554831, 0.256640811047583, 0.267651122786174,
    0.280224842133838, 0.279971796758598, 0.298066313514827,
    0.314749952182947, 0.33006493572479, 0.343210265083255,
    0.351366633436041, 0.362353378760175, 0.369426004804981,
    0.382825370910827, 0.40201134915115, 0.416131400590514,
    0.443041998906594, 0.453063344833092, 0.451435060306367,
    0.452779314754435, 0.432702751758146, 0.411955027777123,
    0.370880666443607, 0.310652664808393, 0.265327782215652, 0.2247260561156,
    0.263029915756367, 0.266947287025747, 0.296322723516394,
    0.308662654500799, 0.288600599239451, 0.180472574147917,
    0.163883864828559, 0.125231393380142, 0.154961388994689,
    0.146057388032206, 0.136944259693473, 0.237380508574376,
    0.305121627034509, 0.517771283751996, 0.568027172739538,
    0.50736289058438, 0.482317351376341, 0.413385954256456,
    0.400374229553073, 0.370092052409071, 0.336648494634005,
    0.300564612619546, 0.283338773802275, 0.341238177586638,
    0.261152207285838, 0.224116424328721, 0.215924232511202,
    0.189917101300733, 0.199322285712027, 0.157243532127691,
    0.19359710752422, 0.187451240086616, 0.196309269556824,
    0.198743901054553, 0.215083947166383, 0.272523138480303,
    0.274216651200398, 0.29467402005329, 0.276077397930394,
    0.228904980612239, 0.234033125425608, 0.258624364207327,
    0.311573093934831, 0.303495080141041, 0.25989005987661,
    0.218364635421538, 0.207690646375217, 0.212594973673417,
    0.215970389337089, 0.219594971489457, 0.21399290992621,
    0.213915985996377, 0.192268435554861, 0.174167760364215,
    0.140072079656223, 0.138048994889202, 0.0790837518180875,
    0.0403957897147722, 0.0827031933453741, 0.0864850134003294,
    0.170175184707559, 0.23613873730674, 0.146922292088942,
    0.211363181392024, 0.243766474618606, 0.296856229449771,
    0.397668207915114, 0.450776965678785, 0.508171170652395, 0.523651537042418,
  0.0820358681750062, 0.157450881891468, 0.277328270778228,

We rely on the model initializing certain surface properties for many variables. If we don't want to do this anymore, then we are opening a can of worms because we will have to check for hundreds of variables. The compiler flags ensure that uustar cannot be uninitialized (but they can't make sure that the variable isn't set to zero).

qstar(i)=0.0
QFX(i)=0.
HFX(i)=0.
QFLX(i)=0.
HFLX(i)=0.
if ( LSM == LSM_RUC ) then
Expand Down Expand Up @@ -461,12 +466,12 @@ SUBROUTINE SFCLAY_mynn( &
PSIM,PSIH, &
HFLX,HFX,QFLX,QFX,LH,FLHC,FLQC, &
QGH,QSFC,U10,V10,TH2,T2,Q2, &
GZ1OZ0,WSPD,wstar, &
GZ1OZ0,WSPD,wstar,qstar, &
spp_pbl,rstoch1D, &
ids,ide, jds,jde, kds,kde, &
ims,ime, jms,jme, kms,kme, &
its,ite, jts,jte, kts,kte &
)
its,ite, jts,jte, kts,kte, &
errmsg, errflg )

END SUBROUTINE SFCLAY_MYNN

Expand Down Expand Up @@ -509,12 +514,12 @@ SUBROUTINE SFCLAY1D_mynn(flag_iter, &
HFLX,HFX,QFLX,QFX,LH,FLHC,FLQC, &
QGH,QSFC, &
U10,V10,TH2,T2,Q2, &
GZ1OZ0,WSPD,wstar, &
GZ1OZ0,WSPD,wstar,qstar, &
spp_pbl,rstoch1D, &
ids,ide, jds,jde, kds,kde, &
ims,ime, jms,jme, kms,kme, &
its,ite, jts,jte, kts,kte &
)
its,ite, jts,jte, kts,kte, &
errmsg, errflg )

!-------------------------------------------------------------------
IMPLICIT NONE
Expand Down Expand Up @@ -563,9 +568,10 @@ SUBROUTINE SFCLAY1D_mynn(flag_iter, &
dz8w1d, &
dz2w1d

REAL, DIMENSION( ims:ime ), INTENT(INOUT) :: HFLX,HFX, &
QFLX,QFX,LH, &
MOL,RMOL, &
REAL, DIMENSION( ims:ime ), INTENT(OUT) :: QFX,HFX, &
RMOL
REAL, DIMENSION( ims:ime ), INTENT(INOUT) :: HFLX,QFLX, &
LH,MOL, &
QGH,QSFC, &
ZNT, &
ZOL, &
Expand Down Expand Up @@ -610,8 +616,13 @@ SUBROUTINE SFCLAY1D_mynn(flag_iter, &

!--------------------------------------------
!JOE-additinal output
REAL, DIMENSION( ims:ime ) :: wstar,qstar
REAL, DIMENSION( ims:ime ), INTENT(OUT) :: wstar,qstar
!JOE-end

! CCPP error handling
character(len=*), intent(inout) :: errmsg
integer, intent(inout) :: errflg

!----------------------------------------------------------------
! LOCAL VARS
!----------------------------------------------------------------
Expand Down Expand Up @@ -661,8 +672,8 @@ SUBROUTINE SFCLAY1D_mynn(flag_iter, &
!-------------------------------------------------------------------
DO I=its,ite

! PSFC ( in cmb) is used later in saturation checks
PSFC(I)=PSFCPA(I)/1000.
! PSFC ( in cmb) is used later in saturation checks
PSFC(I)=PSFCPA(I)/1000.
!tgs - do computations if flag_iter(i) = .true.
if ( flag_iter(i) ) then

Expand Down Expand Up @@ -1224,6 +1235,11 @@ SUBROUTINE SFCLAY1D_mynn(flag_iter, &
CALL zilitinkevich_1995(ZNTstoch_lnd(i),ZT_lnd(i),ZQ_lnd(i),restar,&
UST_lnd(I),KARMAN,1.0,IZ0TLND,spp_pbl,rstoch1D(i))
ELSEIF ( IZ0TLND .EQ. 2 ) THEN
! DH note - at this point, qstar is either not initialized
! or initialized to zero, but certainly not set correctly
errmsg = 'Logic error: qstar is not set correctly when calling Yang_2008'
errflg = 1
return
CALL Yang_2008(ZNTSTOCH_lnd(i),ZT_lnd(i),ZQ_lnd(i),UST_lnd(i),MOL(I),&
qstar(I),restar,visc)
ELSEIF ( IZ0TLND .EQ. 3 ) THEN
Expand Down