Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion sorc/global_cycle.fd/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@ set(lib_src
set(exe_src cycle.f90)

if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel)$")
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -r8")
# set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -r8")
# Yuan Xue: set flag to suppress long name detector
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -diag-disable:5462 -r8")
elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$")
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8")
if(CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10)
Expand Down
130 changes: 104 additions & 26 deletions sorc/global_cycle.fd/cycle.f90
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,9 @@
!! -DO_SFCCYCLE Call sfccycle routine to update surface fields
!! -DO_LNDINC Read in land increment files, and add increments to
!! relevant states.
!! -DO_SOI_INC Do land increments to soil states.
!! -DO_SNO_INC Do land increments to snow states.
!! -DO_SOI_INC_GSI Do land increments (on the Guassian grid) to soil states.
!! -DO_SOI_INC_JEDI Do land increments (on the cubed sphere tiles) to soil.
!! -DO_SNO_INC Do land increments (on the cubed sphere tiles) to snow states.
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For consistency with the new soil increment flags, please change this to DO_SNO_INC_JEDI.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will do.

!! - ISOT Use statsgo soil type when '1'. Use zobler when '0'.
!! - IVEGSRC Use igbp veg type when '1'. Use sib when '2'.
!! - ZSEA1/2_MM When running with NSST model, this is the lower/
Expand Down Expand Up @@ -302,10 +303,14 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, &
USE READ_WRITE_DATA
use machine
USE MPI
USE LAND_INCREMENTS, ONLY: ADD_INCREMENT_SOIL, &
USE NETCDF
USE LAND_INCREMENTS, ONLY: ADD_GSI_INCREMENT_SOIL, &
ADD_JEDI_INCREMENT_SOIL_TEMP, &
ADD_JEDI_INCREMENT_SOIL_MOISTURE, &
ADD_INCREMENT_SNOW, &
CALCULATE_LANDINC_MASK, &
APPLY_LAND_DA_ADJUSTMENTS_SOIL, &
APPLY_LAND_DA_ADJUSTMENTS_SOIL_TEMP, &
APPLY_LAND_DA_ADJUSTMENTS_SOIL_MOISTURE, &
APPLY_LAND_DA_ADJUSTMENTS_SND, &
LSM_NOAH, LSM_NOAHMP

Expand All @@ -328,7 +333,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, &
CHARACTER(LEN=500) :: LND_SOI_FILE
CHARACTER(LEN=4) :: INPUT_NML_FILE(SZ_NML)

INTEGER :: I, IERR
INTEGER :: I, IERR, NCID, ID_VAR
INTEGER :: I_INDEX(LENSFC), J_INDEX(LENSFC)
INTEGER :: IDUM(IDIM,JDIM)
integer :: num_parthds, num_threads
Expand Down Expand Up @@ -366,6 +371,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, &
REAL, ALLOCATABLE :: SLIFCS_FG(:)
INTEGER, ALLOCATABLE :: LANDINC_MASK_FG(:), LANDINC_MASK(:)
REAL, ALLOCATABLE :: SND_BCK(:), SND_INC(:), SWE_BCK(:)
REAL, ALLOCATABLE :: SLCINC(:,:), STCINC(:,:)
REAL(KIND=KIND_IO8), ALLOCATABLE :: SLMASKL(:), SLMASKW(:)

TYPE(NSST_DATA) :: NSST
Expand All @@ -374,7 +380,9 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, &
INTEGER :: veg_type_landice
INTEGER, DIMENSION(LENSFC) :: STC_UPDATED, SLC_UPDATED

LOGICAL :: FILE_EXISTS, DO_SOI_INC, DO_SNO_INC
REAL, DIMENSION(LENSFC,LSOIL) :: STC_INC_TMP, SLC_INC_TMP

LOGICAL :: FILE_EXISTS, DO_SOI_INC_GSI, DO_SOI_INC_JEDI, DO_SNO_INC

!--------------------------------------------------------------------------------
! NST_FILE is the path/name of the gaussian GSI file which contains NSST
Expand All @@ -387,7 +395,8 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, &
DATA LND_SOI_FILE/'NULL'/

DO_SNO_INC = .FALSE.
DO_SOI_INC = .FALSE.
DO_SOI_INC_GSI = .FALSE.
DO_SOI_INC_JEDI = .FALSE.


SIG1T = 0.0 ! Not a dead start!
Expand Down Expand Up @@ -446,13 +455,28 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, &
IF (DO_LNDINC) THEN
! identify variables to be updates, and allocate arrays.
IF (TRIM(LND_SOI_FILE) .NE. "NULL") THEN
DO_SOI_INC = .TRUE.
DO_SOI_INC_GSI = .TRUE.
PRINT*
PRINT*," APPLYING SOIL INCREMENTS FROM THE GSI"
ALLOCATE(STC_BCK(LENSFC, LSOIL), SMC_BCK(LENSFC, LSOIL), SLC_BCK(LENSFC,LSOIL))
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I find the logic here a bit contrived. I seem to have coded this to have an option where DO_LNDINC is on, but then we skip it if the file name is not specified in the namelist. I'm not sure why I did this, but it doesn't make a lot of sense to me now. I'd also like to have a more consistent methods for both options (JEDI, GSI). I suggest changing the namelist to have two input options: DO_SOI_INC_GSI and DO_SOI_INC_JEDI, add then just assume standard file names for each (the GSI one is called, lnd_incr). Add a check to make sure both aren't specified as true, and have the code crash if the file is missing.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Per discussion, the DO_LNDINC will be kept as it is. Separate DO_SOI_INC_GSI and DO_SOI_INC_JEDI. Add options in the namelist file.

ALLOCATE(LANDINC_MASK_FG(LENSFC))
ELSE
INQUIRE(FILE='xainc.001', EXIST=file_exists)
IF (file_exists) then
IERR=NF90_OPEN('xainc.001',NF90_NOWRITE,NCID)
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks like it's only opening the JEDI increment from tile number 1 (right?). This will need to be generalized to open the appropriate file (with the number coming form the task numbers - copy how it's done for opening the background).

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Explained.

IERR=NF90_INQ_VARID(NCID,"stc_inc",ID_VAR)
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why would stc_inc not be on the file, and why do we specifically check for it?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Explained.

IF (IERR == 0) THEN
DO_SOI_INC_JEDI = .TRUE.
PRINT*
PRINT*," APPLYING SOIL INCREMENTS FROM THE JEDI"
ALLOCATE(SLCINC(LENSFC, LSOIL), STCINC(LENSFC, LSOIL))
ENDIF
ENDIF
ENDIF
ALLOCATE(STC_BCK(LENSFC, LSOIL), SMC_BCK(LENSFC, LSOIL), SLC_BCK(LENSFC,LSOIL))
ALLOCATE(LANDINC_MASK_FG(LENSFC))
! FOR NOW, CODE SO CAN DO BOTH, BUT MIGHT NEED TO THINK ABOUT THIS SOME MORE.
! NOTE: BOTH snow and soil related increment file are named as "xainc.00N"
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This answers my earlier question! If we're going to assume file names here, it would be best to use two different names. The increment file name is specified in the JEDI yaml, so easy to change at that end.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will do.

! This may create issues if doing soil and snow DA together
! For now, just put an additional inquiry above to distinguish - Yuan Xue
IF (DO_SNO_INC) THEN
! ideally, would check here that sfcsub snow DA update is not also requested
! but latter is controlled by fnsol, which is read in within that routine.
Expand All @@ -474,6 +498,8 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, &
! READ THE INPUT SURFACE DATA ON THE CUBED-SPHERE TILE.
!--------------------------------------------------------------------------------

IF (.NOT. DO_SOI_INC_JEDI) THEN

CALL READ_DATA(LSOIL,LENSFC,DO_NSST,.false.,IS_NOAHMP=IS_NOAHMP, &
TSFFCS=TSFFCS,SMCFCS=SMCFCS, &
SWEFCS=SWEFCS,STCFCS=STCFCS,TG3FCS=TG3FCS,ZORFCS=ZORFCS, &
Expand All @@ -485,6 +511,13 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, &
VMNFCS=VMNFCS,VMXFCS=VMXFCS,SLCFCS=SLCFCS,SLPFCS=SLPFCS, &
ABSFCS=ABSFCS,T2M=T2M ,Q2M=Q2M ,SLMASK=SLMASK, &
ZSOIL=ZSOIL, NSST=NSST)
ELSE
CALL READ_DATA(LSOIL,LENSFC,DO_NSST,.false.,IS_NOAHMP=IS_NOAHMP, &
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need a different read statement for the JEDI soil increment case?

VEGFCS=VEGFCS, SOTFCS=SOTFCS, &
SWEFCS=SWEFCS,SLIFCS=SLIFCS,VETFCS=VETFCS,SNDFCS=SNDFCS,&
SMCFCS=SMCFCS,SLCFCS=SLCFCS,STCFCS=STCFCS,SLMASK=SLMASK,ZSOIL=ZSOIL)

ENDIF

IF (IS_NOAHMP) THEN
LSM=LSM_NOAHMP
Expand Down Expand Up @@ -519,7 +552,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, &

! CALCULATE MASK FOR LAND INCREMENTS
IF (DO_LNDINC) &
CALL CALCULATE_LANDINC_MASK(SMCFCS(:,1),SWEFCS, VETFCS, &
CALL CALCULATE_LANDINC_MASK(SWEFCS, VETFCS, SOTFCS, &
LENSFC,VEG_TYPE_LANDICE, LANDINC_MASK)

!--------------------------------------------------------------------------------
Expand Down Expand Up @@ -648,7 +681,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, &
ENDIF

! SOIL INCREMENTS
IF (DO_SOI_INC) THEN
IF (DO_SOI_INC_GSI .OR. DO_SOI_INC_JEDI) THEN

!--------------------------------------------------------------------------------
! re-calculate soilsnow mask if snow has been updated.
Expand All @@ -657,23 +690,26 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, &
LANDINC_MASK_FG = LANDINC_MASK

IF (DO_SFCCYCLE .OR. DO_SNO_INC) THEN
CALL CALCULATE_LANDINC_MASK(SMCFCS(:,1),SWEFCS, VETFCS, LENSFC, &
CALL CALCULATE_LANDINC_MASK(SWEFCS, VETFCS, SOTFCS, LENSFC, &
VEG_TYPE_LANDICE, LANDINC_MASK )
ENDIF

!--------------------------------------------------------------------------------
! read increments in
!--------------------------------------------------------------------------------

INQUIRE(FILE=trim(LND_SOI_FILE), EXIST=file_exists)
IF (.not. file_exists) then
IF (DO_SOI_INC_GSI) THEN
INQUIRE(FILE=trim(LND_SOI_FILE), EXIST=file_exists)
IF (.not. file_exists) then
print *, 'FATAL ERROR: land increment update requested, but file does not exist: ', &
trim(lnd_soi_file)
call MPI_ABORT(MPI_COMM_WORLD, 10, IERR)
ENDIF
CALL READ_GSI_DATA(LND_SOI_FILE, 'LND', LSOIL=LSOIL)
ELSE IF (DO_SOI_INC_JEDI) THEN
CALL READ_DATA(LSOIL,LENSFC,.false.,.true.,SLCINC=SLCINC,STCINC=STCINC)
ENDIF

CALL READ_GSI_DATA(LND_SOI_FILE, 'LND', LSOIL=LSOIL)

!--------------------------------------------------------------------------------
! add increments to state vars
!--------------------------------------------------------------------------------
Expand All @@ -688,25 +724,65 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, &
SLC_BCK = SLCFCS

! below updates [STC/SMC/STC]FCS to hold the analysis
CALL ADD_INCREMENT_SOIL(RLA,RLO,STCFCS,SMCFCS,SLCFCS,STC_UPDATED, SLC_UPDATED, &
! ONLY isolate soil temperature and soil moisture ingest/adjust routines in
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need to have the same process for both increment types (since the intention of the JEDI increments is to test what we should do for the GSI), so either need to code the new method for the GSI, or make the JEDI method the same as the GSI.

If "c) ingest soil moisture incr. based on newly-adjusted soil state due to b)" implies that JEDI is being called twice (as in the second call uses the updated soil moisture from b, then we definitely don't want to do it this way (changes to the soil moisture from a and b will be small, and certainly doesn't justify the cost/difficulty of calling JEDI twice).

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just saw below that JEDI is not re-called, which is good.

! the jedi-related portion. In gsi-related part, it is *not* updated yet.
! The gsi-related part is still using the old assimilation sequence
! (i.e., soil moisture incr. ingest based on background state, not the
! analysis state as edited in the new assimilation sequence)
! The new assimilation/adjustment sequence in jedi is:
! a) ingest soil temp incr.
! b) adjust soil ice/liquid moisture as needed
! c) ingest soil moisure incr. based on newly-adjusted soil state due to b)
! d) ingest soil moisture incr.
! e) adjust/bound soil moisture
IF (DO_SOI_INC_GSI) THEN
CALL ADD_GSI_INCREMENT_SOIL(RLA,RLO,STCFCS,SMCFCS,SLCFCS,STC_UPDATED, SLC_UPDATED, &
STC_INC_TMP, SLC_INC_TMP, &
LANDINC_MASK,LANDINC_MASK_FG,LENSFC,LSOIL,IDIM,JDIM,LSM,MYRANK)

!--------------------------------------------------------------------------------
! make any necessary adjustments to dependent variables
!--------------------------------------------------------------------------------

CALL APPLY_LAND_DA_ADJUSTMENTS_SOIL_TEMP(LSM, ISOT, IVEGSRC, LENSFC,&
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure these need to be separated, but lets sort the issue above first.

LSOIL, SOTFCS, LANDINC_MASK_FG,&
STC_BCK, STCFCS, SMCFCS, SLCFCS,&
STC_UPDATED)
CALL APPLY_LAND_DA_ADJUSTMENTS_SOIL_MOISTURE(LSM, LENSFC, &
LSOIL, SMCFCS, SLCFCS,&
SLC_UPDATED, ZSOIL)

! if read in GSI-based increments, save interpolated fields onto cubed sphere tiles
CALL WRITE_DATA(LENSFC,IDIM,JDIM,LSOIL,DO_NSST,.true.,NSST, &
SLC_INC_TMP=SLC_INC_TMP,STC_INC_TMP=STC_INC_TMP)

ELSE IF (DO_SOI_INC_JEDI) THEN
CALL ADD_JEDI_INCREMENT_SOIL_TEMP(STCINC, STCFCS, &
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it possible (or I really mean "easy") to generalize the program that adds the increments to take either input, then interpolate the GSI increment if necessary?

STC_UPDATED, &
LANDINC_MASK, LANDINC_MASK_FG, LENSFC,&
LSOIL, LSM, MYRANK)
CALL APPLY_LAND_DA_ADJUSTMENTS_SOIL_TEMP(LSM, ISOT, IVEGSRC, LENSFC,&
LSOIL, SOTFCS, LANDINC_MASK_FG,&
STC_BCK, STCFCS, SMCFCS, SLCFCS,&
STC_UPDATED)

CALL ADD_JEDI_INCREMENT_SOIL_MOISTURE(SLCINC, STCFCS, SMCFCS, SLCFCS,&
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see the three-step process now. So it's not re-calling JEDI. Let's talk about why you did this, but my impulse is that it makes it more complicated, without really adding any advantages.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will maintain the original logic.

SLC_UPDATED, &
LANDINC_MASK, LANDINC_MASK_FG, LENSFC,&
LSOIL, LSM, MYRANK)
CALL APPLY_LAND_DA_ADJUSTMENTS_SOIL_MOISTURE(LSM, LENSFC, &
LSOIL, SMCFCS, SLCFCS,&
SLC_UPDATED, ZSOIL)

ENDIF ! switch between reading gsi and jedi based increments

CALL APPLY_LAND_DA_ADJUSTMENTS_SOIL(LSM, ISOT, IVEGSRC,LENSFC, LSOIL, &
SOTFCS, LANDINC_MASK_FG, STC_BCK, STCFCS, SMCFCS, SLCFCS, STC_UPDATED, &
SLC_UPDATED,ZSOIL)

ENDIF ! soil increments

!--------------------------------------------------------------------------------
! clean up
!--------------------------------------------------------------------------------

! to do - save and write out STC_INC? (soil temperature increments)
IF(ALLOCATED(LANDINC_MASK_FG)) DEALLOCATE(LANDINC_MASK_FG)
IF(ALLOCATED(LANDINC_MASK)) DEALLOCATE(LANDINC_MASK)
IF(ALLOCATED(STC_BCK)) DEALLOCATE(STC_BCK)
Expand All @@ -715,6 +791,8 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, &
IF(ALLOCATED(SND_BCK)) DEALLOCATE(SND_BCK)
IF(ALLOCATED(SWE_BCK)) DEALLOCATE(SWE_BCK)
IF(ALLOCATED(SND_INC)) DEALLOCATE(SND_INC)
IF(ALLOCATED(SLCINC)) DEALLOCATE(SLCINC)
IF(ALLOCATED(STCINC)) DEALLOCATE(STCINC)

ENDIF
!--------------------------------------------------------------------------------
Expand All @@ -723,13 +801,13 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, &

IF (LSM==LSM_NOAHMP) THEN

CALL WRITE_DATA(LENSFC,IDIM,JDIM,LSOIL,DO_NSST,NSST,VEGFCS=VEGFCS, &
CALL WRITE_DATA(LENSFC,IDIM,JDIM,LSOIL,DO_NSST,.false.,NSST,VEGFCS=VEGFCS, &
SLCFCS=SLCFCS,SMCFCS=SMCFCS,STCFCS=STCFCS)

ELSEIF (LSM==LSM_NOAH) THEN

CALL WRITE_DATA(LENSFC,IDIM,JDIM,LSOIL, &
DO_NSST,NSST,SLIFCS=SLIFCS,TSFFCS=TSFFCS,VEGFCS=VEGFCS, &
DO_NSST,.false.,NSST,SLIFCS=SLIFCS,TSFFCS=TSFFCS,VEGFCS=VEGFCS, &
SWEFCS=SWEFCS,TG3FCS=TG3FCS,ZORFCS=ZORFCS, &
ALBFCS=ALBFCS,ALFFCS=ALFFCS,CNPFCS=CNPFCS, &
F10M=F10M,T2M=T2M,Q2M=Q2M,VETFCS=VETFCS, &
Expand Down Expand Up @@ -831,7 +909,7 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,&
INTEGER :: ISTART, IEND, JSTART, JEND
INTEGER :: MASK_TILE, MASK_FG_TILE
INTEGER :: ITILE, JTILE
INTEGER :: MAX_SEARCH, J, IERR
INTEGER :: MAX_SEARCH, J, IERR
INTEGER :: IGAUSP1, JGAUSP1
integer :: nintp,nsearched,nice,nland
integer :: nfill,nfill_tice,nfill_clm
Expand Down
Loading