From 69f7acf998eb1dd6f6dd21289a51564e8dd2818d Mon Sep 17 00:00:00 2001 From: George Gayno Date: Tue, 23 Aug 2022 18:55:38 +0000 Subject: [PATCH 01/38] Add new routine for fractional grids. When running in fractional mode, only a few surface fields will be updated. This routine will only update those records from a pre-existing restart file. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 21 ++++++----- sorc/global_cycle.fd/read_write_data.f90 | 44 ++++++++++++++++++++++++ ush/global_cycle_driver.sh | 6 ++++ 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index f126aef4b..d47a66546 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -712,14 +712,19 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & ! WRITE OUT UPDATED SURFACE DATA ON THE CUBED-SPHERE TILE. !-------------------------------------------------------------------------------- - CALL WRITE_DATA(SLIFCS,TSFFCS,SWEFCS,TG3FCS,ZORFCS, & - ALBFCS,ALFFCS,VEGFCS,CNPFCS,F10M, & - T2M,Q2M,VETFCS,SOTFCS,USTAR,FMM,FHH, & - SICFCS,SIHFCS,SITFCS, & - TPRCP,SRFLAG,SNDFCS, & - VMNFCS,VMXFCS,SLPFCS,ABSFCS, & - SLCFCS,SMCFCS,STCFCS, & - IDIM,JDIM,LENSFC,LSOIL,DO_NSST,NSST) +! Will keep this for non-fractional grid. Comment out for now. +!CALL WRITE_DATA(SLIFCS,TSFFCS,SWEFCS,TG3FCS,ZORFCS, & +! ALBFCS,ALFFCS,VEGFCS,CNPFCS,F10M, & +! T2M,Q2M,VETFCS,SOTFCS,USTAR,FMM,FHH, & +! SICFCS,SIHFCS,SITFCS, & +! TPRCP,SRFLAG,SNDFCS, & +! VMNFCS,VMXFCS,SLPFCS,ABSFCS, & +! SLCFCS,SMCFCS,STCFCS, & +! IDIM,JDIM,LENSFC,LSOIL,DO_NSST,NSST) + + +!vegfcs=99. + call write_data_frac_grid(vegfcs,lensfc,idim,jdim) IF (DO_NSST) THEN DEALLOCATE(NSST%C_0) diff --git a/sorc/global_cycle.fd/read_write_data.f90 b/sorc/global_cycle.fd/read_write_data.f90 index 340d511b7..1dcf4b7d6 100644 --- a/sorc/global_cycle.fd/read_write_data.f90 +++ b/sorc/global_cycle.fd/read_write_data.f90 @@ -58,11 +58,55 @@ MODULE READ_WRITE_DATA PUBLIC :: READ_GSI_DATA PUBLIC :: READ_LAT_LON_OROG PUBLIC :: WRITE_DATA + PUBLIC :: write_data_frac_grid public :: read_tf_clim_grb,get_tf_clm_dim public :: read_salclm_gfs_nc,get_dim_nc CONTAINS + subroutine write_data_frac_grid(vegfcs, lensfc,idim,jdim) + + use mpi + + implicit none + + integer, intent(in) :: lensfc + integer, intent(in) :: idim, jdim + + real, intent(in) :: vegfcs(lensfc) + + real :: dum2d(idim,jdim) + + character(len=50) :: fnbgso + character(len=3) :: rankch + + integer :: myrank, error, ncid, id_var + + + call mpi_comm_rank(mpi_comm_world, myrank, error) + + write(rankch, '(i3.3)') (myrank+1) + + fnbgso = "./fnbgso." // rankch + + print* + print*,"update OUTPUT SFC DATA TO: ",trim(fnbgso) + + + ERROR=NF90_OPEN(TRIM(fnbgso),NF90_WRITE,NCID) + CALL NETCDF_ERR(ERROR, 'OPENING FILE: '//TRIM(fnbgso) ) + + ERROR=NF90_INQ_VARID(NCID, "vfrac", ID_VAR) + CALL NETCDF_ERR(ERROR, 'READING vfrac ID' ) + + dum2d = reshape(vegfcs, (/idim,jdim/)) + error = nf90_put_var( ncid, id_var, dum2d) + call netcdf_err(error, 'WRITING vegfcs RECORD' ) + + error = nf90_close(ncid) + + end subroutine write_data_frac_grid + !> Write out all surface records - and nsst records if selected - !! on a single cubed-sphere tile to a model restart file (in netcdf). !! diff --git a/ush/global_cycle_driver.sh b/ush/global_cycle_driver.sh index e5327970b..2546df07c 100755 --- a/ush/global_cycle_driver.sh +++ b/ush/global_cycle_driver.sh @@ -70,8 +70,14 @@ export DATA=${DATA:-$pwd/rundir$$} rm -fr $DATA mkdir -p $DATA +# Make a copy of the input restart file as fnbgso. For fractional grids, +# only a few fields will be updated by sfcsub.F. Only these records +# will be overwritten in fnbgso. + for n in $(seq 1 $ntiles); do ln -fs $COMIN/$PDY.${cyc}0000.sfc_data.tile${n}.nc $DATA/fnbgsi.00$n + cp $COMIN/$PDY.${cyc}0000.sfc_data.tile${n}.nc $COMOUT/$PDY.${cyc}0000.sfcanl_data.tile${n}.nc + chmod 644 $COMOUT/$PDY.${cyc}0000.sfcanl_data.tile${n}.nc ln -fs $COMOUT/$PDY.${cyc}0000.sfcanl_data.tile${n}.nc $DATA/fnbgso.00$n ln -fs $FIXfv3/C${CRES}/C${CRES}_grid.tile${n}.nc $DATA/fngrid.00$n ln -fs $FIXfv3/C${CRES}/C${CRES}_oro_data.tile${n}.nc $DATA/fnorog.00$n From 5e95fec3e88423debde9146e34676faf5e48cc7a Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 24 Aug 2022 16:02:33 +0000 Subject: [PATCH 02/38] Add rudimentary doxygen to new routine to prevent unit test failures. Fixes #549. --- sorc/global_cycle.fd/read_write_data.f90 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sorc/global_cycle.fd/read_write_data.f90 b/sorc/global_cycle.fd/read_write_data.f90 index 1dcf4b7d6..123b5d200 100644 --- a/sorc/global_cycle.fd/read_write_data.f90 +++ b/sorc/global_cycle.fd/read_write_data.f90 @@ -64,6 +64,8 @@ MODULE READ_WRITE_DATA CONTAINS +!> Write out surface records for fractional grids +!! to a pre-existing model restart file (in netcdf). subroutine write_data_frac_grid(vegfcs, lensfc,idim,jdim) use mpi From 7086a4763b209b2e6246e6eb087fe7b088e97d03 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 25 Aug 2022 20:45:35 +0000 Subject: [PATCH 03/38] Add read of land fraction record from orography file. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 24 ++++++++++++++++++++++-- sorc/global_cycle.fd/read_write_data.f90 | 11 ++++++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index d47a66546..7ba5a1eb3 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -339,6 +339,8 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & INTEGER :: IDUM(IDIM,JDIM) integer :: num_parthds, num_threads + logical :: frac_grid + real(kind=kind_io8) :: min_ice(lensfc) REAL :: SLMASK(LENSFC), OROG(LENSFC) @@ -369,7 +371,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & REAL, ALLOCATABLE :: SLIFCS_FG(:) INTEGER, ALLOCATABLE :: LANDINC_MASK_FG(:), LANDINC_MASK(:) REAL, ALLOCATABLE :: SND_BCK(:), SND_INC(:), SWE_BCK(:) - REAL(KIND=KIND_IO8), ALLOCATABLE :: SLMASKL(:), SLMASKW(:) + REAL(KIND=KIND_IO8), ALLOCATABLE :: SLMASKL(:), SLMASKW(:), LANDFRAC(:) TYPE(NSST_DATA) :: NSST real, dimension(idim,jdim) :: tf_clm,tf_trd,sal_clm @@ -390,6 +392,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & DO_SNO_INC = .FALSE. DO_SOI_INC = .FALSE. + frac_grid=.true. SIG1T = 0.0 ! Not a dead start! @@ -405,7 +408,12 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & ! READ THE OROGRAPHY AND GRID POINT LAT/LONS FOR THE CUBED-SPHERE TILE. !-------------------------------------------------------------------------------- - CALL READ_LAT_LON_OROG(RLA,RLO,OROG,OROG_UF,TILE_NUM,IDIM,JDIM,LENSFC) + IF(FRAC_GRID) THEN + ALLOCATE(LANDFRAC(LENSFC)) + CALL READ_LAT_LON_OROG(RLA,RLO,OROG,OROG_UF,TILE_NUM,IDIM,JDIM,LENSFC,LANDFRAC=LANDFRAC) + ELSE + CALL READ_LAT_LON_OROG(RLA,RLO,OROG,OROG_UF,TILE_NUM,IDIM,JDIM,LENSFC) + ENDIF DO I = 1, IDIM IDUM(I,:) = I @@ -486,6 +494,18 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & ABSFCS=ABSFCS,T2M=T2M ,Q2M=Q2M ,SLMASK=SLMASK, & ZSOIL=ZSOIL, NSST=NSST) + + print*,'vegfcs check ',maxval(vegfcs),minval(vegfcs) + do i=1,lensfc + if(landfrac(i) > 0.0 .and. vegfcs(i) == 0.0) then + print*,'bad point 1 ',i,landfrac(i),vegfcs(i) + endif + if(landfrac(i) == 0.0 .and. vegfcs(i) > 0.0) then + print*,'bad point 2 ',i,landfrac(i),vegfcs(i) + endif + enddo + stop + IF (USE_UFO) THEN PRINT* PRINT*,'USE UNFILTERED OROGRAPHY.' diff --git a/sorc/global_cycle.fd/read_write_data.f90 b/sorc/global_cycle.fd/read_write_data.f90 index 123b5d200..16c65d5ab 100644 --- a/sorc/global_cycle.fd/read_write_data.f90 +++ b/sorc/global_cycle.fd/read_write_data.f90 @@ -964,7 +964,7 @@ end subroutine write_data !! @param[out] TILE_NUM Cubed-sphere tile number !! @author George Gayno NOAA/EMC SUBROUTINE READ_LAT_LON_OROG(RLA,RLO,OROG,OROG_UF,& - TILE_NUM,IDIM,JDIM,IJDIM) + TILE_NUM,IDIM,JDIM,IJDIM,LANDFRAC) USE MPI @@ -976,6 +976,7 @@ SUBROUTINE READ_LAT_LON_OROG(RLA,RLO,OROG,OROG_UF,& REAL, INTENT(OUT) :: RLA(IJDIM),RLO(IJDIM) REAL, INTENT(OUT) :: OROG(IJDIM),OROG_UF(IJDIM) + REAL, INTENT(OUT), OPTIONAL :: LANDFRAC(IJDIM) CHARACTER(LEN=50) :: FNOROG, FNGRID CHARACTER(LEN=3) :: RANKCH @@ -1085,6 +1086,14 @@ SUBROUTINE READ_LAT_LON_OROG(RLA,RLO,OROG,OROG_UF,& CALL NETCDF_ERR(ERROR, 'ERROR READING orog_filt RECORD' ) OROG = RESHAPE(DUMMY4, (/IJDIM/)) + IF(PRESENT(LANDFRAC))THEN + ERROR=NF90_INQ_VARID(NCID_OROG, 'land_frac', ID_VAR) + CALL NETCDF_ERR(ERROR, 'ERROR READING land_frac ID' ) + ERROR=NF90_GET_VAR(NCID_OROG, ID_VAR, DUMMY4) + CALL NETCDF_ERR(ERROR, 'ERROR READING land_frac RECORD' ) + LANDFRAC = RESHAPE(DUMMY4, (/IJDIM/)) + ENDIF + DEALLOCATE(DUMMY4) ERROR = NF90_CLOSE(NCID_OROG) From 84f65e1b430484430f88e90ebc133e498bdd6dea Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 25 Aug 2022 20:52:17 +0000 Subject: [PATCH 04/38] Update ccpp-physics. Fixes #549. --- ccpp-physics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp-physics b/ccpp-physics index 3405ff171..12c115e99 160000 --- a/ccpp-physics +++ b/ccpp-physics @@ -1 +1 @@ -Subproject commit 3405ff171d7fa55d6aa7fb45b316146516c6e3ca +Subproject commit 12c115e992d3a265eaaa67d72fcbdb3a6f21195f From 540e1e10908b9aca669057114b163ae854dd2e88 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Fri, 2 Sep 2022 19:05:31 +0000 Subject: [PATCH 05/38] Add logic to set 'slmaskl' and 'slmaskw' for fractional grids. Fixes #549 --- sorc/global_cycle.fd/cycle.f90 | 40 ++++++++++++++++++++++-- sorc/global_cycle.fd/read_write_data.f90 | 3 +- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index 7ba5a1eb3..b6ba8ce47 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -497,14 +497,13 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & print*,'vegfcs check ',maxval(vegfcs),minval(vegfcs) do i=1,lensfc - if(landfrac(i) > 0.0 .and. vegfcs(i) == 0.0) then + if(landfrac(i) > 0.0_kind_io8 .and. vegfcs(i) == 0.0) then print*,'bad point 1 ',i,landfrac(i),vegfcs(i) endif - if(landfrac(i) == 0.0 .and. vegfcs(i) > 0.0) then + if(landfrac(i) == 0.0_kind_io8 .and. vegfcs(i) > 0.0) then print*,'bad point 2 ',i,landfrac(i),vegfcs(i) endif enddo - stop IF (USE_UFO) THEN PRINT* @@ -546,6 +545,39 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & IF (DO_SFCCYCLE) THEN ALLOCATE(SLMASKL(LENSFC), SLMASKW(LENSFC)) + IF (FRAC_GRID) THEN + + DO I=1,LENSFC + IF(LANDFRAC(I) > 0.0_KIND_IO8) THEN + SLMASKL(I) = CEILING(LANDFRAC(I)-1.0E-6_KIND_IO8) + SLMASKW(I) = FLOOR(LANDFRAC(I)+1.0E-6_KIND_IO8) + ELSE + IF(NINT(SLMASK(I)) == 1) THEN ! if landfrac is zero, this should not happen. + ! is this a band aid? + print*,'first if' + SLMASKL(I) = 1.0_KIND_io8 + SLMASKW(I) = 1.0_KIND_io8 + ELSE ! if landfrac is zero, this is expected. +! print*,'second if' + SLMASKL(I) = 0.0_KIND_io8 + SLMASKW(I) = 0.0_KIND_io8 + ENDIF + ENDIF + + if(nint(slmaskl(i)) == 1 .and. vegfcs(i) == 0.0) then + print*,'bad point 3 ',i,landfrac(i),slmaskl(i),vegfcs(i) + endif + if(nint(slmaskl(i)) == 0 .and. vegfcs(i) > 0.0) then + print*,'bad point 4 ',i,landfrac(i),slmaskw(i),vegfcs(i) + endif + + + ENDDO + + print*,'got here' + stop + + ELSE ! for running uncoupled (non-fractional grid) DO I=1,LENSFC IF(NINT(SLMASK(I)) == 1) THEN @@ -561,6 +593,8 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & min_ice(i) = 0.0_KIND_io8 endif ENDDO + + ENDIF ! frac_grid num_threads = num_parthds() PRINT* PRINT*,"CALL SFCCYCLE TO UPDATE SURFACE FIELDS." diff --git a/sorc/global_cycle.fd/read_write_data.f90 b/sorc/global_cycle.fd/read_write_data.f90 index 16c65d5ab..df5487879 100644 --- a/sorc/global_cycle.fd/read_write_data.f90 +++ b/sorc/global_cycle.fd/read_write_data.f90 @@ -967,6 +967,7 @@ SUBROUTINE READ_LAT_LON_OROG(RLA,RLO,OROG,OROG_UF,& TILE_NUM,IDIM,JDIM,IJDIM,LANDFRAC) USE MPI + USE MACHINE IMPLICIT NONE @@ -976,7 +977,7 @@ SUBROUTINE READ_LAT_LON_OROG(RLA,RLO,OROG,OROG_UF,& REAL, INTENT(OUT) :: RLA(IJDIM),RLO(IJDIM) REAL, INTENT(OUT) :: OROG(IJDIM),OROG_UF(IJDIM) - REAL, INTENT(OUT), OPTIONAL :: LANDFRAC(IJDIM) + REAL(KIND=KIND_IO8), INTENT(OUT), OPTIONAL :: LANDFRAC(IJDIM) CHARACTER(LEN=50) :: FNOROG, FNGRID CHARACTER(LEN=3) :: RANKCH From b51b685042f6d64922f79de0289514f35ac234d5 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 14 Sep 2022 13:03:37 +0000 Subject: [PATCH 06/38] Add call to write routine for fractional grids. Add some diagnostics. Fixes #549 --- sorc/global_cycle.fd/cycle.f90 | 74 ++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index b6ba8ce47..b444af208 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -571,30 +571,30 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & print*,'bad point 4 ',i,landfrac(i),slmaskw(i),vegfcs(i) endif - ENDDO - print*,'got here' - stop - ELSE ! for running uncoupled (non-fractional grid) + DO I=1,LENSFC + IF(NINT(SLMASK(I)) == 1) THEN + SLMASKL(I) = 1.0_KIND_io8 + SLMASKW(I) = 1.0_KIND_io8 + ELSE + SLMASKL(I) = 0.0_KIND_io8 + SLMASKW(I) = 0.0_KIND_io8 + ENDIF + ENDDO + + ENDIF ! frac_grid + DO I=1,LENSFC - IF(NINT(SLMASK(I)) == 1) THEN - SLMASKL(I) = 1.0_KIND_io8 - SLMASKW(I) = 1.0_KIND_io8 - ELSE - SLMASKL(I) = 0.0_KIND_io8 - SLMASKW(I) = 0.0_KIND_io8 - ENDIF if(nint(slmask(i)) == 0) then min_ice(i) = 0.15_KIND_io8 else min_ice(i) = 0.0_KIND_io8 endif - ENDDO + ENDDO - ENDIF ! frac_grid num_threads = num_parthds() PRINT* PRINT*,"CALL SFCCYCLE TO UPDATE SURFACE FIELDS." @@ -610,6 +610,27 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & SZ_NML, INPUT_NML_FILE, & min_ice, & IALB,ISOT,IVEGSRC,TILE_NUM,I_INDEX,J_INDEX) + + print*,'after call to sfccycle ' + + DO I=1,LENSFC + + if(nint(slmaskl(i)) == 1 .and. vegfcs(i) == 0.0) then + print*,'bad point 5 ',i,landfrac(i),slmaskl(i),vegfcs(i) + endif + if(nint(slmaskl(i)) == 0 .and. vegfcs(i) > 0.0) then + print*,'bad point 6 ',i,landfrac(i),slmaskw(i),vegfcs(i) + endif + + if(landfrac(i) > 0.0_kind_io8 .and. vegfcs(i) == 0.0) then + print*,'bad point 7 ',i,landfrac(i),vegfcs(i) + endif + if(landfrac(i) == 0.0_kind_io8 .and. vegfcs(i) > 0.0) then + print*,'bad point 8 ',i,landfrac(i),vegfcs(i) + endif + + ENDDO + DEALLOCATE(SLMASKL, SLMASKW) ENDIF @@ -767,18 +788,21 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & !-------------------------------------------------------------------------------- ! Will keep this for non-fractional grid. Comment out for now. -!CALL WRITE_DATA(SLIFCS,TSFFCS,SWEFCS,TG3FCS,ZORFCS, & -! ALBFCS,ALFFCS,VEGFCS,CNPFCS,F10M, & -! T2M,Q2M,VETFCS,SOTFCS,USTAR,FMM,FHH, & -! SICFCS,SIHFCS,SITFCS, & -! TPRCP,SRFLAG,SNDFCS, & -! VMNFCS,VMXFCS,SLPFCS,ABSFCS, & -! SLCFCS,SMCFCS,STCFCS, & -! IDIM,JDIM,LENSFC,LSOIL,DO_NSST,NSST) - - -!vegfcs=99. - call write_data_frac_grid(vegfcs,lensfc,idim,jdim) + if (.not. frac_grid) then + CALL WRITE_DATA(SLIFCS,TSFFCS,SWEFCS,TG3FCS,ZORFCS, & + ALBFCS,ALFFCS,VEGFCS,CNPFCS,F10M, & + T2M,Q2M,VETFCS,SOTFCS,USTAR,FMM,FHH, & + SICFCS,SIHFCS,SITFCS, & + TPRCP,SRFLAG,SNDFCS, & + VMNFCS,VMXFCS,SLPFCS,ABSFCS, & + SLCFCS,SMCFCS,STCFCS, & + IDIM,JDIM,LENSFC,LSOIL,DO_NSST,NSST) + + else + + print*,'write out fractional grid' + call write_data_frac_grid(vegfcs,lensfc,idim,jdim) + endif IF (DO_NSST) THEN DEALLOCATE(NSST%C_0) From a4db62a32a2e159cef6f3891e6013bc6b0b8e6f4 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Tue, 20 Sep 2022 15:16:55 +0000 Subject: [PATCH 07/38] Add some diagnostic print. Add write of albedo to new write routine. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 11 ++++++++- sorc/global_cycle.fd/read_write_data.f90 | 31 +++++++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index b444af208..863880ec2 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -338,6 +338,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & INTEGER :: I_INDEX(LENSFC), J_INDEX(LENSFC) INTEGER :: IDUM(IDIM,JDIM) integer :: num_parthds, num_threads + integer :: ichk, jchk, ijchk logical :: frac_grid @@ -494,6 +495,13 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & ABSFCS=ABSFCS,T2M=T2M ,Q2M=Q2M ,SLMASK=SLMASK, & ZSOIL=ZSOIL, NSST=NSST) + ALBFCS=-9. + + ichk = 282 + jchk = 362 + ijchk = (jchk-1) * idim + ichk + + print*,'chk point after read_data ',landfrac(ijchk),slmask(ijchk),vegfcs(ijchk) print*,'vegfcs check ',maxval(vegfcs),minval(vegfcs) do i=1,lensfc @@ -613,6 +621,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & print*,'after call to sfccycle ' + print*,'chk point after sfccycle ',landfrac(ijchk),slmask(ijchk),vegfcs(ijchk) DO I=1,LENSFC if(nint(slmaskl(i)) == 1 .and. vegfcs(i) == 0.0) then @@ -801,7 +810,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & else print*,'write out fractional grid' - call write_data_frac_grid(vegfcs,lensfc,idim,jdim) + call write_data_frac_grid(vegfcs,albfcs,lensfc,idim,jdim) endif IF (DO_NSST) THEN diff --git a/sorc/global_cycle.fd/read_write_data.f90 b/sorc/global_cycle.fd/read_write_data.f90 index df5487879..cd5a6b3b6 100644 --- a/sorc/global_cycle.fd/read_write_data.f90 +++ b/sorc/global_cycle.fd/read_write_data.f90 @@ -66,7 +66,7 @@ MODULE READ_WRITE_DATA !> Write out surface records for fractional grids !! to a pre-existing model restart file (in netcdf). - subroutine write_data_frac_grid(vegfcs, lensfc,idim,jdim) + subroutine write_data_frac_grid(vegfcs,albfcs, lensfc,idim,jdim) use mpi @@ -75,6 +75,7 @@ subroutine write_data_frac_grid(vegfcs, lensfc,idim,jdim) integer, intent(in) :: lensfc integer, intent(in) :: idim, jdim + real, intent(in) :: albfcs(lensfc,4) real, intent(in) :: vegfcs(lensfc) real :: dum2d(idim,jdim) @@ -105,6 +106,34 @@ subroutine write_data_frac_grid(vegfcs, lensfc,idim,jdim) error = nf90_put_var( ncid, id_var, dum2d) call netcdf_err(error, 'WRITING vegfcs RECORD' ) + ERROR=NF90_INQ_VARID(NCID, "alvsf", ID_VAR) + CALL NETCDF_ERR(ERROR, 'READING alvsf ID' ) + + dum2d = reshape(albfcs(:,1), (/idim,jdim/)) + error = nf90_put_var( ncid, id_var, dum2d) + call netcdf_err(error, 'WRITING alvsf RECORD' ) + + ERROR=NF90_INQ_VARID(NCID, "alvwf", ID_VAR) + CALL NETCDF_ERR(ERROR, 'READING alvwf ID' ) + + dum2d = reshape(albfcs(:,2), (/idim,jdim/)) + error = nf90_put_var( ncid, id_var, dum2d) + call netcdf_err(error, 'WRITING alvwf RECORD' ) + + ERROR=NF90_INQ_VARID(NCID, "alnsf", ID_VAR) + CALL NETCDF_ERR(ERROR, 'READING alnsf ID' ) + + dum2d = reshape(albfcs(:,3), (/idim,jdim/)) + error = nf90_put_var( ncid, id_var, dum2d) + call netcdf_err(error, 'WRITING alnsf RECORD' ) + + ERROR=NF90_INQ_VARID(NCID, "alnwf", ID_VAR) + CALL NETCDF_ERR(ERROR, 'READING alnwf ID' ) + + dum2d = reshape(albfcs(:,4), (/idim,jdim/)) + error = nf90_put_var( ncid, id_var, dum2d) + call netcdf_err(error, 'WRITING alnwf RECORD' ) + error = nf90_close(ncid) end subroutine write_data_frac_grid From 511c3c770065d63110a1c325c1f53af605100c07 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Tue, 27 Sep 2022 13:57:37 +0000 Subject: [PATCH 08/38] Remove write of albedo, which noah-mp does not need from cycle. Adjust diagnostic print for greenness. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 28 +++++++++++++-------- sorc/global_cycle.fd/read_write_data.f90 | 32 +----------------------- 2 files changed, 18 insertions(+), 42 deletions(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index 863880ec2..9cbfee999 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -501,15 +501,17 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & jchk = 362 ijchk = (jchk-1) * idim + ichk - print*,'chk point after read_data ',landfrac(ijchk),slmask(ijchk),vegfcs(ijchk) + ijchk=116571 + + print*,'chk point after read_data ',tile_num,landfrac(ijchk),slmask(ijchk),vetfcs(ijchk),vegfcs(ijchk) print*,'vegfcs check ',maxval(vegfcs),minval(vegfcs) do i=1,lensfc if(landfrac(i) > 0.0_kind_io8 .and. vegfcs(i) == 0.0) then - print*,'bad point 1 ',i,landfrac(i),vegfcs(i) + print*,'bad point 1 ',tile_num,i,landfrac(i),vegfcs(i) endif if(landfrac(i) == 0.0_kind_io8 .and. vegfcs(i) > 0.0) then - print*,'bad point 2 ',i,landfrac(i),vegfcs(i) + print*,'bad point 2 ',tile_num,i,landfrac(i),vegfcs(i) endif enddo @@ -573,10 +575,10 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & ENDIF if(nint(slmaskl(i)) == 1 .and. vegfcs(i) == 0.0) then - print*,'bad point 3 ',i,landfrac(i),slmaskl(i),vegfcs(i) + print*,'bad point 3 ',tile_num,i,landfrac(i),slmaskl(i),vegfcs(i) endif if(nint(slmaskl(i)) == 0 .and. vegfcs(i) > 0.0) then - print*,'bad point 4 ',i,landfrac(i),slmaskw(i),vegfcs(i) + print*,'bad point 4 ',tile_num,i,landfrac(i),slmaskw(i),vegfcs(i) endif ENDDO @@ -621,21 +623,25 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & print*,'after call to sfccycle ' - print*,'chk point after sfccycle ',landfrac(ijchk),slmask(ijchk),vegfcs(ijchk) + print*,'chk point after sfccycle ',tile_num,landfrac(ijchk),slmask(ijchk),vetfcs(ijchk),vegfcs(ijchk) DO I=1,LENSFC if(nint(slmaskl(i)) == 1 .and. vegfcs(i) == 0.0) then - print*,'bad point 5 ',i,landfrac(i),slmaskl(i),vegfcs(i) + if (nint(vetfcs(i)) /= 15) then + print*,'bad point 5 ',tile_num,i,landfrac(i),slmaskl(i),vegfcs(i) + endif endif if(nint(slmaskl(i)) == 0 .and. vegfcs(i) > 0.0) then - print*,'bad point 6 ',i,landfrac(i),slmaskw(i),vegfcs(i) + print*,'bad point 6 ',tile_num,i,landfrac(i),slmaskw(i),vegfcs(i) endif if(landfrac(i) > 0.0_kind_io8 .and. vegfcs(i) == 0.0) then - print*,'bad point 7 ',i,landfrac(i),vegfcs(i) + if (nint(vetfcs(i)) /= 15) then + print*,'bad point 7 ',tile_num,i,landfrac(i),vegfcs(i) + endif endif if(landfrac(i) == 0.0_kind_io8 .and. vegfcs(i) > 0.0) then - print*,'bad point 8 ',i,landfrac(i),vegfcs(i) + print*,'bad point 8 ',tile_num,i,landfrac(i),vegfcs(i) endif ENDDO @@ -810,7 +816,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & else print*,'write out fractional grid' - call write_data_frac_grid(vegfcs,albfcs,lensfc,idim,jdim) + call write_data_frac_grid(vegfcs,lensfc,idim,jdim) endif IF (DO_NSST) THEN diff --git a/sorc/global_cycle.fd/read_write_data.f90 b/sorc/global_cycle.fd/read_write_data.f90 index cd5a6b3b6..82eddb496 100644 --- a/sorc/global_cycle.fd/read_write_data.f90 +++ b/sorc/global_cycle.fd/read_write_data.f90 @@ -66,7 +66,7 @@ MODULE READ_WRITE_DATA !> Write out surface records for fractional grids !! to a pre-existing model restart file (in netcdf). - subroutine write_data_frac_grid(vegfcs,albfcs, lensfc,idim,jdim) + subroutine write_data_frac_grid(vegfcs,lensfc,idim,jdim) use mpi @@ -75,7 +75,6 @@ subroutine write_data_frac_grid(vegfcs,albfcs, lensfc,idim,jdim) integer, intent(in) :: lensfc integer, intent(in) :: idim, jdim - real, intent(in) :: albfcs(lensfc,4) real, intent(in) :: vegfcs(lensfc) real :: dum2d(idim,jdim) @@ -94,7 +93,6 @@ subroutine write_data_frac_grid(vegfcs,albfcs, lensfc,idim,jdim) print* print*,"update OUTPUT SFC DATA TO: ",trim(fnbgso) - ERROR=NF90_OPEN(TRIM(fnbgso),NF90_WRITE,NCID) CALL NETCDF_ERR(ERROR, 'OPENING FILE: '//TRIM(fnbgso) ) @@ -106,34 +104,6 @@ subroutine write_data_frac_grid(vegfcs,albfcs, lensfc,idim,jdim) error = nf90_put_var( ncid, id_var, dum2d) call netcdf_err(error, 'WRITING vegfcs RECORD' ) - ERROR=NF90_INQ_VARID(NCID, "alvsf", ID_VAR) - CALL NETCDF_ERR(ERROR, 'READING alvsf ID' ) - - dum2d = reshape(albfcs(:,1), (/idim,jdim/)) - error = nf90_put_var( ncid, id_var, dum2d) - call netcdf_err(error, 'WRITING alvsf RECORD' ) - - ERROR=NF90_INQ_VARID(NCID, "alvwf", ID_VAR) - CALL NETCDF_ERR(ERROR, 'READING alvwf ID' ) - - dum2d = reshape(albfcs(:,2), (/idim,jdim/)) - error = nf90_put_var( ncid, id_var, dum2d) - call netcdf_err(error, 'WRITING alvwf RECORD' ) - - ERROR=NF90_INQ_VARID(NCID, "alnsf", ID_VAR) - CALL NETCDF_ERR(ERROR, 'READING alnsf ID' ) - - dum2d = reshape(albfcs(:,3), (/idim,jdim/)) - error = nf90_put_var( ncid, id_var, dum2d) - call netcdf_err(error, 'WRITING alnsf RECORD' ) - - ERROR=NF90_INQ_VARID(NCID, "alnwf", ID_VAR) - CALL NETCDF_ERR(ERROR, 'READING alnwf ID' ) - - dum2d = reshape(albfcs(:,4), (/idim,jdim/)) - error = nf90_put_var( ncid, id_var, dum2d) - call netcdf_err(error, 'WRITING alnwf RECORD' ) - error = nf90_close(ncid) end subroutine write_data_frac_grid From 4cbed6543b5f4be9ae23616d6a6fda94a45bf3d8 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 5 Oct 2022 19:34:12 +0000 Subject: [PATCH 09/38] Add processing of ice fraction. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 6 +++++- sorc/global_cycle.fd/read_write_data.f90 | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index 9cbfee999..10b79f64a 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -644,6 +644,10 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & print*,'bad point 8 ',tile_num,i,landfrac(i),vegfcs(i) endif + if(nint(slmaskw(i)) == 1 .and. sicfcs(i) > 0.0) then + print*,'bad ice point ',tile_num,i,landfrac(i),sicfcs(i) + endif + ENDDO DEALLOCATE(SLMASKL, SLMASKW) @@ -816,7 +820,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & else print*,'write out fractional grid' - call write_data_frac_grid(vegfcs,lensfc,idim,jdim) + call write_data_frac_grid(vegfcs,sicfcs,lensfc,idim,jdim) endif IF (DO_NSST) THEN diff --git a/sorc/global_cycle.fd/read_write_data.f90 b/sorc/global_cycle.fd/read_write_data.f90 index 82eddb496..ee643230f 100644 --- a/sorc/global_cycle.fd/read_write_data.f90 +++ b/sorc/global_cycle.fd/read_write_data.f90 @@ -66,7 +66,7 @@ MODULE READ_WRITE_DATA !> Write out surface records for fractional grids !! to a pre-existing model restart file (in netcdf). - subroutine write_data_frac_grid(vegfcs,lensfc,idim,jdim) + subroutine write_data_frac_grid(vegfcs,sicfcs,lensfc,idim,jdim) use mpi @@ -75,6 +75,7 @@ subroutine write_data_frac_grid(vegfcs,lensfc,idim,jdim) integer, intent(in) :: lensfc integer, intent(in) :: idim, jdim + real, intent(in) :: sicfcs(lensfc) real, intent(in) :: vegfcs(lensfc) real :: dum2d(idim,jdim) @@ -104,6 +105,13 @@ subroutine write_data_frac_grid(vegfcs,lensfc,idim,jdim) error = nf90_put_var( ncid, id_var, dum2d) call netcdf_err(error, 'WRITING vegfcs RECORD' ) + ERROR=NF90_INQ_VARID(NCID, "fice", ID_VAR) + CALL NETCDF_ERR(ERROR, 'READING fice ID' ) + + dum2d = reshape(sicfcs, (/idim,jdim/)) + error = nf90_put_var( ncid, id_var, dum2d) + call netcdf_err(error, 'WRITING fice RECORD' ) + error = nf90_close(ncid) end subroutine write_data_frac_grid From aa3bb0f72de2e3a3b0fe5f2f7ea3a730ff09b401 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Fri, 14 Apr 2023 14:36:01 +0000 Subject: [PATCH 10/38] Add logic to stop processing for cases that have not been implemented. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 61 ++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index 3ef744ab8..2f2182cf4 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -338,7 +338,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & INTEGER :: I_INDEX(LENSFC), J_INDEX(LENSFC) INTEGER :: IDUM(IDIM,JDIM) integer :: num_parthds, num_threads - integer :: ichk, jchk, ijchk +! integer :: ichk, jchk, ijchk logical :: frac_grid @@ -396,6 +396,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & DO_SOI_INC = .FALSE. frac_grid=.true. +!frac_grid=.false. SIG1T = 0.0 ! Not a dead start! @@ -456,6 +457,10 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & ENDIF IF (DO_LNDINC) THEN + IF (FRAC_GRID) THEN + print *, 'FATAL ERROR: land increment update does not work with fractional grids.' + call MPI_ABORT(MPI_COMM_WORLD, 17, IERR) + ENDIF ! identify variables to be updates, and allocate arrays. IF (TRIM(LND_SOI_FILE) .NE. "NULL") THEN DO_SOI_INC = .TRUE. @@ -498,25 +503,30 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & ABSFCS=ABSFCS,T2M=T2M ,Q2M=Q2M ,SLMASK=SLMASK, & ZSOIL=ZSOIL, NSST=NSST) - ALBFCS=-9. - ichk = 282 - jchk = 362 - ijchk = (jchk-1) * idim + ichk + print*,'is noahmp/fract grid ',is_noahmp, frac_grid - ijchk=116571 + IF (FRAC_GRID .AND. .NOT. IS_NOAHMP) THEN + print *, 'FATAL ERROR: NOAH lsm update does not work with fractional grids.' + call MPI_ABORT(MPI_COMM_WORLD, 18, IERR) + ENDIF - print*,'chk point after read_data ',tile_num,landfrac(ijchk),slmask(ijchk),vetfcs(ijchk),vegfcs(ijchk) + ALBFCS=-9. - print*,'vegfcs check ',maxval(vegfcs),minval(vegfcs) - do i=1,lensfc - if(landfrac(i) > 0.0_kind_io8 .and. vegfcs(i) == 0.0) then - print*,'bad point 1 ',tile_num,i,landfrac(i),vegfcs(i) - endif - if(landfrac(i) == 0.0_kind_io8 .and. vegfcs(i) > 0.0) then - print*,'bad point 2 ',tile_num,i,landfrac(i),vegfcs(i) - endif - enddo +!ichk = 282 +!jchk = 362 +!ijchk = (jchk-1) * idim + ichk +!print*,'chk point after read_data ',tile_num,landfrac(ijchk),slmask(ijchk),vetfcs(ijchk),vegfcs(ijchk) + +!print*,'vegfcs check ',maxval(vegfcs),minval(vegfcs) +!do i=1,lensfc +! if(landfrac(i) > 0.0_kind_io8 .and. vegfcs(i) == 0.0) then +! print*,'bad point 1 ',tile_num,i,landfrac(i),vegfcs(i) +! endif +! if(landfrac(i) == 0.0_kind_io8 .and. vegfcs(i) > 0.0) then +! print*,'bad point 2 ',tile_num,i,landfrac(i),vegfcs(i) +! endif +!enddo IF (USE_UFO) THEN PRINT* @@ -624,20 +634,22 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & min_ice, & IALB,ISOT,IVEGSRC,TILE_NUM,I_INDEX,J_INDEX) - print*,'after call to sfccycle ' +! print*,'after call to sfccycle ' - print*,'chk point after sfccycle ',tile_num,landfrac(ijchk),slmask(ijchk),vetfcs(ijchk),vegfcs(ijchk) - DO I=1,LENSFC +! print*,'chk point after sfccycle ',tile_num,landfrac(ijchk),slmask(ijchk),vetfcs(ijchk),vegfcs(ijchk) + + goto 20 + DO I=1,LENSFC if(nint(slmaskl(i)) == 1 .and. vegfcs(i) == 0.0) then if (nint(vetfcs(i)) /= 15) then print*,'bad point 5 ',tile_num,i,landfrac(i),slmaskl(i),vegfcs(i) endif endif + print*,'in loop after first if' if(nint(slmaskl(i)) == 0 .and. vegfcs(i) > 0.0) then print*,'bad point 6 ',tile_num,i,landfrac(i),slmaskw(i),vegfcs(i) endif - if(landfrac(i) > 0.0_kind_io8 .and. vegfcs(i) == 0.0) then if (nint(vetfcs(i)) /= 15) then print*,'bad point 7 ',tile_num,i,landfrac(i),vegfcs(i) @@ -646,16 +658,19 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & if(landfrac(i) == 0.0_kind_io8 .and. vegfcs(i) > 0.0) then print*,'bad point 8 ',tile_num,i,landfrac(i),vegfcs(i) endif - if(nint(slmaskw(i)) == 1 .and. sicfcs(i) > 0.0) then print*,'bad ice point ',tile_num,i,landfrac(i),sicfcs(i) endif ENDDO + 20 continue + DEALLOCATE(SLMASKL, SLMASKW) ENDIF + IF(FRAC_GRID) DEALLOCATE(LANDFRAC) + !-------------------------------------------------------------------------------- ! IF RUNNING WITH NSST, READ IN GSI FILE WITH THE UPDATED INCREMENTS (ON THE ! GAUSSIAN GRID), INTERPOLATE INCREMENTS TO THE CUBED-SPHERE TILE, AND PERFORM @@ -813,10 +828,6 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & CALL WRITE_DATA(LENSFC,IDIM,JDIM,LSOIL,DO_NSST,NSST,VEGFCS=VEGFCS) -! add this to remove conflict. comment out for now. -!elseif (frac_grid) then -! call write_data(lensfc,idim,jdim,lsoil,do_nsst,nsst, & -! vegfcs=vegfcs,sicfcs=sicfcs,lensfc,idim,jdim) ELSE From 03b3d1fdc5ae38486c913b959b8aa545fadef425 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Fri, 14 Apr 2023 17:43:37 +0000 Subject: [PATCH 11/38] Pass in FRAC_GRID environment variable used by global workflow. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 20 +++++++++----------- ush/global_cycle.sh | 4 +++- ush/global_cycle_driver.sh | 1 + 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index 2f2182cf4..40fa66774 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -114,12 +114,12 @@ PROGRAM SFC_DRV INTEGER :: ISOT, IVEGSRC, LENSFC, ZSEA1_MM, ZSEA2_MM, IERR INTEGER :: NPROCS, MYRANK, NUM_THREADS, NUM_PARTHDS, MAX_TASKS REAL :: FH, DELTSFC, ZSEA1, ZSEA2 - LOGICAL :: USE_UFO, DO_NSST, DO_LNDINC, DO_SFCCYCLE + LOGICAL :: USE_UFO, DO_NSST, DO_LNDINC, DO_SFCCYCLE, FRAC_GRID ! NAMELIST/NAMCYC/ IDIM,JDIM,LSM,LSOIL,LUGB,IY,IM,ID,IH,FH,& DELTSFC,IALB,USE_UFO,DONST, & DO_SFCCYCLE,ISOT,IVEGSRC,ZSEA1_MM, & - ZSEA2_MM, MAX_TASKS, DO_LNDINC + ZSEA2_MM, MAX_TASKS, DO_LNDINC, FRAC_GRID ! DATA IDIM,JDIM,LSM,LSOIL/96,96,1,4/ DATA IY,IM,ID,IH,FH/1997,8,2,0,0./ @@ -143,13 +143,14 @@ PROGRAM SFC_DRV DONST = "NO" DO_LNDINC = .FALSE. DO_SFCCYCLE = .TRUE. + FRAC_GRID = .FALSE. PRINT* PRINT*,"READ NAMCYC NAMELIST." CALL BAOPENR(36, "fort.36", IERR) READ(36, NML=NAMCYC) - !IF (MYRANK==0) WRITE(6,NAMCYC) +!IF (MYRANK==0) WRITE(6,NAMCYC) IF (MAX_TASKS < 99999 .AND. MYRANK > (MAX_TASKS - 1)) THEN PRINT*,"USER SPECIFIED MAX NUMBER OF TASKS: ", MAX_TASKS @@ -175,7 +176,7 @@ PROGRAM SFC_DRV CALL SFCDRV(LUGB,IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & IY,IM,ID,IH,FH,IALB, & USE_UFO,DO_NSST,DO_SFCCYCLE,DO_LNDINC, & - ZSEA1,ZSEA2,ISOT,IVEGSRC,MYRANK) + FRAC_GRID,ZSEA1,ZSEA2,ISOT,IVEGSRC,MYRANK) PRINT* PRINT*,'CYCLE PROGRAM COMPLETED NORMALLY ON RANK: ', MYRANK @@ -293,6 +294,7 @@ END PROGRAM SFC_DRV !! @param[in] DO_SFCCYCLE Call sfccycle routine to update surface fields !! @param[in] DO_LNDINC Read in land increment files, and add increments to !! requested states. + !! @param[in] FRAC_GRID When true, run with fractional grid. !! @param[in] ZSEA1 When running NSST model, this is the lower bound !! of depth of sea temperature. In whole mm. !! @param[in] ZSEA2 When running NSST model, this is the upper bound @@ -304,7 +306,7 @@ END PROGRAM SFC_DRV SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & IY,IM,ID,IH,FH,IALB, & USE_UFO,DO_NSST,DO_SFCCYCLE,DO_LNDINC,& - ZSEA1,ZSEA2,ISOT,IVEGSRC,MYRANK) + FRAC_GRID,ZSEA1,ZSEA2,ISOT,IVEGSRC,MYRANK) ! USE READ_WRITE_DATA use machine @@ -322,7 +324,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & INTEGER, INTENT(IN) :: ISOT, IVEGSRC, MYRANK LOGICAL, INTENT(IN) :: USE_UFO, DO_NSST,DO_SFCCYCLE - LOGICAL, INTENT(IN) :: DO_LNDINC + LOGICAL, INTENT(IN) :: DO_LNDINC, FRAC_GRID REAL, INTENT(IN) :: FH, DELTSFC, ZSEA1, ZSEA2 @@ -340,8 +342,6 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & integer :: num_parthds, num_threads ! integer :: ichk, jchk, ijchk - logical :: frac_grid - LOGICAL :: IS_NOAHMP=.FALSE. real(kind=kind_io8) :: min_ice(lensfc) @@ -395,9 +395,6 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & DO_SNO_INC = .FALSE. DO_SOI_INC = .FALSE. - frac_grid=.true. -!frac_grid=.false. - SIG1T = 0.0 ! Not a dead start! INPUT_NML_FILE = "NULL" @@ -413,6 +410,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & !-------------------------------------------------------------------------------- IF(FRAC_GRID) THEN + PRINT*,'- RUNNING WITH FRACTIONAL GRID.' ALLOCATE(LANDFRAC(LENSFC)) CALL READ_LAT_LON_OROG(RLA,RLO,OROG,OROG_UF,TILE_NUM,IDIM,JDIM,LENSFC,LANDFRAC=LANDFRAC) ELSE diff --git a/ush/global_cycle.sh b/ush/global_cycle.sh index ec3c1da97..a5022272e 100755 --- a/ush/global_cycle.sh +++ b/ush/global_cycle.sh @@ -272,6 +272,7 @@ DO_SNO_INC=${DO_SNO_INC:-.false.} zsea1=${zsea1:-0} zsea2=${zsea2:-0} MAX_TASKS_CY=${MAX_TASKS_CY:-99999} +FRAC_GRID=${FRAC_GRID:-.false.} FNGLAC=${FNGLAC:-${FIXam}/global_glacier.2x2.grb} FNMXIC=${FNMXIC:-${FIXam}/global_maxice.2x2.grb} @@ -384,7 +385,8 @@ cat << EOF > fort.36 iy=$iy, im=$im, id=$id, ih=$ih, fh=$FHOUR, deltsfc=$DELTSFC,ialb=$IALB,use_ufo=$use_ufo,donst=$DONST, do_sfccycle=$DO_SFCCYCLE,do_lndinc=$DO_LNDINC,isot=$ISOT,ivegsrc=$IVEGSRC, - zsea1_mm=$zsea1,zsea2_mm=$zsea2,MAX_TASKS=$MAX_TASKS_CY + zsea1_mm=$zsea1,zsea2_mm=$zsea2,MAX_TASKS=$MAX_TASKS_CY, + frac_grid=$FRAC_GRID / EOF diff --git a/ush/global_cycle_driver.sh b/ush/global_cycle_driver.sh index c5de95137..17324897a 100755 --- a/ush/global_cycle_driver.sh +++ b/ush/global_cycle_driver.sh @@ -53,6 +53,7 @@ export DO_SFCCYLE=${DO_SFCCYCLE:-".true."} export DO_LNDINC=${DO_LNDINC:-".false."} export LND_SOI_FILE=${LND_SOI_FILE:-"NULL"} export DO_SNO_INC=${DO_SNO_INC:-".false."} +export FRAC_GRID=${FRAC_GRID:-".false."} CRES=$(echo $CASE | cut -c 2-) JCAP_CASE=$((2*CRES-2)) From 14bae405d0a5679fd1d24fba7ff63f6b02c9b7d2 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Mon, 17 Apr 2023 12:14:05 +0000 Subject: [PATCH 12/38] Remove some test logic. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 2 -- 1 file changed, 2 deletions(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index 40fa66774..f38f93a0a 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -509,8 +509,6 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LSM,LENSFC,LSOIL,DELTSFC, & call MPI_ABORT(MPI_COMM_WORLD, 18, IERR) ENDIF - ALBFCS=-9. - !ichk = 282 !jchk = 362 !ijchk = (jchk-1) * idim + ichk From 19c7fa09272bf908c0b8fb90ff48319ee861ad10 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 24 Aug 2023 18:16:11 +0000 Subject: [PATCH 13/38] Fix doxygen in read_write_data.f90. Fixes #549. --- sorc/global_cycle.fd/read_write_data.f90 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sorc/global_cycle.fd/read_write_data.f90 b/sorc/global_cycle.fd/read_write_data.f90 index 815539961..5d95a3184 100644 --- a/sorc/global_cycle.fd/read_write_data.f90 +++ b/sorc/global_cycle.fd/read_write_data.f90 @@ -664,8 +664,8 @@ subroutine remove_checksum(ncid, id_var) end subroutine remove_checksum !> Read latitude and longitude for the cubed-sphere tile from the - !! 'grid' file. Read the filtered and unfiltered orography from - !! the 'orography' file. + !! 'grid' file. Read the filtered and unfiltered orography and + !! optionally the land fraction from the 'orography' file. !! !! @param[in] IDIM 'i' dimension of cubed-sphere tile. !! @param[in] JDIM 'j' dimension of cubed-sphere tile. @@ -675,6 +675,7 @@ end subroutine remove_checksum !! @param[out] OROG Filtered orography. !! @param[out] OROG_UF Unfiltered orography. !! @param[out] TILE_NUM Cubed-sphere tile number + !! @param[out] LANDFRAC Land fraction. !! @author George Gayno NOAA/EMC SUBROUTINE READ_LAT_LON_OROG(RLA,RLO,OROG,OROG_UF,& TILE_NUM,IDIM,JDIM,IJDIM,LANDFRAC) From aacd8db52676337475570bd3715ea7f84ef2d4fc Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 24 Aug 2023 18:22:42 +0000 Subject: [PATCH 14/38] Add some diagnostic print. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index 54bbea3db..e2df64215 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -348,6 +348,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & REAL :: ALBFCS(LENSFC,4), TG3FCS(LENSFC) REAL :: CNPFCS(LENSFC), SMCFCS(LENSFC,LSOIL) REAL :: STCFCS(LENSFC,LSOIL), SLIFCS(LENSFC) +!REAL :: STCFCS_SAVE(LENSFC,LSOIL) REAL :: AISFCS(LENSFC), F10M(LENSFC) REAL :: VEGFCS(LENSFC), VETFCS(LENSFC) REAL :: SOTFCS(LENSFC), ALFFCS(LENSFC,2) @@ -498,6 +499,8 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & ABSFCS=ABSFCS,T2M=T2M ,Q2M=Q2M ,SLMASK=SLMASK, & ZSOIL=ZSOIL, NSST=NSST) +!STCFCS_SAVE = STCFCS + print*,'is noahmp/fract grid ',is_noahmp, frac_grid IF (FRAC_GRID .AND. .NOT. IS_NOAHMP) THEN @@ -644,7 +647,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & print*,'bad point 5 ',tile_num,i,landfrac(i),slmaskl(i),vegfcs(i) endif endif - print*,'in loop after first if' +! print*,'in loop after first if' if(nint(slmaskl(i)) == 0 .and. vegfcs(i) > 0.0) then print*,'bad point 6 ',tile_num,i,landfrac(i),slmaskw(i),vegfcs(i) endif @@ -660,6 +663,19 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & print*,'bad ice point ',tile_num,i,landfrac(i),sicfcs(i) endif +! if(landfrac(i) > 0.0_kind_io8 .and. abs(stcfcs(i,1)-stcfcs_save(i,1)) > 0.001) then +! print*,'bad stc 1 point ',tile_num,i,stcfcs(i,1),stcfcs_save(i,1) +! endif +! if(landfrac(i) > 0.0_kind_io8 .and. abs(stcfcs(i,2)-stcfcs_save(i,2)) > 0.001) then +! print*,'bad stc 2 point ',tile_num,i,stcfcs(i,2),stcfcs_save(i,2) +! endif +! if(landfrac(i) > 0.0_kind_io8 .and. abs(stcfcs(i,3)-stcfcs_save(i,3)) > 0.001) then +! print*,'bad stc 3 point ',tile_num,i,stcfcs(i,3),stcfcs_save(i,3) +! endif +! if(landfrac(i) > 0.0_kind_io8 .and. abs(stcfcs(i,4)-stcfcs_save(i,4)) > 0.001) then +! print*,'bad stc 4 point ',tile_num,i,stcfcs(i,4),stcfcs_save(i,4) +! endif + ENDDO 20 continue From 42ab7cd9d4e30eca3dd11273e02cc8f1e0b19f00 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 24 Aug 2023 18:37:39 +0000 Subject: [PATCH 15/38] Add error handling for case when slmsk does not match the land fraction. Fixes #549. --- cmake/mpiexec.hera | 2 +- sorc/global_cycle.fd/cycle.f90 | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/cmake/mpiexec.hera b/cmake/mpiexec.hera index e82f1e9da..23bec1047 100755 --- a/cmake/mpiexec.hera +++ b/cmake/mpiexec.hera @@ -6,7 +6,7 @@ # $2+ - Executable and its arguments # -ACCOUNT=gsd-fv3 +ACCOUNT=fv3-cpu QOS=debug NP=$1 diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index e2df64215..69086bd5e 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -569,20 +569,19 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & IF (DO_SFCCYCLE) THEN ALLOCATE(SLMASKL(LENSFC), SLMASKW(LENSFC)) - IF (FRAC_GRID) THEN + + SET_MASK : IF (FRAC_GRID) THEN DO I=1,LENSFC IF(LANDFRAC(I) > 0.0_KIND_IO8) THEN SLMASKL(I) = CEILING(LANDFRAC(I)-1.0E-6_KIND_IO8) SLMASKW(I) = FLOOR(LANDFRAC(I)+1.0E-6_KIND_IO8) ELSE - IF(NINT(SLMASK(I)) == 1) THEN ! if landfrac is zero, this should not happen. - ! is this a band aid? - print*,'first if' - SLMASKL(I) = 1.0_KIND_io8 - SLMASKW(I) = 1.0_KIND_io8 - ELSE ! if landfrac is zero, this is expected. -! print*,'second if' + IF(NINT(SLMASK(I)) == 1) THEN ! If landfrac is zero, this should not happen. + ! So, stop processing. + PRINT*, 'FATAL ERROR: LAND FRAC AND SLMASK MISMATCH.' + CALL MPI_ABORT(MPI_COMM_WORLD, 27, IERR) + ELSE SLMASKL(I) = 0.0_KIND_io8 SLMASKW(I) = 0.0_KIND_io8 ENDIF @@ -598,7 +597,9 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & ENDDO ELSE -! for running uncoupled (non-fractional grid) + +! For running uncoupled (non-fractional grid). + DO I=1,LENSFC IF(NINT(SLMASK(I)) == 1) THEN SLMASKL(I) = 1.0_KIND_io8 @@ -609,7 +610,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & ENDIF ENDDO - ENDIF ! frac_grid + ENDIF SET_MASK DO I=1,LENSFC if(nint(slmask(i)) == 0) then From e67a305a764e5d99cf910753aebe0aae80d7342e Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 24 Aug 2023 19:19:18 +0000 Subject: [PATCH 16/38] Remove some unused variables. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 20 ++++++++------------ sorc/global_cycle.fd/read_write_data.f90 | 12 +++--------- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index 69086bd5e..41ddb6352 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -724,7 +724,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & ! update foundation & surface temperature for NSST ! CALL ADJUST_NSST(RLA,RLO,SLIFCS,SLIFCS_FG,TSFFCS,SITFCS,SICFCS,STCFCS, & - NSST,LENSFC,LSOIL,IDIM,JDIM,ZSEA1,ZSEA2,IM,ID,DELTSFC, & + NSST,LENSFC,LSOIL,IDIM,JDIM,ZSEA1,ZSEA2, & tf_clm_tile,tf_trd_tile,sal_clm_tile) ENDIF ENDIF @@ -910,9 +910,6 @@ END SUBROUTINE SFCDRV !! depth of sea temperature. In whole mm. !! @param[in] ZSEA2 When running nsst model, this is the upper bound of !! depth of sea temperature. In whole mm. - !! @param[in] MON Month - !! @param[in] DAY Day - !! @param[in] DELTSFC Cycling frequency in hours !! @param[in] tf_clm_tile Climatological reference temperature on the !! cubed-sphere tile. !! @param[in] tf_trd_tile Climatolocial reference temperature trend on the @@ -922,7 +919,7 @@ END SUBROUTINE SFCDRV !! @author Xu Li, George Gayno SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& SICET_TILE,sice_tile,SOILT_TILE,NSST,LENSFC,LSOIL, & - IDIM,JDIM,ZSEA1,ZSEA2,MON,DAY,DELTSFC, & + IDIM,JDIM,ZSEA1,ZSEA2, & tf_clm_tile,tf_trd_tile,sal_clm_tile) USE UTILS @@ -935,11 +932,11 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& IMPLICIT NONE - INTEGER, INTENT(IN) :: LENSFC, LSOIL, IDIM, JDIM, MON, DAY + INTEGER, INTENT(IN) :: LENSFC, LSOIL, IDIM, JDIM REAL, INTENT(IN) :: SLMSK_TILE(LENSFC), SLMSK_FG_TILE(LENSFC) real, intent(in) :: tf_clm_tile(lensfc),tf_trd_tile(lensfc),sal_clm_tile(lensfc) - REAL, INTENT(IN) :: ZSEA1, ZSEA2, DELTSFC + REAL, INTENT(IN) :: ZSEA1, ZSEA2 REAL, INTENT(INOUT) :: RLA(LENSFC), RLO(LENSFC), SKINT_TILE(LENSFC) REAL, INTENT(INOUT) :: SICET_TILE(LENSFC),sice_tile(lensfc),SOILT_TILE(LENSFC,LSOIL) @@ -1710,7 +1707,7 @@ subroutine get_tf_clm(xlats_ij,xlons_ij,ny,nx,iy,im,id,ih,tf_clm,tf_trd) real, dimension(nx*ny) :: tf_clm_ij ! sst climatology at target grids (nx*ny) real, dimension(nx*ny) :: tf_trd_ij ! 6-hourly sst climatology tendency real :: wei1,wei2 - integer :: nxc,nyc,mon1,mon2,i,j + integer :: nxc,nyc,mon1,mon2 character (len=6), parameter :: fin_tf_clm='sstclm' ! sst climatology file name ! ! get which two months used and their weights from atime @@ -1823,7 +1820,7 @@ subroutine get_sal_clm(xlats_ij,xlons_ij,ny,nx,iy,im,id,ih,sal_clm) real, dimension(nx*ny) :: sal_clm_ij ! salinity climatology at target grids (nx*ny) real :: wei1,wei2 - integer :: nxc,nyc,mon1,mon2,i,j + integer :: nxc,nyc,mon1,mon2 character (len=6), parameter :: fin_sal_clm='salclm' ! salinity climatology file name ! ! get which two months used and their weights from atime @@ -1933,11 +1930,10 @@ subroutine intp_tile(tf_lalo,dlats_lalo,dlons_lalo,jdim_lalo,idim_lalo, & real, parameter :: deg2rad=3.1415926/180.0 real, dimension(jdim_lalo) :: xlats_lalo real, dimension(idim_lalo) :: xlons_lalo - real :: tf,wsum,res_km + real :: wsum integer :: itile,jtile - integer :: ii,jj,ij,iii,jjj + integer :: ij integer :: ilalo,jlalo,ilalop1,jlalop1 - integer :: istart,iend,jstart,jend,krad integer, allocatable, dimension(:,:) :: id1,id2,jdc real, allocatable, dimension(:,:,:) :: agrid,s2c diff --git a/sorc/global_cycle.fd/read_write_data.f90 b/sorc/global_cycle.fd/read_write_data.f90 index 5d95a3184..71dabb465 100644 --- a/sorc/global_cycle.fd/read_write_data.f90 +++ b/sorc/global_cycle.fd/read_write_data.f90 @@ -872,7 +872,7 @@ SUBROUTINE READ_GSI_DATA(GSI_FILE, FILE_TYPE, LSOIL) CHARACTER(LEN=1) :: K_CH CHARACTER(LEN=10) :: INCVAR CHARACTER(LEN=80) :: err_msg - INTEGER :: K, I + INTEGER :: K PRINT* PRINT*, "READ INPUT GSI DATA FROM: "//TRIM(GSI_FILE) @@ -1705,7 +1705,6 @@ subroutine get_tf_clm_dim(file_sst,mlat_sst,mlon_sst) integer,parameter:: lu_sst = 21 ! fortran unit number of grib sst file integer :: iret - integer :: mscan,kb1 integer :: kf,kg,k,j,ierr integer, dimension(22):: jgds,kgds integer, dimension(25):: jpds,kpds @@ -1763,15 +1762,14 @@ subroutine read_salclm_gfs_nc(filename,sal,xlats,xlons,nlat,nlon,itime) real, dimension(nlon), intent(out) :: xlons real, dimension(nlon,nlat), intent(out) :: sal ! Local variables - integer :: ncid,ntime + integer :: ncid integer, parameter :: ndims = 3 character (len = *), parameter :: lat_name = "latitude" character (len = *), parameter :: lon_name = "longitude" character (len = *), parameter :: t_name = "time" character (len = *), parameter :: sal_name="sal" - integer :: no_fill,fill_value - integer :: time_varid,lon_varid, lat_varid, z_varid, sal_varid + integer :: time_varid,lon_varid, lat_varid, sal_varid ! The start and count arrays will tell the netCDF library where to read our data. integer, dimension(ndims) :: start, count @@ -1783,10 +1781,6 @@ subroutine read_salclm_gfs_nc(filename,sal,xlats,xlons,nlat,nlon,itime) character (len = *), parameter :: lat_units = "degrees_north" character (len = *), parameter :: lon_units = "degrees_east" - integer :: missv -! Loop indices - integer :: i,j - ! Open the file. call nc_check( nf90_open(filename, nf90_nowrite, ncid) ) From 64e138b1304f132174ecc4526fab82ae9bdb5c5d Mon Sep 17 00:00:00 2001 From: George Gayno Date: Mon, 18 Sep 2023 21:00:24 +0000 Subject: [PATCH 17/38] Add logic to halt snow increment update for noah-mp or fractional grids. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index 41ddb6352..8e200e84a 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -508,6 +508,16 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & call MPI_ABORT(MPI_COMM_WORLD, 18, IERR) ENDIF + IF (FRAC_GRID .AND. DO_SNO_INC) THEN + print *, 'FATAL ERROR: Snow increment update does not work with fractional grids.' + call MPI_ABORT(MPI_COMM_WORLD, 19, IERR) + ENDIF + + IF (IS_NOAHMP .AND. DO_SNO_INC) THEN + print *, 'FATAL ERROR: Snow increment update does not work with NOAH_MP.' + call MPI_ABORT(MPI_COMM_WORLD, 29, IERR) + ENDIF + !ichk = 282 !jchk = 362 !ijchk = (jchk-1) * idim + ichk From 99ea7fadd331bce79573e0ca1aee36762fed98eb Mon Sep 17 00:00:00 2001 From: George Gayno Date: Mon, 23 Oct 2023 15:28:50 +0000 Subject: [PATCH 18/38] Update script variables to be consistent with the current workflow. Fixes #549. --- ush/global_cycle.sh | 35 +++++++++++++++-------------------- ush/global_cycle_driver.sh | 11 +++++++---- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/ush/global_cycle.sh b/ush/global_cycle.sh index a5022272e..c71eb5efd 100755 --- a/ush/global_cycle.sh +++ b/ush/global_cycle.sh @@ -29,12 +29,8 @@ # Default is /nwprod2. # HOMEgfs Directory for gfs. Default is # $BASEDIR/gfs.v15.0.0. -# FIXSUBDA Sub-directory where fixed climatology files reside. -# Defaults to fix/am. # FIXam Directory for the global fixed climatology files. # Defaults to $HOMEgfs/fix/am -# FIXfv3 Directory for the model grid and orography netcdf -# files. Defaults to $HOMEgfs/fix/orog/${CASE} # EXECgfs Directory of the program executable. Defaults to # $HOMEgfs/exec # DATA Working directory @@ -228,14 +224,17 @@ if [[ "$VERBOSE" = "YES" ]] ; then fi CASE=${CASE:-C768} +OCNRES=${OCNRES:-100} # Directories. gfs_ver=${gfs_ver:-v15.0.0} BASEDIR=${BASEDIR:-${NWROOT:-/nwprod2}} HOMEgfs=${HOMEgfs:-$BASEDIR/gfs_ver.${gfs_ver}} EXECgfs=${EXECgfs:-$HOMEgfs/exec} -FIXfv3=${FIXfv3:-$HOMEgfs/fix/orog/$CASE} -FIXam=${FIXam:-$HOMEgfs/fix/am} +FIX_DIR=${FIX_DIR:-$HOMEgfs/fix} +FIXam=${FIXam:-$FIX_DIR/am} +OROFIX=${OROFIX:-$FIX_DIR/orog/${CASE}.mx${OCNRES}_frac} +FIX_SFC=${FIX_SFC:-$OROFIX/sfc} DATA=${DATA:-$(pwd)} COMIN=${COMIN:-$(pwd)} COMOUT=${COMOUT:-$(pwd)} @@ -280,18 +279,18 @@ FNTSFC=${FNTSFC:-${FIXam}/RTGSST.1982.2012.monthly.clim.grb} FNSALC=${FNSALC:-${FIXam}/global_salclm.t1534.3072.1536.nc} FNSNOC=${FNSNOC:-${FIXam}/global_snoclim.1.875.grb} FNZORC=${FNZORC:-igbp} -FNALBC2=${FNALBC2:-${FIXam}/global_albedo4.1x1.grb} FNAISC=${FNAISC:-${FIXam}/IMS-NIC.blended.ice.monthly.clim.grb} -FNTG3C=${FNTG3C:-${FIXam}/global_tg3clim.2.6x1.5.grb} -FNVEGC=${FNVEGC:-${FIXam}/global_vegfrac.0.144.decpercent.grb} -FNALBC=${FNALBC:-${FIXam}/global_snowfree_albedo.bosu.t$JCAP_CASE.$LONB_CASE.$LATB_CASE.rg.grb} -FNVETC=${FNVETC:-${FIXam}/global_vegtype.igbp.t$JCAP_CASE.$LONB_CASE.$LATB_CASE.rg.grb} -FNSOTC=${FNSOTC:-${FIXam}/global_soiltype.statsgo.t$JCAP_CASE.$LONB_CASE.$LATB_CASE.rg.grb} FNSMCC=${FNSMCC:-${FIXam}/global_soilmgldas.statsgo.t$JCAP_CASE.$LONB_CASE.$LATB_CASE.grb} -FNABSC=${FNABSC:-${FIXam}/global_mxsnoalb.uariz.t$JCAP_CASE.$LONB_CASE.$LATB_CASE.rg.grb} -FNVMNC=${FNVMNC:-${FIXam}/global_shdmin.0.144x0.144.grb} -FNVMXC=${FNVMXC:-${FIXam}/global_shdmax.0.144x0.144.grb} -FNSLPC=${FNSLPC:-${FIXam}/global_slope.1x1.grb} +FNALBC2=${FNALBC2:-${FIX_SFC}/${CASE}.mx${OCNRES}.facsf.tileX.nc} +FNTG3C=${FNTG3C:-${FIX_SFC}/${CASE}.mx${OCNRES}.substrate_temperature.tileX.nc} +FNVEGC=${FNVEGC:-${FIX_SFC}/${CASE}.mx${OCNRES}.vegetation_greenness.tileX.nc} +FNALBC=${FNALBC:-${FIX_SFC}/${CASE}.mx${OCNRES}.snowfree_albedo.tileX.nc} +FNVETC=${FNVETC:-${FIX_SFC}/${CASE}.mx${OCNRES}.vegetation_type.tileX.nc} +FNSOTC=${FNSOTC:-${FIX_SFC}/${CASE}.mx${OCNRES}.soil_type.tileX.nc} +FNABSC=${FNABSC:-${FIX_SFC}/${CASE}.mx${OCNRES}.maximum_snow_albedo.tileX.nc} +FNVMNC=${FNVMNC:-${FIX_SFC}/${CASE}.mx${OCNRES}.vegetation_greenness.tileX.nc} +FNVMXC=${FNVMXC:-${FIX_SFC}/${CASE}.mx${OCNRES}.vegetation_greenness.tileX.nc} +FNSLPC=${FNSLPC:-${FIX_SFC}/${CASE}.mx${OCNRES}.slope_type.tileX.nc} FNMSKH=${FNMSKH:-${FIXam}/global_slmask.t1534.3072.1536.grb} NST_FILE=${NST_FILE:-"NULL"} LND_SOI_FILE=${LND_SOI_FILE:-"NULL"} @@ -326,10 +325,6 @@ ln -fs $FNTSFC sstclm ln -fs $FNSALC salclm # If the appropriate resolution fix file is not present, use the highest resolution available (T1534) -[[ ! -f $FNALBC ]] && FNALBC="$FIXam/global_snowfree_albedo.bosu.t1534.3072.1536.rg.grb" -[[ ! -f $FNVETC ]] && FNVETC="$FIXam/global_vegtype.igbp.t1534.3072.1536.rg.grb" -[[ ! -f $FNSOTC ]] && FNSOTC="$FIXam/global_soiltype.statsgo.t1534.3072.1536.rg.grb" -[[ ! -f $FNABSC ]] && FNABSC="$FIXam/global_mxsnoalb.uariz.t1534.3072.1536.rg.grb" [[ ! -f $FNSMCC ]] && FNSMCC="$FIXam/global_soilmgldas.statsgo.t1534.3072.1536.grb" ################################################################################ diff --git a/ush/global_cycle_driver.sh b/ush/global_cycle_driver.sh index 17324897a..2358c6782 100755 --- a/ush/global_cycle_driver.sh +++ b/ush/global_cycle_driver.sh @@ -9,6 +9,7 @@ set -eux #------------------------------------------------------------------------------------------------- export CASE=${CASE:-C768} # resolution of tile: 48, 96, 192, 384, 768, 1152, 3072 +export OCNRES=${OCNRES:-100} export CDATE=${CDATE:-${cdate:-2017031900}} # format yyyymmddhh yyyymmddhh ... export CDUMP=${CDUMP:-gfs} # gfs or gdas export COMPONENT=${COMPONENT:-atmos} @@ -17,8 +18,10 @@ pwd=$(pwd) export NWPROD=${NWPROD:-$pwd} export DMPDIR=${DMPDIR:-$NWPROD} export HOMEgfs=${HOMEgfs:-$NWPROD/gfs.v15.0.0} -export FIXam=${FIXam:-$HOMEgfs/fix/am} -export FIXfv3=${FIXfv3:-$HOMEgfs/fix/orog} +export FIX_DIR=${FIX_DIR:-$HOMEgfs/fix} +export FIXam=${FIXam:-$FIX_DIR/am} +export OROFIX=${OROFIX:-$FIX_DIR/orog/${CASE}.mx${OCNRES}_frac} +export FIX_SFC=${FIX_SFC:-$OROFIX/sfc} ntiles=${ntiles:-6} DONST=${DONST:-"NO"} @@ -85,8 +88,8 @@ for n in $(seq 1 $ntiles); do chmod 644 $COMOUT/$PDY.${cyc}0000.sfcanl_data.tile${n}.nc ln -fs $COMOUT/$PDY.${cyc}0000.sfcanl_data.tile${n}.nc $DATA/fnbgso.00$n - ln -fs $FIXfv3/C${CRES}/C${CRES}_grid.tile${n}.nc $DATA/fngrid.00$n - ln -fs $FIXfv3/C${CRES}/C${CRES}_oro_data.tile${n}.nc $DATA/fnorog.00$n + ln -fs ${OROFIX}/C${CRES}_grid.tile${n}.nc $DATA/fngrid.00$n + ln -fs ${OROFIX}/oro_C${CRES}.mx${OCNRES}.tile${n}.nc $DATA/fnorog.00$n if [[ "$DO_SNO_INC" == ".true." ]] ; then ln -fs $COMIN/$PDY.${cyc}0000.xainc.tile${n}.nc $DATA/xainc.00$n fi From 5b65787b491656360b11b2f394cc2a45321cf426 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Mon, 20 Nov 2023 20:34:30 +0000 Subject: [PATCH 19/38] Fix intent attribute for adjust_nsst mask argument. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index 8e200e84a..fc8a0861a 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -909,7 +909,7 @@ END SUBROUTINE SFCDRV !! @param[in] SLMSK_FG_TILE First guess land-sea mask on the cubed-sphere tile !! @param[inout] SKINT_TILE Skin temperature on the cubed-sphere tile !! @param[inout] SICET_TILE Ice temperature on the cubed-sphere tile - !! @param[inout] sice_tile Ice concentration on the cubed-sphere tile + !! @param[in] sice_tile Ice concentration on the cubed-sphere tile !! @param[inout] SOILT_TILE Soil temperature on the cubed-sphere tile !! @param[in] NSST Data structure holding nsst fields !! @param[in] LENSFC Number of points on a tile @@ -946,9 +946,9 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& REAL, INTENT(IN) :: SLMSK_TILE(LENSFC), SLMSK_FG_TILE(LENSFC) real, intent(in) :: tf_clm_tile(lensfc),tf_trd_tile(lensfc),sal_clm_tile(lensfc) - REAL, INTENT(IN) :: ZSEA1, ZSEA2 + REAL, INTENT(IN) :: ZSEA1, ZSEA2,sice_tile(lensfc) REAL, INTENT(INOUT) :: RLA(LENSFC), RLO(LENSFC), SKINT_TILE(LENSFC) - REAL, INTENT(INOUT) :: SICET_TILE(LENSFC),sice_tile(lensfc),SOILT_TILE(LENSFC,LSOIL) + REAL, INTENT(INOUT) :: SICET_TILE(LENSFC),SOILT_TILE(LENSFC,LSOIL) TYPE(NSST_DATA) :: NSST @@ -1092,6 +1092,10 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& MASK_TILE = NINT(SLMSK_TILE(IJ)) MASK_FG_TILE = NINT(SLMSK_FG_TILE(IJ)) + if (sice_tile(ij) > 0. .and. mask_tile /= 2) then + print*,'bad ice point ',ij,sice_tile(ij),mask_tile + endif + ! ! when sea ice exists, get salinity dependent water temperature ! From 5c467b42a045941c5cf6607ff655ce28b07e074d Mon Sep 17 00:00:00 2001 From: George Gayno Date: Mon, 20 Nov 2023 20:38:30 +0000 Subject: [PATCH 20/38] Point to latest version of orog files. Fixes #549. --- fix/link_fixdirs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fix/link_fixdirs.sh b/fix/link_fixdirs.sh index 7e9d28025..436cba34e 100755 --- a/fix/link_fixdirs.sh +++ b/fix/link_fixdirs.sh @@ -57,7 +57,7 @@ elif [ $machine = "s4" ]; then fi am_ver=${am_ver:-20220805} -orog_ver=${orog_ver:-20220805} +orog_ver=${orog_ver:-20230615} sfc_climo_ver=${sfc_climo_ver:-20230925} for dir in am orog orog_raw sfc_climo; do From 138cb47fefd47317adf231399575390bef891d8e Mon Sep 17 00:00:00 2001 From: George Gayno Date: Tue, 21 Nov 2023 20:59:05 +0000 Subject: [PATCH 21/38] Updates so the first regression test can run successfully. Fixes #549. --- reg_tests/global_cycle/C768.fv3gfs.sh | 15 +++++++++++++++ ush/global_cycle_driver.sh | 4 +++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/reg_tests/global_cycle/C768.fv3gfs.sh b/reg_tests/global_cycle/C768.fv3gfs.sh index 268c18f44..4210063c6 100755 --- a/reg_tests/global_cycle/C768.fv3gfs.sh +++ b/reg_tests/global_cycle/C768.fv3gfs.sh @@ -32,7 +32,22 @@ export JCAP=1534 export LONB=3072 export LATB=1536 +export OROFIX=$BASE_GSM/fix/orog/$CASE +export OROFILE=${CASE}_oro_data + export FIXgsm=$BASE_GSM/fix/am + +export FNALBC2=$FIXgsm/global_albedo4.1x1.grb +export FNALBC=$FIXgsm/global_snowfree_albedo.bosu.t1534.3072.1536.rg.grb +export FNALBC=$FIXgsm/global_snowfree_albedo.bosu.t1534.3072.1536.rg.grb +export FNTG3C=$FIXgsm/global_tg3clim.2.6x1.5.grb +export FNVETC=$FIXgsm/global_vegtype.igbp.t1534.3072.1536.rg.grb +export FNSOTC=$FIXgsm/global_soiltype.statsgo.t1534.3072.1536.rg.grb +export FNVEGC=$FIXgsm/global_vegfrac.0.144.decpercent.grb +export FNVMNC=$FIXgsm/global_shdmin.0.144x0.144.grb +export FNVMXC=$FIXgsm/global_shdmax.0.144x0.144.grb +export FNSLPC=$FIXgsm/global_slope.1x1.grb +export FNABSC=$FIXgsm/global_mxsnoalb.uariz.t1534.3072.1536.rg.grb export FNAISC=$FIXgsm/CFSR.SEAICE.1982.2012.monthly.clim.grb export DONST="YES" diff --git a/ush/global_cycle_driver.sh b/ush/global_cycle_driver.sh index 2358c6782..b8ae3b1c0 100755 --- a/ush/global_cycle_driver.sh +++ b/ush/global_cycle_driver.sh @@ -23,6 +23,8 @@ export FIXam=${FIXam:-$FIX_DIR/am} export OROFIX=${OROFIX:-$FIX_DIR/orog/${CASE}.mx${OCNRES}_frac} export FIX_SFC=${FIX_SFC:-$OROFIX/sfc} +OROFILE=${OROFILE:-oro_${CASE}.mx${OCNRES}} + ntiles=${ntiles:-6} DONST=${DONST:-"NO"} COMIN=${COMIN:-$pwd} @@ -89,7 +91,7 @@ for n in $(seq 1 $ntiles); do ln -fs $COMOUT/$PDY.${cyc}0000.sfcanl_data.tile${n}.nc $DATA/fnbgso.00$n ln -fs ${OROFIX}/C${CRES}_grid.tile${n}.nc $DATA/fngrid.00$n - ln -fs ${OROFIX}/oro_C${CRES}.mx${OCNRES}.tile${n}.nc $DATA/fnorog.00$n + ln -fs ${OROFIX}/${OROFILE}.tile${n}.nc $DATA/fnorog.00$n if [[ "$DO_SNO_INC" == ".true." ]] ; then ln -fs $COMIN/$PDY.${cyc}0000.xainc.tile${n}.nc $DATA/xainc.00$n fi From 1f1c34a202dbf5ad76425e7d8408e1d403436800 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Tue, 21 Nov 2023 21:14:05 +0000 Subject: [PATCH 22/38] Fix the 2nd regression test. Fixes #549. --- reg_tests/global_cycle/C192.lndincsoilnoahmp.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/reg_tests/global_cycle/C192.lndincsoilnoahmp.sh b/reg_tests/global_cycle/C192.lndincsoilnoahmp.sh index 0c34fa82f..a2268e61d 100755 --- a/reg_tests/global_cycle/C192.lndincsoilnoahmp.sh +++ b/reg_tests/global_cycle/C192.lndincsoilnoahmp.sh @@ -33,6 +33,8 @@ export JCAP=1534 export LONB=3072 export LATB=1536 +export OROFIX=$BASE_GSM/fix/orog/$CASE +export OROFILE=${CASE}_oro_data export FIXgsm=$BASE_GSM/fix/am export DONST="NO" From 1d2d42540ee6049fa70eee0acd60b976423276c1 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Tue, 21 Nov 2023 21:21:48 +0000 Subject: [PATCH 23/38] Fix final regression test. Fixes #549. --- reg_tests/global_cycle/C768.lndincsnow.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/reg_tests/global_cycle/C768.lndincsnow.sh b/reg_tests/global_cycle/C768.lndincsnow.sh index b0fe41a46..9ba369f77 100755 --- a/reg_tests/global_cycle/C768.lndincsnow.sh +++ b/reg_tests/global_cycle/C768.lndincsnow.sh @@ -34,6 +34,8 @@ export JCAP=1534 export LONB=3072 export LATB=1536 +export OROFIX=$BASE_GSM/fix/orog/$CASE +export OROFILE=${CASE}_oro_data export FIXgsm=$BASE_GSM/fix/am export DONST="NO" From ee19c3c1af02b415efe84af7a3a955bb1535274c Mon Sep 17 00:00:00 2001 From: George Gayno Date: Tue, 28 Nov 2023 21:07:11 +0000 Subject: [PATCH 24/38] Begin updates for fractional grids and nsst. First, update some logic so that it will work with either fractional or non-frac grids. All regression tests passed. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 85 ++++++++++++++++++++++++++-------- 1 file changed, 65 insertions(+), 20 deletions(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index fc8a0861a..c9e8048ac 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -367,7 +367,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & !! dead start. Set to zero for non-dead !! start. REAL, ALLOCATABLE :: STC_BCK(:,:), SMC_BCK(:,:), SLC_BCK(:,:) - REAL, ALLOCATABLE :: SLIFCS_FG(:) + REAL, ALLOCATABLE :: SLIFCS_FG(:), SICFCS_FG(:) INTEGER, ALLOCATABLE :: LANDINC_MASK_FG(:), LANDINC_MASK(:) REAL, ALLOCATABLE :: SND_BCK(:), SND_INC(:), SWE_BCK(:) REAL(KIND=KIND_IO8), ALLOCATABLE :: SLMASKL(:), SLMASKW(:), LANDFRAC(:) @@ -450,6 +450,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & ALLOCATE(NSST%Z_C(LENSFC)) ALLOCATE(NSST%ZM(LENSFC)) ALLOCATE(SLIFCS_FG(LENSFC)) + ALLOCATE(SICFCS_FG(LENSFC)) ENDIF IF (DO_LNDINC) THEN @@ -501,6 +502,17 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & !STCFCS_SAVE = STCFCS +!if (tile_num == 'tile6') then +! print*,'remove ice' +! slifcs(434252) = 0. +! sicfcs(434252) = 0. +! sihfcs(434252) = 0. +! print*,'add ice' +! slifcs(491001) = 2. +! sicfcs(491001) = .9 +! sihfcs(491001) = 1. +!endif + print*,'is noahmp/fract grid ',is_noahmp, frac_grid IF (FRAC_GRID .AND. .NOT. IS_NOAHMP) THEN @@ -552,6 +564,15 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & ENDDO IF (DO_NSST) THEN + SICFCS_FG=SICFCS + do i=1,lensfc + if (slifcs(i) == 2.0 .and. sicfcs(i) < .01) then + print*,'bad ice 1 ',i,slifcs(i),sicfcs(i) + endif + if (slifcs(i) == 0.0 .and. sicfcs(i) > 0.) then + print*,'bad ice 2 ',i,slifcs(i),sicfcs(i) + endif + enddo IF (.NOT. DO_SFCCYCLE ) THEN PRINT* PRINT*,"FIRST GUESS MASK ADJUSTED BY IFD RECORD" @@ -696,18 +717,33 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & IF(FRAC_GRID) DEALLOCATE(LANDFRAC) + if (tile_num == 'tile6') then + print*,'remove ice after cycle ', slifcs(434252),sicfcs(434252),sihfcs(434252) + print*,'add ice after cycle ', slifcs(491001),sicfcs(491001),sihfcs(491001) + endif + !-------------------------------------------------------------------------------- ! IF RUNNING WITH NSST, READ IN GSI FILE WITH THE UPDATED INCREMENTS (ON THE ! GAUSSIAN GRID), INTERPOLATE INCREMENTS TO THE CUBED-SPHERE TILE, AND PERFORM ! REQUIRED ADJUSTMENTS AND QC. !-------------------------------------------------------------------------------- + if (do_nsst) then + print*,'ice check 2', tile_num + do i = 1, lensfc + if (sicfcs(i) /= sicfcs_fg(i)) then + print*,' ice update ',i,sicfcs(i),sicfcs_fg(i) + endif + enddo + endif + IF (DO_NSST) THEN IF (NST_FILE == "NULL") THEN PRINT* PRINT*,"NO GSI FILE. ADJUST IFD FOR FORMER ICE POINTS." DO I = 1, LENSFC - IF (NINT(SLIFCS_FG(I)) == 2 .AND. NINT(SLIFCS(I)) == 0) THEN +! IF (NINT(SLIFCS_FG(I)) == 2 .AND. NINT(SLIFCS(I)) == 0) THEN + IF (SICFCS_FG(I) > 0.0 .AND. SICFCS(I) == 0) THEN NSST%IFD(I) = 3.0 ENDIF ENDDO @@ -733,8 +769,8 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & ! ! update foundation & surface temperature for NSST ! - CALL ADJUST_NSST(RLA,RLO,SLIFCS,SLIFCS_FG,TSFFCS,SITFCS,SICFCS,STCFCS, & - NSST,LENSFC,LSOIL,IDIM,JDIM,ZSEA1,ZSEA2, & + CALL ADJUST_NSST(RLA,RLO,SLIFCS,SLIFCS_FG,TSFFCS,SITFCS,SICFCS,SICFCS_FG,& + STCFCS,NSST,LENSFC,LSOIL,IDIM,JDIM,ZSEA1,ZSEA2, & tf_clm_tile,tf_trd_tile,sal_clm_tile) ENDIF ENDIF @@ -893,6 +929,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & DEALLOCATE(NSST%Z_C) DEALLOCATE(NSST%ZM) DEALLOCATE(SLIFCS_FG) + DEALLOCATE(SICFCS_FG) ENDIF RETURN @@ -903,8 +940,8 @@ END SUBROUTINE SFCDRV !! grid), interpolate increments to the cubed-sphere tile, and !! perform required nsst adjustments and qc. !! - !! @param[inout] RLA Latitude on the cubed-sphere tile - !! @param[inout] RLO Longitude on the cubed-sphere tile + !! @param[in] RLA Latitude on the cubed-sphere tile + !! @param[in] RLO Longitude on the cubed-sphere tile !! @param[in] SLMSK_TILE Land-sea mask on the cubed-sphere tile !! @param[in] SLMSK_FG_TILE First guess land-sea mask on the cubed-sphere tile !! @param[inout] SKINT_TILE Skin temperature on the cubed-sphere tile @@ -928,8 +965,8 @@ END SUBROUTINE SFCDRV !! !! @author Xu Li, George Gayno SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& - SICET_TILE,sice_tile,SOILT_TILE,NSST,LENSFC,LSOIL, & - IDIM,JDIM,ZSEA1,ZSEA2, & + SICET_TILE,sice_tile,sice_fg_tile,SOILT_TILE,NSST, & + LENSFC,LSOIL,IDIM,JDIM,ZSEA1,ZSEA2, & tf_clm_tile,tf_trd_tile,sal_clm_tile) USE UTILS @@ -946,8 +983,9 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& REAL, INTENT(IN) :: SLMSK_TILE(LENSFC), SLMSK_FG_TILE(LENSFC) real, intent(in) :: tf_clm_tile(lensfc),tf_trd_tile(lensfc),sal_clm_tile(lensfc) - REAL, INTENT(IN) :: ZSEA1, ZSEA2,sice_tile(lensfc) - REAL, INTENT(INOUT) :: RLA(LENSFC), RLO(LENSFC), SKINT_TILE(LENSFC) + REAL, INTENT(IN) :: ZSEA1, ZSEA2,sice_tile(lensfc),sice_fg_tile(lensfc) + REAL, INTENT(IN) :: RLA(LENSFC), RLO(LENSFC) + REAL, INTENT(INOUT) :: SKINT_TILE(LENSFC) REAL, INTENT(INOUT) :: SICET_TILE(LENSFC),SOILT_TILE(LENSFC,LSOIL) TYPE(NSST_DATA) :: NSST @@ -957,7 +995,8 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& INTEGER :: IOPT, NRET, KGDS_GAUS(200) INTEGER :: IGAUS, JGAUS, IJ, II, JJ, III, JJJ, KRAD INTEGER :: ISTART, IEND, JSTART, JEND - INTEGER :: MASK_TILE, MASK_FG_TILE +!INTEGER :: MASK_TILE, MASK_FG_TILE + INTEGER,allocatable :: MASK_TILE(:),MASK_FG_TILE(:) INTEGER :: ITILE, JTILE INTEGER :: MAX_SEARCH, J, IERR INTEGER :: IGAUSP1, JGAUSP1 @@ -1087,13 +1126,19 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& NSST%TFINC = 0.0 + allocate(mask_tile(lensfc)) + allocate(mask_fg_tile(lensfc)) + + MASK_TILE = NINT(SLMSK_TILE) + MASK_FG_TILE = NINT(SLMSK_FG_TILE) + IJ_LOOP : DO IJ = 1, LENSFC - MASK_TILE = NINT(SLMSK_TILE(IJ)) - MASK_FG_TILE = NINT(SLMSK_FG_TILE(IJ)) +! MASK_TILE = NINT(SLMSK_TILE(IJ)) +! MASK_FG_TILE = NINT(SLMSK_FG_TILE(IJ)) - if (sice_tile(ij) > 0. .and. mask_tile /= 2) then - print*,'bad ice point ',ij,sice_tile(ij),mask_tile + if (sice_tile(ij) > 0. .and. mask_tile(ij) /= 2) then + print*,'bad ice point ',ij,sice_tile(ij),mask_tile(ij) endif ! @@ -1104,7 +1149,7 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& ! SKIP LAND POINTS. NSST NOT APPLIED AT LAND. !---------------------------------------------------------------------- - IF (MASK_TILE == 1) THEN + IF (MASK_TILE(ij) == 1) THEN nland = nland + 1 CYCLE IJ_LOOP ENDIF @@ -1112,7 +1157,7 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& ! ! these are ice points. set tref to tf_ice and update tmpsfc. ! - if (mask_tile == 2) then + if (mask_tile(ij) == 2) then nsst%tref(ij)=tf_ice ! water part tmp set skint_tile(ij)=(1.0-sice_tile(ij))*nsst%tref(ij)+sice_tile(ij)*sicet_tile(ij) nice = nice + 1 @@ -1132,11 +1177,11 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& ! weighted average of tf_ice and tf_clm. For NSST vars, set xz TO '30' AND ALL OTHER FIELDS TO ZERO. !---------------------------------------------------------------------- - IF (MASK_FG_TILE == 2 .AND. MASK_TILE == 0) THEN + IF (mask_fg_tile(ij) == 2 .AND. mask_tile(ij) == 0) THEN ! ! set background for the thaw (just melted water) situation ! - call tf_thaw_set(nsst%tref,nint(slmsk_fg_tile),itile,jtile,tf_ice,tf_clm_tile(ij),tf_thaw,idim,jdim, & + call tf_thaw_set(nsst%tref,mask_fg_tile,itile,jtile,tf_ice,tf_clm_tile(ij),tf_thaw,idim,jdim, & nset_thaw_s,nset_thaw_i,nset_thaw_c) call nsst_water_reset(nsst,ij,tf_thaw) nset_thaw = nset_thaw + 1 @@ -1326,7 +1371,7 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& write(*,'(a,I8)') ' nice = ',nice write(*,'(a,I8)') ' nland = ',nland - DEALLOCATE(ID1, ID2, JDC, S2C) + DEALLOCATE(ID1, ID2, JDC, S2C, mask_tile, mask_fg_tile) END SUBROUTINE ADJUST_NSST From 6cea9abdb0ae9d4c62db0ffe374060e19d127b53 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 28 Dec 2023 15:16:05 +0000 Subject: [PATCH 25/38] Comment out diagnostic code. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index c9e8048ac..6cb5bf252 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -717,10 +717,10 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & IF(FRAC_GRID) DEALLOCATE(LANDFRAC) - if (tile_num == 'tile6') then - print*,'remove ice after cycle ', slifcs(434252),sicfcs(434252),sihfcs(434252) - print*,'add ice after cycle ', slifcs(491001),sicfcs(491001),sihfcs(491001) - endif +!if (tile_num == 'tile6') then +! print*,'remove ice after cycle ', slifcs(434252),sicfcs(434252),sihfcs(434252) +! print*,'add ice after cycle ', slifcs(491001),sicfcs(491001),sihfcs(491001) +!endif !-------------------------------------------------------------------------------- ! IF RUNNING WITH NSST, READ IN GSI FILE WITH THE UPDATED INCREMENTS (ON THE @@ -1837,6 +1837,7 @@ subroutine get_tf_clm_ta(tf_clm_ta,tf_clm_trend,xlats,xlons,nlat,nlon,mon1,mon2, ! ! read in rtg sst climatology without bitmap (surface mask) for mon1 and mon2 ! + print*,'before read_tf_clim_grb',trim(fin_tf_clm) call read_tf_clim_grb(trim(fin_tf_clm),tf_clm1,xlats,xlons,nlat,nlon,mon1) call read_tf_clim_grb(trim(fin_tf_clm),tf_clm2,xlats,xlons,nlat,nlon,mon2) ! From 0984fbd1d63b876653881027202b1df8c3e7fe17 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 28 Dec 2023 21:07:34 +0000 Subject: [PATCH 26/38] Update adjust_nsst routine for fractional grids. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 41 +++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index 6cb5bf252..0778a2a79 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -407,12 +407,14 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & ! READ THE OROGRAPHY AND GRID POINT LAT/LONS FOR THE CUBED-SPHERE TILE. !-------------------------------------------------------------------------------- + ALLOCATE(LANDFRAC(LENSFC)) IF(FRAC_GRID) THEN PRINT*,'- RUNNING WITH FRACTIONAL GRID.' - ALLOCATE(LANDFRAC(LENSFC)) +! ALLOCATE(LANDFRAC(LENSFC)) CALL READ_LAT_LON_OROG(RLA,RLO,OROG,OROG_UF,TILE_NUM,IDIM,JDIM,LENSFC,LANDFRAC=LANDFRAC) ELSE CALL READ_LAT_LON_OROG(RLA,RLO,OROG,OROG_UF,TILE_NUM,IDIM,JDIM,LENSFC) + LANDFRAC=-999.9 ENDIF DO I = 1, IDIM @@ -715,7 +717,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & DEALLOCATE(SLMASKL, SLMASKW) ENDIF - IF(FRAC_GRID) DEALLOCATE(LANDFRAC) +!IF(FRAC_GRID) DEALLOCATE(LANDFRAC) !if (tile_num == 'tile6') then ! print*,'remove ice after cycle ', slifcs(434252),sicfcs(434252),sihfcs(434252) @@ -771,7 +773,7 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & ! CALL ADJUST_NSST(RLA,RLO,SLIFCS,SLIFCS_FG,TSFFCS,SITFCS,SICFCS,SICFCS_FG,& STCFCS,NSST,LENSFC,LSOIL,IDIM,JDIM,ZSEA1,ZSEA2, & - tf_clm_tile,tf_trd_tile,sal_clm_tile) + tf_clm_tile,tf_trd_tile,sal_clm_tile,landfrac,frac_grid) ENDIF ENDIF @@ -967,7 +969,8 @@ END SUBROUTINE SFCDRV SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& SICET_TILE,sice_tile,sice_fg_tile,SOILT_TILE,NSST, & LENSFC,LSOIL,IDIM,JDIM,ZSEA1,ZSEA2, & - tf_clm_tile,tf_trd_tile,sal_clm_tile) + tf_clm_tile,tf_trd_tile,sal_clm_tile,LANDFRAC, & + FRAC_GRID) USE UTILS USE GDSWZD_MOD @@ -981,7 +984,9 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& INTEGER, INTENT(IN) :: LENSFC, LSOIL, IDIM, JDIM - REAL, INTENT(IN) :: SLMSK_TILE(LENSFC), SLMSK_FG_TILE(LENSFC) + LOGICAL, INTENT(IN) :: FRAC_GRID + + REAL, INTENT(IN) :: SLMSK_TILE(LENSFC), SLMSK_FG_TILE(LENSFC), LANDFRAC(LENSFC) real, intent(in) :: tf_clm_tile(lensfc),tf_trd_tile(lensfc),sal_clm_tile(lensfc) REAL, INTENT(IN) :: ZSEA1, ZSEA2,sice_tile(lensfc),sice_fg_tile(lensfc) REAL, INTENT(IN) :: RLA(LENSFC), RLO(LENSFC) @@ -1128,9 +1133,18 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& allocate(mask_tile(lensfc)) allocate(mask_fg_tile(lensfc)) - - MASK_TILE = NINT(SLMSK_TILE) - MASK_FG_TILE = NINT(SLMSK_FG_TILE) + + IF(.NOT. FRAC_GRID) THEN + MASK_TILE = NINT(SLMSK_TILE) + MASK_FG_TILE = NINT(SLMSK_FG_TILE) + ELSE + MASK_TILE=0 + WHERE(SICE_TILE > 0.0) MASK_TILE=2 + WHERE(LANDFRAC == 1.0) MASK_TILE=1 + MASK_FG_TILE=0 + WHERE(SICE_FG_TILE > 0.0) MASK_FG_TILE=2 + WHERE(LANDFRAC == 1.0) MASK_FG_TILE=1 + ENDIF IJ_LOOP : DO IJ = 1, LENSFC @@ -1141,6 +1155,9 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& print*,'bad ice point ',ij,sice_tile(ij),mask_tile(ij) endif + if (sice_fg_tile(ij) > 0. .and. mask_fg_tile(ij) /= 2) then + print*,'bad fg ice point ',ij,sice_fg_tile(ij),mask_fg_tile(ij) + endif ! ! when sea ice exists, get salinity dependent water temperature ! @@ -1250,7 +1267,9 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& SKINT_TILE(IJ) = MIN(SKINT_TILE(IJ), TMAX) SICET_TILE(IJ) = SKINT_TILE(IJ) - SOILT_TILE(IJ,:) = SKINT_TILE(IJ) +! Under fractional grids, soilt is used at points with at +! least some land. + IF(.NOT. FRAC_GRID) SOILT_TILE(IJ,:) = SKINT_TILE(IJ) !---------------------------------------------------------------------- ! NO NEARBY GSI/GAUSSIAN OPEN WATER POINTS. PERFORM A SPIRAL SEARCH TO @@ -1313,7 +1332,7 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& SKINT_TILE(IJ) = MIN(SKINT_TILE(IJ), TMAX) SICET_TILE(IJ) = SKINT_TILE(IJ) - SOILT_TILE(IJ,:) = SKINT_TILE(IJ) + IF(.NOT. FRAC_GRID) SOILT_TILE(IJ,:) = SKINT_TILE(IJ) CYCLE IJ_LOOP ENDIF ! GSI/Gaussian mask is open water @@ -1357,7 +1376,7 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& SKINT_TILE(IJ) = MIN(SKINT_TILE(IJ), TMAX) SICET_TILE(IJ) = SKINT_TILE(IJ) - SOILT_TILE(IJ,:) = SKINT_TILE(IJ) + IF (.NOT. FRAC_GRID) SOILT_TILE(IJ,:) = SKINT_TILE(IJ) ENDIF ! NEARBY GAUSSIAN POINTS ARE OPEN WATER? From ed8c24db88047baa1f74aff5b6e4e357f1fa9862 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 3 Jan 2024 16:31:17 +0000 Subject: [PATCH 27/38] Output ice fields as nsst update is ice dependent. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index 0778a2a79..4ee4064ce 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -893,7 +893,8 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & IF (LSM==LSM_NOAHMP) THEN CALL WRITE_DATA(LENSFC,IDIM,JDIM,LSOIL,DO_NSST,NSST,VEGFCS=VEGFCS, & - SLCFCS=SLCFCS,SMCFCS=SMCFCS,STCFCS=STCFCS) + SLCFCS=SLCFCS,SMCFCS=SMCFCS,STCFCS=STCFCS,& + SICFCS=SICFCS,SIHFCS=SIHFCS) ELSEIF (LSM==LSM_NOAH) THEN From 5a870df8f9db05174b44818733ead64806fc3354 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 3 Jan 2024 21:18:55 +0000 Subject: [PATCH 28/38] Minor script fixes for the first cycle regression test required after the recent merge. Fixes #549. --- reg_tests/global_cycle/C768.fv3gfs.sh | 6 ++---- reg_tests/global_cycle/driver.hera.sh | 4 +++- ush/global_cycle_driver.sh | 11 +++++------ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/reg_tests/global_cycle/C768.fv3gfs.sh b/reg_tests/global_cycle/C768.fv3gfs.sh index 0096bc2c6..8dd6d303c 100755 --- a/reg_tests/global_cycle/C768.fv3gfs.sh +++ b/reg_tests/global_cycle/C768.fv3gfs.sh @@ -21,6 +21,7 @@ export FHOUR=00 export DELTSFC=6 export CASE=C768 +export OCNRES=99999 export COMIN=$HOMEreg/input_data export FNTSFA=$COMIN/gdas.t00z.rtgssthr.grb @@ -32,8 +33,7 @@ export JCAP=1534 export LONB=3072 export LATB=1536 -export OROFIX=$BASE_GSM/fix/orog/$CASE -export OROFILE=${CASE}_oro_data +export OROFIX=$HOMEreg/fix/$CASE export FIXgsm=$BASE_GSM/fix/am @@ -50,8 +50,6 @@ export FNSLPC=$FIXgsm/global_slope.1x1.grb export FNABSC=$FIXgsm/global_mxsnoalb.uariz.t1534.3072.1536.rg.grb export FNAISC=$FIXgsm/CFSR.SEAICE.1982.2012.monthly.clim.grb -export FIXfv3=$HOMEreg/fix - export DONST="YES" export use_ufo=.true. diff --git a/reg_tests/global_cycle/driver.hera.sh b/reg_tests/global_cycle/driver.hera.sh index 9ea901726..bbfb8f211 100755 --- a/reg_tests/global_cycle/driver.hera.sh +++ b/reg_tests/global_cycle/driver.hera.sh @@ -30,7 +30,7 @@ module list WORK_DIR="${WORK_DIR:-/scratch2/NCEPDEV/stmp1/$LOGNAME}" PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}" -QUEUE="${QUEUE:-batch}" +QUEUE="${QUEUE:-debug}" #----------------------------------------------------------------------------- # Should not have to change anything below. @@ -61,6 +61,8 @@ export COMOUT=$DATA TEST1=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c768.fv3gfs \ -o $LOG_FILE -e $LOG_FILE ./C768.fv3gfs.sh) +exit + LOG_FILE=consistency.log02 export DATA="${DATA_DIR}/test2" export COMOUT=$DATA diff --git a/ush/global_cycle_driver.sh b/ush/global_cycle_driver.sh index 100db8bac..ce4d1756a 100755 --- a/ush/global_cycle_driver.sh +++ b/ush/global_cycle_driver.sh @@ -23,8 +23,6 @@ export FIXam=${FIXam:-$FIX_DIR/am} export OROFIX=${OROFIX:-$FIX_DIR/orog/${CASE}.mx${OCNRES}_frac} export FIX_SFC=${FIX_SFC:-$OROFIX/sfc} -OROFILE=${OROFILE:-oro_${CASE}.mx${OCNRES}} - ntiles=${ntiles:-6} DONST=${DONST:-"NO"} COMIN=${COMIN:-$pwd} @@ -90,11 +88,11 @@ for n in $(seq 1 $ntiles); do chmod 644 $COMOUT/$PDY.${cyc}0000.sfcanl_data.tile${n}.nc ln -fs $COMOUT/$PDY.${cyc}0000.sfcanl_data.tile${n}.nc $DATA/fnbgso.00$n - ln -fs $FIXfv3/C${CRES}/C${CRES}_grid.tile${n}.nc $DATA/fngrid.00$n - if [ -z "${ocn}" ];then - ln -fs $FIXfv3/C${CRES}/C${CRES}_oro_data.tile${n}.nc $DATA/fnorog.00$n + ln -fs $OROFIX/C${CRES}_grid.tile${n}.nc $DATA/fngrid.00$n + if (( OCNRES > 9999 ));then + ln -fs $OROFIX/C${CRES}_oro_data.tile${n}.nc $DATA/fnorog.00$n else - ln -fs $FIXfv3/C${CRES}/C${CRES}.mx${ocn}_oro_data.tile${n}.nc $DATA/fnorog.00$n + ln -fs $OROFIX/C${CRES}.mx${OCNRES}_oro_data.tile${n}.nc $DATA/fnorog.00$n fi if [[ "$DO_SNO_INC" == ".true." ]] ; then @@ -103,6 +101,7 @@ for n in $(seq 1 $ntiles); do done $CYCLESH + rc=$? if [[ $rc -ne 0 ]] ; then echo "***ERROR*** rc= $rc" From b17a90d44f452e789a11ed8fbbca0005b512a12d Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 4 Jan 2024 14:35:03 +0000 Subject: [PATCH 29/38] Minor script changes for the remaining regression tests. Required after the latest merge from develop. Fixes #549. --- reg_tests/global_cycle/C192.lndincsoilnoahmp.sh | 6 ++---- reg_tests/global_cycle/C768.lndincsnow.sh | 6 ++---- reg_tests/global_cycle/driver.hera.sh | 4 +--- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/reg_tests/global_cycle/C192.lndincsoilnoahmp.sh b/reg_tests/global_cycle/C192.lndincsoilnoahmp.sh index 4788e69b5..9e676ef8b 100755 --- a/reg_tests/global_cycle/C192.lndincsoilnoahmp.sh +++ b/reg_tests/global_cycle/C192.lndincsoilnoahmp.sh @@ -24,6 +24,7 @@ export FHOUR=00 export DELTSFC=6 export CASE=C192 +export OCNRES=99999 export COMIN=$HOMEreg/input_data_noahmp @@ -33,12 +34,9 @@ export JCAP=1534 export LONB=3072 export LATB=1536 -export OROFIX=$BASE_GSM/fix/orog/$CASE -export OROFILE=${CASE}_oro_data +export OROFIX=$HOMEreg/fix/$CASE export FIXgsm=$BASE_GSM/fix/am -export FIXfv3=$HOMEreg/fix - export DONST="NO" export use_ufo=.true. diff --git a/reg_tests/global_cycle/C768.lndincsnow.sh b/reg_tests/global_cycle/C768.lndincsnow.sh index f668daa53..86a2a7815 100755 --- a/reg_tests/global_cycle/C768.lndincsnow.sh +++ b/reg_tests/global_cycle/C768.lndincsnow.sh @@ -22,6 +22,7 @@ export FHOUR=00 export DELTSFC=6 export CASE=C768 +export OCNRES=99999 export COMIN=$HOMEreg/input_data export FNTSFA=$COMIN/gdas.t00z.rtgssthr.grb @@ -34,12 +35,9 @@ export JCAP=1534 export LONB=3072 export LATB=1536 -export OROFIX=$BASE_GSM/fix/orog/$CASE -export OROFILE=${CASE}_oro_data +export OROFIX=$HOMEreg/fix/$CASE export FIXgsm=$BASE_GSM/fix/am -export FIXfv3=$HOMEreg/fix - export DONST="NO" export use_ufo=.true. diff --git a/reg_tests/global_cycle/driver.hera.sh b/reg_tests/global_cycle/driver.hera.sh index bbfb8f211..9ea901726 100755 --- a/reg_tests/global_cycle/driver.hera.sh +++ b/reg_tests/global_cycle/driver.hera.sh @@ -30,7 +30,7 @@ module list WORK_DIR="${WORK_DIR:-/scratch2/NCEPDEV/stmp1/$LOGNAME}" PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}" -QUEUE="${QUEUE:-debug}" +QUEUE="${QUEUE:-batch}" #----------------------------------------------------------------------------- # Should not have to change anything below. @@ -61,8 +61,6 @@ export COMOUT=$DATA TEST1=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c768.fv3gfs \ -o $LOG_FILE -e $LOG_FILE ./C768.fv3gfs.sh) -exit - LOG_FILE=consistency.log02 export DATA="${DATA_DIR}/test2" export COMOUT=$DATA From fc13873c219f64dd21014dd563b2e1687fc30cca Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 11 Jan 2024 16:09:28 +0000 Subject: [PATCH 30/38] Fix definition of OROFIX in the cycle driver script. Fixes #549. --- ush/global_cycle_driver.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/global_cycle_driver.sh b/ush/global_cycle_driver.sh index ce4d1756a..84699c306 100755 --- a/ush/global_cycle_driver.sh +++ b/ush/global_cycle_driver.sh @@ -20,7 +20,7 @@ export DMPDIR=${DMPDIR:-$NWPROD} export HOMEgfs=${HOMEgfs:-$NWPROD/gfs.v15.0.0} export FIX_DIR=${FIX_DIR:-$HOMEgfs/fix} export FIXam=${FIXam:-$FIX_DIR/am} -export OROFIX=${OROFIX:-$FIX_DIR/orog/${CASE}.mx${OCNRES}_frac} +export OROFIX=${OROFIX:-$FIX_DIR/orog/${CASE}} export FIX_SFC=${FIX_SFC:-$OROFIX/sfc} ntiles=${ntiles:-6} From 8af4329e42235a218cde9e5e5df636fc6497f2ac Mon Sep 17 00:00:00 2001 From: George Gayno Date: Tue, 16 Jan 2024 21:25:51 +0000 Subject: [PATCH 31/38] Remove some diagnostic print. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 100 --------------------------------- 1 file changed, 100 deletions(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index 4ee4064ce..bb57c8e60 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -334,7 +334,6 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & INTEGER :: I_INDEX(LENSFC), J_INDEX(LENSFC) INTEGER :: IDUM(IDIM,JDIM) integer :: num_parthds, num_threads -! integer :: ichk, jchk, ijchk LOGICAL :: IS_NOAHMP INTEGER :: LSM @@ -348,7 +347,6 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & REAL :: ALBFCS(LENSFC,4), TG3FCS(LENSFC) REAL :: CNPFCS(LENSFC), SMCFCS(LENSFC,LSOIL) REAL :: STCFCS(LENSFC,LSOIL), SLIFCS(LENSFC) -!REAL :: STCFCS_SAVE(LENSFC,LSOIL) REAL :: AISFCS(LENSFC), F10M(LENSFC) REAL :: VEGFCS(LENSFC), VETFCS(LENSFC) REAL :: SOTFCS(LENSFC), ALFFCS(LENSFC,2) @@ -410,7 +408,6 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & ALLOCATE(LANDFRAC(LENSFC)) IF(FRAC_GRID) THEN PRINT*,'- RUNNING WITH FRACTIONAL GRID.' -! ALLOCATE(LANDFRAC(LENSFC)) CALL READ_LAT_LON_OROG(RLA,RLO,OROG,OROG_UF,TILE_NUM,IDIM,JDIM,LENSFC,LANDFRAC=LANDFRAC) ELSE CALL READ_LAT_LON_OROG(RLA,RLO,OROG,OROG_UF,TILE_NUM,IDIM,JDIM,LENSFC) @@ -502,21 +499,6 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & ABSFCS=ABSFCS,T2M=T2M ,Q2M=Q2M ,SLMASK=SLMASK, & ZSOIL=ZSOIL, NSST=NSST) -!STCFCS_SAVE = STCFCS - -!if (tile_num == 'tile6') then -! print*,'remove ice' -! slifcs(434252) = 0. -! sicfcs(434252) = 0. -! sihfcs(434252) = 0. -! print*,'add ice' -! slifcs(491001) = 2. -! sicfcs(491001) = .9 -! sihfcs(491001) = 1. -!endif - - print*,'is noahmp/fract grid ',is_noahmp, frac_grid - IF (FRAC_GRID .AND. .NOT. IS_NOAHMP) THEN print *, 'FATAL ERROR: NOAH lsm update does not work with fractional grids.' call MPI_ABORT(MPI_COMM_WORLD, 18, IERR) @@ -532,21 +514,6 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & call MPI_ABORT(MPI_COMM_WORLD, 29, IERR) ENDIF -!ichk = 282 -!jchk = 362 -!ijchk = (jchk-1) * idim + ichk -!print*,'chk point after read_data ',tile_num,landfrac(ijchk),slmask(ijchk),vetfcs(ijchk),vegfcs(ijchk) - -!print*,'vegfcs check ',maxval(vegfcs),minval(vegfcs) -!do i=1,lensfc -! if(landfrac(i) > 0.0_kind_io8 .and. vegfcs(i) == 0.0) then -! print*,'bad point 1 ',tile_num,i,landfrac(i),vegfcs(i) -! endif -! if(landfrac(i) == 0.0_kind_io8 .and. vegfcs(i) > 0.0) then -! print*,'bad point 2 ',tile_num,i,landfrac(i),vegfcs(i) -! endif -!enddo - IF (IS_NOAHMP) THEN LSM=LSM_NOAHMP ELSE @@ -567,14 +534,6 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & IF (DO_NSST) THEN SICFCS_FG=SICFCS - do i=1,lensfc - if (slifcs(i) == 2.0 .and. sicfcs(i) < .01) then - print*,'bad ice 1 ',i,slifcs(i),sicfcs(i) - endif - if (slifcs(i) == 0.0 .and. sicfcs(i) > 0.) then - print*,'bad ice 2 ',i,slifcs(i),sicfcs(i) - endif - enddo IF (.NOT. DO_SFCCYCLE ) THEN PRINT* PRINT*,"FIRST GUESS MASK ADJUSTED BY IFD RECORD" @@ -620,13 +579,6 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & ENDIF ENDIF - if(nint(slmaskl(i)) == 1 .and. vegfcs(i) == 0.0) then - print*,'bad point 3 ',tile_num,i,landfrac(i),slmaskl(i),vegfcs(i) - endif - if(nint(slmaskl(i)) == 0 .and. vegfcs(i) > 0.0) then - print*,'bad point 4 ',tile_num,i,landfrac(i),slmaskw(i),vegfcs(i) - endif - ENDDO ELSE @@ -669,61 +621,9 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & min_ice, & IALB,ISOT,IVEGSRC,TILE_NUM,I_INDEX,J_INDEX) -! print*,'after call to sfccycle ' - -! print*,'chk point after sfccycle ',tile_num,landfrac(ijchk),slmask(ijchk),vetfcs(ijchk),vegfcs(ijchk) - - goto 20 - - DO I=1,LENSFC - if(nint(slmaskl(i)) == 1 .and. vegfcs(i) == 0.0) then - if (nint(vetfcs(i)) /= 15) then - print*,'bad point 5 ',tile_num,i,landfrac(i),slmaskl(i),vegfcs(i) - endif - endif -! print*,'in loop after first if' - if(nint(slmaskl(i)) == 0 .and. vegfcs(i) > 0.0) then - print*,'bad point 6 ',tile_num,i,landfrac(i),slmaskw(i),vegfcs(i) - endif - if(landfrac(i) > 0.0_kind_io8 .and. vegfcs(i) == 0.0) then - if (nint(vetfcs(i)) /= 15) then - print*,'bad point 7 ',tile_num,i,landfrac(i),vegfcs(i) - endif - endif - if(landfrac(i) == 0.0_kind_io8 .and. vegfcs(i) > 0.0) then - print*,'bad point 8 ',tile_num,i,landfrac(i),vegfcs(i) - endif - if(nint(slmaskw(i)) == 1 .and. sicfcs(i) > 0.0) then - print*,'bad ice point ',tile_num,i,landfrac(i),sicfcs(i) - endif - -! if(landfrac(i) > 0.0_kind_io8 .and. abs(stcfcs(i,1)-stcfcs_save(i,1)) > 0.001) then -! print*,'bad stc 1 point ',tile_num,i,stcfcs(i,1),stcfcs_save(i,1) -! endif -! if(landfrac(i) > 0.0_kind_io8 .and. abs(stcfcs(i,2)-stcfcs_save(i,2)) > 0.001) then -! print*,'bad stc 2 point ',tile_num,i,stcfcs(i,2),stcfcs_save(i,2) -! endif -! if(landfrac(i) > 0.0_kind_io8 .and. abs(stcfcs(i,3)-stcfcs_save(i,3)) > 0.001) then -! print*,'bad stc 3 point ',tile_num,i,stcfcs(i,3),stcfcs_save(i,3) -! endif -! if(landfrac(i) > 0.0_kind_io8 .and. abs(stcfcs(i,4)-stcfcs_save(i,4)) > 0.001) then -! print*,'bad stc 4 point ',tile_num,i,stcfcs(i,4),stcfcs_save(i,4) -! endif - - ENDDO - - 20 continue - DEALLOCATE(SLMASKL, SLMASKW) ENDIF -!IF(FRAC_GRID) DEALLOCATE(LANDFRAC) - -!if (tile_num == 'tile6') then -! print*,'remove ice after cycle ', slifcs(434252),sicfcs(434252),sihfcs(434252) -! print*,'add ice after cycle ', slifcs(491001),sicfcs(491001),sihfcs(491001) -!endif - !-------------------------------------------------------------------------------- ! IF RUNNING WITH NSST, READ IN GSI FILE WITH THE UPDATED INCREMENTS (ON THE ! GAUSSIAN GRID), INTERPOLATE INCREMENTS TO THE CUBED-SPHERE TILE, AND PERFORM From cf37243f0f47e77dec8c390c98803c8805274e9d Mon Sep 17 00:00:00 2001 From: George Gayno Date: Tue, 16 Jan 2024 21:34:37 +0000 Subject: [PATCH 32/38] Remove more diagnostic print. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index bb57c8e60..4b314db58 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -630,21 +630,11 @@ SUBROUTINE SFCDRV(LUGB, IDIM,JDIM,LENSFC,LSOIL,DELTSFC, & ! REQUIRED ADJUSTMENTS AND QC. !-------------------------------------------------------------------------------- - if (do_nsst) then - print*,'ice check 2', tile_num - do i = 1, lensfc - if (sicfcs(i) /= sicfcs_fg(i)) then - print*,' ice update ',i,sicfcs(i),sicfcs_fg(i) - endif - enddo - endif - IF (DO_NSST) THEN IF (NST_FILE == "NULL") THEN PRINT* PRINT*,"NO GSI FILE. ADJUST IFD FOR FORMER ICE POINTS." DO I = 1, LENSFC -! IF (NINT(SLIFCS_FG(I)) == 2 .AND. NINT(SLIFCS(I)) == 0) THEN IF (SICFCS_FG(I) > 0.0 .AND. SICFCS(I) == 0) THEN NSST%IFD(I) = 3.0 ENDIF @@ -1049,16 +1039,6 @@ SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,& IJ_LOOP : DO IJ = 1, LENSFC -! MASK_TILE = NINT(SLMSK_TILE(IJ)) -! MASK_FG_TILE = NINT(SLMSK_FG_TILE(IJ)) - - if (sice_tile(ij) > 0. .and. mask_tile(ij) /= 2) then - print*,'bad ice point ',ij,sice_tile(ij),mask_tile(ij) - endif - - if (sice_fg_tile(ij) > 0. .and. mask_fg_tile(ij) /= 2) then - print*,'bad fg ice point ',ij,sice_fg_tile(ij),mask_fg_tile(ij) - endif ! ! when sea ice exists, get salinity dependent water temperature ! @@ -1757,7 +1737,6 @@ subroutine get_tf_clm_ta(tf_clm_ta,tf_clm_trend,xlats,xlons,nlat,nlon,mon1,mon2, ! ! read in rtg sst climatology without bitmap (surface mask) for mon1 and mon2 ! - print*,'before read_tf_clim_grb',trim(fin_tf_clm) call read_tf_clim_grb(trim(fin_tf_clm),tf_clm1,xlats,xlons,nlat,nlon,mon1) call read_tf_clim_grb(trim(fin_tf_clm),tf_clm2,xlats,xlons,nlat,nlon,mon2) ! From fcc0b31813e366e39004a8095027e3a6fc34e2e5 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 17 Jan 2024 15:27:38 +0000 Subject: [PATCH 33/38] Create new cycle consistency test based on Cathy's c48 noahmp fractional grid workflow test. Fixes #549. --- reg_tests/global_cycle/C48.noahmp.fracgrid.sh | 92 +++++++++++++++++++ reg_tests/global_cycle/driver.hera.sh | 8 +- 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100755 reg_tests/global_cycle/C48.noahmp.fracgrid.sh diff --git a/reg_tests/global_cycle/C48.noahmp.fracgrid.sh b/reg_tests/global_cycle/C48.noahmp.fracgrid.sh new file mode 100755 index 000000000..ec6d03d37 --- /dev/null +++ b/reg_tests/global_cycle/C48.noahmp.fracgrid.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +#------------------------------------------------------------------ +# Run global_cycle for a C48 case that tests the NOAHMP and +# fractional grid options. +# +# Compare output to a baseline set of files using the 'nccmp' +# utility. +#------------------------------------------------------------------ + +set -x + +NCCMP=${NCCMP:-$(which nccmp)} + +export MAX_TASKS_CY=6 + +export HOMEgfs=$NWPROD +export BASE_GSM=$NWPROD + +export CYCLEXEC=$BASE_GSM/exec/global_cycle + +export CDATE=2021032406 +export FHOUR=00 +export DELTSFC=6 + +export CASE=C48 +export OCNRES=500 + +export COMIN=$HOMEreg/input_data_c48.noahmp.frac.grid +export FNACNA=$COMIN/gdas.t06z.seaice.5min.blend.grb +export FNTSFA=" " +export FNSNOA=" " +export NST_FILE=$COMIN/gdas.t06z.dtfanl.nc + +export JCAP=1534 +export LONB=3072 +export LATB=1536 + +export OROFIX=$HOMEreg/fix/$CASE + +export FIXgsm=$BASE_GSM/fix/am + +export FNAISC=$FIXgsm/IMS-NIC.blended.ice.monthly.clim.grb + +export DONST="YES" +export use_ufo=.true. +export FRAC_GRID=.true. + +export VERBOSE=YES +export CYCLVARS=FSNOL=99999.,FSNOS=99999., + +$BASE_GSM/ush/global_cycle_driver.sh + +iret=$? +if [ $iret -ne 0 ]; then + set +x + echo "<<< C48 NOAHMP FRAC GRID TEST FAILED. >>>" + exit $iret +fi + +test_failed=0 + +cd $DATA +for files in *tile*.nc +do + if [ -f $files ]; then + echo CHECK $files + $NCCMP -dmfqS $files $HOMEreg/baseline_data/c48.noahmp.fracgrid/$files + iret=$? + if [ $iret -ne 0 ]; then + test_failed=1 + fi + fi +done + +set +x +if [ $test_failed -ne 0 ]; then + echo + echo "******************************************" + echo "<<< C48 NOAHMP FRAC GRID TEST FAILED. >>>" + echo "******************************************" + if [ "$UPDATE_BASELINE" = "TRUE" ]; then + $BASE_GSM/reg_tests/update_baseline.sh $HOMEreg "c48.noahmp.fracgrid" $commit_num + fi +else + echo + echo "*****************************************" + echo "<<< C48 NOAHMP FRAC GRID TEST PASSED. >>>" + echo "*****************************************" +fi + +exit diff --git a/reg_tests/global_cycle/driver.hera.sh b/reg_tests/global_cycle/driver.hera.sh index 9ea901726..db23cc6f1 100755 --- a/reg_tests/global_cycle/driver.hera.sh +++ b/reg_tests/global_cycle/driver.hera.sh @@ -73,10 +73,16 @@ export COMOUT=$DATA TEST3=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c768.lndincsnow \ -o $LOG_FILE -e $LOG_FILE ./C768.lndincsnow.sh) +LOG_FILE=consistency.log04 +export DATA="${DATA_DIR}/test4" +export COMOUT=$DATA +TEST4=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c48.noahmp.frac \ + -o $LOG_FILE -e $LOG_FILE ./C48.noahmp.fracgrid.sh) + LOG_FILE=consistency.log sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J chgres_summary -o $LOG_FILE -e $LOG_FILE \ --open-mode=append -q $QUEUE -d\ - afterok:$TEST1:$TEST2:$TEST3 << EOF + afterok:$TEST1:$TEST2:$TEST3:$TEST4 << EOF #!/bin/bash grep -a '<<<' ${LOG_FILE}* > summary.log EOF From 3685f24a3c33bd33281b7a0487dca459d312f1f6 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 17 Jan 2024 16:06:23 +0000 Subject: [PATCH 34/38] Update Jet regression test driver script for new c48 test. Fixes #549. --- reg_tests/global_cycle/driver.jet.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/reg_tests/global_cycle/driver.jet.sh b/reg_tests/global_cycle/driver.jet.sh index 75fc6b343..d680dc023 100755 --- a/reg_tests/global_cycle/driver.jet.sh +++ b/reg_tests/global_cycle/driver.jet.sh @@ -71,10 +71,16 @@ export COMOUT=$DATA TEST3=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c768.lndincsnow \ --partition=xjet -o $LOG_FILE -e $LOG_FILE ./C768.lndincsnow.sh) +LOG_FILE=consistency.log04 +export DATA="${DATA_DIR}/test4" +export COMOUT=$DATA +TEST4=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c48.noahmp.frac \ + --partition=xjet -o $LOG_FILE -e $LOG_FILE ./C48.noahmp.fracgrid.sh) + LOG_FILE=consistency.log sbatch --partition=xjet --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J summary -o $LOG_FILE -e $LOG_FILE \ --open-mode=append -q $QUEUE -d\ - afterok:$TEST1:$TEST2:$TEST3 << EOF + afterok:$TEST1:$TEST2:$TEST3:$TEST4 << EOF #!/bin/bash grep -a '<<<' ${LOG_FILE}* > ./summary.log EOF From d63dac532643bf60eb53911d12cf1a7bbe5d559f Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 17 Jan 2024 10:45:10 -0600 Subject: [PATCH 35/38] Update Orion driver script for new c48 regression test. Fixes #549 --- reg_tests/global_cycle/driver.orion.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/reg_tests/global_cycle/driver.orion.sh b/reg_tests/global_cycle/driver.orion.sh index 6fbd15fd2..d3f4f6415 100755 --- a/reg_tests/global_cycle/driver.orion.sh +++ b/reg_tests/global_cycle/driver.orion.sh @@ -71,10 +71,16 @@ export COMOUT=$DATA TEST3=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c768.lndincsnow \ -o $LOG_FILE -e $LOG_FILE ./C768.lndincsnow.sh) +LOG_FILE=consistency.log04 +export DATA="${DATA_DIR}/test4" +export COMOUT=$DATA +TEST4=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c48.noahmp.frac \ + -o $LOG_FILE -e $LOG_FILE ./C48.noahmp.fracgrid.sh) + LOG_FILE=consistency.log sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J chgres_summary -o $LOG_FILE -e $LOG_FILE \ --open-mode=append -q $QUEUE -d\ - afterok:$TEST1:$TEST2:$TEST3 << EOF + afterok:$TEST1:$TEST2:$TEST3:$TEST4 << EOF #!/bin/bash grep -a '<<<' ${LOG_FILE}* > summary.log EOF From 066feef92c14b5e8163ce8fdfe3112100a1f17c2 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Wed, 17 Jan 2024 13:32:45 -0600 Subject: [PATCH 36/38] Update hercules and wcoss2 test driver scripts for new c48 test. Fixes #549. --- reg_tests/global_cycle/driver.hercules.sh | 10 ++++++++-- reg_tests/global_cycle/driver.wcoss2.sh | 7 ++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/reg_tests/global_cycle/driver.hercules.sh b/reg_tests/global_cycle/driver.hercules.sh index d8cb1048d..b522ad1bc 100755 --- a/reg_tests/global_cycle/driver.hercules.sh +++ b/reg_tests/global_cycle/driver.hercules.sh @@ -27,7 +27,7 @@ module list ulimit -s unlimited -export WORK_DIR="${WORK_DIR:-/work/noaa/stmp/$LOGNAME}" +export WORK_DIR="${WORK_DIR:-/work2/noaa/stmp/$LOGNAME}" PROJECT_CODE="${PROJECT_CODE:-fv3-cpu}" QUEUE="${QUEUE:-batch}" @@ -73,10 +73,16 @@ export COMOUT=$DATA TEST3=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c768.lndincsnow \ -o $LOG_FILE -e $LOG_FILE ./C768.lndincsnow.sh) +LOG_FILE=consistency.log04 +export DATA="${DATA_DIR}/test4" +export COMOUT=$DATA +TEST4=$(sbatch --parsable --ntasks-per-node=6 --nodes=1 -t 0:05:00 -A $PROJECT_CODE -q $QUEUE -J c48.noahmp.frac \ + -o $LOG_FILE -e $LOG_FILE ./C48.noahmp.fracgrid.sh) + LOG_FILE=consistency.log sbatch --nodes=1 -t 0:01:00 -A $PROJECT_CODE -J chgres_summary -o $LOG_FILE -e $LOG_FILE \ --open-mode=append -q $QUEUE -d\ - afterok:$TEST1:$TEST2:$TEST3 << EOF + afterok:$TEST1:$TEST2:$TEST3:$TEST4 << EOF #!/bin/bash grep -a '<<<' ${LOG_FILE}* > summary.log EOF diff --git a/reg_tests/global_cycle/driver.wcoss2.sh b/reg_tests/global_cycle/driver.wcoss2.sh index 1facda522..077896cd3 100755 --- a/reg_tests/global_cycle/driver.wcoss2.sh +++ b/reg_tests/global_cycle/driver.wcoss2.sh @@ -76,8 +76,13 @@ export COMOUT=$DATA TEST3=$(qsub -V -o ${LOG_FILE}03 -e ${LOG_FILE}03 -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ -N c768.lndincsnow -l select=1:ncpus=12:mem=8GB $PWD/C768.lndincsnow.sh) +export DATA="${DATA_DIR}/test4" +export COMOUT=$DATA +TEST4=$(qsub -V -o ${LOG_FILE}04 -e ${LOG_FILE}04 -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \ + -N c48.noahmp.frac -l select=1:ncpus=12:mem=8GB $PWD/C48.noahmp.fracgrid.sh) + qsub -V -o ${LOG_FILE} -e ${LOG_FILE} -q $QUEUE -A $PROJECT_CODE -l walltime=00:01:00 \ - -N cycle_summary -l select=1:ncpus=1:mem=100MB -W depend=afterok:$TEST1:$TEST2:$TEST3 << EOF + -N cycle_summary -l select=1:ncpus=1:mem=100MB -W depend=afterok:$TEST1:$TEST2:$TEST3:$TEST4 << EOF #!/bin/bash cd $reg_dir grep -a '<<<' ${LOG_FILE}?? | grep -v echo > summary.log From 29489412b8933bc70b3025aede3ca66dcd023e19 Mon Sep 17 00:00:00 2001 From: George Gayno Date: Thu, 18 Jan 2024 08:22:03 -0600 Subject: [PATCH 37/38] Fix some diagnostic prin in C192.lndincsoilnoahmp.sh. Fixes #549. --- reg_tests/global_cycle/C192.lndincsoilnoahmp.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/reg_tests/global_cycle/C192.lndincsoilnoahmp.sh b/reg_tests/global_cycle/C192.lndincsoilnoahmp.sh index 9e676ef8b..b5780eb64 100755 --- a/reg_tests/global_cycle/C192.lndincsoilnoahmp.sh +++ b/reg_tests/global_cycle/C192.lndincsoilnoahmp.sh @@ -73,9 +73,9 @@ done set +x if [ $test_failed -ne 0 ]; then echo - echo "*****************************************" - echo "<<< C192 LANDINC SOIL-NOAHP CYCLE TEST FAILED. >>>" - echo "*****************************************" + echo "**********************************************" + echo "<<< C192 LANDINC SOIL-NOAHMP CYCLE TEST FAILED. >>>" + echo "**********************************************" if [ "$UPDATE_BASELINE" = "TRUE" ]; then $BASE_GSM/reg_tests/update_baseline.sh $HOMEreg "c192.lndincsoilnoahmp" $commit_num fi From 8f1d5601ee38ec9414c5626c380338ff430ddfad Mon Sep 17 00:00:00 2001 From: George Gayno Date: Fri, 19 Jan 2024 19:34:07 +0000 Subject: [PATCH 38/38] Update doxygen. Fixes #549. --- sorc/global_cycle.fd/cycle.f90 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sorc/global_cycle.fd/cycle.f90 b/sorc/global_cycle.fd/cycle.f90 index dee0d3bd9..ab983bc04 100644 --- a/sorc/global_cycle.fd/cycle.f90 +++ b/sorc/global_cycle.fd/cycle.f90 @@ -845,6 +845,7 @@ END SUBROUTINE SFCDRV !! @param[inout] SKINT_TILE Skin temperature on the cubed-sphere tile !! @param[inout] SICET_TILE Ice temperature on the cubed-sphere tile !! @param[in] sice_tile Ice concentration on the cubed-sphere tile + !! @param[in] sice_fg_tile First guess ice concentration on the cubed-sphere tile !! @param[inout] SOILT_TILE Soil temperature on the cubed-sphere tile !! @param[in] NSST Data structure holding nsst fields !! @param[in] LENSFC Number of points on a tile @@ -860,6 +861,8 @@ END SUBROUTINE SFCDRV !! @param[in] tf_trd_tile Climatolocial reference temperature trend on the !! cubed-sphere tile. !! @param[in] sal_clm_tile Climatological salinity on the cubed-sphere tile. + !! @param[in] LANDFRAC Land fraction + !! @param[in] FRAC_GRID Process fractional grid when true. !! !! @author Xu Li, George Gayno SUBROUTINE ADJUST_NSST(RLA,RLO,SLMSK_TILE,SLMSK_FG_TILE,SKINT_TILE,&