From b7da671df64636fc38e2a33ade6eefbe9f60ee18 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Tue, 8 Jun 2021 16:12:30 +0000 Subject: [PATCH 01/39] Checkout and build ufs-weather-model app integrating prognostic aerosols into the UFS, replacing original FV3-GOCART system. A forked repository is temporarily used to download the updated ufs-weather-model app. --- sorc/build_ufs_coupled.sh | 7 ++++--- sorc/checkout.sh | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/sorc/build_ufs_coupled.sh b/sorc/build_ufs_coupled.sh index e18e6fd3e88..0d6be6a0e19 100755 --- a/sorc/build_ufs_coupled.sh +++ b/sorc/build_ufs_coupled.sh @@ -13,15 +13,16 @@ if [ $target = hera ]; then target=hera.intel ; fi if [ $target = orion ]; then target=orion.intel ; fi if [ $target = stampede ]; then target=stampede.intel ; fi -MOD_PATH=$cwd/ufs_coupled.fd/modulefiles/$target +MOD_PATH=$cwd/ufs_coupled.fd/modulefiles module purge module use $MOD_PATH -module load fv3 +module load ufs_${target} cd ufs_coupled.fd/ if [[ -d build ]]; then rm -Rf build; fi if [[ -d GOCART ]]; then - ./build.sh + module load ufs_aerosols_${target} + CMAKE_FLAGS="-DAPP=ATMAERO" CCPP_SUITES="FV3_GFS_v16" ./build.sh else CMAKE_FLAGS="-DS2S=ON -DWW3=ON" CCPP_SUITES="FV3_GFS_v15p2_coupled,FV3_GFS_v16_coupled,FV3_GFS_v16" ./build.sh fi diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 796ccd79121..fc35a12e375 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -46,9 +46,9 @@ if [ ${COUPLED:-"NO"} = "NO" ]; then else if [[ ! -d ufs_coupled.fd ]] ; then if [ "${AEROSOL}" = "YES" ] ; then - git clone https://github.com/NOAA-EMC/FV3-GOCART ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1 + git clone https://github.com/rmontuoro/ufs-weather-model.git ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1 cd ufs_coupled.fd - git checkout develop + git checkout feature/gocart else git clone https://github.com/ufs-community/ufs-weather-model ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1 cd ufs_coupled.fd From 5c8e0781b058f600c7c8826566ff3e67ee577310 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Tue, 8 Jun 2021 16:28:55 +0000 Subject: [PATCH 02/39] Use proper modulefiles for updated ufs-weather-model app with prognostic aerosols. --- ush/load_fv3gfs_modules.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index 2cb57f0a04f..19c7a2664ec 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -24,10 +24,10 @@ elif [[ -d /lfs3 ]] ; then module load module_base.jet elif [[ -d /scratch1 ]] ; then # We are on NOAA Hera - module load module_base.hera + module use "$HOMEgfs/sorc/ufs_coupled.fd/modulefiles" + module load ufs_hera.intel if [[ -d $HOMEgfs/sorc/ufs_coupled.fd/GOCART ]] ; then - module use "$HOMEgfs/sorc/ufs_coupled.fd/modulefiles/hera.intel" - module load fv3 + module load ufs_aerosols_hera.intel fi elif [[ -d /work ]] ; then # We are on MSU Orion From e2f52cce042cd08b7b76cf23ecda231ce2d41303 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Tue, 8 Jun 2021 16:30:26 +0000 Subject: [PATCH 03/39] Update parsing function for FV3 namelist to work with latest FV3 revision and include settings for FV3_GFS_v16 and aerosol scavenging factors. --- ush/parsing_namelists_FV3.sh | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/ush/parsing_namelists_FV3.sh b/ush/parsing_namelists_FV3.sh index ddcca3a2412..0053478275d 100755 --- a/ush/parsing_namelists_FV3.sh +++ b/ush/parsing_namelists_FV3.sh @@ -191,13 +191,14 @@ cat > input.nml <> input.nml << EOF oz_phys = .false. oz_phys_2015 = .true. EOF -elif [ $CCPP_SUITE = "FV3_GSD_v0" ]; then + ;; + "FV3_GSD_v0") cat >> input.nml << EOF iovr = ${iovr:-"3"} ltaerosol = ${ltaerosol:-".F."} ! In config.fcst @@ -215,7 +216,8 @@ elif [ $CCPP_SUITE = "FV3_GSD_v0" ]; then min_lakeice = ${min_lakeice:-"0.15"} min_seaice = ${min_seaice:-"0.15"} EOF -elif [ $CCPP_SUITE = "FV3_GFS_v16_coupled" ]; then + ;; + "FV3_GFS_v16_coupled") cat >> input.nml << EOF iovr = ${iovr:-"3"} ltaerosol = ${ltaerosol:-".false."} @@ -233,12 +235,25 @@ elif [ $CCPP_SUITE = "FV3_GFS_v16_coupled" ]; then min_lakeice = ${min_lakeice:-"0.15"} min_seaice = ${min_seaice:-"0.15"} EOF -else + ;; + "FV3_GFS_v16") + cat >> input.nml << EOF + iovr = ${iovr:-"3"} + ltaerosol = ${ltaerosol:-".false."} + oz_phys = ${oz_phys:-".false."} + oz_phys_2015 = ${oz_phys_2015:-".true."} + lsoil = ${lsoil:-"4"} + icloud_bl = ${icloud_bl:-"1"} + min_lakeice = ${min_lakeice:-"0.15"} + min_seaice = ${min_seaice:-"0.15"} +EOF + ;; + *) cat >> input.nml << EOF - iovr_lw = ${iovr_lw:-"3"} - iovr_sw = ${iovr_sw:-"3"} + iovr = ${iovr:-"3"} EOF -fi + ;; +esac cat >> input.nml <> input.nml <> input.nml << EOF + cplchm = ${cplchem:-".false."} cplflx = $cplflx cplwav2atm = ${cplwav2atm} ! CROW configured EOF From ff971a2974e9e08723c875c18879fef11c8ad674 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Tue, 8 Jun 2021 16:34:48 +0000 Subject: [PATCH 04/39] Add aerosol scavenging factors (fscav_aero) to FV3 settings. This item is implemented as a YAML list of strings, which allows to set scavenging factors for each tracer. --- workflow/defaults/fv3_gfs.yaml | 1 + workflow/schema/fv3.yaml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/workflow/defaults/fv3_gfs.yaml b/workflow/defaults/fv3_gfs.yaml index 8b7a7af7f4b..53dd7d436df 100644 --- a/workflow/defaults/fv3_gfs.yaml +++ b/workflow/defaults/fv3_gfs.yaml @@ -82,6 +82,7 @@ fv3_gfs_defaults: &fv3_gfs_defaults # WRTTASK_PER_GROUP: layout.WGRP_NTASKS # WRTIOBUF: layout.WRTIOBUF + fscav_aero: [ '*:0.0' ] new_o3force: yes h2o_phys: yes do_vort_damp: yes diff --git a/workflow/schema/fv3.yaml b/workflow/schema/fv3.yaml index 61aaba6eb3f..a8bce77d52f 100644 --- a/workflow/schema/fv3.yaml +++ b/workflow/schema/fv3.yaml @@ -32,6 +32,10 @@ fv3_settings_template: !Template &fv3_settings_template agrid_vel_rst: type: bool description: "write velocity restarts on A grid?" + fscav_aero: + type: string list + description: aerosol convective scavenging coefficients + optional: true # cal_pre: { type: bool } # do_sat_adjust: { type: bool } From d16630f3b34671e2350e6a88b9df749e4fd58ece Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Tue, 8 Jun 2021 16:39:55 +0000 Subject: [PATCH 05/39] Properly set NST_SPINUP flag in FV3 namelist. --- workflow/config/nsst.yaml | 2 +- workflow/defaults/nsst.yaml | 2 +- workflow/schema/nsst.yaml | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/workflow/config/nsst.yaml b/workflow/config/nsst.yaml index d37114b33d8..52a8d6a2d8a 100644 --- a/workflow/config/nsst.yaml +++ b/workflow/config/nsst.yaml @@ -6,7 +6,7 @@ config_nsst: filename: config.nsst nst_spinup_logic: !FirstTrue - when: !calc doc.nsst.get("NST_SPINUP","") - do: !expand "export NST_SPINUP=$NST_SPINUP" + do: !expand "export NST_SPINUP={doc.nsst.NST_SPINUP}" - otherwise: | export NST_SPINUP=0 #if [[ "$CDATE" = $SDATE ]]; then diff --git a/workflow/defaults/nsst.yaml b/workflow/defaults/nsst.yaml index 1ac69093986..f426364e9dc 100644 --- a/workflow/defaults/nsst.yaml +++ b/workflow/defaults/nsst.yaml @@ -1,8 +1,8 @@ nsst_defaults: &nsst_defaults NST_MODEL: 2 + NST_SPINUP: 0 NST_RESV: 0 ZSEA1: 0 ZSEA2: 0 NST_GSI: 3 NSTINFO: 0 - NST_SPINUP: 0 diff --git a/workflow/schema/nsst.yaml b/workflow/schema/nsst.yaml index 1efcc8eb05a..7ffd0b319bc 100644 --- a/workflow/schema/nsst.yaml +++ b/workflow/schema/nsst.yaml @@ -5,6 +5,10 @@ nsst_schema: &nsst_schema !Template description: >- nstf_name(1) : NST_MODEL (NSST Model) : 0 = OFF, 1 = ON but uncoupled, 2 = ON and coupled + NST_SPINUP: + type: int + allowed: [ 0, 1] + optional: yes NST_RESV: type: int allowed: [ 0, 1 ] @@ -24,7 +28,3 @@ nsst_schema: &nsst_schema !Template NSTINFO: type: int description: number of elements added in obs. data array - NST_SPINUP: - type: int - allowed: [ 0, 1] - optional: yes From c26a0dd7256ce7993e15a3c19bf77049309ef7c6 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Tue, 8 Jun 2021 16:44:03 +0000 Subject: [PATCH 06/39] Restore `cplchem` as the flag for coupled aerosols, removing all references to `cplgocart`. --- scripts/exglobal_forecast.sh | 8 +------- ush/cplvalidate.sh | 2 +- ush/forecast_postdet.sh | 8 ++------ workflow/config/base.yaml | 4 ++-- workflow/config/fcst.yaml | 6 ++++-- workflow/defaults/chem.yaml | 6 +++--- workflow/defaults/settings.yaml | 3 --- workflow/schema/chem.yaml | 2 +- 8 files changed, 14 insertions(+), 25 deletions(-) diff --git a/scripts/exglobal_forecast.sh b/scripts/exglobal_forecast.sh index 9efed80ef46..fdfebc350a6 100755 --- a/scripts/exglobal_forecast.sh +++ b/scripts/exglobal_forecast.sh @@ -102,7 +102,6 @@ cpl=${cpl:-.false.} cplflx=${cplflx:-.false.} # default off,import from outside source cplwav=${cplwav:-.false.} # ? how to control 1-way/2-way? cplchem=${cplchem:-.false.} # Chemistry model -cplgocart=${cplgocart:-.false.} # Chemistry model cplice=${cplice:-.false.} # ICE model OCNTIM=${OCNTIM:-3600} @@ -134,7 +133,6 @@ esac [[ $cplflx = .true. ]] && MOM6_predet #[[ $cplwav = .true. ]] && WW3_predet #no WW3_predet at this time [[ $cplice = .true. ]] && CICE_predet -[[ $cplchem = .true. ]] && GSD_predet case $RUN in 'gfs') FV3_GFS_det;; @@ -144,7 +142,6 @@ esac #no run type determination for data atmosphere [[ $cplflx = .true. ]] && MOM6_det [[ $cplwav = .true. ]] && WW3_det [[ $cplice = .true. ]] && CICE_det -[[ $cplchem = .true. ]] && GSD_det echo "MAIN: RUN Type Determined" @@ -159,7 +156,6 @@ esac #no post determination set up for data atmosphere [[ $cplflx = .true. ]] && MOM6_postdet [[ $cplwav = .true. ]] && WW3_postdet [[ $cplice = .true. ]] && CICE_postdet -[[ $cplchem = .true. ]] && GSD_postdet echo "MAIN: Post-determination set up of run type finished" echo "MAIN: Writing name lists and model configuration" @@ -172,8 +168,7 @@ esac #no namelist for data atmosphere [[ $cplflx = .true. ]] && MOM6_nml [[ $cplwav = .true. ]] && WW3_nml [[ $cplice = .true. ]] && CICE_nml -[[ $cplchem = .true. ]] && GSD_nml -[[ $cplgocart = .true. ]] && GOCART_rc +[[ $cplchem = .true. ]] && GOCART_rc case $RUN in 'data') DATM_model_configure;; @@ -216,7 +211,6 @@ if [ $machine != 'sandbox' ]; then [[ $cplflx = .true. ]] && MOM6_out [[ $cplwav = .true. ]] && WW3_out [[ $cplice = .true. ]] && CICE_out - [[ $cplchem = .true. ]] && GSD_out [[ $esmf_profile = .true. ]] && CPL_out else echo "MAIN: Running on sandbox mode, no output linking" diff --git a/ush/cplvalidate.sh b/ush/cplvalidate.sh index ec481cd8f23..46f861dbe12 100755 --- a/ush/cplvalidate.sh +++ b/ush/cplvalidate.sh @@ -29,7 +29,7 @@ case $confignamevarfornems in *) echo "SUB cplvalidate: Combination not supported" exit 1 ;; esac -control=$cpl$cplflx$cplice$cplwav$cplgocart +control=$cpl$cplflx$cplice$cplwav$cplchem #echo $control if [ $control != $combination ]; then echo "SUB cplvalidate: inconsistent cpl setting!" diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 7bdde50324f..4acbd1c2fba 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -498,11 +498,7 @@ FV3_GFS_nml(){ echo "MAIN: !!!Sandbox mode, writing to current directory!!!" fi # Call child scripts in current script directory - if [ $cplgocart = .true. ]; then - source $SCRIPTDIR/parsing_namelists_FV3_GOCART.sh - else - source $SCRIPTDIR/parsing_namelists_FV3.sh - fi + source $SCRIPTDIR/parsing_namelists_FV3.sh FV3_namelists echo SUB ${FUNCNAME[0]}: FV3 name lists and model configure file created } @@ -868,7 +864,7 @@ GSD_in() { GSD_nml() { echo "SUB ${FUNCNAME[0]}: Creating name list for GSD" - sh parsing_namelists_GSD.sh + # generate GSD-specific input } GSD_out() { diff --git a/workflow/config/base.yaml b/workflow/config/base.yaml index 92c24578579..24be461e5f5 100644 --- a/workflow/config/base.yaml +++ b/workflow/config/base.yaml @@ -26,13 +26,13 @@ config_base: # Coupling settings export cpl="{doc.settings.cpl}" export cplflx="{doc.settings.cplflx}" - export cplgocart="{doc.settings.cplgocart}" + export cplchem="{doc.settings.cplchem}" export cplice="{doc.settings.cplice}" export cplwav="{doc.settings.cplwav}" export cplwav2atm="{doc.settings.cplwav2atm}" # Temporarily override settings for aerosol coupling - if [ $cplgocart = ".true." ]; then + if [ $cplchem = ".true." ]; then export cplflx=".false." export cplice=".false." export cplocn=".false." diff --git a/workflow/config/fcst.yaml b/workflow/config/fcst.yaml index d96dcac912a..7f1175b7325 100644 --- a/workflow/config/fcst.yaml +++ b/workflow/config/fcst.yaml @@ -5,7 +5,7 @@ config_fcst: filename: config.fcst FCSTEXEC: !FirstTrue - - when: !calc ( doc.settings.cplflx==".true." or doc.settings.cplgocart==".true." ) + - when: !calc ( doc.settings.cplflx==".true." or doc.settings.cplchem==".true." ) do: "ufs_model" - otherwise: "global_fv3gfs.x" more_exports_for_microphys: !FirstTrue @@ -123,6 +123,8 @@ config_fcst: export dddmp={doc.fv3_gfs_settings.dddmp} # gfs_physics_nml section + export fscav_aero="{tools.fort(doc.fv3_gfs_settings.fscav_aero)}" + export h2o_phys="{tools.fort(doc.fv3_gfs_settings.h2o_phys)}" export ncld={doc.fv3_gfs_settings.phy_dependent_var.ncld} @@ -225,7 +227,7 @@ config_fcst: fi - if [ $cplgocart = .true. ]; then # temporary settings for aerosol coupling + if [ $cplchem = .true. ]; then # temporary settings for aerosol coupling export DIAG_TABLE="$HOMEgfs/parm/parm_fv3diag/diag_table_aer" export FIELD_TABLE="$HOMEgfs/parm/parm_fv3diag/field_table_aer" export CHM_CFGDIR="{doc.chem_settings.CFGDIR}" diff --git a/workflow/defaults/chem.yaml b/workflow/defaults/chem.yaml index 58f45cd8aab..f26ab117202 100644 --- a/workflow/defaults/chem.yaml +++ b/workflow/defaults/chem.yaml @@ -1,13 +1,13 @@ # default settings for chemistry model core chem_defaults: &chem_defaults - MOD: gsdchem + MOD: gocart ntdiag: 0 CFGDIR: !FirstTrue - - when: !calc (doc.settings.cplgocart != ".true.") + - when: !calc (doc.settings.cplchem != ".true.") do: "not_set" - otherwise: INPDIR: !FirstTrue - - when: !calc (doc.settings.cplgocart != ".true.") + - when: !calc (doc.settings.cplchem != ".true.") do: "not_set" - otherwise: diff --git a/workflow/defaults/settings.yaml b/workflow/defaults/settings.yaml index f6e52d3742c..be713b9478d 100644 --- a/workflow/defaults/settings.yaml +++ b/workflow/defaults/settings.yaml @@ -55,7 +55,6 @@ default_settings: &default_settings cplwav: .false. cplwav2atm: .false. cplchem: .false. - cplgocart: .false. cpl: !FirstTrue - when: !calc doc.settings.cplflx==".true." do: .true. @@ -65,8 +64,6 @@ default_settings: &default_settings do: .true. - when: !calc doc.settings.cplchem==".true." do: .true. - - when: !calc doc.settings.cplgocart==".true." - do: .true. - otherwise: .false. ic_source: coupled_ic # choices: coupled_ic, fv3_ic; default: coupled_ic. This option controls which diff --git a/workflow/schema/chem.yaml b/workflow/schema/chem.yaml index 64b10900a33..eb49adf9f53 100644 --- a/workflow/schema/chem.yaml +++ b/workflow/schema/chem.yaml @@ -5,7 +5,7 @@ chem_settings_template: !Template &chem_settings_template MOD: type: string - allowed: [ gsdchem ] + allowed: [ gocart ] description: "model selection for aerosols" CFGDIR: type: string From 9661d1eaade20a015b8268a4ba20b1147ba2897d Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Tue, 8 Jun 2021 16:46:23 +0000 Subject: [PATCH 07/39] - Set NEMS identifier for coupled aerosol component to 'gocart'. - Copy NUOPC field dictionary to run directory by default. --- ush/nems_configure.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/nems_configure.sh b/ush/nems_configure.sh index 8e446271458..ed7bc1d50de 100755 --- a/ush/nems_configure.sh +++ b/ush/nems_configure.sh @@ -36,7 +36,7 @@ ATM_model=${ATM_model:-'fv3'} OCN_model=${OCN_model:-'mom6'} ICE_model=${ICE_model:-'cice'} WAV_model=${WAV_model:-'ww3'} -CHM_model=${CHM_model:-'gsdchem'} +CHM_model=${CHM_model:-'gocart'} ATMPETS=${ATMPETS:-8} MEDPETS=${MEDPETS:-8} @@ -90,7 +90,7 @@ if [ $cplice = .true. ]; then sed -i -e "s;@\[MESHICE\];$MESHICE;g" tmp1 sed -i -e "s;@\[FHMAX\];$FHMAX_GFS;g" tmp1 fi -if [ $cplchem = .true. -o $cplgocart = .true. ]; then +if [ $cplchem = .true. ]; then sed -i -e "s;@\[chm_model\];$CHM_model;g" tmp1 sed -i -e "s;@\[chm_petlist_bounds\];$chm_petlist_bounds;g" tmp1 sed -i -e "s;@\[coupling_interval_fast_sec\];$CPL_FAST;g" tmp1 @@ -155,10 +155,10 @@ EOF echo "$(cat med_modelio.nml)" -cp $HOMEgfs/sorc/ufs_coupled.fd/CMEPS-interface/CMEPS/mediator/fd_nems.yaml fd_nems.yaml - fi +cp $HOMEgfs/sorc/ufs_coupled.fd/tests/parm/fd_nems.yaml fd_nems.yaml + echo "SUB ${FUNCNAME[0]}: Nems configured for $confignamevarfornems" } From 52dbd17e193944b35a7836dc89b0510652a2a789 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Tue, 8 Jun 2021 16:48:37 +0000 Subject: [PATCH 08/39] Update test cases and input files for coupled aerosols runs. --- parm/parm_fv3diag/diag_table_aer | 6 +- parm/parm_fv3diag/field_table_aer | 257 +++++++++++---------- workflow/cases/aerosol_firex_forecast.yaml | 12 +- workflow/cases/aerosol_free_forecast.yaml | 9 +- 4 files changed, 150 insertions(+), 134 deletions(-) diff --git a/parm/parm_fv3diag/diag_table_aer b/parm/parm_fv3diag/diag_table_aer index 2a7ba17e0db..f2617c1c560 100644 --- a/parm/parm_fv3diag/diag_table_aer +++ b/parm/parm_fv3diag/diag_table_aer @@ -123,7 +123,7 @@ # chemical tracers advected by FV3 ### "gfs_dyn", "so2", "so2", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "sulf", "sulf", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "so4", "so4", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "dms", "dms", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "msa", "msa", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "bc1", "bc1", "fv3_history", "all", .false., "none", 2 @@ -145,8 +145,8 @@ "gfs_dyn", "no3an1", "no3an1", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "no3an2", "no3an2", "fv3_history", "all", .false., "none", 2 "gfs_dyn", "no3an3", "no3an3", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "pp25", "pp25", "fv3_history", "all", .false., "none", 2 -"gfs_dyn", "pp10", "pp10", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "pm25", "pm25", "fv3_history", "all", .false., "none", 2 +"gfs_dyn", "pm10", "pm10", "fv3_history", "all", .false., "none", 2 "gfs_phys", "ALBDO_ave", "albdo_ave", "fv3_history2d", "all", .false., "none", 2 "gfs_phys", "cnvprcp_ave", "cprat_ave", "fv3_history2d", "all", .false., "none", 2 diff --git a/parm/parm_fv3diag/field_table_aer b/parm/parm_fv3diag/field_table_aer index 0bfa937980d..c31d4b157ad 100644 --- a/parm/parm_fv3diag/field_table_aer +++ b/parm/parm_fv3diag/field_table_aer @@ -1,158 +1,169 @@ # added by FRE: sphum must be present in atmos # specific humidity for moist runs "TRACER", "atmos_mod", "sphum" - "longname", "specific humidity" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "longname", "specific humidity" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / # prognostic cloud water mixing ratio "TRACER", "atmos_mod", "liq_wat" - "longname", "cloud water mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "longname", "cloud water mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / "TRACER", "atmos_mod", "rainwat" - "longname", "rain mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "longname", "rain mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / "TRACER", "atmos_mod", "ice_wat" - "longname", "cloud ice mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "longname", "cloud ice mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / "TRACER", "atmos_mod", "snowwat" - "longname", "snow mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "longname", "snow mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / "TRACER", "atmos_mod", "graupel" - "longname", "graupel mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / + "longname", "graupel mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / # prognostic ozone mixing ratio tracer "TRACER", "atmos_mod", "o3mr" - "longname", "ozone mixing ratio" - "units", "kg/kg" - "profile_type", "fixed", "surface_value=1.e30" / -# prognostic so2 mixing ratio tracer + "longname", "ozone mixing ratio" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic subgrid scale turbulent kinetic energy + "TRACER", "atmos_mod", "sgs_tke" + "longname", "subgrid scale turbulent kinetic energy" + "units", "m2/s2" + "profile_type", "fixed", "surface_value=1.e30" / +# prognostic chemistry tracers "TRACER", "atmos_mod", "so2" - "longname", "so2 mixing ratio" - "units", "ppm" - "profile_type", "fixed", "surface_value=5.e-6" / -# prognostic sulfate mixing ratio tracer - "TRACER", "atmos_mod", "sulf" - "longname", "sulfate mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=3.e-6" / -# prognostic DMS mixing ratio tracer + "longname", "so2 mixing ratio" + "units", "ppm" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=5.e-6" / + "TRACER", "atmos_mod", "so4" + "longname", "sulfate mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=3.e-6" / "TRACER", "atmos_mod", "dms" - "longname", "DMS mixing ratio" - "units", "ppm" - "profile_type", "fixed", "surface_value=1.e-7" / -# prognostic msa mixing ratio tracer + "longname", "DMS mixing ratio" + "units", "ppm" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e-7" / "TRACER", "atmos_mod", "msa" - "longname", "msa mixing ratio" - "units", "ppm" - "profile_type", "fixed", "surface_value=1.e-7" / -# prognostic bc1 mixing ratio tracer + "longname", "msa mixing ratio" + "units", "ppm" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e-7" / "TRACER", "atmos_mod", "bc1" - "longname", "hydrophobic black carbon mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e-7" / -# prognostic bc2 mixing ratio tracer + "longname", "hydrophobic black carbon mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e-7" / "TRACER", "atmos_mod", "bc2" - "longname", "hydrophillic black carbon mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e-7" / -# prognostic oc1 mixing ratio tracer + "longname", "hydrophillic black carbon mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e-7" / "TRACER", "atmos_mod", "oc1" - "longname", "hydrophobic organic carbon mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e-7" / -# prognostic oc2 mixing ratio tracer + "longname", "hydrophobic organic carbon mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e-7" / "TRACER", "atmos_mod", "oc2" - "longname", "hydrophillic organic carbon mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e-7" / -# prognostic dust1 mixing ratio tracer + "longname", "hydrophillic organic carbon mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e-7" / "TRACER", "atmos_mod", "dust1" - "longname", "fine dust1 mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e-7" / -# prognostic dust2 mixing ratio tracer + "longname", "fine dust1 mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e-7" / "TRACER", "atmos_mod", "dust2" - "longname", "fine dust2 mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e-7" / -# prognostic dust3 mixing ratio tracer + "longname", "fine dust2 mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e-7" / "TRACER", "atmos_mod", "dust3" - "longname", "coarse dust3 mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e-7" / -# prognostic dust4 mixing ratio tracer + "longname", "coarse dust3 mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e-7" / "TRACER", "atmos_mod", "dust4" - "longname", "coarse dust4 mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e-7" / -# prognostic dust5 mixing ratio tracer + "longname", "coarse dust4 mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e-7" / "TRACER", "atmos_mod", "dust5" - "longname", "coarse dust5 mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e-7" / -# prognostic seas1 mixing ratio tracer + "longname", "coarse dust5 mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e-7" / "TRACER", "atmos_mod", "seas1" - "longname", "seasalt1 mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e-7" / -# prognostic seas2 mixing ratio tracer + "longname", "seasalt1 mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e-7" / "TRACER", "atmos_mod", "seas2" - "longname", "seasalt2 mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e-7" / -# prognostic seas3 mixing ratio tracer + "longname", "seasalt2 mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e-7" / "TRACER", "atmos_mod", "seas3" - "longname", "seasalt3 mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e-7" / -# prognostic seas4 mixing ratio tracer + "longname", "seasalt3 mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e-7" / "TRACER", "atmos_mod", "seas4" - "longname", "seasalt4 mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e-7" / -# prognostic seas5 mixing ratio tracer + "longname", "seasalt4 mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e-7" / "TRACER", "atmos_mod", "seas5" - "longname", "seasalt5 mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e-7" / -# prognostic NI mixing ratio tracer + "longname", "seasalt5 mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e-7" / "TRACER", "atmos_mod", "nh3" - "longname", "primary NH3 mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e+0" / + "longname", "primary NH3 mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e+0" / "TRACER", "atmos_mod", "nh4a" - "longname", "primary NH4a mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e+0" / + "longname", "primary NH4a mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e+0" / "TRACER", "atmos_mod", "no3an1" - "longname", "primary NO3an1 mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e+0" / + "longname", "primary NO3an1 mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e+0" / "TRACER", "atmos_mod", "no3an2" - "longname", "primary NO3an2 mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e+0" / + "longname", "primary NO3an2 mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e+0" / "TRACER", "atmos_mod", "no3an3" - "longname", "primary NO3an3 mixing ratio" - "units", "ug/kg" - "profile_type", "fixed", "surface_value=1.e+0" / -# diagnostic pm2.5 tracer - "TRACER", "atmos_mod", "pp25" - "longname", "primary PM25 mixing ratio" - "units", "ug/m3" - "profile_type", "fixed", "surface_value=1.e+0" / -# diagnostic pm10 tracer - "TRACER", "atmos_mod", "pp10" - "longname", "primary PM10 mixing ratio" - "units", "ug/m3" - "profile_type", "fixed", "surface_value=1.e+0" / + "longname", "primary NO3an3 mixing ratio" + "units", "ug/kg" + "tracer_usage", "chemistry" + "profile_type", "fixed", "surface_value=1.e+0" / +# diagnostic chemistry tracers + "TRACER", "atmos_mod", "pm25" + "longname", "primary PM25 mixing ratio" + "units", "ug/m3" + "tracer_usage", "chemistry", "type=diagnostic" + "profile_type", "fixed", "surface_value=1.e+0" / + "TRACER", "atmos_mod", "pm10" + "longname", "primary PM10 mixing ratio" + "units", "ug/m3" + "tracer_usage", "chemistry", "type=diagnostic" + "profile_type", "fixed", "surface_value=1.e+0" / # non-prognostic cloud amount "TRACER", "atmos_mod", "cld_amt" - "longname", "cloud amount" - "units", "1" - "profile_type", "fixed", "surface_value=1.e30" / + "longname", "cloud amount" + "units", "1" + "profile_type", "fixed", "surface_value=1.e30" / diff --git a/workflow/cases/aerosol_firex_forecast.yaml b/workflow/cases/aerosol_firex_forecast.yaml index 4a881d25929..8feea0d05a4 100644 --- a/workflow/cases/aerosol_firex_forecast.yaml +++ b/workflow/cases/aerosol_firex_forecast.yaml @@ -5,8 +5,8 @@ case: places: workflow_file: layout/free_forecast_gfs.yaml BASE_CPLIC: /scratch1/NCEPDEV/nems/Raffaele.Montuoro/data - OROFIX: /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200914/INTEL/FV3_input_data_c384/INPUT - GRDFIX: /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200914/INTEL/FV3_input_data_c384/INPUT + OROFIX: /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/input-data-20210528/FV3_input_data384/INPUT + GRDFIX: /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/input-data-20210528/FV3_input_data384/INPUT settings: SDATE: 2019-07-01t00:00:00 @@ -15,7 +15,7 @@ case: gfs_cyc: 1 - cplgocart: .true. + cplchem: .true. print_esmf: .true. nems_temp: 'atm_aer' @@ -28,7 +28,8 @@ case: KEEPDATA: Yes nsst: - NST_MODEL: 0 + NST_MODEL: 2 + NST_SPINUP: 1 output_settings: FHOUT_GFS: 6 @@ -49,9 +50,10 @@ case: WGRP_NTASKS: 24 WRTIOBUF: "32M" CPL_ATMIC: FV3ICS + CCPP_SUITE: FV3_GFS_v16 chem_settings: - MOD: gsdchem + MOD: gocart ntdiag: 2 CFGDIR: /scratch1/NCEPDEV/nems/Raffaele.Montuoro/data/NASA/rc.firex INPDIR: /scratch1/NCEPDEV/nems/Raffaele.Montuoro/data/NASA/ExtData diff --git a/workflow/cases/aerosol_free_forecast.yaml b/workflow/cases/aerosol_free_forecast.yaml index 28923dd2c7a..9d4c77bed9d 100644 --- a/workflow/cases/aerosol_free_forecast.yaml +++ b/workflow/cases/aerosol_free_forecast.yaml @@ -7,7 +7,7 @@ case: EDATE: 2013-04-01t00:00:00 max_job_tries: 1 - cplgocart: .true. + cplchem: .true. print_esmf: .true. nems_temp: 'atm_aer' nems_temp_cold: 'atm_aer' @@ -21,7 +21,8 @@ case: KEEPDATA: Yes nsst: - NST_MODEL: 0 + NST_MODEL: 2 + NST_SPINUP: 1 output_settings: FHOUT_GFS: 6 @@ -41,9 +42,11 @@ case: WGRP: 1 WGRP_NTASKS: 24 WRTIOBUF: "32M" + CCPP_SUITE: FV3_GFS_v16 + fscav_aero: [ '*:0.6', 'seas1:1.0', 'seas2:1.0' ] chem_settings: - MOD: gsdchem + MOD: gocart ntdiag: 2 # On Hera CFGDIR: /scratch1/NCEPDEV/nems/Raffaele.Montuoro/data/NASA/rc From bc7c20a880bf29a303ff3ac9446c7628c9f73d0d Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Tue, 8 Jun 2021 22:16:57 +0000 Subject: [PATCH 09/39] Add scavenging factors to aerosol test cases. --- workflow/cases/aerosol_firex_forecast.yaml | 1 + workflow/cases/aerosol_free_forecast.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/workflow/cases/aerosol_firex_forecast.yaml b/workflow/cases/aerosol_firex_forecast.yaml index 8feea0d05a4..d5d4b5ab27e 100644 --- a/workflow/cases/aerosol_firex_forecast.yaml +++ b/workflow/cases/aerosol_firex_forecast.yaml @@ -51,6 +51,7 @@ case: WRTIOBUF: "32M" CPL_ATMIC: FV3ICS CCPP_SUITE: FV3_GFS_v16 + fscav_aero: [ '*:0.6', 'seas1:1.0', 'seas2:1.0', 'seas3:1.0', 'seas4:1.0', 'seas5:1.0' ] chem_settings: MOD: gocart diff --git a/workflow/cases/aerosol_free_forecast.yaml b/workflow/cases/aerosol_free_forecast.yaml index 9d4c77bed9d..13547e87068 100644 --- a/workflow/cases/aerosol_free_forecast.yaml +++ b/workflow/cases/aerosol_free_forecast.yaml @@ -43,7 +43,7 @@ case: WGRP_NTASKS: 24 WRTIOBUF: "32M" CCPP_SUITE: FV3_GFS_v16 - fscav_aero: [ '*:0.6', 'seas1:1.0', 'seas2:1.0' ] + fscav_aero: [ '*:0.6', 'seas1:1.0', 'seas2:1.0', 'seas3:1.0', 'seas4:1.0', 'seas5:1.0' ] chem_settings: MOD: gocart From f81741bdb3a8cc221f6e0846c706cfe3cc9c4010 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Thu, 10 Jun 2021 03:44:06 +0000 Subject: [PATCH 10/39] Use updated modulefile names only when building the ufs-weather-model with prognostic aerosols. This option is recognized by means of a hidden file created in the model's repository when running checkout.sh with option -a. --- sorc/build_ufs_coupled.sh | 10 ++++++---- sorc/checkout.sh | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/sorc/build_ufs_coupled.sh b/sorc/build_ufs_coupled.sh index 0d6be6a0e19..66de17a7fda 100755 --- a/sorc/build_ufs_coupled.sh +++ b/sorc/build_ufs_coupled.sh @@ -16,13 +16,15 @@ if [ $target = stampede ]; then target=stampede.intel ; fi MOD_PATH=$cwd/ufs_coupled.fd/modulefiles module purge -module use $MOD_PATH -module load ufs_${target} + cd ufs_coupled.fd/ if [[ -d build ]]; then rm -Rf build; fi -if [[ -d GOCART ]]; then - module load ufs_aerosols_${target} +if [[ -e .build_aerosols ]] ; then + module use ${MOD_PATH} + module load ufs_${target} ufs_aerosols_${target} CMAKE_FLAGS="-DAPP=ATMAERO" CCPP_SUITES="FV3_GFS_v16" ./build.sh else + module use ${MOD_PATH}/${target} + module load fv3 CMAKE_FLAGS="-DS2S=ON -DWW3=ON" CCPP_SUITES="FV3_GFS_v15p2_coupled,FV3_GFS_v16_coupled,FV3_GFS_v16" ./build.sh fi diff --git a/sorc/checkout.sh b/sorc/checkout.sh index fc35a12e375..10f79ceb849 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -49,6 +49,7 @@ else git clone https://github.com/rmontuoro/ufs-weather-model.git ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1 cd ufs_coupled.fd git checkout feature/gocart + echo > .build_aerosols else git clone https://github.com/ufs-community/ufs-weather-model ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1 cd ufs_coupled.fd From 0307896af1d309718d6b08d89631f8c080c575e7 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Thu, 10 Jun 2021 03:55:52 +0000 Subject: [PATCH 11/39] Load updated modulefiles for coupled aerosols on Hera/Intel. A new shell function (get_platform) is introduced to increase modularity. This fuction auto-detects the local platform and returns a string identifier. As a result, load_fv3gfs_modules.sh is refactored. --- ush/get_platform.sh | 44 ++++++++++++++++++++ ush/load_fv3gfs_modules.sh | 82 +++++++++++++++++--------------------- 2 files changed, 80 insertions(+), 46 deletions(-) create mode 100644 ush/get_platform.sh diff --git a/ush/get_platform.sh b/ush/get_platform.sh new file mode 100644 index 00000000000..7a2442ec1c6 --- /dev/null +++ b/ush/get_platform.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +############################################################### +# Auto-detect platform and return string identifier +############################################################### + +get_platform() { + + local hname=`hostname -d` + local pname="unknown" + + if [[ $hname = 'stampede2.tacc.utexas.edu' ]] ; then + # We are on Xsede stampede2 + pname="stampede" + elif [[ $hname = 'hpc.msstate.edu' ]] ; then + # We are on MSU Orion + pname="orion" + elif [[ -d /lfs3 ]] ; then + # We are on NOAA Jet + pname="jet" + elif [[ -d /scratch1 ]] ; then + # We are on NOAA Hera + pname="hera.intel" + elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then + # We are on NOAA Luna or Surge + pname="wcoss_c" + elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then + # We are on NOAA Mars or Venus + pname="wcoss_dell_p3" + elif [[ -d /dcom && -d /hwrf ]] ; then + # We are on NOAA Tide or Gyre + pname="wcoss" + elif [[ -d /glade ]] ; then + # We are on NCAR Cheyenne + pname="cheyenne" + elif [[ -d /lustre && -d /ncrc ]] ; then + # We are on GAEA. + pname="gaea" + else + echo WARNING: UNKNOWN PLATFORM + fi + + echo "${pname}" +} diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index 19c7a2664ec..a1e39ec7689 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -1,59 +1,49 @@ #!/bin/sh -#set -x ############################################################### # Setup runtime environment by loading modules +############################################################### + ulimit_s=$( ulimit -S -s ) -#ulimit -S -s 10000 # Find module command and purge: source "$HOMEgfs/modulefiles/module-setup.sh.inc" +source "$HOMEgfs/ush/get_platform.sh" -# Load our modules: -module use "$HOMEgfs/modulefiles" -hname=`hostname -d` +platform=$( get_platform ) -if [[ $hname = 'stampede2.tacc.utexas.edu' ]] ; then - # We are on Xsede stampede2 - module load module_base.stampede -elif [[ $hname = 'hpc.msstate.edu' ]] ; then - # We are on MSU Orion - module load module_base.orion -elif [[ -d /lfs3 ]] ; then - # We are on NOAA Jet - module load module_base.jet -elif [[ -d /scratch1 ]] ; then - # We are on NOAA Hera - module use "$HOMEgfs/sorc/ufs_coupled.fd/modulefiles" - module load ufs_hera.intel - if [[ -d $HOMEgfs/sorc/ufs_coupled.fd/GOCART ]] ; then - module load ufs_aerosols_hera.intel - fi -elif [[ -d /work ]] ; then - # We are on MSU Orion - module load module_base.orion - if [[ -d $HOMEgfs/sorc/ufs_coupled.fd/GOCART ]] ; then - module use "$HOMEgfs/sorc/ufs_coupled.fd/modulefiles/orion.intel" - module load fv3 - fi -elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then - # We are on NOAA Luna or Surge - module load module_base.wcoss_c -elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then - # We are on NOAA Mars or Venus - module load module_base.wcoss_dell_p3 -elif [[ -d /dcom && -d /hwrf ]] ; then - # We are on NOAA Tide or Gyre - module load module_base.wcoss -elif [[ -d /glade ]] ; then - # We are on NCAR Yellowstone - module load module_base.cheyenne -elif [[ -d /lustre && -d /ncrc ]] ; then - # We are on GAEA. - module load module_base.gaea -else - echo WARNING: UNKNOWN PLATFORM -fi +moduledir= +modulelist= +case "${platform}" in + hera.intel) + moduledir="$HOMEgfs/sorc/ufs_coupled.fd/modulefiles" + if [[ -r ${moduledir}/${platform}/fv3 ]] ; then + moduledir="$HOMEgfs/sorc/ufs_coupled.fd/modulefiles/${platform}" + modulelist=fv3 + elif [[ -r ${moduledir}/ufs_${platform} ]] ; then + modulelist=ufs_${platform} + if [[ "$( grep UFS_GOCART ${HOMEgfs}/sorc/ufs_coupled.fd/build/CMakeCache.txt 2>/dev/null | cut -d= -f2 )" = "ON" ]] ; then + # add aerosols modulefile + modulelist="${modulelist} ufs_aerosols_${platform}" + fi + else + echo "Unable to load modulefiles on ${platform}" + exit 1 + fi + ;; + unknown) + echo "Unknown platform" + exit 1 + ;; + *) + moduledir="$HOMEgfs/modulefiles" + modulelist="module_base.${platform}" + ;; +esac + +# Load our modules: +module use ${moduledir} +module load ${modulelist} # Restore stack soft limit: ulimit -S -s "$ulimit_s" From 6b19a39b6f086ff7eb077c60a3acbc40d6dae5a1 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Thu, 10 Jun 2021 04:04:51 +0000 Subject: [PATCH 12/39] Remove GSD-specific functions in forecast postdet. --- ush/forecast_postdet.sh | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 4acbd1c2fba..545a09d2929 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -856,21 +856,3 @@ GOCART_rc() { [[ $status -ne 0 ]] && exit $status fi } - -GSD_in() { - echo "SUB ${FUNCNAME[0]}: Linking input data for GSD" - # soft link commands insert here -} - -GSD_nml() { - echo "SUB ${FUNCNAME[0]}: Creating name list for GSD" - # generate GSD-specific input -} - -GSD_out() { - echo "SUB ${FUNCNAME[0]}: Copying output data for GSD" - # soft link commands insert here -} - - - From 8a8194f7fe2210de4a70de8322de2f4384d3098a Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Thu, 10 Jun 2021 04:05:57 +0000 Subject: [PATCH 13/39] Remove no longer needed parsing script used to generate the FV3 namelist file for FV3-GOCART. --- ush/parsing_namelists_FV3_GOCART.sh | 364 ---------------------------- 1 file changed, 364 deletions(-) delete mode 100755 ush/parsing_namelists_FV3_GOCART.sh diff --git a/ush/parsing_namelists_FV3_GOCART.sh b/ush/parsing_namelists_FV3_GOCART.sh deleted file mode 100755 index 0d2390515c3..00000000000 --- a/ush/parsing_namelists_FV3_GOCART.sh +++ /dev/null @@ -1,364 +0,0 @@ -#! /bin/sh -##### -## "parsing_namelist_FV3.sh" -## This script writes namelist for FV3 model -## -## This is the child script of ex-global forecast, -## writing namelist for FV3 -## This script is a direct execution. -##### - -FV3_namelists(){ - -# copy over the tables -DIAG_TABLE=${DIAG_TABLE:-$PARM_FV3DIAG/diag_table_aer} -DATA_TABLE=${DATA_TABLE:-$PARM_FV3DIAG/data_table} -FIELD_TABLE=${FIELD_TABLE:-$PARM_FV3DIAG/field_table_aer} - -# build the diag_table with the experiment name and date stamp -if [ $DOIAU = "YES" ]; then -cat > diag_table << EOF -FV3 Forecast -${gPDY:0:4} ${gPDY:4:2} ${gPDY:6:2} ${gcyc} 0 0 -EOF -cat $DIAG_TABLE >> diag_table -else -cat > diag_table << EOF -FV3 Forecast -${sPDY:0:4} ${sPDY:4:2} ${sPDY:6:2} ${scyc} 0 0 -EOF -cat $DIAG_TABLE >> diag_table -fi - -$NCP $DATA_TABLE data_table -$NCP $FIELD_TABLE field_table - -cat > input.nml <> input.nml <> input.nml - -cat >> input.nml <> input.nml << EOF -&nam_stochy -/ -&nam_sfcperts -/ -EOF - -echo "$(cat input.nml)" -} From a821a1aef761b386beace4efab476495e209adbb Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Thu, 10 Jun 2021 05:32:41 +0000 Subject: [PATCH 14/39] Return proper name for Hera platform and load correct modules. --- ush/get_platform.sh | 2 +- ush/load_fv3gfs_modules.sh | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/ush/get_platform.sh b/ush/get_platform.sh index 7a2442ec1c6..417f6ce920f 100644 --- a/ush/get_platform.sh +++ b/ush/get_platform.sh @@ -20,7 +20,7 @@ get_platform() { pname="jet" elif [[ -d /scratch1 ]] ; then # We are on NOAA Hera - pname="hera.intel" + pname="hera" elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then # We are on NOAA Luna or Surge pname="wcoss_c" diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index a1e39ec7689..0a5f69e983b 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -15,20 +15,18 @@ platform=$( get_platform ) moduledir= modulelist= case "${platform}" in - hera.intel) + hera) moduledir="$HOMEgfs/sorc/ufs_coupled.fd/modulefiles" - if [[ -r ${moduledir}/${platform}/fv3 ]] ; then - moduledir="$HOMEgfs/sorc/ufs_coupled.fd/modulefiles/${platform}" - modulelist=fv3 - elif [[ -r ${moduledir}/ufs_${platform} ]] ; then - modulelist=ufs_${platform} + target=${platform}.intel + if [[ -r ${moduledir}/ufs_${target} ]] ; then + modulelist=ufs_${target} if [[ "$( grep UFS_GOCART ${HOMEgfs}/sorc/ufs_coupled.fd/build/CMakeCache.txt 2>/dev/null | cut -d= -f2 )" = "ON" ]] ; then # add aerosols modulefile - modulelist="${modulelist} ufs_aerosols_${platform}" + modulelist="${modulelist} ufs_aerosols_${target}" fi else - echo "Unable to load modulefiles on ${platform}" - exit 1 + moduledir="$HOMEgfs/modulefiles" + modulelist="module_base.${platform}" fi ;; unknown) From c5456b5c6373254769d3e655be4deb34d7805495 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Sun, 13 Jun 2021 02:31:59 +0000 Subject: [PATCH 15/39] Load aerosol-specific modulefiles only for the forecast task. --- ush/load_fv3gfs_modules.sh | 19 ++++++++++++++++++- workflow/layout/free_forecast_gfs.yaml | 4 ++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index 0a5f69e983b..d9ae4f0e9ad 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -2,6 +2,10 @@ ############################################################### # Setup runtime environment by loading modules +# +# This script accepts an optional argument for the task type. +# Acceptable types are: exclusive, forecast, service, shared +# ############################################################### ulimit_s=$( ulimit -S -s ) @@ -10,6 +14,19 @@ ulimit_s=$( ulimit -S -s ) source "$HOMEgfs/modulefiles/module-setup.sh.inc" source "$HOMEgfs/ush/get_platform.sh" +tasktype= +if [ $# -eq 1 ] ; then + case "${1}" in + exclusive|forecast|service|shared) + tasktype="${1}" + ;; + *) + echo "Unsupported task type" + exit 1 + ;; + esac +fi + platform=$( get_platform ) moduledir= @@ -18,7 +35,7 @@ case "${platform}" in hera) moduledir="$HOMEgfs/sorc/ufs_coupled.fd/modulefiles" target=${platform}.intel - if [[ -r ${moduledir}/ufs_${target} ]] ; then + if [[ "${tasktype}" = "forecast" && -r ${moduledir}/ufs_${target} ]] ; then modulelist=ufs_${target} if [[ "$( grep UFS_GOCART ${HOMEgfs}/sorc/ufs_coupled.fd/build/CMakeCache.txt 2>/dev/null | cut -d= -f2 )" = "ON" ]] ; then # add aerosols modulefile diff --git a/workflow/layout/free_forecast_gfs.yaml b/workflow/layout/free_forecast_gfs.yaml index 2d8f64da654..f45b752263a 100644 --- a/workflow/layout/free_forecast_gfs.yaml +++ b/workflow/layout/free_forecast_gfs.yaml @@ -69,7 +69,7 @@ suite: !Cycle forecast: !Family medcold: !Task - <<: *exclusive_task_template + <<: *forecast_task_template Disable: !calc doc.settings.cplflx == '.false.' or doc.settings.onestep == '.true.' Trigger: !FirstTrue - when: !calc doc.settings.cplwav=='.true.' @@ -80,7 +80,7 @@ suite: !Cycle J_JOB: JGLOBAL_FORECAST_MEDCOLD highres: !Task - <<: *exclusive_task_template + <<: *forecast_task_template Trigger: !FirstTrue - when: !calc doc.settings.cplflx=='.true.' and doc.settings.onestep=='.false.' do: !Depend medcold From bfe150ae29d18c5ddf3353bd9ac773accdf82613 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Mon, 14 Jun 2021 18:57:29 +0000 Subject: [PATCH 16/39] Replace legacy 'cplgocart' flag with 'cplchem'. --- workflow/config/fcst.yaml | 2 +- workflow/layout/free_forecast_gfs.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow/config/fcst.yaml b/workflow/config/fcst.yaml index bd46c040b73..f424cfc7c11 100644 --- a/workflow/config/fcst.yaml +++ b/workflow/config/fcst.yaml @@ -40,7 +40,7 @@ config_fcst: - otherwise: !calc doc.partition_common.resources.run_gfsfcst restart_interval_gfs: !FirstTrue - - when: !calc doc.settings.cplgocart == ".false." + - when: !calc doc.settings.cplchem == ".false." do: !calc doc.fv3_gfs_settings.restart_interval - otherwise: !FirstTrue - when: !calc doc.fv3_gfs_settings.restart_interval == "0" diff --git a/workflow/layout/free_forecast_gfs.yaml b/workflow/layout/free_forecast_gfs.yaml index d6046a95625..624202e7fc1 100644 --- a/workflow/layout/free_forecast_gfs.yaml +++ b/workflow/layout/free_forecast_gfs.yaml @@ -47,7 +47,7 @@ suite: !Cycle jgfs_init_chem: !Task <<: *exclusive_task_template RUN: 'gfs' - Disable: !calc doc.settings.cplgocart != '.true.' + Disable: !calc doc.settings.cplchem != '.true.' Trigger: !Depend jgfs_emc_mom6ic & up.gfs.forecast.at('-' + tools.dt_to_HMS(doc.settings.STEP_GFS)) resources: !calc partition.resources.run_init_chem J_JOB: rocoto/init_chem.sh @@ -99,7 +99,7 @@ suite: !Cycle do: !Depend medcold - when: !calc doc.settings.cplwav=='.true.' do: !Depend up.prep - - when: !calc doc.settings.cplgocart=='.true.' + - when: !calc doc.settings.cplchem=='.true.' do: !Depend up.up.ics.jgfs_init_chem & up.up.ics.jgfs_emc_mom6ic - otherwise: !Depend up.up.ics.jgfs_emc_mom6ic resources_remap: !JobRequest [ { <<: *remap_resource_template } ] From 88f0358b7f09713f11cdf9606c2b71a68def94fe Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Mon, 14 Jun 2021 18:58:16 +0000 Subject: [PATCH 17/39] Update tracer list. --- parm/chem/gocart_tracer.list | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parm/chem/gocart_tracer.list b/parm/chem/gocart_tracer.list index 871a41eedb7..0ac80b32ff4 100644 --- a/parm/chem/gocart_tracer.list +++ b/parm/chem/gocart_tracer.list @@ -1,9 +1,9 @@ so2 -sulf +so4 dms msa -pp25 -pp10 +pm25 +pm10 bc1 bc2 oc1 @@ -17,4 +17,4 @@ seas1 seas2 seas3 seas4 -seas5 \ No newline at end of file +seas5 From 2c40cdcdece2c06b35e3d3d9c20dac61596c5bf7 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Mon, 14 Jun 2021 22:45:22 +0000 Subject: [PATCH 18/39] Restore original path to modulefiles in module_base.hera. --- modulefiles/module_base.hera | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera index aa910eb6dba..f42986068ca 100644 --- a/modulefiles/module_base.hera +++ b/modulefiles/module_base.hera @@ -4,8 +4,7 @@ ## # Modules availble through hpc-stack -# module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack -module use /scratch1/NCEPDEV/nems/Raffaele.Montuoro/dev/nasa/libs/hpc-stack/1.1.0-nasa/modulefiles/stack +module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack module load hpc/1.1.0 module load hpc-intel/18.0.5.274 From 7e84ea45bc6207572695f13bae2278ed10d659b9 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Tue, 22 Jun 2021 17:01:41 +0000 Subject: [PATCH 19/39] Update aerosol_firex_forecast test case. --- workflow/cases/aerosol_firex_forecast.yaml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/workflow/cases/aerosol_firex_forecast.yaml b/workflow/cases/aerosol_firex_forecast.yaml index 0b240126482..057b9c86159 100644 --- a/workflow/cases/aerosol_firex_forecast.yaml +++ b/workflow/cases/aerosol_firex_forecast.yaml @@ -4,14 +4,12 @@ case: places: workflow_file: layout/free_forecast_gfs.yaml - BASE_CPLIC: /scratch1/NCEPDEV/nems/Raffaele.Montuoro/data - OROFIX: /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/input-data-20210528/FV3_input_data384/INPUT - GRDFIX: /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/input-data-20210528/FV3_input_data384/INPUT + BASE_CPLIC: /scratch1/NCEPDEV/nems/Raffaele.Montuoro/data/ICS settings: SDATE: 2019-07-01t00:00:00 - EDATE: 2019-07-15t00:00:00 - STEP_GFS: !timedelta "24:00:00" + EDATE: 2019-07-02t00:00:00 + STEP_GFS: !timedelta "06:00:00" STEP_DA: !timedelta "00:00:00" max_job_tries: 1 @@ -34,7 +32,7 @@ case: output_settings: FHOUT_GFS: 6 FHMIN_GFS: 0 - FHMAX_GFS: 24 + FHMAX_GFS: 6 FHMAX_HF_GFS: 0 FHOUT_HF_GFS: -1 @@ -45,7 +43,7 @@ case: layout: x: 6 y: 8 - nth: 1 + nth: 2 WGRP: 1 WGRP_NTASKS: 24 WRTIOBUF: "32M" From 165c2b716cfe29ba3fb0009e9b39b114316b8e4b Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Wed, 23 Jun 2021 13:41:40 +0000 Subject: [PATCH 20/39] update wave ww3_multi.inp file Issue #325 --- scripts/exgfs_wave_prep.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/exgfs_wave_prep.sh b/scripts/exgfs_wave_prep.sh index 6cae1b2539d..f0533b5f8a3 100755 --- a/scripts/exgfs_wave_prep.sh +++ b/scripts/exgfs_wave_prep.sh @@ -888,7 +888,7 @@ case ${WW3ATMINP} in 'YES' ) NFGRIDS=`expr $NFGRIDS + 1` - WINDLINE=" '$WAVEWND_FID' F F T F F F F" + WINDLINE=" '$WAVEWND_FID' F F T F F F F F F" WINDFLAG="$WAVEWND_FID" ;; 'CPL' ) @@ -896,7 +896,7 @@ if [ ${waveesmfGRD} ] then WINDFLAG="CPL:${waveesmfGRD}" - CPLILINE=" '${waveesmfGRD}' F F T F F F F" + CPLILINE=" '${waveesmfGRD}' F F T F F F F F F" else WINDFLAG="CPL:native" fi @@ -907,7 +907,7 @@ 'YES' ) NFGRIDS=`expr $NFGRIDS + 1` ICEIFLAG='T' - ICELINE=" '$WAVEICE_FID' F F F T F F F" + ICELINE=" '$WAVEICE_FID' F F F T F F F F F" ICEFLAG="$WAVEICE_FID" ;; 'CPL' ) @@ -915,7 +915,7 @@ if [ ${waveesmfGRD} ] then ICEFLAG="CPL:${waveesmfGRD}" - CPLILINE=" '${waveesmfGRD}' F F ${WNDIFLAG} T F F F" + CPLILINE=" '${waveesmfGRD}' F F ${WNDIFLAG} T F F F F F" else ICEFLAG="CPL:native" fi @@ -926,10 +926,10 @@ 'YES' ) if [ "$WAVECUR_FID" != "$WAVEICE_FID" ]; then NFGRIDS=`expr $NFGRIDS + 1` - CURRLINE=" '$WAVECUR_FID' F T F F F F F" + CURRLINE=" '$WAVECUR_FID' F T F F F F F F F" CURRFLAG="$WAVECUR_FID" else # cur fields share the same grid as ice grid - ICELINE=" '$WAVEICE_FID' F T F ${ICEIFLAG} F F F" + ICELINE=" '$WAVEICE_FID' F T F ${ICEIFLAG} F F F F F" CURRFLAG="$WAVEICE_FID" fi ;; @@ -938,7 +938,7 @@ if [ ${waveesmfGRD} ] then CURRFLAG="CPL:${waveesmfGRD}" - CPLILINE=" '${waveesmfGRD}' F T ${WNDIFLAG} ${ICEFLAG} F F F" + CPLILINE=" '${waveesmfGRD}' F T ${WNDIFLAG} ${ICEFLAG} F F F F F" else CURRFLAG="CPL:native" fi @@ -957,7 +957,7 @@ NMGRIDS=`expr $NMGRIDS + 1` gridN=`echo $waveGRDN | awk -v i=$GRDN '{print $i}'` gridG=`echo $waveGRDG | awk -v i=$GRDN '{print $i}'` - gline="${gline}'${grid}' 'no' 'CURRFLAG' 'WINDFLAG' 'ICEFLAG' 'no' 'no' 'no' ${gridN} ${gridG} 0.00 1.00 F\n" + gline="${gline}'${grid}' 'no' 'CURRFLAG' 'WINDFLAG' 'ICEFLAG' 'no' 'no' 'no' 'no' 'no' ${gridN} ${gridG} 0.00 1.00 F\n" done gline="${gline}\$" echo $gline From 6f4a3f26656c9725ef8a2d3eb2dd5b69c59cb157 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Tue, 22 Jun 2021 21:44:53 +0000 Subject: [PATCH 21/39] Update UFS model hash Updates the UFS model to the current tip of develop. Refs: #324 --- sorc/checkout.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/checkout.sh b/sorc/checkout.sh index f13d99a833f..6be7d40aeca 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -52,7 +52,7 @@ else else git clone https://github.com/ufs-community/ufs-weather-model ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1 cd ufs_coupled.fd - git checkout 3e46f5b7050e18884a0bed13691823ad88d443c3 + git checkout aa2ab44 fi git submodule update --init --recursive cd ${topdir} From d0de272a3a6f5f19f0c821e5eb299572fc1bf8c8 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Tue, 22 Jun 2021 21:46:06 +0000 Subject: [PATCH 22/39] Update MOM_input with landmask change option Adds the ALLOW_LANDMASK_CHANGES option to the MOM6 input templates and allows the setting via MOM6_ALLOW_LANDMASK_CHANGES (default False). Refs: #324 --- parm/mom6/MOM_input_template_025 | 2 ++ parm/mom6/MOM_input_template_050 | 2 ++ parm/mom6/MOM_input_template_100 | 2 ++ ush/parsing_namelists_MOM6.sh | 6 ++++-- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/parm/mom6/MOM_input_template_025 b/parm/mom6/MOM_input_template_025 index aa1da780298..1340db5b66b 100644 --- a/parm/mom6/MOM_input_template_025 +++ b/parm/mom6/MOM_input_template_025 @@ -148,6 +148,8 @@ TOPO_FILE = "ocean_topog.nc" ! default = "topog.nc" ! The file from which the bathymetry is read. TOPO_EDITS_FILE = "All_edits.nc" ! default = "" ! The file from which to read a list of i,j,z topography overrides. +ALLOW_LANDMASK_CHANGES = MOM6_ALLOW_LANDMASK_CHANGES ! default = "False" + ! If true, allow topography overrides to change ocean points to land MAXIMUM_DEPTH = 6500.0 ! [m] ! The maximum depth of the ocean. MINIMUM_DEPTH = 9.5 ! [m] default = 0.0 diff --git a/parm/mom6/MOM_input_template_050 b/parm/mom6/MOM_input_template_050 index 46780dfc822..ceff6e98d43 100644 --- a/parm/mom6/MOM_input_template_050 +++ b/parm/mom6/MOM_input_template_050 @@ -147,6 +147,8 @@ TOPO_CONFIG = "file" ! ! USER - call a user modified routine. TOPO_FILE = "ocean_topog.nc" ! default = "topog.nc" ! The file from which the bathymetry is read. +ALLOW_LANDMASK_CHANGES = MOM6_ALLOW_LANDMASK_CHANGES ! default = "False" + ! If true, allow topography overrides to change ocean points to land MAXIMUM_DEPTH = 6500.0 ! [m] ! The maximum depth of the ocean. MINIMUM_DEPTH = 9.5 ! [m] default = 0.0 diff --git a/parm/mom6/MOM_input_template_100 b/parm/mom6/MOM_input_template_100 index d5af4cccd21..24ea3c7c47a 100644 --- a/parm/mom6/MOM_input_template_100 +++ b/parm/mom6/MOM_input_template_100 @@ -140,6 +140,8 @@ TOPO_CONFIG = "file" ! ! USER - call a user modified routine. TOPO_EDITS_FILE = "topo_edits_011818.nc" ! default = "" ! The file from which to read a list of i,j,z topography overrides. +ALLOW_LANDMASK_CHANGES = MOM6_ALLOW_LANDMASK_CHANGES ! default = "False" + ! If true, allow topography overrides to change ocean points to land MAXIMUM_DEPTH = 6500.0 ! [m] ! The maximum depth of the ocean. MINIMUM_DEPTH = 9.5 ! [m] default = 0.0 diff --git a/ush/parsing_namelists_MOM6.sh b/ush/parsing_namelists_MOM6.sh index 2af5e1d457d..e7ab58a0ae0 100644 --- a/ush/parsing_namelists_MOM6.sh +++ b/ush/parsing_namelists_MOM6.sh @@ -9,8 +9,9 @@ MOM_INPUT=MOM_input_template_$OCNRES #TODO: Make these variables configurable #Set to False for restart reproducibility -MOM6_REPRO_LA='True' -MOM6_THERMO_SPAN='False' +MOM6_REPRO_LA=${MOM6_REPRO_LA:-'True'} +MOM6_THERMO_SPAN=${MOM6_THERMO_SPAN:-'False'} +MOM6_ALLOW_LANDMASK_CHANGES=${MOM6_ALLOW_LANDMASK_CHANGES:-'False'} if [ $cplwav = ".true." ] ; then MOM6_USE_WAVES='True' @@ -74,6 +75,7 @@ sed -e "s/DT_THERM_MOM6/$DT_THERM_MOM6/g" \ -e "s/MOM6_THERMO_SPAN/$MOM6_THERMO_SPAN/g" \ -e "s/MOM6_REPRO_LA/$MOM6_REPRO_LA/g" \ -e "s/MOM6_USE_WAVES/$MOM6_USE_WAVES/g" \ + -e "s/MOM6_ALLOW_LANDMASK_CHANGES/$MOM6_ALLOW_LANDMASK_CHANGES/g" \ -e "s/NX_GLB/$NX_GLB/g" \ -e "s/NY_GLB/$NY_GLB/g" \ -e "s/CHLCLIM/$CHLCLIM/g" $DATA/INPUT/MOM_input_template_$OCNRES > $DATA/INPUT/MOM_input From 56d559757fc9257db81d0a6e07e1a423f2ce146d Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Tue, 22 Jun 2021 22:26:54 +0000 Subject: [PATCH 23/39] Update CICE inputs for new model version The newest version of CICE replaces istep0 with the normal month and day, so the namelist is updated to reflect that. ktherm and tfrz_option are updated to allow alternate settings, following similar changes to the UFS test harness. The default values are the previous settings. Settings for tr_FY, sw_redist, f_mlt_onset, and f_frx_onset are updated to new values in the UFS test harness. Additionally, many namelist options have been removed following the same deletions in the UFS test harness. Refs: #324 --- ush/forecast_postdet.sh | 6 +- ush/parsing_namelists_CICE.sh | 261 ++-------------------------------- 2 files changed, 19 insertions(+), 248 deletions(-) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 7bdde50324f..7abd83a92c9 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -729,9 +729,11 @@ CICE_postdet() { echo "SUB ${FUNCNAME[0]}: CICE after run type determination" year=$(echo $CDATE|cut -c 1-4) + month=$(echo $CDATE|cut -c 5-6) + day=$(echo $CDATE|cut -c 7-8) + stepsperhr=$((3600/$ICETIM)) nhours=$($NHOUR $CDATE ${year}010100) - istep0=$((nhours*stepsperhr)) steps=$((nhours*stepsperhr)) npt=$((FHMAX*$stepsperhr)) # Need this in order for dump_last to work @@ -746,6 +748,8 @@ CICE_postdet() { cice_hist_avg=${cice_hist_avg:-".true."} FRAZIL_FWSALT=${FRAZIL_FWSALT:-".true."} + ktherm=${ktherm:-1} + tfrz_option=${tfrz_option:-"linear_salt"} tr_pond_lvl=${tr_pond_lvl:-".true."} # Use level melt ponds tr_pond_lvl=true # restart_pond_lvl (if tr_pond_lvl=true): diff --git a/ush/parsing_namelists_CICE.sh b/ush/parsing_namelists_CICE.sh index 60baa023e39..d19f0c41722 100644 --- a/ush/parsing_namelists_CICE.sh +++ b/ush/parsing_namelists_CICE.sh @@ -14,12 +14,13 @@ cat > ice_in < ice_in < ice_in < ice_in < ice_in < ice_in < ice_in < ice_in < ice_in < ice_in < ice_in < ice_in < ice_in < Date: Tue, 22 Jun 2021 23:13:36 +0000 Subject: [PATCH 24/39] Update UFS build script Some changes to the UFS require updates to the build script. The modulefile organization has been updated to be modulefiles/ufs_${target} rather than modulefiles/${target}/fv3, so the module paths in the build script are updated correspondingly. A cmake switch DAPP is now required in order to build UFS. Non- aerosol coupled is using S2SW for the time being. This replaces setting DS2S and DWW3 to YES. The old v15 suite is removed from the CCPP suite list. Refs: #324 --- sorc/build_ufs_coupled.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sorc/build_ufs_coupled.sh b/sorc/build_ufs_coupled.sh index e18e6fd3e88..e2176967178 100755 --- a/sorc/build_ufs_coupled.sh +++ b/sorc/build_ufs_coupled.sh @@ -13,15 +13,17 @@ if [ $target = hera ]; then target=hera.intel ; fi if [ $target = orion ]; then target=orion.intel ; fi if [ $target = stampede ]; then target=stampede.intel ; fi -MOD_PATH=$cwd/ufs_coupled.fd/modulefiles/$target +MOD_PATH=$cwd/ufs_coupled.fd/modulefiles module purge module use $MOD_PATH -module load fv3 +module load ufs_${target} cd ufs_coupled.fd/ if [[ -d build ]]; then rm -Rf build; fi if [[ -d GOCART ]]; then + module load ufs_aerosols_${target} + CMAKE_FLAGS="-DAPP=ATMAERO" CCPP_SUITES="FV3_GFS_v16" ./build.sh ./build.sh else - CMAKE_FLAGS="-DS2S=ON -DWW3=ON" CCPP_SUITES="FV3_GFS_v15p2_coupled,FV3_GFS_v16_coupled,FV3_GFS_v16" ./build.sh + CMAKE_FLAGS="-DAPP=S2SW" CCPP_SUITES="FV3_GFS_v16_coupled,FV3_GFS_v16" ./build.sh fi From e86db2317aab6a98d56d3ae4d63696f82ad765f7 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Tue, 22 Jun 2021 23:24:12 +0000 Subject: [PATCH 25/39] Update source of fd_nems.yaml The location of the fd_nems.yaml file changed in the UFS repository, so updated nems_configure to copy from the new location. Refs: #324 --- ush/nems_configure.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/nems_configure.sh b/ush/nems_configure.sh index 8e446271458..402e8598d70 100755 --- a/ush/nems_configure.sh +++ b/ush/nems_configure.sh @@ -155,9 +155,9 @@ EOF echo "$(cat med_modelio.nml)" -cp $HOMEgfs/sorc/ufs_coupled.fd/CMEPS-interface/CMEPS/mediator/fd_nems.yaml fd_nems.yaml +cp $HOMEgfs/sorc/ufs_coupled.fd/tests/parm/fd_nems.yaml fd_nems.yaml -fi +fi echo "SUB ${FUNCNAME[0]}: Nems configured for $confignamevarfornems" From 12050b8edfebd20efaa7c35904e26dbf2f8243b3 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Wed, 23 Jun 2021 00:19:36 +0000 Subject: [PATCH 26/39] Update templates to set ocean mesh file The latest version of UFS adds a setting to define the ocean mesh file, so the nems.configure templates are updated to define them. They are identical to the ice mesh file, so the MESHICE variable is renamed to MESH_OCN_ICE and used for both. Additionally, the use_mommesh variable is added and defaults to true. Refs: #324 --- ush/forecast_postdet.sh | 2 +- ush/nems.configure.cpld.IN | 4 +++- ush/nems.configure.cpld_wave.IN | 4 +++- ush/nems.configure.medcold_atm_ocn_ice.IN | 3 ++- ush/nems.configure.medcold_atm_ocn_ice_wav.IN | 3 ++- ush/nems_configure.sh | 5 ++++- workflow/config/fcst.yaml | 2 +- 7 files changed, 16 insertions(+), 7 deletions(-) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 7abd83a92c9..1a56c89754b 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -789,7 +789,7 @@ CICE_postdet() { echo "Link CICE fixed files" $NLN -sf $FIXcice/$ICERES/${ice_grid_file} $DATA/ $NLN -sf $FIXcice/$ICERES/${ice_kmt_file} $DATA/ - $NLN -sf $FIXcice/$ICERES/$MESHICE $DATA/ + $NLN -sf $FIXcice/$ICERES/$MESH_OCN_ICE $DATA/ } CICE_nml() { diff --git a/ush/nems.configure.cpld.IN b/ush/nems.configure.cpld.IN index 08d40044c09..69b6c737f3a 100644 --- a/ush/nems.configure.cpld.IN +++ b/ush/nems.configure.cpld.IN @@ -31,6 +31,7 @@ OCN_attributes:: DumpFields = @[DumpFields] ProfileMemory = false OverwriteSlice = true + mesh_ocn = @[MESH_OCN_ICE] :: # ICE # @@ -41,7 +42,7 @@ ICE_attributes:: DumpFields = @[DumpFields] ProfileMemory = false OverwriteSlice = true - mesh_ice = @[MESHICE] + mesh_ice = @[MESH_OCN_ICE] stop_n = @[RESTART_N] stop_option = nhours stop_ymd = -999 @@ -99,4 +100,5 @@ ALLCOMP_attributes:: restart_ymd = -999 dbug_flag = @[cap_dbug_flag] use_coldstart = @[use_coldstart] + use_mommesh = @[use_mommesh] :: diff --git a/ush/nems.configure.cpld_wave.IN b/ush/nems.configure.cpld_wave.IN index ee4443f2d90..e4d9eca31d2 100644 --- a/ush/nems.configure.cpld_wave.IN +++ b/ush/nems.configure.cpld_wave.IN @@ -31,6 +31,7 @@ OCN_attributes:: DumpFields = @[DumpFields] ProfileMemory = false OverwriteSlice = true + mesh_ocn = @[MESH_OCN_ICE] :: # ICE # @@ -41,7 +42,7 @@ ICE_attributes:: DumpFields = @[DumpFields] ProfileMemory = false OverwriteSlice = true - mesh_ice = @[MESHICE] + mesh_ice = @[MESH_OCN_ICE] stop_n = @[RESTART_N] stop_option = nhours stop_ymd = -999 @@ -113,4 +114,5 @@ ALLCOMP_attributes:: restart_ymd = -999 dbug_flag = @[cap_dbug_flag] use_coldstart = @[use_coldstart] + use_mommesh = @[use_mommesh] :: diff --git a/ush/nems.configure.medcold_atm_ocn_ice.IN b/ush/nems.configure.medcold_atm_ocn_ice.IN index d54f6596a6f..fe4030274e5 100644 --- a/ush/nems.configure.medcold_atm_ocn_ice.IN +++ b/ush/nems.configure.medcold_atm_ocn_ice.IN @@ -32,6 +32,7 @@ OCN_attributes:: ProfileMemory = false OverwriteSlice = true dbug_flag = 0 + mesh_ocn = @[MESH_OCN_ICE] :: # ICE # @@ -42,7 +43,7 @@ ICE_attributes:: DumpFields = @[DumpFields] ProfileMemory = false OverwriteSlice = true - mesh_ice = @[MESHICE] + mesh_ice = @[MESH_OCN_ICE] stop_n = @[FHMAX] stop_option = nhours stop_ymd = -999 diff --git a/ush/nems.configure.medcold_atm_ocn_ice_wav.IN b/ush/nems.configure.medcold_atm_ocn_ice_wav.IN index 53cfc8c09d4..fa8ab4e4ff1 100644 --- a/ush/nems.configure.medcold_atm_ocn_ice_wav.IN +++ b/ush/nems.configure.medcold_atm_ocn_ice_wav.IN @@ -31,6 +31,7 @@ OCN_attributes:: DumpFields = @[DumpFields] ProfileMemory = false OverwriteSlice = true + mesh_ocn = @[MESH_OCN_ICE] :: # ICE # @@ -41,7 +42,7 @@ ICE_attributes:: DumpFields = @[DumpFields] ProfileMemory = false OverwriteSlice = true - mesh_ice = @[MESHICE] + mesh_ice = @[MESH_OCN_ICE] stop_n = @[FHMAX] stop_option = nhours stop_ymd = -999 diff --git a/ush/nems_configure.sh b/ush/nems_configure.sh index 402e8598d70..58ecef74a52 100755 --- a/ush/nems_configure.sh +++ b/ush/nems_configure.sh @@ -45,6 +45,8 @@ ICEPETS=${ICEPETS:-8} WAVPETS=${WAVPETS:-8} CHMPETS=${CHMPETS:-${ATMPETS}} +USE_MOMMESH=${USE_MOMMESH:-"true"} + rm -f $DATA/nems.configure med_petlist_bounds=${med_petlist_bounds:-"0 $(( $MEDPETS-1 ))"} @@ -79,6 +81,7 @@ if [ $cplflx = .true. ]; then sed -i -e "s;@\[coupling_interval_slow_sec\];$CPL_SLOW;g" tmp1 sed -i -e "s;@\[coupling_interval_fast_sec\];$CPL_FAST;g" tmp1 sed -i -e "s;@\[RESTART_N\];$restart_interval_nems;g" tmp1 + sed -i -e "s;@\[use_mommesh\];$USE_MOMMESH;g" tmp1 fi if [ $cplwav = .true. ]; then sed -i -e "s;@\[wav_model\];ww3;g" tmp1 @@ -87,7 +90,7 @@ fi if [ $cplice = .true. ]; then sed -i -e "s;@\[ice_model\];$ICE_model;g" tmp1 sed -i -e "s;@\[ice_petlist_bounds\];$ice_petlist_bounds;g" tmp1 - sed -i -e "s;@\[MESHICE\];$MESHICE;g" tmp1 + sed -i -e "s;@\[MESH_OCN_ICE\];$MESH_OCN_ICE;g" tmp1 sed -i -e "s;@\[FHMAX\];$FHMAX_GFS;g" tmp1 fi if [ $cplchem = .true. -o $cplgocart = .true. ]; then diff --git a/workflow/config/fcst.yaml b/workflow/config/fcst.yaml index a89a9e2e84d..42ccca3c064 100644 --- a/workflow/config/fcst.yaml +++ b/workflow/config/fcst.yaml @@ -110,7 +110,7 @@ config_fcst: # CICE parameters export NX_GLB="{doc.ice_settings.NX_GLB}" export NY_GLB="{doc.ice_settings.NY_GLB}" - export MESHICE="mesh.mx{doc.ice_settings.ICERES}.nc" + export MESH_OCN_ICE="mesh.mx{doc.ice_settings.ICERES}.nc" if [[ "$machine" == "WCOSS_C" ]]; then export memory_fcst="1024M" From d4910b5df107c3c311d365113202bbcfa2f29f22 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Wed, 23 Jun 2021 01:29:52 +0000 Subject: [PATCH 27/39] Update Ww3 prepost build The WW3 pre and post components now need NetCDF to build, so NetCDF is added back to the modulelist, and previously removed code in the build script is restored. Refs: #324 --- modulefiles/modulefile.ww3.hera | 2 ++ modulefiles/modulefile.ww3.orion | 2 ++ modulefiles/modulefile.ww3.wcoss_dell_p3 | 2 ++ sorc/build_ww3prepost.sh | 5 +++-- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/modulefiles/modulefile.ww3.hera b/modulefiles/modulefile.ww3.hera index 59bacd17471..462d7a9dac5 100644 --- a/modulefiles/modulefile.ww3.hera +++ b/modulefiles/modulefile.ww3.hera @@ -15,3 +15,5 @@ module load png/1.6.35 module load bacio/2.4.1 module load g2/3.4.1 + +module load netcdf/4.7.4 diff --git a/modulefiles/modulefile.ww3.orion b/modulefiles/modulefile.ww3.orion index 36774389891..1702532b716 100644 --- a/modulefiles/modulefile.ww3.orion +++ b/modulefiles/modulefile.ww3.orion @@ -16,3 +16,5 @@ module load png/1.6.35 module load bacio/2.4.1 module load g2/3.4.1 + +module load netcdf/4.7.4 diff --git a/modulefiles/modulefile.ww3.wcoss_dell_p3 b/modulefiles/modulefile.ww3.wcoss_dell_p3 index ab43f602bf5..edb57b9fd24 100644 --- a/modulefiles/modulefile.ww3.wcoss_dell_p3 +++ b/modulefiles/modulefile.ww3.wcoss_dell_p3 @@ -11,3 +11,5 @@ module load png/1.6.35 module load bacio/2.4.1 module load g2/3.4.1 + +module load netcdf/4.7.4 diff --git a/sorc/build_ww3prepost.sh b/sorc/build_ww3prepost.sh index 339f050ba61..f17d7bcbeb0 100755 --- a/sorc/build_ww3prepost.sh +++ b/sorc/build_ww3prepost.sh @@ -35,8 +35,8 @@ export WWATCH3_ENV=${WW3_BINDIR}/wwatch3.env export PNG_LIB=${PNG_LIB:-$PNG_ROOT/lib64/libpng.a} export Z_LIB=${Z_LIB:-$ZLIB_ROOT/lib/libz.a} export JASPER_LIB=${JASPER_LIB:-$JASPER_ROOT/lib64/libjasper.a} -# export WWATCH3_NETCDF=NC4 -# export NETCDF_CONFIG=$NETCDF_ROOT/bin/nc-config +export WWATCH3_NETCDF=NC4 +export NETCDF_CONFIG=$NETCDF_ROOT/bin/nc-config export PRINTER=${PRINTER:-""} rm $WWATCH3_ENV @@ -50,6 +50,7 @@ echo "WWATCH3_F90 $WW3_F90" >> $WWATCH3_ENV echo "WWATCH3_CC $WW3_CC" >> $WWATCH3_ENV echo "WWATCH3_DIR $WW3_DIR" >> $WWATCH3_ENV echo "WWATCH3_TMP $WW3_TMPDIR" >> $WWATCH3_ENV +echo "WWATCH3_NETCDF $WWATCH3_NETCDF" >> $WWATCH3_ENV echo 'WWATCH3_SOURCE yes' >> $WWATCH3_ENV echo 'WWATCH3_LIST yes' >> $WWATCH3_ENV echo '' >> $WWATCH3_ENV From 351f5aa04be1fcd40ce31e1af09c6a6b0cb6ffbc Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Thu, 24 Jun 2021 03:10:25 +0000 Subject: [PATCH 28/39] Update meditaor name In the latest UFS, the mediator name was changed from nems to cmeps, so the nems.configure is updated to match. Refs: #324 --- ush/nems_configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/nems_configure.sh b/ush/nems_configure.sh index 58ecef74a52..c904948f999 100755 --- a/ush/nems_configure.sh +++ b/ush/nems_configure.sh @@ -58,7 +58,7 @@ chm_petlist_bounds=${chm_petlist_bounds:-"0 $(( $CHMPETS-1 ))"} # Copy the selected template into run directory cp $SCRIPTDIR/nems.configure.$confignamevarfornems.IN tmp1 -sed -i -e "s;@\[med_model\];nems;g" tmp1 +sed -i -e "s;@\[med_model\];cmeps;g" tmp1 sed -i -e "s;@\[atm_model\];$ATM_model;g" tmp1 sed -i -e "s;@\[med_petlist_bounds\];$med_petlist_bounds;g" tmp1 sed -i -e "s;@\[atm_petlist_bounds\];$atm_petlist_bounds;g" tmp1 From 88fb063ffd501a4de66f3477e3772294b405384d Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Thu, 24 Jun 2021 03:16:19 +0000 Subject: [PATCH 29/39] Update wave IC Update wave IC to new set compatible with current UFS version. Refs: #324 --- workflow/cases/coupled_free_forecast_wave.yaml | 2 +- workflow/schema/wave.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/cases/coupled_free_forecast_wave.yaml b/workflow/cases/coupled_free_forecast_wave.yaml index acf39609daf..90fbd8b46ee 100644 --- a/workflow/cases/coupled_free_forecast_wave.yaml +++ b/workflow/cases/coupled_free_forecast_wave.yaml @@ -66,7 +66,7 @@ case: wave_settings: WAVPETS: 160 - CPL_WAVIC: CFSRwave20200925 + CPL_WAVIC: GEFSwave20210623 waveGRD: 'gwes_30m' wavepostGRD: 'gwes_30m' waveesmfGRD: '' diff --git a/workflow/schema/wave.yaml b/workflow/schema/wave.yaml index 47df41acbf1..1da284f41ec 100644 --- a/workflow/schema/wave.yaml +++ b/workflow/schema/wave.yaml @@ -9,4 +9,4 @@ wave_settings_template: !Template &wave_settings_template description: "model selection for wave" CPL_WAVIC: type: string - allowed: [ 'CFSR', 'CFSRwave20200925' ] + allowed: [ 'CFSR', 'CFSRwave20200925', 'GEFSwave20210623' ] From 1957bb9d4b5bb81420715ef5f613fc9ab888fce6 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Thu, 24 Jun 2021 05:43:57 +0000 Subject: [PATCH 30/39] Update modules to match UFS Update module versions to match those in the UFS common file. Refs: #324 --- modulefiles/module_base.hera | 6 +++--- modulefiles/module_base.orion | 6 +++--- modulefiles/module_base.wcoss_dell_p3 | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera index 5f845a7caab..51eb9c343bc 100644 --- a/modulefiles/module_base.hera +++ b/modulefiles/module_base.hera @@ -12,8 +12,8 @@ module load hpc-intel/18.0.5.274 module load hpc-impi/2018.0.4 # Non-MPI -module load g2/3.4.1 -module load g2tmpl/1.9.1 +module load g2/3.4.2 +module load g2tmpl/1.10.0 module load grib_util/1.2.2 module load crtm/2.3.0 module load prod_util/1.2.2 @@ -31,7 +31,7 @@ module load png/1.6.35 # MPI module load netcdf/4.7.4 module load hdf5/1.10.6 -module load esmf/8_1_0_beta_snapshot_27 +module load esmf/8_1_1 module load w3emc/2.7.3 module load wgrib2/2.0.8 setenv "WGRIB2" "wgrib2" diff --git a/modulefiles/module_base.orion b/modulefiles/module_base.orion index 9fbc6a081b4..86a8ce419ae 100644 --- a/modulefiles/module_base.orion +++ b/modulefiles/module_base.orion @@ -20,8 +20,8 @@ module load hpc-intel/2018.4 module load hpc-impi/2018.4 # Non-MPI -module load g2/3.4.1 -module load g2tmpl/1.9.1 +module load g2/3.4.2 +module load g2tmpl/1.10.0 module load grib_util/1.2.2 module load crtm 2.3.0 module load prod_util/1.2.2 @@ -38,7 +38,7 @@ module load png/1.6.35 # MPI module load netcdf/4.7.4 module load hdf5/1.10.6 -module load esmf/8_1_0_beta_snapshot_27 +module load esmf/8_1_1 module load w3emc/2.7.3 module load wgrib2/2.0.8 setenv WGRIB2 wgrib2 diff --git a/modulefiles/module_base.wcoss_dell_p3 b/modulefiles/module_base.wcoss_dell_p3 index a651cb86c0c..e53199967c8 100755 --- a/modulefiles/module_base.wcoss_dell_p3 +++ b/modulefiles/module_base.wcoss_dell_p3 @@ -11,8 +11,8 @@ module load CFP/2.0.2 setenv USE_CFP YES # Non-MPI -module load g2/3.4.1 -module load g2tmpl/1.9.1 +module load g2/3.4.2 +module load g2tmpl/1.10.0 module load grib_util/1.2.2 module load crtm 2.3.0 module load prod_util/1.2.2 @@ -28,7 +28,7 @@ module load png/1.6.35 # MPI module load netcdf/4.7.4 module load hdf5/1.10.6 -module load esmf/8_1_0_beta_snapshot_27 +module load esmf/8_1_1 module load w3emc/2.7.3 module load wgrib2/2.0.8 setenv WGRIB2 wgrib2 From 2dcf8f2b1039b0ca1b706d1465e141894209ad98 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Thu, 24 Jun 2021 19:07:57 +0000 Subject: [PATCH 31/39] Update ice configuration to match UFS There were still a number of settings in the ice input file that did not match the setting in UFS (and one setting that was misplaced). Refs: #324 --- ush/parsing_namelists_CICE.sh | 54 +++++++++++++++++------------------ 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/ush/parsing_namelists_CICE.sh b/ush/parsing_namelists_CICE.sh index d19f0c41722..62cde8494b7 100644 --- a/ush/parsing_namelists_CICE.sh +++ b/ush/parsing_namelists_CICE.sh @@ -120,7 +120,6 @@ cat > ice_in < ice_in < ice_in < ice_in < ice_in < ice_in < Date: Thu, 1 Jul 2021 22:46:28 +0000 Subject: [PATCH 32/39] Add ammonia and nitrates to the list of cycled GOCART tracers. --- parm/chem/gocart_tracer.list | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/parm/chem/gocart_tracer.list b/parm/chem/gocart_tracer.list index 0ac80b32ff4..8b0202e2c4e 100644 --- a/parm/chem/gocart_tracer.list +++ b/parm/chem/gocart_tracer.list @@ -2,8 +2,6 @@ so2 so4 dms msa -pm25 -pm10 bc1 bc2 oc1 @@ -18,3 +16,10 @@ seas2 seas3 seas4 seas5 +nh3 +nh4a +no3an1 +no3an2 +no3an3 +pm25 +pm10 From acdc6a69194eadcba05c115219dfb5a4afabdd00 Mon Sep 17 00:00:00 2001 From: Raffaele Montuoro Date: Thu, 1 Jul 2021 22:48:37 +0000 Subject: [PATCH 33/39] Update checkout and build script to use the authoritative GitHub repository for the UFS weather model only. This repo now contains UFS-Aerosols, the aerosol component based on NASA's GOCART. Since we are using a single repository for both S2S and coupled aerosols, the -a option was removed from checkout.sh and added to build_all.sh. This option is passed to build_ufs_coupled.sh and partial_build.sh, which have been streamlined and now accept both short and long options: build_ufs_coupled.sh: -a, --aerosols -c, --coupled partial_build.sh: -a, --aerosols -c, --coupled -h, --help -v, --verbose --- sorc/build_all.sh | 17 ++++++++-- sorc/build_ufs_coupled.sh | 41 ++++++++++++++++-------- sorc/checkout.sh | 18 ++--------- sorc/partial_build.sh | 67 ++++++++++++++++++--------------------- 4 files changed, 76 insertions(+), 67 deletions(-) diff --git a/sorc/build_all.sh b/sorc/build_all.sh index 577f42df052..a47eb8455ab 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -9,14 +9,25 @@ set +x # Anything other than "true" will use libraries locally. #------------------------------------ -while getopts "c" option; +_build_ufs_options="" + +while getopts "ac" option; do case $option in + a) + echo "Received -a flag, build ufs-weather-model develop branch with aerosols and CCPP physics" + echo "setting coupled=yes and skipping builds not needed for prototype runs" + _build_ufs_options=-a + RUN_CCPP="YES" + COUPLED="YES" + break + ;; c) echo "Received -c flag, build ufs-weather-model develop branch with CCPP physics" echo "setting coupled=yes and skipping builds not needed for prototype runs" RUN_CCPP="YES" COUPLED="YES" + break ;; esac done @@ -49,7 +60,7 @@ source ./machine-setup.sh > /dev/null 2>&1 #------------------------------------ # INCLUDE PARTIAL BUILD #------------------------------------ -. ./partial_build.sh +. ./partial_build.sh $@ #------------------------------------ # Exception Handling Init @@ -87,7 +98,7 @@ if [[ $rc -ne 0 ]] ; then fi ((err+=$rc)) else -./build_ufs_coupled.sh > $logs_dir/build_ufs_coupled.log 2>&1 +./build_ufs_coupled.sh ${_build_ufs_options} > $logs_dir/build_ufs_coupled.log 2>&1 rc=$? if [[ $rc -ne 0 ]] ; then echo "Fatal error in building ufs coupled forecast model." diff --git a/sorc/build_ufs_coupled.sh b/sorc/build_ufs_coupled.sh index 66de17a7fda..75bec964fa3 100755 --- a/sorc/build_ufs_coupled.sh +++ b/sorc/build_ufs_coupled.sh @@ -1,6 +1,25 @@ #! /usr/bin/env bash set -eux +# Build S2S by default +CMAKE_FLAGS="-DS2S=ON -DWW3=ON" +CCPP_SUITES="FV3_GFS_v15p2_coupled,FV3_GFS_v16_coupled,FV3_GFS_v16" + +while [ ${#} -gt 0 ] +do + case "${1}" in + -a|--aerosols) + CMAKE_FLAGS="-DAPP=ATMAERO" + CCPP_SUITES="FV3_GFS_v16" + shift + ;; + *) + echo "Unrecognized option: ${1}" + exit 1 + ;; + esac +done + source ./machine-setup.sh > /dev/null 2>&1 cwd=`pwd` @@ -9,22 +28,18 @@ if [ ! -d "../exec" ]; then mkdir ../exec fi -if [ $target = hera ]; then target=hera.intel ; fi -if [ $target = orion ]; then target=orion.intel ; fi -if [ $target = stampede ]; then target=stampede.intel ; fi +# Set target platform +case "${target}" in + hera|orion|stampede) + target=${target}.intel + ;; +esac MOD_PATH=$cwd/ufs_coupled.fd/modulefiles module purge cd ufs_coupled.fd/ -if [[ -d build ]]; then rm -Rf build; fi -if [[ -e .build_aerosols ]] ; then - module use ${MOD_PATH} - module load ufs_${target} ufs_aerosols_${target} - CMAKE_FLAGS="-DAPP=ATMAERO" CCPP_SUITES="FV3_GFS_v16" ./build.sh -else - module use ${MOD_PATH}/${target} - module load fv3 - CMAKE_FLAGS="-DS2S=ON -DWW3=ON" CCPP_SUITES="FV3_GFS_v15p2_coupled,FV3_GFS_v16_coupled,FV3_GFS_v16" ./build.sh -fi +module use ${MOD_PATH} +module load ufs_${target} +CMAKE_FLAGS="${CMAKE_FLAGS}" CCPP_SUITES="${CCPP_SUITES}" ./build.sh diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 078cd64a887..3ed89732c05 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -13,11 +13,6 @@ do echo "Received -c flag, running coupled model" COUPLED="YES" ;; - a) - echo "Received -a flag, running coupled aerosols model" - AEROSOL="YES" - COUPLED="YES" - ;; :) echo "option -$OPTARG needs an argument" ;; @@ -45,16 +40,9 @@ if [ ${COUPLED:-"NO"} = "NO" ]; then fi else if [[ ! -d ufs_coupled.fd ]] ; then - if [ "${AEROSOL}" = "YES" ] ; then - git clone https://github.com/rmontuoro/ufs-weather-model.git ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1 - cd ufs_coupled.fd - git checkout feature/gocart - echo > .build_aerosols - else - git clone https://github.com/ufs-community/ufs-weather-model ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1 - cd ufs_coupled.fd - git checkout 3e46f5b7050e18884a0bed13691823ad88d443c3 - fi + git clone https://github.com/ufs-community/ufs-weather-model ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1 + cd ufs_coupled.fd + git checkout a842d57f9c352acd3771fbccd8d8eb044558d090 git submodule update --init --recursive cd ${topdir} else diff --git a/sorc/partial_build.sh b/sorc/partial_build.sh index 671ca6da2da..24971568fa9 100755 --- a/sorc/partial_build.sh +++ b/sorc/partial_build.sh @@ -129,54 +129,49 @@ fi } + + usage() { + echo "Usage: $0 [ALL|config=config_file|[select=][prog1[,prog2[,...]]]" 2>&1 + } + # # read command line arguments; processing config file # + declare -a parse_argv=() + coupled=false verbose=false - num_arg=$# - (( num_arg > 1 )) && { - [[ ${1,,} == "--verbose" ]] && { - verbose=true - } || { - echo "Usage: $0 [ALL|config=config_file|[select=][prog1[,prog2[,...]]]" 2>&1 - exit 1 - } - } - (( num_arg == 1 )) && { - ( [[ $1 == "-h" ]] || [[ $1 == "--help" ]] ) && { - echo "Usage: $0 [ALL|config=config_file|[select=][prog1[,prog2[,...]]]" 2>&1 + while [ ${#} -ne 0 ]; do + case "${1}" in + -v|--verbose) + verbose=true + parse_argv+=( "${1}" ) + shift + ;; + -h|--help) + usage exit 2 - } - ( [[ $1 == "-v" ]] || [[ ${1,,} == "--verbose" ]] || [[ $1 == "-c" ]] ) && { - if [[ $1 == "-v" ]]; then - verbose=true - fi - if [[ $1 == "-c" ]]; then - coupled=true - fi - num_arg=0 - } || { - echo "Usage: $0 [ALL|config=config_file|[select=][prog1[,prog2[,...]]]" 2>&1 + ;; + -a|--aerosols|-c|--coupled) + coupled=true + shift + ;; + *) + usage exit 3 - } - } + ;; + esac + done - if (( num_arg == 0 )); then -# -# set default values for partial build -# - if [[ $coupled ]]; then - parse_cfg 1 "config=cpl_build.cfg" ${Build_prg[@]} - else - parse_cfg 1 "config=fv3gfs_build.cfg" ${Build_prg[@]} - fi + if [ "${coupled}" = true ]; then + parse_argv+=( "config=cpl_build.cfg" ) else + parse_argv+=( "config=fv3gfs_build.cfg" ) + fi # # call arguments retriever/config parser # - parse_cfg $num_arg "$@" ${Build_prg[@]} - fi + parse_cfg ${#parse_argv[@]} "${parse_argv[@]}" ${Build_prg[@]} # # print values of build array From a51d4ef77e36995e437f481a10cdef4eea14f35b Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 8 Jul 2021 20:39:17 +0000 Subject: [PATCH 34/39] Add spaces to indent FV3 build section for better readability Refs: #230 --- sorc/build_all.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sorc/build_all.sh b/sorc/build_all.sh index 577f42df052..242e1c002e0 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -78,22 +78,22 @@ fi $Build_fv3gfs && { echo " .... Building forecast model .... " if [ ${COUPLED:-"NO"} = "NO" ]; then -export RUN_CCPP=${RUN_CCPP:-"NO"} -./build_fv3.sh > $logs_dir/build_fv3.log 2>&1 -rc=$? -if [[ $rc -ne 0 ]] ; then + export RUN_CCPP=${RUN_CCPP:-"NO"} + ./build_fv3.sh > $logs_dir/build_fv3.log 2>&1 + rc=$? + if [[ $rc -ne 0 ]] ; then echo "Fatal error in building fv3." echo "The log file is in $logs_dir/build_fv3.log" -fi -((err+=$rc)) + fi + ((err+=$rc)) else -./build_ufs_coupled.sh > $logs_dir/build_ufs_coupled.log 2>&1 -rc=$? -if [[ $rc -ne 0 ]] ; then + ./build_ufs_coupled.sh > $logs_dir/build_ufs_coupled.log 2>&1 + rc=$? + if [[ $rc -ne 0 ]] ; then echo "Fatal error in building ufs coupled forecast model." echo "The log file is in $logs_dir/build_ufs_coupled.log" -fi -((err+=$rc)) + fi + ((err+=$rc)) fi } From 5bec2a725271d7423690d1050771eaf83db562d3 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 8 Jul 2021 20:41:14 +0000 Subject: [PATCH 35/39] Add FV3_GFS_v16_couplednsst CCPP suite option to UFS coupled build and FV3 namelist parser Refs: #230 --- sorc/build_ufs_coupled.sh | 2 +- ush/parsing_namelists_FV3.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sorc/build_ufs_coupled.sh b/sorc/build_ufs_coupled.sh index e2176967178..9d368fff95a 100755 --- a/sorc/build_ufs_coupled.sh +++ b/sorc/build_ufs_coupled.sh @@ -25,5 +25,5 @@ if [[ -d GOCART ]]; then CMAKE_FLAGS="-DAPP=ATMAERO" CCPP_SUITES="FV3_GFS_v16" ./build.sh ./build.sh else - CMAKE_FLAGS="-DAPP=S2SW" CCPP_SUITES="FV3_GFS_v16_coupled,FV3_GFS_v16" ./build.sh + CMAKE_FLAGS="-DAPP=S2SW" CCPP_SUITES="FV3_GFS_v16_coupled,FV3_GFS_v16_couplednsst,FV3_GFS_v16" ./build.sh fi diff --git a/ush/parsing_namelists_FV3.sh b/ush/parsing_namelists_FV3.sh index ddcca3a2412..b14eec96d51 100755 --- a/ush/parsing_namelists_FV3.sh +++ b/ush/parsing_namelists_FV3.sh @@ -215,7 +215,7 @@ elif [ $CCPP_SUITE = "FV3_GSD_v0" ]; then min_lakeice = ${min_lakeice:-"0.15"} min_seaice = ${min_seaice:-"0.15"} EOF -elif [ $CCPP_SUITE = "FV3_GFS_v16_coupled" ]; then +elif [ $CCPP_SUITE = "FV3_GFS_v16_coupled" -o $CCPP_SUITE = "FV3_GFS_v16_couplednsst" ]; then cat >> input.nml << EOF iovr = ${iovr:-"3"} ltaerosol = ${ltaerosol:-".false."} From bbcad18ba4e91a6c242a0276f7c52525c056f772 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 8 Jul 2021 20:42:46 +0000 Subject: [PATCH 36/39] Replace CROW yaml nst_spinup_logic in nsst.yaml with NST_SPINUP variable - removed CROW FirstTrue check when building config.nsst from nsst.yaml - replace nst_spinup_logic with NST_SPINUP variable that is parsed from variable dictionary - nst_spinup_logic not needed currently Refs: #230 --- workflow/config/nsst.yaml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/workflow/config/nsst.yaml b/workflow/config/nsst.yaml index 6a188a03771..bc965adf4b5 100644 --- a/workflow/config/nsst.yaml +++ b/workflow/config/nsst.yaml @@ -4,14 +4,6 @@ config_nsst: filename: config.nsst - nst_spinup_logic: !FirstTrue - - when: !calc doc.nsst.get("NST_SPINUP","") - do: !expand "export NST_SPINUP={doc.nsst.NST_SPINUP}" - - otherwise: | - export NST_SPINUP=0 - #if [[ "$CDATE" = $SDATE ]]; then - # export NST_SPINUP=1 - #fi content: !expand | #!/bin/ksh -x @@ -30,7 +22,7 @@ config_nsst: export NST_MODEL={doc.nsst.NST_MODEL} # nstf_name(2) : NST_SPINUP : 0 = OFF, 1 = ON, - {nst_spinup_logic} + export NST_SPINUP={doc.nsst.NST_SPINUP} # nstf_name(3) : NST_RESV (Reserved, NSST Analysis) : 0 = OFF, 1 = ON export NST_RESV={doc.nsst.NST_RESV} From cec61887b6c5635828045b422877de7ae61ca1c0 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Fri, 9 Jul 2021 13:13:48 +0000 Subject: [PATCH 37/39] Add NSST settings to the coupled_free_forecast_wave.yaml case file - set NST_MODEL=2 (turns it on) - add additional NSST namelist variables: NST_SPINUP, NST_RESV, ZSEA1, ZSEA2 - set NST_SPINUP=1 (NSST will spinup) - change CCPP_SUITE from FV3_GFS_v16_coupled to FV3_GFS_v16_couplednsst Refs: #230 --- workflow/cases/coupled_free_forecast_wave.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/workflow/cases/coupled_free_forecast_wave.yaml b/workflow/cases/coupled_free_forecast_wave.yaml index 90fbd8b46ee..c6329bfd313 100644 --- a/workflow/cases/coupled_free_forecast_wave.yaml +++ b/workflow/cases/coupled_free_forecast_wave.yaml @@ -22,7 +22,11 @@ case: KEEPDATA: NO nsst: - NST_MODEL: 0 + NST_MODEL: 2 + NST_SPINUP: 1 + NST_RESV: 0 + ZSEA1: 0 + ZSEA2: 0 output_settings: OCN_INTERVAL: 24 @@ -39,7 +43,7 @@ case: min_lakeice: 0.15 min_seaice: 1.0e-11 SEEDLET: 10 - CCPP_SUITE: FV3_GFS_v16_coupled + CCPP_SUITE: FV3_GFS_v16_couplednsst CPL_ATMIC: CFSRfracL127 nst_anl: yes psm_bc: 1 From d3039974cd7887ed3f984f4e7c586408bbda3678 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Wed, 14 Jul 2021 04:22:59 +0000 Subject: [PATCH 38/39] Update nems.configure to inclue eps_imesh Adds the eps_imesh setting to the nems.configure files when CICE is on. Refs: #324, #333 --- ush/nems.configure.cpld.IN | 1 + ush/nems.configure.cpld_wave.IN | 1 + ush/nems.configure.medcold_atm_ocn_ice.IN | 1 + ush/nems.configure.medcold_atm_ocn_ice_wav.IN | 1 + ush/nems_configure.sh | 2 ++ 5 files changed, 6 insertions(+) diff --git a/ush/nems.configure.cpld.IN b/ush/nems.configure.cpld.IN index 69b6c737f3a..8a070787f51 100644 --- a/ush/nems.configure.cpld.IN +++ b/ush/nems.configure.cpld.IN @@ -101,4 +101,5 @@ ALLCOMP_attributes:: dbug_flag = @[cap_dbug_flag] use_coldstart = @[use_coldstart] use_mommesh = @[use_mommesh] + eps_imesh = @[eps_imesh] :: diff --git a/ush/nems.configure.cpld_wave.IN b/ush/nems.configure.cpld_wave.IN index e4d9eca31d2..2b8a49e56b6 100644 --- a/ush/nems.configure.cpld_wave.IN +++ b/ush/nems.configure.cpld_wave.IN @@ -115,4 +115,5 @@ ALLCOMP_attributes:: dbug_flag = @[cap_dbug_flag] use_coldstart = @[use_coldstart] use_mommesh = @[use_mommesh] + eps_imesh = @[eps_imesh] :: diff --git a/ush/nems.configure.medcold_atm_ocn_ice.IN b/ush/nems.configure.medcold_atm_ocn_ice.IN index fe4030274e5..80a1527b9aa 100644 --- a/ush/nems.configure.medcold_atm_ocn_ice.IN +++ b/ush/nems.configure.medcold_atm_ocn_ice.IN @@ -100,4 +100,5 @@ ALLCOMP_attributes:: restart_option = nhours restart_ymd = -999 dbug_flag = 0 + eps_imesh = @[eps_imesh] :: diff --git a/ush/nems.configure.medcold_atm_ocn_ice_wav.IN b/ush/nems.configure.medcold_atm_ocn_ice_wav.IN index fa8ab4e4ff1..2ef5f706bf3 100644 --- a/ush/nems.configure.medcold_atm_ocn_ice_wav.IN +++ b/ush/nems.configure.medcold_atm_ocn_ice_wav.IN @@ -113,4 +113,5 @@ ALLCOMP_attributes:: restart_option = nhours restart_ymd = -999 dbug_flag = 0 + eps_imesh = @[eps_imesh] :: diff --git a/ush/nems_configure.sh b/ush/nems_configure.sh index c904948f999..7916a6a6cc7 100755 --- a/ush/nems_configure.sh +++ b/ush/nems_configure.sh @@ -47,6 +47,7 @@ CHMPETS=${CHMPETS:-${ATMPETS}} USE_MOMMESH=${USE_MOMMESH:-"true"} + rm -f $DATA/nems.configure med_petlist_bounds=${med_petlist_bounds:-"0 $(( $MEDPETS-1 ))"} @@ -82,6 +83,7 @@ if [ $cplflx = .true. ]; then sed -i -e "s;@\[coupling_interval_fast_sec\];$CPL_FAST;g" tmp1 sed -i -e "s;@\[RESTART_N\];$restart_interval_nems;g" tmp1 sed -i -e "s;@\[use_mommesh\];$USE_MOMMESH;g" tmp1 + sed -i -e "s;@\[eps_imesh\];$ICERESdec;g" tmp1 fi if [ $cplwav = .true. ]; then sed -i -e "s;@\[wav_model\];ww3;g" tmp1 From dc970f9f3644ed67624141d08ca4b9841eccea00 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Wed, 14 Jul 2021 16:21:32 +0000 Subject: [PATCH 39/39] Change forecast module back to module_base The modulefile in UFS doesn't load some modules needed for the forecast script at runtime, notably produtil, which provides err_chk. The module for the forecast job is changed back to module_base for the time being. Refs: #336 --- modulefiles/module_base.hera | 2 ++ ush/load_fv3gfs_modules.sh | 28 ++-------------------------- 2 files changed, 4 insertions(+), 26 deletions(-) diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera index a14ab46cb8c..de89fbf9547 100644 --- a/modulefiles/module_base.hera +++ b/modulefiles/module_base.hera @@ -9,6 +9,8 @@ module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack module load hpc/1.1.0 module load hpc-intel/18.0.5.274 module load hpc-impi/2018.0.4 +# use shared memory and OpenFabrics Alliance (OFA) fabric with Intel MPI to circumvent RDMA-related bug in DAPL. +setenv I_MPI_FABRICS shm:ofa # Non-MPI module load g2/3.4.2 diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index d9ae4f0e9ad..521d0eccfb0 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -29,32 +29,8 @@ fi platform=$( get_platform ) -moduledir= -modulelist= -case "${platform}" in - hera) - moduledir="$HOMEgfs/sorc/ufs_coupled.fd/modulefiles" - target=${platform}.intel - if [[ "${tasktype}" = "forecast" && -r ${moduledir}/ufs_${target} ]] ; then - modulelist=ufs_${target} - if [[ "$( grep UFS_GOCART ${HOMEgfs}/sorc/ufs_coupled.fd/build/CMakeCache.txt 2>/dev/null | cut -d= -f2 )" = "ON" ]] ; then - # add aerosols modulefile - modulelist="${modulelist} ufs_aerosols_${target}" - fi - else - moduledir="$HOMEgfs/modulefiles" - modulelist="module_base.${platform}" - fi - ;; - unknown) - echo "Unknown platform" - exit 1 - ;; - *) - moduledir="$HOMEgfs/modulefiles" - modulelist="module_base.${platform}" - ;; -esac +moduledir="$HOMEgfs/modulefiles" +modulelist="module_base.${platform}" # Load our modules: module use ${moduledir}