Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
a4575c6
replace @TEMPLATE@ with {{ TEMPLATE }} jinja
aerorahul Feb 25, 2025
ceb2dff
do the same for gefs and sfs configs
aerorahul Feb 26, 2025
5dd518a
update setup_expt.py to render jinja templates when filling expdir. a…
aerorahul Feb 27, 2025
44952d8
Merge branch 'develop' into feature/jinja-configs
aerorahul Feb 27, 2025
0183c1a
improve logging of setup scripts
aerorahul Feb 28, 2025
1f05869
fix pynorms
aerorahul Feb 28, 2025
441c67e
fix stray quotes
aerorahul Feb 28, 2025
55c6144
Merge branch 'develop' into feature/jinja-configs
aerorahul Feb 28, 2025
05c288a
Update sorc/link_workflow.sh
aerorahul Mar 3, 2025
801e978
Merge branch 'develop' into feature/jinja-configs
aerorahul Mar 4, 2025
2979d39
update configs to allow for a default value for the resources and upd…
aerorahul Mar 4, 2025
62b28b9
remove ACCOUNT from hosts/ yamls as it is an argument to setup_expt.py
aerorahul Mar 4, 2025
53d91e4
Merge branch 'develop' into feature/jinja-configs
aerorahul Mar 22, 2025
ef28763
rearrange options into groups
aerorahul Mar 22, 2025
57c1f0c
hosts/container.yaml update
aerorahul Mar 22, 2025
a4eb403
update hosts.py to accept values set by detect_machine.sh
aerorahul Mar 22, 2025
380ab2f
pass account to setup_expt
aerorahul Mar 23, 2025
aff38e3
EOL missing in these files
aerorahul Mar 23, 2025
51f2a48
account must be specified in create_experiment.yaml, not base section
aerorahul Mar 23, 2025
b0e5583
satisfy the request in issue #3439
aerorahul Mar 23, 2025
16f097b
fix shellcheck identified issues
aerorahul Mar 23, 2025
5763cc1
make CASE_ENS consistent w/ setup scripts and use elsewhere
aerorahul Mar 23, 2025
ca1fc05
Merge branch 'develop' into feature/jinja-configs
DavidHuber-NOAA Mar 24, 2025
ebf932f
Enable METP on C6
aerorahul Mar 24, 2025
80437df
Update sorc/link_workflow.sh
aerorahul Mar 24, 2025
d2ab888
Merge branch 'develop' into feature/jinja-configs
aerorahul Mar 24, 2025
1d0aeea
reduce the number of logging calls by disabling the decorator. When …
aerorahul Mar 24, 2025
4a68871
Update workflow/setup_expt.py
aerorahul Mar 24, 2025
23e607b
fix pynorm stuff
aerorahul Mar 24, 2025
76fcfd1
enable ufs_hybatmDA test on c6
aerorahul Mar 24, 2025
5619847
make --account required in build_compute.py
aerorahul Mar 25, 2025
9b502c4
build_compute.sh will provide account to the python script
aerorahul Mar 25, 2025
8c600d9
update comment
aerorahul Mar 25, 2025
ab72460
Merge branch 'develop' into feature/jinja-configs
aerorahul Mar 25, 2025
2205a77
Merge branch 'develop' into feature/jinja-configs
aerorahul Mar 26, 2025
fdf8046
require -A when invoking build_compute.sh
aerorahul Mar 26, 2025
4f5c85d
Merge branch 'develop' into feature/jinja-configs
aerorahul Mar 27, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ nohup.out
exec/
build*/
install*/
venv/

# Ignore fix directory symlinks
#------------------------------
Expand Down
1 change: 0 additions & 1 deletion parm/config/gefs/config.aero

This file was deleted.

1 change: 1 addition & 0 deletions parm/config/gefs/config.aero.j2
114 changes: 57 additions & 57 deletions parm/config/gefs/config.base → parm/config/gefs/config.base.j2
Original file line number Diff line number Diff line change
Expand Up @@ -6,24 +6,24 @@
echo "BEGIN: config.base"

# Machine environment
export machine="@MACHINE@"
export machine="{{ MACHINE }}"

# EMC parallel or NCO production
export RUN_ENVIR="emc"

# Account, queue, etc.
export ACCOUNT="@ACCOUNT@"
export QUEUE="@QUEUE@"
export QUEUE_SERVICE="@QUEUE_SERVICE@"
export PARTITION_BATCH="@PARTITION_BATCH@"
export PARTITION_SERVICE="@PARTITION_SERVICE@"
export RESERVATION="@RESERVATION@"
export ACCOUNT="{{ ACCOUNT }}"
export QUEUE="{{ QUEUE }}"
export QUEUE_SERVICE="{{ QUEUE_SERVICE }}"
export PARTITION_BATCH="{{ PARTITION_BATCH }}"
export PARTITION_SERVICE="{{ PARTITION_SERVICE }}"
export RESERVATION="{{ RESERVATION }}"

# Project to use in mass store:
export HPSS_PROJECT="@HPSS_PROJECT@"
export HPSS_PROJECT="{{ HPSS_PROJECT }}"

# Directories relative to installation areas:
export HOMEgfs=@HOMEgfs@
export HOMEgfs="{{ HOMEgfs }}"
export EXECgfs=${HOMEgfs}/exec
export FIXgfs=${HOMEgfs}/fix
export PARMgfs=${HOMEgfs}/parm
Expand All @@ -35,26 +35,26 @@ export FIXugwd=${FIXgfs}/ugwd
########################################################################

# GLOBAL static environment parameters
export PACKAGEROOT="@PACKAGEROOT@" # TODO: set via prod_envir in Ops
export COMROOT="@COMROOT@" # TODO: set via prod_envir in Ops
export COMINsyn="@COMINsyn@"
export PACKAGEROOT="{{ PACKAGEROOT }}" # TODO: set via prod_envir in Ops
export COMROOT="{{ COMROOT }}" # TODO: set via prod_envir in Ops
export COMINsyn="{{ COMINsyn }}"

# USER specific paths
export HOMEDIR="@HOMEDIR@"
export STMP="@STMP@"
export PTMP="@PTMP@"
export NOSCRUB="@NOSCRUB@"
export HOMEDIR="{{ HOMEDIR }}"
export STMP="{{ STMP }}"
export PTMP="{{ PTMP }}"
export NOSCRUB="{{ NOSCRUB }}"

# Base directories for various builds
export BASE_GIT="@BASE_GIT@"
export BASE_GIT="{{ BASE_GIT }}"

# Base directory for staged data
export BASE_DATA="@BASE_DATA@"
export BASE_DATA="{{ BASE_DATA }}"

# Toggle to turn on/off GFS downstream processing.
export DO_BUFRSND="@DO_BUFRSND@" # BUFR sounding products
export DO_GEMPAK="@DO_GEMPAK@" # GEMPAK products
export DO_AWIPS="@DO_AWIPS@" # AWIPS products
export DO_BUFRSND="{{ DO_BUFRSND }}" # BUFR sounding products
export DO_GEMPAK="{{ DO_GEMPAK }}" # GEMPAK products
export DO_AWIPS="{{ DO_AWIPS }}" # AWIPS products

# NO for retrospective parallel; YES for real-time parallel
# arch.sh uses REALTIME for MOS. Need to set REALTIME=YES
Expand All @@ -63,8 +63,8 @@ export DO_AWIPS="@DO_AWIPS@" # AWIPS products
export REALTIME="YES"

# Experiment mode (cycled or forecast-only)
export MODE="@MODE@" # cycled/forecast-only
export DO_TEST_MODE="@DO_TEST_MODE@" # option to change configuration for automated testing
export MODE="{{ MODE }}" # cycled/forecast-only
export DO_TEST_MODE="{{ DO_TEST_MODE }}" # option to change configuration for automated testing

####################################################
# DO NOT ADD MACHINE DEPENDENT STUFF BELOW THIS LINE
Expand All @@ -80,10 +80,10 @@ export NCP="/bin/cp -p"
export NMV="/bin/mv"
export NLN="/bin/ln -sf"
export VERBOSE="YES"
export KEEPDATA="@KEEPDATA@"
export KEEPDATA="{{ KEEPDATA }}"
export DEBUG_POSTSCRIPT="NO" # PBS only; sets debug=true
export CHGRP_RSTPROD="@CHGRP_RSTPROD@"
export CHGRP_CMD="@CHGRP_CMD@"
export CHGRP_RSTPROD="{{ CHGRP_RSTPROD }}"
export CHGRP_CMD="{{ CHGRP_CMD }}"
export NCDUMP="${NETCDF:-${netcdf_c_ROOT:-}}/bin/ncdump"
export NCLEN="${HOMEgfs}/ush/getncdimlen"

Expand All @@ -92,17 +92,17 @@ export BASE_ENV="${HOMEgfs}/env"
export BASE_JOB="${HOMEgfs}/jobs/rocoto"

# EXPERIMENT specific environment parameters
export SDATE=@SDATE@
export EDATE=@EDATE@
export EXP_WARM_START="@EXP_WARM_START@"
export SDATE="{{ SDATE }}"
export EDATE="{{ EDATE }}"
export EXP_WARM_START="{{ EXP_WARM_START }}"
export assim_freq=6
export PSLOT="@PSLOT@"
export EXPDIR="@EXPDIR@/${PSLOT}"
export ROTDIR="@COMROOT@/${PSLOT}"
export PSLOT="{{ PSLOT }}"
export EXPDIR="{{ EXPDIR }}/${PSLOT}"
export ROTDIR="{{ COMROOT }}/${PSLOT}"

export ARCDIR="${NOSCRUB}/archive/${PSLOT}"
export ATARDIR="@ATARDIR@"
export FETCHDIR="@FETCHDIR@" # HPSS or local directory where IC tarball(s) can be found.
export ATARDIR="{{ ATARDIR }}"
export FETCHDIR="{{ FETCHDIR }}" # HPSS or local directory where IC tarball(s) can be found.

# Commonly defined parameters in JJOBS
export envir=${envir:-"prod"}
Expand All @@ -127,15 +127,15 @@ export SENDDBN=${SENDDBN:-"NO"}
export DBNROOT=${DBNROOT:-${UTILROOT:-}/fakedbn}

# APP settings
export APP=@APP@
export APP="{{ APP }}"

# Defaults:
export DO_ATM="YES"
export DO_COUPLED="NO"
export DO_WAVE="NO"
export DO_OCN="NO"
export DO_ICE="NO"
export DO_EXTRACTVARS="@DO_EXTRACTVARS@" # Option to process and extract a subset of products to save on disk
export DO_EXTRACTVARS="{{ DO_EXTRACTVARS }}" # Option to process and extract a subset of products to save on disk
export DO_AERO_FCST="NO"
export DOBNDPNT_WAVE="NO" # The GEFS buoys file does not currently have any boundary points
export DOIBP_WAV="NO" # Option to create point outputs from input boundary points
Expand All @@ -153,9 +153,9 @@ export OPS_RES="C768" # Do not change

# Resolution specific parameters
export LEVS=128
export CASE="@CASECTL@" # CASE is required in GEFS to determine ocean/ice/wave resolutions
export CASE_ENS="@CASEENS@"
export OCNRES="@OCNRES@"
export CASE="{{ CASECTL }}" # CASE is required in GEFS to determine ocean/ice/wave resolutions
export CASE_ENS="{{ CASEENS }}"
export OCNRES="{{ OCNRES }}"
export ICERES="${OCNRES}"
# These are the currently recommended grid-combinations
case "${CASE}" in
Expand Down Expand Up @@ -220,34 +220,34 @@ export FHOUT_ICE=3
export FHOUT_AERO=3

# GFS cycle info
export INTERVAL_GFS=@INTERVAL_GFS@ # Frequency of GFS forecast
export SDATE_GFS=@SDATE_GFS@
export INTERVAL_GFS="{{ INTERVAL_GFS }}" # Frequency of GFS forecast
export SDATE_GFS="{{ SDATE_GFS }}"

# set variables needed for use with REPLAY ICs
export REPLAY_ICS=@REPLAY_ICS@
export REPLAY_ICS="{{ REPLAY_ICS }}"
if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
export OFFSET_START_HOUR=$(( assim_freq / 2 ))
echo "WARNING: Replay ICs require perturbation files, ignoring any previous settings"
export USE_OCN_ENS_PERTURB_FILES="YES"
export USE_ATM_ENS_PERTURB_FILES="YES"
else
export OFFSET_START_HOUR=0
export USE_OCN_ENS_PERTURB_FILES=@USE_OCN_ENS_PERTURB_FILES@
export USE_ATM_ENS_PERTURB_FILES=@USE_ATM_ENS_PERTURB_FILES@
export USE_OCN_ENS_PERTURB_FILES="{{ USE_OCN_ENS_PERTURB_FILES }}"
export USE_ATM_ENS_PERTURB_FILES="{{ USE_ATM_ENS_PERTURB_FILES }}"
fi

# GFS output and frequency
export FHMIN_GFS=0
export FHMAX_GFS="@FHMAX_GFS@"
export FHMAX_GFS="{{ FHMAX_GFS }}"
# Intermediate times to stop forecast when running in segments
breakpnts="@FCST_BREAKPOINTS@"
breakpnts="{{ FCST_BREAKPOINTS }}"
export FCST_SEGMENTS="${FHMIN_GFS},${breakpnts:+${breakpnts},}${FHMAX_GFS}"

export FHOUT_GFS=@FHOUT_GFS@
export FHMAX_HF_GFS=@FHMAX_HF_GFS@
export FHOUT_HF_GFS=@FHOUT_HF_GFS@
export FHOUT_OCN_GFS=@FHOUT_OCN_GFS@
export FHOUT_ICE_GFS=@FHOUT_ICE_GFS@
export FHOUT_GFS="{{ FHOUT_GFS }}"
export FHMAX_HF_GFS="{{ FHMAX_HF_GFS }}"
export FHOUT_HF_GFS="{{ FHOUT_HF_GFS }}"
export FHOUT_OCN_GFS="{{ FHOUT_OCN_GFS }}"
export FHOUT_ICE_GFS="{{ FHOUT_ICE_GFS }}"
export FHMIN_WAV=${OFFSET_START_HOUR:-0}
export FHOUT_WAV=3
export FHMAX_HF_WAV=120
Expand Down Expand Up @@ -290,7 +290,7 @@ export DO_MERGENSST="NO"
export DO_STARTMEM_FROM_JEDIICE="NO"

# Hybrid related
export NMEM_ENS=@NMEM_ENS@
export NMEM_ENS="{{ NMEM_ENS }}"

# set default member number memdir for control
# this will be overwridden for the perturbed members
Expand Down Expand Up @@ -323,19 +323,19 @@ if [[ ${DONST} = "YES" ]]; then export FNTSFA=" "; fi
export nst_anl=.true.

# Make the nsstbufr file on the fly or use the GDA version
export MAKE_NSSTBUFR="@MAKE_NSSTBUFR@"
export MAKE_NSSTBUFR="{{ MAKE_NSSTBUFR }}"

# Make the aircraft prepbufr file on the fly or use the GDA version
export MAKE_ACFTBUFR="@MAKE_ACFTBUFR@"
export MAKE_ACFTBUFR="{{ MAKE_ACFTBUFR }}"

# Verification options
export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config.metp
export DO_FIT2OBS="NO" # Run fit to observations package

# Archiving options
export VRFYARCH="@VRFYARCH@" # save verification data locally
export HPSSARCH="@HPSSARCH@" # save data to HPSS archive
export LOCALARCH="@LOCALARCH@" # save data to local archive
export VRFYARCH="{{ VRFYARCH }}" # save verification data locally
export HPSSARCH="{{ HPSSARCH }}" # save data to HPSS archive
export LOCALARCH="{{ LOCALARCH }}" # save data to local archive
if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then
echo "Both HPSS and local archiving selected. Please choose one or the other."
exit 3
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ export FHOUT=${FHOUT_GFS}
export FHOUT_HF=${FHOUT_HF_GFS}
export FHOUT_OCN=${FHOUT_OCN_GFS}
export FHOUT_ICE=${FHOUT_ICE_GFS}
export FHZER=@FHZER@
export FHZER="{{ FHZER }}"

# Get task specific resources
source "${EXPDIR}/config.resources" fcst
Expand All @@ -55,8 +55,8 @@ export FCSTEXEC="${NET}_model.x"

#######################################################################
# Model configuration
export TYPE=@TYPE@
export MONO=@MONO@
export TYPE="{{ TYPE }}"
export MONO="{{ MONO }}"

# Use stratosphere h2o physics
export h2o_phys=".true."
Expand Down Expand Up @@ -268,7 +268,7 @@ else
export io_layout="1,1"
fi

export reforecast=@reforecast@
export reforecast="{{ reforecast }}"
# Remember config.efcs will over-ride these values for ensemble forecasts
# if these variables are re-defined there.
# Otherwise, the ensemble forecast will inherit from config.fcst
Expand Down
1 change: 0 additions & 1 deletion parm/config/gefs/config.ocn

This file was deleted.

1 change: 1 addition & 0 deletions parm/config/gefs/config.ocn.j2
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ echo "BEGIN: config.stage_ic"
# Get task specific resources
source "${EXPDIR}/config.resources" stage_ic

export ICSDIR="@ICSDIR@" # User provided ICSDIR; blank if not provided
export BASE_IC="@BASE_IC@" # Platform home for staged ICs
export ICSDIR="{{ ICSDIR }}" # User provided ICSDIR; blank if not provided
export BASE_IC="{{ BASE_IC }}" # Platform home for staged ICs

export STAGE_IC_YAML_TMPL="${PARMgfs}/stage/master_gefs.yaml.j2"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# UFS-Aerosols settings

# Path to the input data tree
export AERO_INPUTS_DIR=@AERO_INPUTS_DIR@
export AERO_INPUTS_DIR="{{ AERO_INPUTS_DIR }}"

export AERO_DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table.aero"
export AERO_FIELD_TABLE="${PARMgfs}/ufs/fv3/field_table.aero"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ export JEDI_CONFIG_YAML="${PARMgfs}/gdas/aeroanl_jedi_config.yaml.j2"
export AERO_STAGE_VARIATIONAL_TMPL="${PARMgfs}/gdas/aero_stage_variational.yaml.j2"
export AERO_FINALIZE_VARIATIONAL_TMPL="${PARMgfs}/gdas/aero_finalize_variational.yaml.j2"

export io_layout_x=@IO_LAYOUT_X@
export io_layout_y=@IO_LAYOUT_Y@
export io_layout_x="{{ IO_LAYOUT_X }}"
export io_layout_y="{{ IO_LAYOUT_Y }}"

export JEDIEXE="${EXECgfs}/gdas.x"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@

echo "BEGIN: config.atmanl"

export JCB_ALGO_YAML_VAR=@JCB_ALGO_YAML_VAR@
export JCB_ALGO_YAML_FV3INC=@JCB_ALGO_YAML_FV3INC@
export JCB_ALGO_YAML_VAR="{{ JCB_ALGO_YAML_VAR }}"
export JCB_ALGO_YAML_FV3INC="{{ JCB_ALGO_YAML_FV3INC }}"

export STATICB_TYPE=@STATICB_TYPE@
export STATICB_TYPE="{{ STATICB_TYPE }}"
export LOCALIZATION_TYPE="bump"
export INTERP_METHOD='barycentric'

Expand All @@ -29,10 +29,10 @@ export VAR_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_bkg.yaml.j2"
export BERROR_STAGING_YAML="${PARMgfs}/gdas/staging/atm_berror_${STATICB_TYPE}.yaml.j2"
export FV3ENS_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_fv3ens.yaml.j2"

export layout_x_atmanl=@LAYOUT_X_ATMANL@
export layout_y_atmanl=@LAYOUT_Y_ATMANL@

export io_layout_x=@IO_LAYOUT_X@
export io_layout_y=@IO_LAYOUT_Y@
export layout_x_atmanl="{{ LAYOUT_X_ATMANL }}"
export layout_y_atmanl="{{ LAYOUT_Y_ATMANL }}"
"
export io_layout_x="{{ IO_LAYOUT_X }}"
export io_layout_y="{{ IO_LAYOUT_Y }}"

echo "END: config.atmanl"
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@

echo "BEGIN: config.atmensanl"

export JCB_ALGO_YAML_LETKF=@JCB_ALGO_YAML_LETKF@
export JCB_ALGO_YAML_OBS=@JCB_ALGO_YAML_OBS@
export JCB_ALGO_YAML_SOL=@JCB_ALGO_YAML_SOL@
export JCB_ALGO_YAML_FV3INC=@JCB_ALGO_YAML_FV3INC@
export JCB_ALGO_YAML_LETKF="{{ JCB_ALGO_YAML_LETKF }}"
export JCB_ALGO_YAML_OBS="{{ JCB_ALGO_YAML_OBS }}"
export JCB_ALGO_YAML_SOL="{{ JCB_ALGO_YAML_SOL }}"
export JCB_ALGO_YAML_FV3INC="{{ JCB_ALGO_YAML_FV3INC }}"

export INTERP_METHOD='barycentric'

Expand All @@ -18,10 +18,10 @@ export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atmensanl_jedi_config.yaml.j2"
export LGETKF_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_lgetkf_bkg.yaml.j2"

export layout_x_atmensanl=@LAYOUT_X_ATMENSANL@
export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@
export layout_x_atmensanl="{{ LAYOUT_X_ATMENSANL }}"
export layout_y_atmensanl="{{ LAYOUT_Y_ATMENSANL }}"

export io_layout_x=@IO_LAYOUT_X@
export io_layout_y=@IO_LAYOUT_Y@
export io_layout_x="{{ IO_LAYOUT_X }}"
export io_layout_y="{{ IO_LAYOUT_Y }}"

echo "END: config.atmensanl"
Loading