From f6a3be3a428e3344b210a01f9b4ed35a86c7f97d Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Tue, 6 Oct 2020 15:23:14 -0500 Subject: [PATCH 01/27] Add ORION as a new platform. Common sets of changes: ---------------------- (A) Add ORION stanza to case statement that sets the run command and the resource limits. (B) Edit comments. File-by-file changes: -------------------- jobs/JREGIONAL_MAKE_ICS: (A) jobs/JREGIONAL_MAKE_LBCS: (A) scripts/exregional_make_grid.sh: (A), (B) scripts/exregional_make_orog.sh: (A), (B) scripts/exregional_make_sfc_climo.sh: (A), (B) scripts/exregional_run_fcst.sh: (A) scripts/exregional_run_post.sh: (A) ush/set_extrn_mdl_params.sh: (B) * Add ORION stanzas to some of the case statements that set EXTRN_MDL_SYSBASEDIR_ICS and EXTRN_MDL_SYSBASEDIR_LBCS for certain external models. ush/setup.sh: * Add ORION stanza to case statement that sets the number of cores per node, the scheduler, and various queue-related workflow parameters. * Add ORION stanza to case statement that sets the workflow variables FIXgsm and SFC_CLIMO_INPUT_DIR. ush/valid_param_vals.sh: * Add ORION to list of valid machine names. --- jobs/JREGIONAL_MAKE_ICS | 6 +++ jobs/JREGIONAL_MAKE_LBCS | 5 +++ scripts/exregional_make_grid.sh | 23 +++------- scripts/exregional_make_orog.sh | 15 +++---- scripts/exregional_make_sfc_climo.sh | 6 ++- scripts/exregional_run_fcst.sh | 8 +++- scripts/exregional_run_post.sh | 10 +++-- ush/set_extrn_mdl_params.sh | 64 ++++++++++++++++------------ ush/setup.sh | 24 +++++++---- ush/valid_param_vals.sh | 2 +- 10 files changed, 97 insertions(+), 66 deletions(-) diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS index b8e872837..6e54268ae 100755 --- a/jobs/JREGIONAL_MAKE_ICS +++ b/jobs/JREGIONAL_MAKE_ICS @@ -92,10 +92,16 @@ case "$MACHINE" in nprocs=$(( NNODES_MAKE_ICS*PPN_MAKE_ICS )) APRUN="mpirun -np $nprocs" ;; + "STAMPEDE") APRUN="ibrun" ;; + "ORION") + ulimit -s unlimited + APRUN="srun" + ;; + esac # #----------------------------------------------------------------------- diff --git a/jobs/JREGIONAL_MAKE_LBCS b/jobs/JREGIONAL_MAKE_LBCS index 7b53dab31..5f2cc0be6 100755 --- a/jobs/JREGIONAL_MAKE_LBCS +++ b/jobs/JREGIONAL_MAKE_LBCS @@ -97,6 +97,11 @@ case "$MACHINE" in APRUN="ibrun" ;; + "ORION") + ulimit -s unlimited + APRUN="srun" + ;; + esac # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 48cfb49a2..9426c856a 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -90,19 +90,14 @@ export OMP_STACKSIZE=2048m # #----------------------------------------------------------------------- # -# Load modules and set various computational parameters and directories. -# -# Note: -# These module loads should all be moved to modulefiles. This has been -# done for Hera but must still be done for other machines. +# Set the machine-dependent run command. Also, set resource limits as +# necessary. # #----------------------------------------------------------------------- # case $MACHINE in - "WCOSS_C" | "WCOSS") -# { save_shell_opts; set +x; } > /dev/null 2>&1 . $MODULESHOME/init/sh @@ -119,27 +114,20 @@ case $MACHINE in ulimit -a ;; - "HERA") -# APRUN="time" -# # ulimit -s unlimited # ulimit -a ;; -# "JET") -# APRUN="time" ulimit -a ;; "ODIN") -# export APRUN="srun -n 1" - ulimit -s unlimited ulimit -a ;; @@ -149,12 +137,15 @@ case $MACHINE in ;; "STAMPEDE") -# export APRUN="time" - ulimit -s unlimited ulimit -a ;; + +"ORION") + APRUN="time" + ;; + esac # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index a8a39dcc0..e5d429a65 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -98,9 +98,7 @@ export OMP_STACKSIZE=2048m # case $MACHINE in - "WCOSS_C" | "WCOSS") -# { save_shell_opts; set +x; } > /dev/null 2>&1 . $MODULESHOME/init/sh @@ -117,14 +115,12 @@ case $MACHINE in ulimit -a ;; - "HERA") ulimit -s unlimited ulimit -a APRUN="time" ;; - "JET") ulimit -s unlimited ulimit -a @@ -133,14 +129,11 @@ case $MACHINE in "ODIN") -# export APRUN="srun -n 1" - ulimit -s unlimited ulimit -a ;; - "CHEYENNE") APRUN="time" ;; @@ -150,6 +143,12 @@ case $MACHINE in export topo_dir="/work/00315/tg455890/stampede2/regional_fv3/fix_orog" ;; +"ORION") + ulimit -s unlimited + ulimit -a + APRUN="time" + ;; + esac # #----------------------------------------------------------------------- @@ -322,7 +321,7 @@ ${tmp_dir}" \ ;; -"CHEYENNE" | "HERA" | "JET" | "ODIN" | "STAMPEDE") +"CHEYENNE" | "HERA" | "JET" | "ODIN" | "STAMPEDE" | "ORION") $APRUN "${exec_fp}" < "${input_redirect_fn}" || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates the raw orography file returned diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index 938ede5e9..b7e6d118a 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -122,7 +122,7 @@ EOF # #----------------------------------------------------------------------- # -# Set the run machine-dependent run command. +# Set the machine-dependent run command. # #----------------------------------------------------------------------- # @@ -161,6 +161,10 @@ case $MACHINE in APRUN="ibrun -np ${nprocs}" ;; +"ORION") + APRUN="srun" + ;; + *) print_err_msg_exit "\ Run command has not been specified for this machine: diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 10b7fed07..3f664013a 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -128,7 +128,13 @@ case $MACHINE in APRUN="ibrun -np ${PE_MEMBER01}" #LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ;; - +# +"ORION") + ulimit -s unlimited + ulimit -a + APRUN="srun" +# LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +# esac # #----------------------------------------------------------------------- diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 60758381e..d876ada53 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -79,9 +79,6 @@ print_input_args valid_args # #----------------------------------------------------------------------- # -print_info_msg "$VERBOSE" " -Starting post-processing for fhr = $fhr hr..." - case $MACHINE in "WCOSS_CRAY") @@ -133,6 +130,10 @@ case $MACHINE in APRUN="ibrun -n $nprocs" ;; +"ORION") + APRUN="srun" + ;; + esac # #----------------------------------------------------------------------- @@ -195,6 +196,9 @@ EOF # #----------------------------------------------------------------------- # +print_info_msg "$VERBOSE" " +Starting post-processing for fhr = $fhr hr..." + ${APRUN} ./ncep_post < itag || print_err_msg_exit "\ Call to executable to run post for forecast hour $fhr returned with non- zero exit code." diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index c434f8d60..50ffff61e 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -1,9 +1,9 @@ # #----------------------------------------------------------------------- # -# This file defines and then calls a function that sets parameters rela- -# ting to the external model used for initial conditions (ICs) and the -# one used for lateral boundary conditions (LBCs). +# This file defines and then calls a function that sets parameters +# associated with the external model used for initial conditions (ICs) +# and the one used for lateral boundary conditions (LBCs). # #----------------------------------------------------------------------- # @@ -32,15 +32,14 @@ local func_name="${FUNCNAME[0]}" #----------------------------------------------------------------------- # # Set the system directory (i.e. location on disk, not on HPSS) in which -# the files generated by the external model specified by EXTRN_MDL_- -# NAME_ICS that are necessary for generating initial condition (IC) -# and surface files for the FV3SAR are stored (usually for a limited -# time, e.g. for the GFS external model, 2 weeks on WCOSS and 2 days on -# theia). If for a given cycle these files are available in this system -# directory, they will be copied over to a subdirectory within the cy- -# cle's run directory. If these files are not available in the system -# directory, then we search for them elsewhere, e.g. in the mass store -# (HPSS). +# the files generated by the external model specified by EXTRN_MDL_NAME_ICS +# that are necessary for generating initial condition (IC) and surface +# files for the FV3SAR are stored (usually for a limited time, e.g. for +# the GFS external model, 2 weeks on WCOSS and 2 days on hera). If for +# a given cycle these files are available in this system directory, they +# will be copied over to a subdirectory under the cycle directory. If +# these files are not available in the system directory, then we search +# for them elsewhere, e.g. in the mass store (HPSS). # #----------------------------------------------------------------------- # @@ -76,6 +75,9 @@ else "STAMPEDE") EXTRN_MDL_FILES_SYSBASEDIR_ICS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001" ;; + "ORION") + EXTRN_MDL_FILES_SYSBASEDIR_ICS="" + ;; *) print_err_msg_exit "\ The system directory in which to look for the files generated by the ex- @@ -108,6 +110,9 @@ this machine and external model combination: "CHEYENNE") EXTRN_MDL_FILES_SYSBASEDIR_ICS="/glade/p/ral/jntp/UFS_CAM/COMGFS" ;; + "ORION") + EXTRN_MDL_FILES_SYSBASEDIR_ICS="" + ;; *) print_err_msg_exit "\ The system directory in which to look for the files generated by the ex- @@ -166,9 +171,9 @@ fi # #----------------------------------------------------------------------- # -# Set EXTRN_MDL_LBCS_OFFSET_HRS, which is the number of hours to -# shift the starting time of the external model that provides lateral -# boundary conditions. +# Set EXTRN_MDL_LBCS_OFFSET_HRS, which is the number of hours to shift +# the starting time of the external model that provides lateral boundary +# conditions. # #----------------------------------------------------------------------- # @@ -190,15 +195,14 @@ esac #----------------------------------------------------------------------- # # Set the system directory (i.e. location on disk, not on HPSS) in which -# the files generated by the external model specified by EXTRN_MDL_- -# NAME_LBCS that are necessary for generating lateral boundary condition -# (LBC) files for the FV3SAR are stored (usually for a limited time, -# e.g. for the GFS external model, 2 weeks on WCOSS and 2 days on the- -# ia). If for a given cycle these files are available in this system -# directory, they will be copied over to a subdirectory within the cy- -# cle's run directory. If these files are not available in the system -# directory, then we search for them elsewhere, e.g. in the mass store -# (HPSS). +# the files generated by the external model specified by EXTRN_MDL_NAME_LBCS +# that are necessary for generating lateral boundary condition (LBC) files +# for the FV3SAR are stored (usually for a limited time, e.g. for the GFS +# external model, 2 weeks on WCOSS and 2 days on hera). If for a given +# cycle these files are available in this system directory, they will be +# copied over to a subdirectory under the cycle directory. If these files +# are not available in the system directory, then we search for them +# elsewhere, e.g. in the mass store (HPSS). # #----------------------------------------------------------------------- # @@ -234,6 +238,9 @@ else "STAMPEDE") EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001" ;; + "ORION") + EXTRN_MDL_FILES_SYSBASEDIR_LBCS="" + ;; *) print_err_msg_exit "\ The system directory in which to look for the files generated by the ex- @@ -267,6 +274,9 @@ this machine and external model combination: "CHEYENNE") EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/glade/p/ral/jntp/UFS_CAM/COMGFS" ;; + "ORION") + EXTRN_MDL_FILES_SYSBASEDIR_LBCS="" + ;; *) print_err_msg_exit "\ The system directory in which to look for the files generated by the ex- @@ -300,7 +310,6 @@ this machine and external model combination: ;; - "HRRRX") case $MACHINE in @@ -321,9 +330,8 @@ this machine and external model combination: ;; esac ;; - - - + + esac fi diff --git a/ush/setup.sh b/ush/setup.sh index 507cfbd02..1719031c2 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -318,7 +318,6 @@ check_var_valid_value "MACHINE" "valid_vals_MACHINE" case $MACHINE in # "WCOSS_CRAY") -# NCORES_PER_NODE="24" SCHED="lsfcray" QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} @@ -328,7 +327,6 @@ case $MACHINE in ;; # "WCOSS_DELL_P3") -# NCORES_PER_NODE=24 SCHED="lsf" QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} @@ -338,8 +336,7 @@ case $MACHINE in ;; # "HERA") -# - NCORES_PER_NODE=24 + NCORES_PER_NODE=24 # Should this be 40? Need to check heradocs. SCHED="${SCHED:-slurm}" QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} QUEUE_HPSS=${QUEUE_HPSS:-"service"} @@ -347,7 +344,6 @@ case $MACHINE in ;; # "JET") -# NCORES_PER_NODE=24 SCHED="${SCHED:-slurm}" QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} @@ -356,7 +352,6 @@ case $MACHINE in ;; # "ODIN") -# NCORES_PER_NODE=24 SCHED="${SCHED:-slurm}" QUEUE_DEFAULT=${QUEUE_DEFAULT:-""} @@ -365,7 +360,6 @@ case $MACHINE in ;; # "CHEYENNE") -# NCORES_PER_NODE=36 SCHED="${SCHED:-pbspro}" QUEUE_DEFAULT=${QUEUE_DEFAULT:-"regular"} @@ -375,13 +369,21 @@ case $MACHINE in ;; # "STAMPEDE") -# NCORES_PER_NODE=68 SCHED="slurm" QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"} QUEUE_HPSS=${QUEUE_HPSS:-"development"} QUEUE_FCST=${QUEUE_FCST:-"normal"} ;; +# +"ORION") + NCORES_PER_NODE=40 + SCHED="${SCHED:-slurm}" + QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} + QUEUE_HPSS=${QUEUE_HPSS:-"service"} + QUEUE_FCST=${QUEUE_FCST:-""} + ;; +# esac # #----------------------------------------------------------------------- @@ -741,6 +743,7 @@ case $MACHINE in FIXgsm="/scratch/ywang/fix/theia_fix/fix_am" SFC_CLIMO_INPUT_DIR="/scratch/ywang/fix/climo_fields_netcdf" ;; + "CHEYENNE") FIXgsm="/glade/p/ral/jntp/UFS_CAM/fix/fix_am" SFC_CLIMO_INPUT_DIR="/glade/p/ral/jntp/UFS_CAM/fix/climo_fields_netcdf" @@ -751,6 +754,11 @@ case $MACHINE in SFC_CLIMO_INPUT_DIR="/work/00315/tg455890/stampede2/regional_fv3/climo_fields_netcdf" ;; +"ORION") + FIXgsm="/work/noaa/fv3-cam/emc.campara/fix_fv3cam/fix_am" + SFC_CLIMO_INPUT_DIR="/dont/know/what/this/should/be" + ;; + *) print_err_msg_exit "\ Directories have not been specified for this machine: diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index a4378f03d..0d89c5ed6 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -1,6 +1,6 @@ valid_vals_RUN_ENVIR=("nco" "community") valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "THEIA" "HERA" "JET" "ODIN" "CHEYENNE" "STAMPEDE") +valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "THEIA" "HERA" "JET" "ODIN" "CHEYENNE" "STAMPEDE" "ORION") valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none") valid_vals_PREDEF_GRID_NAME=( \ "EMC_CONUS_3km" \ From 9ccfe46688318b611b5e475aa91e405c9943fdf5 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Tue, 6 Oct 2020 15:51:03 -0500 Subject: [PATCH 02/27] Merge set_extrn_mdl_params.sh manually (even outside of mergetool) to make the merge of develop into feature/orion_port easier. --- ush/set_extrn_mdl_params.sh | 224 +++++++++++++++--------------------- 1 file changed, 91 insertions(+), 133 deletions(-) diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index 50ffff61e..2a4e88ffd 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -45,130 +45,114 @@ local func_name="${FUNCNAME[0]}" # if [ "${RUN_ENVIR}" = "nco" ]; then - EXTRN_MDL_FILES_SYSBASEDIR_ICS="$COMINgfs" + EXTRN_MDL_SYSBASEDIR_ICS="$COMINgfs" else case ${EXTRN_MDL_NAME_ICS} in - - + "GSMGFS") case $MACHINE in "WCOSS_CRAY") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="" + EXTRN_MDL_SYSBASEDIR_ICS="" ;; "WCOSS_DELL_P3") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="" + EXTRN_MDL_SYSBASEDIR_ICS="" ;; "HERA") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="" + EXTRN_MDL_SYSBASEDIR_ICS="" ;; "JET") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="" + EXTRN_MDL_SYSBASEDIR_ICS="" ;; "ODIN") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="/scratch/ywang/EPIC/GDAS/2019053000_mem001" + EXTRN_MDL_SYSBASEDIR_ICS="/scratch/ywang/EPIC/GDAS/2019053000_mem001" ;; "CHEYENNE") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="/glade/p/ral/jntp/UFS_CAM/COMGFS" + EXTRN_MDL_SYSBASEDIR_ICS="/glade/p/ral/jntp/UFS_CAM/COMGFS" ;; "STAMPEDE") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001" + EXTRN_MDL_SYSBASEDIR_ICS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001" ;; "ORION") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="" - ;; - *) - print_err_msg_exit "\ -The system directory in which to look for the files generated by the ex- -ternal model specified by EXTRN_MDL_NAME_ICS has not been specified for -this machine and external model combination: - MACHINE = \"$MACHINE\" - EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" + EXTRN_MDL_SYSBASEDIR_ICS="" ;; esac ;; - - + "FV3GFS") case $MACHINE in "WCOSS_CRAY") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="/gpfs/dell1/nco/ops/com/gfs/prod" + EXTRN_MDL_SYSBASEDIR_ICS="/gpfs/dell1/nco/ops/com/gfs/prod" ;; "WCOSS_DELL_P3") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="/gpfs/dell1/nco/ops/com/gfs/prod" + EXTRN_MDL_SYSBASEDIR_ICS="/gpfs/dell1/nco/ops/com/gfs/prod" ;; "HERA") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="/scratch1/NCEPDEV/rstprod/com/gfs/prod" + EXTRN_MDL_SYSBASEDIR_ICS="/scratch1/NCEPDEV/rstprod/com/gfs/prod" ;; "JET") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="/public/data/grids/gfs/nemsio" + EXTRN_MDL_SYSBASEDIR_ICS="/public/data/grids/gfs/nemsio" ;; "ODIN") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="/scratch/ywang/test_runs/FV3_regional/gfs" + EXTRN_MDL_SYSBASEDIR_ICS="/scratch/ywang/test_runs/FV3_regional/gfs" ;; "CHEYENNE") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="/glade/p/ral/jntp/UFS_CAM/COMGFS" + EXTRN_MDL_SYSBASEDIR_ICS="/glade/p/ral/jntp/UFS_CAM/COMGFS" ;; "ORION") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="" - ;; - *) - print_err_msg_exit "\ -The system directory in which to look for the files generated by the ex- -ternal model specified by EXTRN_MDL_NAME_ICS has not been specified for -this machine and external model combination: - MACHINE = \"$MACHINE\" - EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" + EXTRN_MDL_SYSBASEDIR_ICS="" ;; esac ;; - - + "RAPX") case $MACHINE in "HERA") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="/scratch2/BMC/public/data/gsd/rap/full/wrfnat" + EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/public/data/gsd/rap/full/wrfnat" ;; "JET") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="/misc/whome/rtrr/rap" + EXTRN_MDL_SYSBASEDIR_ICS="/misc/whome/rtrr/rap" ;; - *) - print_err_msg_exit "\ -The system directory in which to look for the files generated by the ex- -ternal model specified by EXTRN_MDL_NAME_ICS has not been specified for -this machine and external model combination: - MACHINE = \"$MACHINE\" - EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" +# This goes with the comment below for the if-statement (-z EXTRN_MDL_SYSBASEDIR_ICS). +# Should not need this case. + "CHEYENNE") + EXTRN_MDL_SYSBASEDIR_ICS="dummy_value" ;; esac ;; - "HRRRX") case $MACHINE in "HERA") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="/scratch2/BMC/public/data/gsd/hrrr/conus/wrfnat" + EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/public/data/gsd/hrrr/conus/wrfnat" ;; "JET") - EXTRN_MDL_FILES_SYSBASEDIR_ICS="/misc/whome/rtrr/hrrr" + EXTRN_MDL_SYSBASEDIR_ICS="/misc/whome/rtrr/hrrr" ;; - *) - print_err_msg_exit "\ -The system directory in which to look for the files generated by the ex- -ternal model specified by EXTRN_MDL_NAME_ICS has not been specified -for this machine and external model combination: - MACHINE = \"$MACHINE\" - EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" + "CHEYENNE") + EXTRN_MDL_SYSBASEDIR_ICS="dummy_value" ;; esac ;; - esac fi # +# If EXTRN_MDL_SYSBASEDIR_ICS has not been set (not even to a null string), +# print out an error message and exit. +# +if [ -z "${EXTRN_MDL_SYSBASEDIR_ICS+x}" ]; then + print_err_msg_exit "\ +The variable EXTRN_MDL_SYSBASEDIR_ICS specifying the system directory +in which to look for the files generated by the external model for ICs +has not been set for the current combination of machine (MACHINE) and +external model (EXTRN_MDL_NAME_ICS): + MACHINE = \"$MACHINE\" + EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" +fi +# #----------------------------------------------------------------------- # # Set EXTRN_MDL_LBCS_OFFSET_HRS, which is the number of hours to shift @@ -178,18 +162,18 @@ fi #----------------------------------------------------------------------- # case ${EXTRN_MDL_NAME_LBCS} in -"GSMGFS") - EXTRN_MDL_LBCS_OFFSET_HRS="0" - ;; -"FV3GFS") - EXTRN_MDL_LBCS_OFFSET_HRS="0" - ;; -"RAPX") - EXTRN_MDL_LBCS_OFFSET_HRS="3" - ;; -"HRRRX") - EXTRN_MDL_LBCS_OFFSET_HRS="0" - ;; + "GSMGFS") + EXTRN_MDL_LBCS_OFFSET_HRS="0" + ;; + "FV3GFS") + EXTRN_MDL_LBCS_OFFSET_HRS="0" + ;; + "RAPX") + EXTRN_MDL_LBCS_OFFSET_HRS="3" + ;; + "HRRRX") + EXTRN_MDL_LBCS_OFFSET_HRS="0" + ;; esac # #----------------------------------------------------------------------- @@ -208,134 +192,108 @@ esac # if [ "${RUN_ENVIR}" = "nco" ]; then - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="$COMINgfs" + EXTRN_MDL_SYSBASEDIR_LBCS="$COMINgfs" else case ${EXTRN_MDL_NAME_LBCS} in - - + "GSMGFS") case $MACHINE in "WCOSS_CRAY") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="" + EXTRN_MDL_SYSBASEDIR_LBCS="" ;; "WCOSS_DELL_P3") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="" + EXTRN_MDL_SYSBASEDIR_LBCS="" ;; "HERA") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="" + EXTRN_MDL_SYSBASEDIR_LBCS="" ;; "JET") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="" + EXTRN_MDL_SYSBASEDIR_LBCS="" ;; "ODIN") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch/ywang/EPIC/GDAS/2019053000_mem001" + EXTRN_MDL_SYSBASEDIR_LBCS="/scratch/ywang/EPIC/GDAS/2019053000_mem001" ;; "CHEYENNE") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/glade/p/ral/jntp/UFS_CAM/COMGFS" + EXTRN_MDL_SYSBASEDIR_LBCS="/glade/p/ral/jntp/UFS_CAM/COMGFS" ;; "STAMPEDE") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001" + EXTRN_MDL_SYSBASEDIR_LBCS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001" ;; "ORION") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="" - ;; - *) - print_err_msg_exit "\ -The system directory in which to look for the files generated by the ex- -ternal model specified by EXTRN_MDL_NAME_LBCS has not been specified for -this machine and external model combination: - MACHINE = \"$MACHINE\" - EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" + EXTRN_MDL_SYSBASEDIR_LBCS="" ;; esac ;; - "FV3GFS") - case $MACHINE in "WCOSS_CRAY") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/gpfs/dell1/nco/ops/com/gfs/prod" + EXTRN_MDL_SYSBASEDIR_LBCS="/gpfs/dell1/nco/ops/com/gfs/prod" ;; "WCOSS_DELL_P3") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/gpfs/dell1/nco/ops/com/gfs/prod" + EXTRN_MDL_SYSBASEDIR_LBCS="/gpfs/dell1/nco/ops/com/gfs/prod" ;; "HERA") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch1/NCEPDEV/rstprod/com/gfs/prod" + EXTRN_MDL_SYSBASEDIR_LBCS="/scratch1/NCEPDEV/rstprod/com/gfs/prod" ;; "JET") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/public/data/grids/gfs/nemsio" + EXTRN_MDL_SYSBASEDIR_LBCS="/public/data/grids/gfs/nemsio" ;; "ODIN") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch/ywang/test_runs/FV3_regional/gfs" + EXTRN_MDL_SYSBASEDIR_LBCS="/scratch/ywang/test_runs/FV3_regional/gfs" ;; "CHEYENNE") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/glade/p/ral/jntp/UFS_CAM/COMGFS" + EXTRN_MDL_SYSBASEDIR_LBCS="/glade/p/ral/jntp/UFS_CAM/COMGFS" ;; "ORION") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="" - ;; - *) - print_err_msg_exit "\ -The system directory in which to look for the files generated by the ex- -ternal model specified by EXTRN_MDL_NAME_LBCS has not been specified for -this machine and external model combination: - MACHINE = \"$MACHINE\" - EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" + EXTRN_MDL_SYSBASEDIR_LBCS="" ;; esac ;; - "RAPX") - case $MACHINE in "HERA") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch2/BMC/public/data/gsd/rap/full/wrfnat" + EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/public/data/gsd/rap/full/wrfnat" ;; "JET") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/misc/whome/rtrr/rap" + EXTRN_MDL_SYSBASEDIR_LBCS="/misc/whome/rtrr/rap" ;; - *) - print_err_msg_exit "\ -The system directory in which to look for the files generated by the ex- -ternal model specified by EXTRN_MDL_NAME_LBCS has not been specified for -this machine and external model combination: - MACHINE = \"$MACHINE\" - EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" + "CHEYENNE") + EXTRN_MDL_SYSBASEDIR_LBCS="dummy_value" ;; esac ;; - - + "HRRRX") - case $MACHINE in "HERA") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/scratch2/BMC/public/data/gsd/hrrr/conus/wrfnat" + EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/public/data/gsd/hrrr/conus/wrfnat" ;; "JET") - EXTRN_MDL_FILES_SYSBASEDIR_LBCS="/misc/whome/rtrr/hrrr" - ;; - - *) - print_err_msg_exit "\ -The system directory in which to look for the files generated by the ex- -ternal model specified by EXTRN_MDL_NAME_LBCS has not been specified for -this machine and external model combination: - MACHINE = \"$MACHINE\" - EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" + EXTRN_MDL_SYSBASEDIR_LBCS="/misc/whome/rtrr/hrrr" ;; esac ;; - esac fi - +# +# If EXTRN_MDL_SYSBASEDIR_LBCS has not been set (not even to a null string), +# print out an error message and exit. +# +if [ -z "${EXTRN_MDL_SYSBASEDIR_LBCS+x}" ]; then + print_err_msg_exit "\ +The variable EXTRN_MDL_SYSBASEDIR_LBCS specifying the system directory +in which to look for the files generated by the external model for LBCs +has not been set for the current combination of machine (MACHINE) and +external model (EXTRN_MDL_NAME_LBCS): + MACHINE = \"$MACHINE\" + EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" +fi } # #----------------------------------------------------------------------- From 81281b06fc934401ca3741313e538e02a9322cbb Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 8 Oct 2020 17:40:19 -0500 Subject: [PATCH 03/27] Bug fixes for Orion port. --- tests/run_experiments.sh | 2 ++ ush/setup.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/run_experiments.sh b/tests/run_experiments.sh index 753e6917e..f71de6902 100755 --- a/tests/run_experiments.sh +++ b/tests/run_experiments.sh @@ -741,6 +741,8 @@ PTMP=\"${PTMP}\"" extrn_mdl_source_basedir="/mnt/lfs1/BMC/fim/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" elif [ "$MACHINE" = "CHEYENNE" ]; then extrn_mdl_source_basedir="/glade/p/ral/jntp/UFS_CAM/staged_extrn_mdl_files" + elif [ "$MACHINE" = "ORION" ]; then + extrn_mdl_source_basedir="/work/noaa/gsd-fv3-dev/gsketefia/UFS/staged_extrn_mdl_files" else print_err_msg_exit "\ The base directory (extrn_mdl_source_basedir) in which the user-staged diff --git a/ush/setup.sh b/ush/setup.sh index ce6c462c5..6b6386cd1 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -403,7 +403,7 @@ case $MACHINE in SCHED="${SCHED:-slurm}" QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} QUEUE_HPSS=${QUEUE_HPSS:-"service"} - QUEUE_FCST=${QUEUE_FCST:-""} + QUEUE_FCST=${QUEUE_FCST:-"batch"} ;; # esac From 92c626a3d36de803f0c5b6c159601f041c545371 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 8 Oct 2020 19:26:56 -0500 Subject: [PATCH 04/27] Add TOPO_DIR for ORION; don't require that the full path except last portion to the EXPT_BASEDIR initially exist (i.e. no -f to readlink) before creating the directory. --- ush/setup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/setup.sh b/ush/setup.sh index 6b6386cd1..81a7d452f 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -779,6 +779,7 @@ case $MACHINE in "ORION") FIXgsm="/work/noaa/fv3-cam/emc.campara/fix_fv3cam/fix_am" + TOPO_DIR="/work/noaa/fv3-cam/emc.campara/fix_fv3cam/fix_am" SFC_CLIMO_INPUT_DIR="/dont/know/what/this/should/be" ;; @@ -1027,7 +1028,7 @@ fi if [ "${EXPT_BASEDIR:0:1}" != "/" ]; then EXPT_BASEDIR="${SR_WX_APP_TOP_DIR}/../expt_dirs/${EXPT_BASEDIR}" fi -EXPT_BASEDIR="$( readlink -f ${EXPT_BASEDIR} )" +EXPT_BASEDIR="$( readlink -m ${EXPT_BASEDIR} )" mkdir_vrfy -p "${EXPT_BASEDIR}" # #----------------------------------------------------------------------- From d684d82a910387d6ace2257eb9c732ce90dd6236 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 9 Oct 2020 16:53:29 -0500 Subject: [PATCH 05/27] More modifications to run workflow on orion. --- ush/get_extrn_mdl_file_dir_info.sh | 15 +++++++++++++++ ush/launch_FV3LAM_wflow.sh | 8 ++++++-- ush/load_modules_run_task.sh | 4 ++++ ush/set_extrn_mdl_params.sh | 8 ++++---- ush/setup.sh | 18 ++++++------------ 5 files changed, 35 insertions(+), 18 deletions(-) diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh index ae1300634..f2055d068 100755 --- a/ush/get_extrn_mdl_file_dir_info.sh +++ b/ush/get_extrn_mdl_file_dir_info.sh @@ -560,6 +560,9 @@ bination of external model (extrn_mdl_name) and analysis or forecast "HERA") sysdir="" ;; + "ORION") + sysdir="$sysbasedir" + ;; "JET") sysdir="" ;; @@ -594,6 +597,9 @@ has not been specified for this external model and machine combination: "HERA") sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}" ;; + "ORION") + sysdir="$sysbasedir" + ;; "JET") sysdir="$sysbasedir" ;; @@ -625,6 +631,9 @@ has not been specified for this external model and machine combination: "HERA") sysdir="$sysbasedir" ;; + "ORION") + sysdir="$sysbasedir" + ;; "JET") sysdir="$sysbasedir/${yyyymmdd}${hh}/postprd" ;; @@ -656,6 +665,9 @@ has not been specified for this external model and machine combination: "HERA") sysdir="$sysbasedir" ;; + "ORION") + sysdir="$sysbasedir" + ;; "JET") sysdir="$sysbasedir/${yyyymmdd}${hh}/postprd" ;; @@ -686,6 +698,9 @@ has not been specified for this external model and machine combination: "HERA") sysdir="$sysbasedir" ;; + "ORION") + sysdir="$sysbasedir" + ;; "JET") sysdir="$sysbasedir" ;; diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 8004e32e4..60910d812 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -92,8 +92,12 @@ expt_name="${EXPT_SUBDIR}" #----------------------------------------------------------------------- # if [ "$MACHINE" != "CHEYENNE" ]; then - module purge - module load rocoto + if [ "$MACHINE" = "ORION" ]; then + module load contrib rocoto + else + module purge + module load rocoto + fi fi # #----------------------------------------------------------------------- diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 6e523ce3e..94ef2039d 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -114,6 +114,10 @@ case "$MACHINE" in "HERA") . /apps/lmod/lmod/init/sh ;; +# + "ORION") + . /apps/lmod/lmod/init/sh + ;; # "JET") . /apps/lmod/lmod/init/sh diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index 2a4e88ffd..4e68bf474 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -75,7 +75,7 @@ else EXTRN_MDL_SYSBASEDIR_ICS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001" ;; "ORION") - EXTRN_MDL_SYSBASEDIR_ICS="" + EXTRN_MDL_SYSBASEDIR_ICS="dummy_value" ;; esac ;; @@ -101,7 +101,7 @@ else EXTRN_MDL_SYSBASEDIR_ICS="/glade/p/ral/jntp/UFS_CAM/COMGFS" ;; "ORION") - EXTRN_MDL_SYSBASEDIR_ICS="" + EXTRN_MDL_SYSBASEDIR_ICS="dummy_value" ;; esac ;; @@ -222,7 +222,7 @@ else EXTRN_MDL_SYSBASEDIR_LBCS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001" ;; "ORION") - EXTRN_MDL_SYSBASEDIR_LBCS="" + EXTRN_MDL_SYSBASEDIR_LBCS="dummy_value" ;; esac ;; @@ -248,7 +248,7 @@ else EXTRN_MDL_SYSBASEDIR_LBCS="/glade/p/ral/jntp/UFS_CAM/COMGFS" ;; "ORION") - EXTRN_MDL_SYSBASEDIR_LBCS="" + EXTRN_MDL_SYSBASEDIR_LBCS="dummy_value" ;; esac ;; diff --git a/ush/setup.sh b/ush/setup.sh index 81a7d452f..c99054e51 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -741,18 +741,18 @@ case $MACHINE in SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-""} ;; -"THEIA") - FIXgsm=${FIXgsm:-"/scratch4/NCEPDEV/global/save/glopara/git/fv3gfs/fix/fix_am"} - TOPO_DIR=${TOPO_DIR:-"/scratch4/NCEPDEV/global/save/glopara/git/fv3gfs/fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch4/NCEPDEV/da/noscrub/George.Gayno/climo_fields_netcdf"} - ;; - "HERA") FIXgsm=${FIXgsm:-"/scratch1/NCEPDEV/global/glopara/fix/fix_am"} TOPO_DIR=${TOPO_DIR:-"/scratch1/NCEPDEV/global/glopara/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch1/NCEPDEV/da/George.Gayno/ufs_utils.git/climo_fields_netcdf"} ;; +"ORION") + FIXgsm=${FIXgsm:-"/work/noaa/fv3-cam/emc.campara/fix_fv3cam/fix_am"} + TOPO_DIR=${TOPO_DIR:-"/work/noaa/gsd-fv3-dev/gsketefia/UFS/fix_orog"} + SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/noaa/gsd-fv3-dev/gsketefia/UFS/climo_fields_netcdf"} + ;; + "JET") FIXgsm=${FIXgsm:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_am"} TOPO_DIR=${TOPO_DIR:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_orog"} @@ -777,12 +777,6 @@ case $MACHINE in SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/climo_fields_netcdf"} ;; -"ORION") - FIXgsm="/work/noaa/fv3-cam/emc.campara/fix_fv3cam/fix_am" - TOPO_DIR="/work/noaa/fv3-cam/emc.campara/fix_fv3cam/fix_am" - SFC_CLIMO_INPUT_DIR="/dont/know/what/this/should/be" - ;; - *) print_err_msg_exit "\ One or more fix file directories have not been specified for this machine: From c3db63f98c6c0eb498b3f898ff929de593a696c5 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 9 Oct 2020 16:58:01 -0500 Subject: [PATCH 06/27] Add module files for orion. --- modulefiles/tasks/orion/get_extrn_ics | 6 ++++++ modulefiles/tasks/orion/get_extrn_lbcs | 6 ++++++ modulefiles/tasks/orion/make_grid.local | 6 ++++++ modulefiles/tasks/orion/make_ics.local | 6 ++++++ modulefiles/tasks/orion/make_lbcs.local | 6 ++++++ modulefiles/tasks/orion/run_fcst.local | 6 ++++++ 6 files changed, 36 insertions(+) create mode 100644 modulefiles/tasks/orion/get_extrn_ics create mode 100644 modulefiles/tasks/orion/get_extrn_lbcs create mode 100644 modulefiles/tasks/orion/make_grid.local create mode 100644 modulefiles/tasks/orion/make_ics.local create mode 100644 modulefiles/tasks/orion/make_lbcs.local create mode 100644 modulefiles/tasks/orion/run_fcst.local diff --git a/modulefiles/tasks/orion/get_extrn_ics b/modulefiles/tasks/orion/get_extrn_ics new file mode 100644 index 000000000..a9d5b4412 --- /dev/null +++ b/modulefiles/tasks/orion/get_extrn_ics @@ -0,0 +1,6 @@ +#%Module##################################################### +## Module file for get_extrn_ics task. +############################################################# + +module purge + diff --git a/modulefiles/tasks/orion/get_extrn_lbcs b/modulefiles/tasks/orion/get_extrn_lbcs new file mode 100644 index 000000000..09f37151a --- /dev/null +++ b/modulefiles/tasks/orion/get_extrn_lbcs @@ -0,0 +1,6 @@ +#%Module##################################################### +## Module file for get_extrn_lbcs task. +############################################################# + +module purge + diff --git a/modulefiles/tasks/orion/make_grid.local b/modulefiles/tasks/orion/make_grid.local new file mode 100644 index 000000000..5c1331242 --- /dev/null +++ b/modulefiles/tasks/orion/make_grid.local @@ -0,0 +1,6 @@ + +#module use -a /contrib/miniconda3/modulefiles +#module load miniconda3 +#if [module-info mode load] { +# system "conda activate regional_workflow" +#} diff --git a/modulefiles/tasks/orion/make_ics.local b/modulefiles/tasks/orion/make_ics.local new file mode 100644 index 000000000..5c1331242 --- /dev/null +++ b/modulefiles/tasks/orion/make_ics.local @@ -0,0 +1,6 @@ + +#module use -a /contrib/miniconda3/modulefiles +#module load miniconda3 +#if [module-info mode load] { +# system "conda activate regional_workflow" +#} diff --git a/modulefiles/tasks/orion/make_lbcs.local b/modulefiles/tasks/orion/make_lbcs.local new file mode 100644 index 000000000..5c1331242 --- /dev/null +++ b/modulefiles/tasks/orion/make_lbcs.local @@ -0,0 +1,6 @@ + +#module use -a /contrib/miniconda3/modulefiles +#module load miniconda3 +#if [module-info mode load] { +# system "conda activate regional_workflow" +#} diff --git a/modulefiles/tasks/orion/run_fcst.local b/modulefiles/tasks/orion/run_fcst.local new file mode 100644 index 000000000..5c1331242 --- /dev/null +++ b/modulefiles/tasks/orion/run_fcst.local @@ -0,0 +1,6 @@ + +#module use -a /contrib/miniconda3/modulefiles +#module load miniconda3 +#if [module-info mode load] { +# system "conda activate regional_workflow" +#} From a2ab5b9f5a0586544a60b0e73cab4042c9127451 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 9 Oct 2020 17:46:07 -0500 Subject: [PATCH 07/27] Changes to orion modulefiles. --- modulefiles/tasks/orion/make_grid.local | 1 + modulefiles/tasks/orion/make_ics.local | 1 + modulefiles/tasks/orion/make_lbcs.local | 1 + modulefiles/tasks/orion/run_fcst.local | 1 + 4 files changed, 4 insertions(+) diff --git a/modulefiles/tasks/orion/make_grid.local b/modulefiles/tasks/orion/make_grid.local index 5c1331242..7b0776c62 100644 --- a/modulefiles/tasks/orion/make_grid.local +++ b/modulefiles/tasks/orion/make_grid.local @@ -4,3 +4,4 @@ #if [module-info mode load] { # system "conda activate regional_workflow" #} +module load python/3.7 diff --git a/modulefiles/tasks/orion/make_ics.local b/modulefiles/tasks/orion/make_ics.local index 5c1331242..7b0776c62 100644 --- a/modulefiles/tasks/orion/make_ics.local +++ b/modulefiles/tasks/orion/make_ics.local @@ -4,3 +4,4 @@ #if [module-info mode load] { # system "conda activate regional_workflow" #} +module load python/3.7 diff --git a/modulefiles/tasks/orion/make_lbcs.local b/modulefiles/tasks/orion/make_lbcs.local index 5c1331242..7b0776c62 100644 --- a/modulefiles/tasks/orion/make_lbcs.local +++ b/modulefiles/tasks/orion/make_lbcs.local @@ -4,3 +4,4 @@ #if [module-info mode load] { # system "conda activate regional_workflow" #} +module load python/3.7 diff --git a/modulefiles/tasks/orion/run_fcst.local b/modulefiles/tasks/orion/run_fcst.local index 5c1331242..2b539c8f0 100644 --- a/modulefiles/tasks/orion/run_fcst.local +++ b/modulefiles/tasks/orion/run_fcst.local @@ -4,3 +4,4 @@ #if [module-info mode load] { # system "conda activate regional_workflow" #} +#module load python/3.7 From 896733860a3924077ddc71d79793eb4c28af094d Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 9 Oct 2020 18:29:36 -0500 Subject: [PATCH 08/27] Remove the "-l" flag to bash so that the script is no longer executed in a login shell. This allows the settings set by a "module load" (e.g. module load python/3.7 that prepends a python path to PATH) to be inherited by the script rather than get overwritten. --- tests/run_experiments.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run_experiments.sh b/tests/run_experiments.sh index f71de6902..ceead7215 100755 --- a/tests/run_experiments.sh +++ b/tests/run_experiments.sh @@ -1,4 +1,4 @@ -#!/bin/bash -l +#!/bin/bash # #----------------------------------------------------------------------- From 4e5f55cf4cf8a7189bf213025f0fee128400bcd8 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 9 Oct 2020 19:02:04 -0500 Subject: [PATCH 09/27] Clean up spacing, etc. --- jobs/JREGIONAL_MAKE_ICS | 13 ++- jobs/JREGIONAL_MAKE_LBCS | 13 ++- scripts/exregional_make_grid.sh | 166 ++++++++++++++------------- scripts/exregional_make_orog.sh | 126 ++++++++++---------- scripts/exregional_make_sfc_climo.sh | 60 +++++----- scripts/exregional_run_fcst.sh | 122 ++++++++++---------- scripts/exregional_run_post.sh | 97 ++++++++-------- 7 files changed, 314 insertions(+), 283 deletions(-) diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS index bb0f71c3c..ff3cc5010 100755 --- a/jobs/JREGIONAL_MAKE_ICS +++ b/jobs/JREGIONAL_MAKE_ICS @@ -70,6 +70,11 @@ case "$MACHINE" in APRUN="srun" ;; + "ORION") + ulimit -s unlimited + APRUN="srun" + ;; + "JET") ulimit -s unlimited APRUN="srun" @@ -88,9 +93,11 @@ case "$MACHINE" in APRUN="ibrun" ;; - "ORION") - ulimit -s unlimited - APRUN="srun" + *) + print_err_msg_exit "\ +Run command has not been specified for this machine: + MACHINE = \"$MACHINE\" + APRUN = \"$APRUN\"" ;; esac diff --git a/jobs/JREGIONAL_MAKE_LBCS b/jobs/JREGIONAL_MAKE_LBCS index 58d53f3ec..c09c17f6c 100755 --- a/jobs/JREGIONAL_MAKE_LBCS +++ b/jobs/JREGIONAL_MAKE_LBCS @@ -70,6 +70,11 @@ case "$MACHINE" in APRUN="srun" ;; + "ORION") + ulimit -s unlimited + APRUN="srun" + ;; + "JET") ulimit -s unlimited APRUN="srun" @@ -88,9 +93,11 @@ case "$MACHINE" in APRUN="ibrun" ;; - "ORION") - ulimit -s unlimited - APRUN="srun" + *) + print_err_msg_exit "\ +Run command has not been specified for this machine: + MACHINE = \"$MACHINE\" + APRUN = \"$APRUN\"" ;; esac diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 4ae93e7b6..4c3a2f6d0 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -99,60 +99,64 @@ export OMP_STACKSIZE=2048m # case $MACHINE in -"WCOSS_C" | "WCOSS") - { save_shell_opts; set +x; } > /dev/null 2>&1 + "WCOSS_C" | "WCOSS") + { save_shell_opts; set +x; } > /dev/null 2>&1 - . $MODULESHOME/init/sh - module load PrgEnv-intel cfp-intel-sandybridge/1.1.0 - module list + . $MODULESHOME/init/sh + module load PrgEnv-intel cfp-intel-sandybridge/1.1.0 + module list - { restore_shell_opts; } > /dev/null 2>&1 + { restore_shell_opts; } > /dev/null 2>&1 - export NODES=1 - export APRUN="aprun -n 1 -N 1 -j 1 -d 1 -cc depth" - export KMP_AFFINITY=disabled + export NODES=1 + export APRUN="aprun -n 1 -N 1 -j 1 -d 1 -cc depth" + export KMP_AFFINITY=disabled - ulimit -s unlimited - ulimit -a - ;; + ulimit -s unlimited + ulimit -a + ;; -"HERA") - APRUN="time" -# ulimit -s unlimited -# ulimit -a - ;; + "HERA") + APRUN="time" + ;; -"JET") - APRUN="time" - ulimit -a - ;; + "ORION") + APRUN="time" + ;; + "JET") + APRUN="time" + ulimit -a + ;; -"ODIN") - export APRUN="srun -n 1" - ulimit -s unlimited - ulimit -a - ;; + "ODIN") + export APRUN="srun -n 1" + ulimit -s unlimited + ulimit -a + ;; -"CHEYENNE") - APRUN="time" - ;; + "CHEYENNE") + APRUN="time" + ;; -"STAMPEDE") - export APRUN="time" - ulimit -s unlimited - ulimit -a - ;; + "STAMPEDE") + export APRUN="time" + ulimit -s unlimited + ulimit -a + ;; -"ORION") - APRUN="time" - ;; + *) + print_err_msg_exit "\ +Run command has not been specified for this machine: + MACHINE = \"$MACHINE\" + APRUN = \"$APRUN\"" + ;; esac # #----------------------------------------------------------------------- # -# Create the (cycle-independent) subdirectories under the experiment +# Create the (cycle-independent) subdirectories under the experiment # directory (EXPTDIR) that are needed by the various steps and substeps # in this script. # @@ -198,12 +202,12 @@ mkdir_vrfy -p "$tmpdir" # size specified by the argument to the --halo flag does not extend be- # yond the boundaries of the parent grid (tile 6). In this case, since # the values passed to the --istart_nest, ..., and --jend_nest flags al- -# ready include a halo (because these arguments are +# ready include a halo (because these arguments are # -# ${ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}, -# ${IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}, +# ${ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}, +# ${IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}, # ${JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}, and -# ${JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}, +# ${JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}, # # i.e. they include "WITH_WIDE_HALO_" in their names), it is reasonable # to pass as the argument to --halo a zero. However, make_hgrid re- @@ -221,7 +225,7 @@ mkdir_vrfy -p "$tmpdir" # --nlon 2*${RES} \ # --grid_name C${RES}_grid \ # --do_schmidt --stretch_factor ${STRETCH_FAC} \ -# --target_lon ${LON_CTR} +# --target_lon ${LON_CTR} # --target_lat ${LAT_CTR} \ # --nest_grid --parent_tile 6 --refine_ratio ${GFDLgrid_REFINE_RATIO} \ # --istart_nest ${ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG} \ @@ -326,7 +330,7 @@ if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then --halo 1 \ --great_circle_algorithm || \ print_err_msg_exit "\ -Call to executable (exec_fp) that generates grid files returned with +Call to executable (exec_fp) that generates grid files returned with nonzero exit code. exec_fp = \"${exec_fp}\"" # @@ -344,15 +348,15 @@ elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then rgnl_grid_nml_fp="$tmpdir/${RGNL_GRID_NML_FN}" print_info_msg "$VERBOSE" " -Creating namelist file (rgnl_grid_nml_fp) to be read in by the grid +Creating namelist file (rgnl_grid_nml_fp) to be read in by the grid generation executable (exec_fp): rgnl_grid_nml_fp = \"${rgnl_grid_nml_fp}\" exec_fp = \"${exec_fp}\"" # -# Create a multiline variable that consists of a yaml-compliant string -# specifying the values that the namelist variables need to be set to -# (one namelist variable per line, plus a header and footer). Below, -# this variable will be passed to a python script that will create the +# Create a multiline variable that consists of a yaml-compliant string +# specifying the values that the namelist variables need to be set to +# (one namelist variable per line, plus a header and footer). Below, +# this variable will be passed to a python script that will create the # namelist file. # settings=" @@ -370,8 +374,8 @@ generation executable (exec_fp): # ${USHDIR}/set_namelist.py -q -u "$settings" -o ${rgnl_grid_nml_fp} || \ print_err_msg_exit "\ -Call to python script set_namelist.py to set the variables in the -regional_esg_grid namelist file failed. Parameters passed to this script +Call to python script set_namelist.py to set the variables in the +regional_esg_grid namelist file failed. Parameters passed to this script are: Full path to output namelist file: rgnl_grid_nml_fp = \"${rgn_grid_nml_fp}\" @@ -385,9 +389,9 @@ $settings" print_err_msg_exit "\ Call to executable (exec_fp) that generates a ESGgrid-type regional grid returned with nonzero exit code: - exec_fp = \"${exec_fp}\"" + exec_fp = \"${exec_fp}\"" # -# Set the name of the regional grid file generated by the above call. +# Set the name of the regional grid file generated by the above call. # This must be the same name as in the regional_esg_grid code. # grid_fn="regional_grid.nc" @@ -475,7 +479,7 @@ mv_vrfy "${grid_fp_orig}" "${grid_fp}" # be using (i.e. if RUN_TASK_MAKE_OROG or RUN_TASK_MAKE_SURF_CLIMO is set # to "FALSE", in which case RES_IN_FIXLAM_FILENAMES will not be set to a # null string), check that the grid resolution contained in the variable -# CRES set above matches the resolution appearing in the names of the +# CRES set above matches the resolution appearing in the names of the # preexisting orography and/or surface climatology files. # #----------------------------------------------------------------------- @@ -484,7 +488,7 @@ if [ ! -z "${RES_IN_FIXLAM_FILENAMES}" ]; then res="${CRES:1}" if [ "$res" -ne "${RES_IN_FIXLAM_FILENAMES}" ]; then print_err_msg_exit "\ -The resolution (res) calculated for the grid does not match the resolution +The resolution (res) calculated for the grid does not match the resolution (RES_IN_FIXLAM_FILENAMES) appearing in the names of the orography and/or surface climatology files: res = \"$res\" @@ -494,10 +498,10 @@ fi # #----------------------------------------------------------------------- # -# Partially "shave" the halo from the grid file having a wide halo to +# Partially "shave" the halo from the grid file having a wide halo to # generate two new grid files -- one with a 3-grid-wide halo and another # with a 4-cell-wide halo. These are needed as inputs by the forecast -# model as well as by the code (chgres_cube) that generates the lateral +# model as well as by the code (chgres_cube) that generates the lateral # boundary condition files. <== Are these also needed by make_sfc_climo??? # #----------------------------------------------------------------------- @@ -508,27 +512,27 @@ exec_fn="shave.x" exec_fp="$EXECDIR/${exec_fn}" if [ ! -f "${exec_fp}" ]; then print_err_msg_exit "\ -The executable (exec_fp) for \"shaving\" down the halo in the grid file +The executable (exec_fp) for \"shaving\" down the halo in the grid file does not exist: exec_fp = \"${exec_fp}\" Please ensure that you've built this executable." fi # # Set the full path to the "unshaved" grid file, i.e. the one with a wide -# halo. This is the input grid file for generating both the grid file +# halo. This is the input grid file for generating both the grid file # with a 3-cell-wide halo and the one with a 4-cell-wide halo. # unshaved_fp="${grid_fp}" # -# We perform the work in tmpdir, so change location to that directory. -# Once it is complete, we will move the resultant file from tmpdir to +# We perform the work in tmpdir, so change location to that directory. +# Once it is complete, we will move the resultant file from tmpdir to # GRID_DIR. # cd_vrfy "$tmpdir" # # Create an input namelist file for the shave executable to generate a -# grid file with a 3-cell-wide halo from the one with a wide halo. Then -# call the shave executable. Finally, move the resultant file to the +# grid file with a 3-cell-wide halo from the one with a wide halo. Then +# call the shave executable. Finally, move the resultant file to the # GRID_DIR directory. # print_info_msg "$VERBOSE" " @@ -553,8 +557,8 @@ The namelist file (nml_fn) used in this call is in directory tmpdir: mv_vrfy ${shaved_fp} ${GRID_DIR} # # Create an input namelist file for the shave executable to generate a -# grid file with a 4-cell-wide halo from the one with a wide halo. Then -# call the shave executable. Finally, move the resultant file to the +# grid file with a 4-cell-wide halo from the one with a wide halo. Then +# call the shave executable. Finally, move the resultant file to the # GRID_DIR directory. # print_info_msg "$VERBOSE" " @@ -626,7 +630,7 @@ halo failed." # #----------------------------------------------------------------------- # -# Create symlinks in the FIXLAM directory to the grid and mosaic files +# Create symlinks in the FIXLAM directory to the grid and mosaic files # generated above in the GRID_DIR directory. # #----------------------------------------------------------------------- @@ -640,28 +644,28 @@ failed." # #----------------------------------------------------------------------- # -# Call a function (set_FV3nml_sfc_climo_filenames) to set the values of -# those variables in the forecast model's namelist file that specify the -# paths to the surface climatology files. These files will either already +# Call a function (set_FV3nml_sfc_climo_filenames) to set the values of +# those variables in the forecast model's namelist file that specify the +# paths to the surface climatology files. These files will either already # be avaialable in a user-specified directory (SFC_CLIMO_DIR) or will be -# generated by the MAKE_SFC_CLIMO_TN task. They (or symlinks to them) +# generated by the MAKE_SFC_CLIMO_TN task. They (or symlinks to them) # will be placed (or wll already exist) in the FIXLAM directory. # -# Also, if running ensemble forecasts, call a function (set_FV3nml_stoch_params) -# to create a new FV3 namelist file for each ensemble member that contains -# a unique set of stochastic parameters (i.e. relative to the namelist +# Also, if running ensemble forecasts, call a function (set_FV3nml_stoch_params) +# to create a new FV3 namelist file for each ensemble member that contains +# a unique set of stochastic parameters (i.e. relative to the namelist # files of the other members). # -# Note that unless RUN_TASK_MAKE_GRID is set to "FALSE", the call to -# set_FV3nml_sfc_climo_filenames has to be performed here instead of -# earlier during experiment generation because the surface climatology -# file names depend on the grid resolution variable CRES, and that may +# Note that unless RUN_TASK_MAKE_GRID is set to "FALSE", the call to +# set_FV3nml_sfc_climo_filenames has to be performed here instead of +# earlier during experiment generation because the surface climatology +# file names depend on the grid resolution variable CRES, and that may # not be available until the above steps in this script have been performed. # -# Similarly, unless RUN_TASK_MAKE_GRID is set to "FALSE", the call to -# set_FV3nml_stoch_params must be performed here because it uses the +# Similarly, unless RUN_TASK_MAKE_GRID is set to "FALSE", the call to +# set_FV3nml_stoch_params must be performed here because it uses the # namelist file generated by the call to set_FV3nml_sfc_climo_filenames -# as a starting point (base) and modifies it to add the stochastic +# as a starting point (base) and modifies it to add the stochastic # parameters. Thus, the changes made by set_FV3nml_sfc_climo_filenames # must already be in the base namelist file. # @@ -678,7 +682,7 @@ for the various ensemble members failed." fi create_diag_table_files || print_err_msg_exit "\ -Call to function to create a diagnostics table file under each cycle +Call to function to create a diagnostics table file under each cycle directory failed." # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index a6c4dbad8..38cd95fdd 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -98,55 +98,61 @@ export OMP_STACKSIZE=2048m # case $MACHINE in -"WCOSS_C" | "WCOSS") - { save_shell_opts; set +x; } > /dev/null 2>&1 - - . $MODULESHOME/init/sh - module load PrgEnv-intel cfp-intel-sandybridge/1.1.0 - module list - - { restore_shell_opts; } > /dev/null 2>&1 - - export NODES=1 - export APRUN="aprun -n 1 -N 1 -j 1 -d 1 -cc depth" - export KMP_AFFINITY=disabled - - ulimit -s unlimited - ulimit -a - ;; - -"HERA") - ulimit -s unlimited - ulimit -a - APRUN="time" - ;; - -"JET") - ulimit -s unlimited - ulimit -a - export APRUN="time" - ;; - - -"ODIN") - export APRUN="srun -n 1" - ulimit -s unlimited - ulimit -a - ;; - -"CHEYENNE") - APRUN="time" - ;; - -"STAMPEDE") - export APRUN="time" - ;; - -"ORION") - ulimit -s unlimited - ulimit -a - APRUN="time" - ;; + "WCOSS_C" | "WCOSS") + { save_shell_opts; set +x; } > /dev/null 2>&1 + + . $MODULESHOME/init/sh + module load PrgEnv-intel cfp-intel-sandybridge/1.1.0 + module list + + { restore_shell_opts; } > /dev/null 2>&1 + + export NODES=1 + export APRUN="aprun -n 1 -N 1 -j 1 -d 1 -cc depth" + export KMP_AFFINITY=disabled + + ulimit -s unlimited + ulimit -a + ;; + + "HERA") + ulimit -s unlimited + ulimit -a + APRUN="time" + ;; + + "ORION") + ulimit -s unlimited + ulimit -a + APRUN="time" + ;; + + "JET") + ulimit -s unlimited + ulimit -a + export APRUN="time" + ;; + + "ODIN") + export APRUN="srun -n 1" + ulimit -s unlimited + ulimit -a + ;; + + "CHEYENNE") + APRUN="time" + ;; + + "STAMPEDE") + export APRUN="time" + ;; + + *) + print_err_msg_exit "\ +Run command has not been specified for this machine: + MACHINE = \"$MACHINE\" + APRUN = \"$APRUN\"" + ;; esac # @@ -191,7 +197,7 @@ fi # # Create a temporary (work) directory in which to generate the raw orography # file and change location to it. -# +# tmp_dir="${raw_dir}/tmp" mkdir_vrfy -p "${tmp_dir}" cd_vrfy "${tmp_dir}" @@ -229,7 +235,7 @@ grid_fp="${FIXLAM}/${grid_fn}" # them to a text file. # # Note that it doesn't matter what lonb and latb are set to below because -# if we specify an input grid file to the executable read in (which is +# if we specify an input grid file to the executable read in (which is # what we do below), then if lonb and latb are not set to the dimensions # of the grid specified in that file (divided by 2 since the grid file # specifies a "supergrid"), then lonb and latb effectively get reset to @@ -263,7 +269,7 @@ cat "${input_redirect_fn}" # #----------------------------------------------------------------------- # -# Call the executable to generate the raw orography file corresponding +# Call the executable to generate the raw orography file corresponding # to tile 7 (the regional domain) only. # # The following will create an orography file named @@ -314,7 +320,7 @@ ${tmp_dir}" \ "CHEYENNE" | "HERA" | "JET" | "ODIN" | "STAMPEDE" | "ORION") $APRUN "${exec_fp}" < "${input_redirect_fn}" || \ print_err_msg_exit "\ -Call to executable (exec_fp) that generates the raw orography file returned +Call to executable (exec_fp) that generates the raw orography file returned with nonzero exit code: exec_fp = \"${exec_fp}\"" ;; @@ -346,9 +352,9 @@ mv_vrfy "${raw_orog_fp_orig}" "${raw_orog_fp}" # # Copy the two orography files needed for the drag suite in the FV3_RRFS_v1beta # physics suite. -# -# Note that the following is a temporary fix. We need a long-term solution -# that calls a script or program to generates the necessary files (instead +# +# Note that the following is a temporary fix. We need a long-term solution +# that calls a script or program to generates the necessary files (instead # of copying them). # #----------------------------------------------------------------------- @@ -365,22 +371,22 @@ Orography file generation complete." # # Note that the orography filtering code assumes that the regional grid # is a GFDLgrid type of grid; it is not designed to handle ESGgrid type -# regional grids. If the flag "regional" in the orography filtering +# regional grids. If the flag "regional" in the orography filtering # namelist file is set to .TRUE. (which it always is will be here; see -# below), then filtering code will first calculate a resolution (i.e. +# below), then filtering code will first calculate a resolution (i.e. # number of grid points) value named res_regional for the assumed GFDLgrid # type regional grid using the formula # # res_regional = res*stretch_fac*real(refine_ratio) # # Here res, stretch_fac, and refine_ratio are the values passed to the -# code via the namelist. res and stretch_fac are assumed to be the +# code via the namelist. res and stretch_fac are assumed to be the # resolution (in terms of number of grid points) and the stretch factor # of the (GFDLgrid type) regional grid's parent global cubed-sphere grid, # and refine_ratio is the ratio of the number of grid cells on the regional # grid to a single cell on tile 6 of the parent global grid. After # calculating res_regional, the code interpolates/extrapolates between/ -# beyond a set of (currently 7) resolution values for which the four +# beyond a set of (currently 7) resolution values for which the four # filtering parameters (n_del2_weak, cd4, max_slope, peak_fac) are provided # (by GFDL) to obtain the corresponding values of these parameters at a # resolution of res_regional. These interpolated/extrapolated values are diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index 9453831ce..0e4067f0f 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -128,49 +128,49 @@ EOF # case $MACHINE in -"WCOSS_C") + "WCOSS_C") # This could be wrong. Just a guess since I don't have access to this machine. - APRUN=${APRUN:-"aprun -j 1 -n 6 -N 6"} - ;; + APRUN=${APRUN:-"aprun -j 1 -n 6 -N 6"} + ;; -"WCOSS") + "WCOSS") # This could be wrong. Just a guess since I don't have access to this machine. - APRUN=${APRUN:-"aprun -j 1 -n 6 -N 6"} - ;; + APRUN=${APRUN:-"aprun -j 1 -n 6 -N 6"} + ;; -"HERA") - APRUN="srun" - ;; + "HERA") + APRUN="srun" + ;; -"JET") - APRUN="srun" - ;; + "ORION") + APRUN="srun" + ;; -"CHEYENNE") - nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO )) - APRUN="mpirun -np $nprocs" - ;; + "JET") + APRUN="srun" + ;; -"ODIN") - nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO )) - APRUN="srun -n $nprocs" - ;; + "CHEYENNE") + nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO )) + APRUN="mpirun -np $nprocs" + ;; -"STAMPEDE") - nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO )) - APRUN="ibrun -np ${nprocs}" - ;; + "ODIN") + nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO )) + APRUN="srun -n $nprocs" + ;; -"ORION") - APRUN="srun" - ;; + "STAMPEDE") + nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO )) + APRUN="ibrun -np ${nprocs}" + ;; -*) - print_err_msg_exit "\ + *) + print_err_msg_exit "\ Run command has not been specified for this machine: MACHINE = \"$MACHINE\" APRUN = \"$APRUN\"" - ;; + ;; esac # diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index a77bdf482..10d899034 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -88,68 +88,68 @@ print_input_args valid_args #----------------------------------------------------------------------- # case $MACHINE in -# -"WCOSS_CRAY") -# - ulimit -s unlimited - ulimit -a - APRUN="aprun -b -j1 -n${PE_MEMBER01} -N24 -d1 -cc depth" - ;; -# -"WCOSS_DELL_P3") - ulimit -s unlimited - ulimit -a - APRUN="mpirun -l -np ${PE_MEMBER01}" - ;; -# -"HERA") - ulimit -s unlimited - ulimit -a - APRUN="srun" - LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - OMP_NUM_THREADS=4 - ;; -# -"JET") - ulimit -s unlimited - ulimit -a - APRUN="srun" - LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - OMP_NUM_THREADS=4 - ;; -# -"ODIN") -# - module list - - ulimit -s unlimited - ulimit -a - APRUN="srun -n ${PE_MEMBER01}" - ;; -# -"CHEYENNE") -# - module list - nprocs=$(( NNODES_RUN_FCST*PPN_RUN_FCST )) - APRUN="mpirun -np $nprocs" - LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - ;; -# - -"STAMPEDE") -# - module list - APRUN="ibrun -np ${PE_MEMBER01}" - #LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - ;; -# -"ORION") - ulimit -s unlimited - ulimit -a - APRUN="srun" -# LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -# + "WCOSS_CRAY") + ulimit -s unlimited + ulimit -a + APRUN="aprun -b -j1 -n${PE_MEMBER01} -N24 -d1 -cc depth" + ;; + + "WCOSS_DELL_P3") + ulimit -s unlimited + ulimit -a + APRUN="mpirun -l -np ${PE_MEMBER01}" + ;; + + "HERA") + ulimit -s unlimited + ulimit -a + APRUN="srun" + LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + OMP_NUM_THREADS=4 + ;; + + "ORION") + ulimit -s unlimited + ulimit -a + APRUN="srun" +# LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + + "JET") + ulimit -s unlimited + ulimit -a + APRUN="srun" + LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + OMP_NUM_THREADS=4 + ;; + + "ODIN") + module list + ulimit -s unlimited + ulimit -a + APRUN="srun -n ${PE_MEMBER01}" + ;; + + "CHEYENNE") + module list + nprocs=$(( NNODES_RUN_FCST*PPN_RUN_FCST )) + APRUN="mpirun -np $nprocs" + LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + ;; + + "STAMPEDE") + module list + APRUN="ibrun -np ${PE_MEMBER01}" +# LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + ;; + + *) + print_err_msg_exit "\ +Run command has not been specified for this machine: + MACHINE = \"$MACHINE\" + APRUN = \"$APRUN\"" + ;; + esac # #----------------------------------------------------------------------- diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 8d799e03a..957c19c21 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -82,58 +82,65 @@ print_input_args valid_args # case $MACHINE in -"WCOSS_CRAY") + "WCOSS_CRAY") # Specify computational resources. - export NODES=2 - export ntasks=48 - export ptile=24 - export threads=1 - export MP_LABELIO=yes - export OMP_NUM_THREADS=$threads + export NODES=2 + export ntasks=48 + export ptile=24 + export threads=1 + export MP_LABELIO=yes + export OMP_NUM_THREADS=$threads - APRUN="aprun -j 1 -n${ntasks} -N${ptile} -d${threads} -cc depth" - ;; + APRUN="aprun -j 1 -n${ntasks} -N${ptile} -d${threads} -cc depth" + ;; -"WCOSS_DELL_P3") + "WCOSS_DELL_P3") # Specify computational resources. - export NODES=2 - export ntasks=48 - export ptile=24 - export threads=1 - export MP_LABELIO=yes - export OMP_NUM_THREADS=$threads - - APRUN="mpirun" - ;; + export NODES=2 + export ntasks=48 + export ptile=24 + export threads=1 + export MP_LABELIO=yes + export OMP_NUM_THREADS=$threads -"HERA") - APRUN="srun" - ;; + APRUN="mpirun" + ;; -"JET") - APRUN="srun" - ;; + "HERA") + APRUN="srun" + ;; -"ODIN") - APRUN="srun -n 1" - ;; + "ORION") + APRUN="srun" + ;; -"CHEYENNE") - module list - nprocs=$(( NNODES_RUN_POST*PPN_RUN_POST )) - APRUN="mpirun -np $nprocs" - ;; + "JET") + APRUN="srun" + ;; -"STAMPEDE") - nprocs=$(( NNODES_RUN_POST*PPN_RUN_POST )) - APRUN="ibrun -n $nprocs" - ;; + "ODIN") + APRUN="srun -n 1" + ;; -"ORION") - APRUN="srun" - ;; + "CHEYENNE") + module list + nprocs=$(( NNODES_RUN_POST*PPN_RUN_POST )) + APRUN="mpirun -np $nprocs" + ;; + + "STAMPEDE") + nprocs=$(( NNODES_RUN_POST*PPN_RUN_POST )) + APRUN="ibrun -n $nprocs" + ;; + + *) + print_err_msg_exit "\ +Run command has not been specified for this machine: + MACHINE = \"$MACHINE\" + APRUN = \"$APRUN\"" + ;; esac # @@ -149,7 +156,7 @@ if [ ${USE_CUSTOM_POST_CONFIG_FILE} = "TRUE" ]; then post_config_fp="${CUSTOM_POST_CONFIG_FP}" print_info_msg " ==================================================================== -Copying the user-defined post flat file specified by CUSTOM_POST_CONFIG_FP +Copying the user-defined post flat file specified by CUSTOM_POST_CONFIG_FP to the post forecast hour directory (fhr_dir): CUSTOM_POST_CONFIG_FP = \"${CUSTOM_POST_CONFIG_FP}\" fhr_dir = \"${fhr_dir}\" @@ -158,7 +165,7 @@ else post_config_fp="${EMC_POST_DIR}/parm/postxconfig-NT-fv3lam.txt" print_info_msg " ==================================================================== -Copying the default post flat file specified by post_config_fp to the post +Copying the default post flat file specified by post_config_fp to the post forecast hour directory (fhr_dir): post_config_fp = \"${post_config_fp}\" fhr_dir = \"${fhr_dir}\" @@ -170,7 +177,7 @@ cp_vrfy ${EXECDIR}/ncep_post . # #----------------------------------------------------------------------- # -# Get the cycle date and hour (in formats of yyyymmdd and hh, respectively) +# Get the cycle date and hour (in formats of yyyymmdd and hh, respectively) # from cdate. # #----------------------------------------------------------------------- @@ -225,14 +232,14 @@ zero exit code." #----------------------------------------------------------------------- # # Move (and rename) the output files from the work directory to their -# final location (postprd_dir). Then delete the work directory. +# final location (postprd_dir). Then delete the work directory. # #----------------------------------------------------------------------- # # #----------------------------------------------------------------------- # -# A separate ${post_fhr} forecast hour variable is required for the post +# A separate ${post_fhr} forecast hour variable is required for the post # files, since they may or may not be three digits long, depending on the # length of the forecast. # From 0077c062cedeace59752aa7ba88e4715bf18ffe6 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 9 Oct 2020 19:08:36 -0500 Subject: [PATCH 10/27] More spacing changes. --- ush/setup.sh | 252 ++++++++++++++++++++-------------------- ush/valid_param_vals.sh | 2 +- 2 files changed, 127 insertions(+), 127 deletions(-) diff --git a/ush/setup.sh b/ush/setup.sh index c99054e51..51ce38c6f 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -338,74 +338,74 @@ check_var_valid_value "MACHINE" "valid_vals_MACHINE" #----------------------------------------------------------------------- # case $MACHINE in -# -"WCOSS_CRAY") - NCORES_PER_NODE="24" - SCHED="lsfcray" - QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} - QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} - QUEUE_HPSS_TAG="queue" # lsfcray does not support "partition" tag - QUEUE_FCST=${QUEUE_FCST:-"dev"} - ;; -# -"WCOSS_DELL_P3") - NCORES_PER_NODE=24 - SCHED="lsf" - QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} - QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} - QUEUE_HPSS_TAG="queue" # lsf does not support "partition" tag - QUEUE_FCST=${QUEUE_FCST:-"dev"} - ;; -# -"HERA") - NCORES_PER_NODE=24 # Should this be 40? Need to check heradocs. - SCHED="${SCHED:-slurm}" - QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} - QUEUE_HPSS=${QUEUE_HPSS:-"service"} - QUEUE_FCST=${QUEUE_FCST:-"batch"} - ;; -# -"JET") - NCORES_PER_NODE=24 - SCHED="${SCHED:-slurm}" - QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} - QUEUE_HPSS=${QUEUE_HPSS:-"service"} - QUEUE_FCST=${QUEUE_FCST:-"batch"} - ;; -# -"ODIN") - NCORES_PER_NODE=24 - SCHED="${SCHED:-slurm}" - QUEUE_DEFAULT=${QUEUE_DEFAULT:-""} - QUEUE_HPSS=${QUEUE_HPSS:-""} - QUEUE_FCST=${QUEUE_FCST:-""} - ;; -# -"CHEYENNE") - NCORES_PER_NODE=36 - SCHED="${SCHED:-pbspro}" - QUEUE_DEFAULT=${QUEUE_DEFAULT:-"regular"} - QUEUE_HPSS=${QUEUE_HPSS:-"regular"} - QUEUE_HPSS_TAG="queue" # pbspro does not support "partition" tag - QUEUE_FCST=${QUEUE_FCST:-"regular"} - ;; -# -"STAMPEDE") - NCORES_PER_NODE=68 - SCHED="slurm" - QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"} - QUEUE_HPSS=${QUEUE_HPSS:-"development"} - QUEUE_FCST=${QUEUE_FCST:-"normal"} - ;; -# -"ORION") - NCORES_PER_NODE=40 - SCHED="${SCHED:-slurm}" - QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} - QUEUE_HPSS=${QUEUE_HPSS:-"service"} - QUEUE_FCST=${QUEUE_FCST:-"batch"} - ;; -# + + "WCOSS_CRAY") + NCORES_PER_NODE="24" + SCHED="lsfcray" + QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} + QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} + QUEUE_HPSS_TAG="queue" # lsfcray does not support "partition" tag + QUEUE_FCST=${QUEUE_FCST:-"dev"} + ;; + + "WCOSS_DELL_P3") + NCORES_PER_NODE=24 + SCHED="lsf" + QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} + QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} + QUEUE_HPSS_TAG="queue" # lsf does not support "partition" tag + QUEUE_FCST=${QUEUE_FCST:-"dev"} + ;; + + "HERA") + NCORES_PER_NODE=24 # Should this be 40? Need to check heradocs. + SCHED="${SCHED:-slurm}" + QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} + QUEUE_HPSS=${QUEUE_HPSS:-"service"} + QUEUE_FCST=${QUEUE_FCST:-"batch"} + ;; + + "ORION") + NCORES_PER_NODE=40 + SCHED="${SCHED:-slurm}" + QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} + QUEUE_HPSS=${QUEUE_HPSS:-"service"} + QUEUE_FCST=${QUEUE_FCST:-"batch"} + ;; + + "JET") + NCORES_PER_NODE=24 + SCHED="${SCHED:-slurm}" + QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} + QUEUE_HPSS=${QUEUE_HPSS:-"service"} + QUEUE_FCST=${QUEUE_FCST:-"batch"} + ;; + + "ODIN") + NCORES_PER_NODE=24 + SCHED="${SCHED:-slurm}" + QUEUE_DEFAULT=${QUEUE_DEFAULT:-""} + QUEUE_HPSS=${QUEUE_HPSS:-""} + QUEUE_FCST=${QUEUE_FCST:-""} + ;; + + "CHEYENNE") + NCORES_PER_NODE=36 + SCHED="${SCHED:-pbspro}" + QUEUE_DEFAULT=${QUEUE_DEFAULT:-"regular"} + QUEUE_HPSS=${QUEUE_HPSS:-"regular"} + QUEUE_HPSS_TAG="queue" # pbspro does not support "partition" tag + QUEUE_FCST=${QUEUE_FCST:-"regular"} + ;; + + "STAMPEDE") + NCORES_PER_NODE=68 + SCHED="slurm" + QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"} + QUEUE_HPSS=${QUEUE_HPSS:-"development"} + QUEUE_FCST=${QUEUE_FCST:-"normal"} + ;; + esac # #----------------------------------------------------------------------- @@ -723,70 +723,70 @@ TEMPLATE_DIR="$USHDIR/templates" case $MACHINE in -"WCOSS_CRAY") - FIXgsm=${FIXgsm:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} - TOPO_DIR=${TOPO_DIR:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-""} - ;; - -"WCOSS_DELL_P3") - FIXgsm=${FIXgsm:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} - TOPO_DIR=${TOPO_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-""} - ;; - -"DELL") - FIXgsm=${FIXgsm:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} - TOPO_DIR=${TOPO_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-""} - ;; - -"HERA") - FIXgsm=${FIXgsm:-"/scratch1/NCEPDEV/global/glopara/fix/fix_am"} - TOPO_DIR=${TOPO_DIR:-"/scratch1/NCEPDEV/global/glopara/fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch1/NCEPDEV/da/George.Gayno/ufs_utils.git/climo_fields_netcdf"} - ;; - -"ORION") - FIXgsm=${FIXgsm:-"/work/noaa/fv3-cam/emc.campara/fix_fv3cam/fix_am"} - TOPO_DIR=${TOPO_DIR:-"/work/noaa/gsd-fv3-dev/gsketefia/UFS/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/noaa/gsd-fv3-dev/gsketefia/UFS/climo_fields_netcdf"} - ;; - -"JET") - FIXgsm=${FIXgsm:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_am"} - TOPO_DIR=${TOPO_DIR:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/lfs1/HFIP/hwrf-data/git/fv3gfs/fix/fix_sfc_climo"} - ;; - -"ODIN") - FIXgsm=${FIXgsm:-"/scratch/ywang/fix/theia_fix/fix_am"} - TOPO_DIR=${TOPO_DIR:-"/scratch/ywang/fix/theia_fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch/ywang/fix/climo_fields_netcdf"} - ;; - -"CHEYENNE") - FIXgsm=${FIXgsm:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_am"} - TOPO_DIR=${TOPO_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/climo_fields_netcdf"} - ;; - -"STAMPEDE") - FIXgsm=${FIXgsm:-"/work/00315/tg455890/stampede2/regional_fv3/fix_am"} - TOPO_DIR=${TOPO_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/climo_fields_netcdf"} - ;; - -*) - print_err_msg_exit "\ + "WCOSS_CRAY") + FIXgsm=${FIXgsm:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} + TOPO_DIR=${TOPO_DIR:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} + SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-""} + ;; + + "WCOSS_DELL_P3") + FIXgsm=${FIXgsm:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} + TOPO_DIR=${TOPO_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} + SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-""} + ;; + + "DELL") + FIXgsm=${FIXgsm:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} + TOPO_DIR=${TOPO_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} + SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-""} + ;; + + "HERA") + FIXgsm=${FIXgsm:-"/scratch1/NCEPDEV/global/glopara/fix/fix_am"} + TOPO_DIR=${TOPO_DIR:-"/scratch1/NCEPDEV/global/glopara/fix/fix_orog"} + SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch1/NCEPDEV/da/George.Gayno/ufs_utils.git/climo_fields_netcdf"} + ;; + + "ORION") + FIXgsm=${FIXgsm:-"/work/noaa/fv3-cam/emc.campara/fix_fv3cam/fix_am"} + TOPO_DIR=${TOPO_DIR:-"/work/noaa/gsd-fv3-dev/gsketefia/UFS/fix_orog"} + SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/noaa/gsd-fv3-dev/gsketefia/UFS/climo_fields_netcdf"} + ;; + + "JET") + FIXgsm=${FIXgsm:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_am"} + TOPO_DIR=${TOPO_DIR:-"/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/fix_orog"} + SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/lfs1/HFIP/hwrf-data/git/fv3gfs/fix/fix_sfc_climo"} + ;; + + "ODIN") + FIXgsm=${FIXgsm:-"/scratch/ywang/fix/theia_fix/fix_am"} + TOPO_DIR=${TOPO_DIR:-"/scratch/ywang/fix/theia_fix/fix_orog"} + SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/scratch/ywang/fix/climo_fields_netcdf"} + ;; + + "CHEYENNE") + FIXgsm=${FIXgsm:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_am"} + TOPO_DIR=${TOPO_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/fix_orog"} + SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/glade/p/ral/jntp/UFS_CAM/fix/climo_fields_netcdf"} + ;; + + "STAMPEDE") + FIXgsm=${FIXgsm:-"/work/00315/tg455890/stampede2/regional_fv3/fix_am"} + TOPO_DIR=${TOPO_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/fix_orog"} + SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/00315/tg455890/stampede2/regional_fv3/climo_fields_netcdf"} + ;; + + *) + print_err_msg_exit "\ One or more fix file directories have not been specified for this machine: MACHINE = \"$MACHINE\" FIXgsm = \"${FIXgsm:-\"\"} TOPO_DIR = \"${TOPO_DIR:-\"\"} SFC_CLIMO_INPUT_DIR = \"${SFC_CLIMO_INPUT_DIR:-\"\"} - You can specify the missing location(s) in config.sh" - ;; + ;; + esac # #----------------------------------------------------------------------- diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 4e9fffcc4..7d3001fef 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -1,6 +1,6 @@ valid_vals_RUN_ENVIR=("nco" "community") valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "THEIA" "HERA" "JET" "ODIN" "CHEYENNE" "STAMPEDE" "ORION") +valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE") valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none") valid_vals_PREDEF_GRID_NAME=( \ "EMC_CONUS_3km" \ From ad04092d523c19e8bd88e9bbb393057aa4888187 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 9 Oct 2020 20:39:42 -0500 Subject: [PATCH 11/27] Bug fix. --- scripts/exregional_run_fcst.sh | 63 +++++++++++++++++----------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 10d899034..5dd04f437 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -94,13 +94,13 @@ case $MACHINE in ulimit -a APRUN="aprun -b -j1 -n${PE_MEMBER01} -N24 -d1 -cc depth" ;; - + "WCOSS_DELL_P3") ulimit -s unlimited ulimit -a APRUN="mpirun -l -np ${PE_MEMBER01}" ;; - + "HERA") ulimit -s unlimited ulimit -a @@ -108,13 +108,14 @@ case $MACHINE in LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" OMP_NUM_THREADS=4 ;; - + "ORION") ulimit -s unlimited ulimit -a APRUN="srun" # LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - + ;; + "JET") ulimit -s unlimited ulimit -a @@ -122,34 +123,34 @@ case $MACHINE in LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" OMP_NUM_THREADS=4 ;; - + "ODIN") module list ulimit -s unlimited ulimit -a APRUN="srun -n ${PE_MEMBER01}" ;; - + "CHEYENNE") module list nprocs=$(( NNODES_RUN_FCST*PPN_RUN_FCST )) APRUN="mpirun -np $nprocs" LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ;; - + "STAMPEDE") module list APRUN="ibrun -np ${PE_MEMBER01}" # LD_LIBRARY_PATH="${UFS_WTHR_MDL_DIR}/FV3/ccpp/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ;; - + *) print_err_msg_exit "\ Run command has not been specified for this machine: MACHINE = \"$MACHINE\" APRUN = \"$APRUN\"" ;; - + esac # #----------------------------------------------------------------------- @@ -162,13 +163,13 @@ run_dir="${cycle_dir}${slash_ensmem_subdir}" # #----------------------------------------------------------------------- # -# Create links in the INPUT subdirectory of the current run directory to +# Create links in the INPUT subdirectory of the current run directory to # the grid and (filtered) orography files. # #----------------------------------------------------------------------- # print_info_msg "$VERBOSE" " -Creating links in the INPUT subdirectory of the current run directory to +Creating links in the INPUT subdirectory of the current run directory to the grid and (filtered) orography files ..." @@ -259,14 +260,14 @@ Cannot create symlink because target does not exist: fi # # If using the FV3_RRFS_v1beta physics suite, there are two files (that -# contain statistics of the orography) that are needed by the drag +# contain statistics of the orography) that are needed by the drag # parameterization in that suite. Below, symlinks to these are created # in the run directory. Note that the symlinks must have specific names -# that the FV3 model is hardcoded to recognize ("${CRES}_" and "halo0" +# that the FV3 model is hardcoded to recognize ("${CRES}_" and "halo0" # must be stripped from the file names). We use those below. # if [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then -# Symlink to orographic statistics fields file with "${CRES}_" and "halo0" +# Symlink to orographic statistics fields file with "${CRES}_" and "halo0" # stripped from name. target="${FIXLAM}/${CRES}${DOT_OR_USCORE}oro_data_ls.tile${TILE_RGNL}.halo${NH0}.nc" symlink="oro_data_ls.nc" @@ -322,7 +323,7 @@ fi # gfs_bndy*.nc # gfs_ctrl.nc # -# Some of these files (gfs_ctrl.nc, gfs_bndy*.nc) already exist, but +# Some of these files (gfs_ctrl.nc, gfs_bndy*.nc) already exist, but # others do not. Thus, create links with these names to the appropriate # files (in this case the initial condition and surface files only). # @@ -363,9 +364,9 @@ fi #----------------------------------------------------------------------- # # Create links in the current run directory to fixed (i.e. static) files -# in the FIXam directory. These links have names that are set to the +# in the FIXam directory. These links have names that are set to the # names of files that the forecast model expects to exist in the current -# working directory when the forecast model executable is called (and +# working directory when the forecast model executable is called (and # that is just the run directory). # #----------------------------------------------------------------------- @@ -373,7 +374,7 @@ fi cd_vrfy ${run_dir} print_info_msg "$VERBOSE" " -Creating links in the current run directory (run_dir) to fixed (i.e. +Creating links in the current run directory (run_dir) to fixed (i.e. static) files in the FIXam directory: FIXam = \"${FIXam}\" run_dir = \"${run_dir}\"" @@ -407,8 +408,8 @@ done # #----------------------------------------------------------------------- # -# If running this cycle/ensemble member combination more than once (e.g. -# using rocotoboot), remove any time stamp file that may exist from the +# If running this cycle/ensemble member combination more than once (e.g. +# using rocotoboot), remove any time stamp file that may exist from the # previous attempt. # #----------------------------------------------------------------------- @@ -419,13 +420,13 @@ rm_vrfy -f time_stamp.out #----------------------------------------------------------------------- # # Create links in the current run directory to cycle-independent (and -# ensemble-member-independent) model input files in the main experiment +# ensemble-member-independent) model input files in the main experiment # directory. # #----------------------------------------------------------------------- # print_info_msg "$VERBOSE" " -Creating links in the current run directory to cycle-independent model +Creating links in the current run directory to cycle-independent model input files in the main experiment directory..." relative_or_null="" @@ -445,7 +446,7 @@ fi if [ "${USE_CCPP}" = "TRUE" ]; then - ln_vrfy -sf ${relative_or_null} ${CCPP_PHYS_SUITE_FP} ${run_dir} + ln_vrfy -sf ${relative_or_null} ${CCPP_PHYS_SUITE_FP} ${run_dir} if [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ] || \ @@ -466,17 +467,17 @@ create_model_configure_file \ cdate="$cdate" \ nthreads=${OMP_NUM_THREADS:-1} \ run_dir="${run_dir}" || print_err_msg_exit "\ -Call to function to create a model configuration file for the current +Call to function to create a model configuration file for the current cycle's (cdate) run directory (run_dir) failed: cdate = \"${cdate}\" run_dir = \"${run_dir}\"" # #----------------------------------------------------------------------- # -# If running ensemble forecasts, create a link to the cycle-specific -# diagnostic tables file in the cycle directory. Note that this link -# should not be made if not running ensemble forecasts because in that -# case, the cycle directory is the run directory (and we would be creating +# If running ensemble forecasts, create a link to the cycle-specific +# diagnostic tables file in the cycle directory. Note that this link +# should not be made if not running ensemble forecasts because in that +# case, the cycle directory is the run directory (and we would be creating # a symlink with the name of a file that already exists). # #----------------------------------------------------------------------- @@ -501,15 +502,15 @@ export OMP_STACKSIZE=1024m #----------------------------------------------------------------------- # # Run the FV3-LAM model. Note that we have to launch the forecast from -# the current cycle's directory because the FV3 executable will look for -# input files in the current directory. Since those files have been +# the current cycle's directory because the FV3 executable will look for +# input files in the current directory. Since those files have been # staged in the cycle directory, the current directory must be the cycle # directory (which it already is). # #----------------------------------------------------------------------- # $APRUN ${FV3_EXEC_FP} || print_err_msg_exit "\ -Call to executable to run FV3-LAM forecast returned with nonzero exit +Call to executable to run FV3-LAM forecast returned with nonzero exit code." # #----------------------------------------------------------------------- From 116086aad84d4b5e70a20cef4c0d9d20df429fbd Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 9 Oct 2020 20:40:18 -0500 Subject: [PATCH 12/27] Rearrange case-statement cases for consistency with case-statements in other scripts. --- ush/set_extrn_mdl_params.sh | 38 +++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index 4e68bf474..9ec1c8707 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -62,6 +62,9 @@ else "HERA") EXTRN_MDL_SYSBASEDIR_ICS="" ;; + "ORION") + EXTRN_MDL_SYSBASEDIR_ICS="" + ;; "JET") EXTRN_MDL_SYSBASEDIR_ICS="" ;; @@ -74,9 +77,6 @@ else "STAMPEDE") EXTRN_MDL_SYSBASEDIR_ICS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001" ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_ICS="dummy_value" - ;; esac ;; @@ -91,6 +91,9 @@ else "HERA") EXTRN_MDL_SYSBASEDIR_ICS="/scratch1/NCEPDEV/rstprod/com/gfs/prod" ;; + "ORION") + EXTRN_MDL_SYSBASEDIR_ICS="" + ;; "JET") EXTRN_MDL_SYSBASEDIR_ICS="/public/data/grids/gfs/nemsio" ;; @@ -100,9 +103,6 @@ else "CHEYENNE") EXTRN_MDL_SYSBASEDIR_ICS="/glade/p/ral/jntp/UFS_CAM/COMGFS" ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_ICS="dummy_value" - ;; esac ;; @@ -111,11 +111,12 @@ else "HERA") EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/public/data/gsd/rap/full/wrfnat" ;; + "ORION") + EXTRN_MDL_SYSBASEDIR_ICS="" + ;; "JET") EXTRN_MDL_SYSBASEDIR_ICS="/misc/whome/rtrr/rap" ;; -# This goes with the comment below for the if-statement (-z EXTRN_MDL_SYSBASEDIR_ICS). -# Should not need this case. "CHEYENNE") EXTRN_MDL_SYSBASEDIR_ICS="dummy_value" ;; @@ -127,6 +128,9 @@ else "HERA") EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/public/data/gsd/hrrr/conus/wrfnat" ;; + "ORION") + EXTRN_MDL_SYSBASEDIR_ICS="" + ;; "JET") EXTRN_MDL_SYSBASEDIR_ICS="/misc/whome/rtrr/hrrr" ;; @@ -209,6 +213,9 @@ else "HERA") EXTRN_MDL_SYSBASEDIR_LBCS="" ;; + "ORION") + EXTRN_MDL_SYSBASEDIR_LBCS="" + ;; "JET") EXTRN_MDL_SYSBASEDIR_LBCS="" ;; @@ -221,9 +228,6 @@ else "STAMPEDE") EXTRN_MDL_SYSBASEDIR_LBCS="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001" ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_LBCS="dummy_value" - ;; esac ;; @@ -238,6 +242,9 @@ else "HERA") EXTRN_MDL_SYSBASEDIR_LBCS="/scratch1/NCEPDEV/rstprod/com/gfs/prod" ;; + "ORION") + EXTRN_MDL_SYSBASEDIR_LBCS="" + ;; "JET") EXTRN_MDL_SYSBASEDIR_LBCS="/public/data/grids/gfs/nemsio" ;; @@ -247,9 +254,6 @@ else "CHEYENNE") EXTRN_MDL_SYSBASEDIR_LBCS="/glade/p/ral/jntp/UFS_CAM/COMGFS" ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_LBCS="dummy_value" - ;; esac ;; @@ -258,6 +262,9 @@ else "HERA") EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/public/data/gsd/rap/full/wrfnat" ;; + "ORION") + EXTRN_MDL_SYSBASEDIR_LBCS="" + ;; "JET") EXTRN_MDL_SYSBASEDIR_LBCS="/misc/whome/rtrr/rap" ;; @@ -272,6 +279,9 @@ else "HERA") EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/public/data/gsd/hrrr/conus/wrfnat" ;; + "ORION") + EXTRN_MDL_SYSBASEDIR_LBCS="" + ;; "JET") EXTRN_MDL_SYSBASEDIR_LBCS="/misc/whome/rtrr/hrrr" ;; From ea5192b295524def194fb317eccbc6d4d5f79dff Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Tue, 13 Oct 2020 16:02:26 -0500 Subject: [PATCH 13/27] Modifications to more easily specify slurm partitions. * Remove the workflow variables QUEUE_DEFAULT_TAG, QUEUE_HPSS_TAG, and QUEUE_FCST_TAG which are currently used to determine whether QUEUE_DEFAULT, QUEUE_HPSS, and QUEUE_FCST specify the names of queue/QOS's or slurm partitions. * Add the workflow variables PARTITION_DEFAULT_TAG, PARTITION_HPSS_TAG, and PARTITION_FCST_TAG. These will be used to specify slurm partitions only, and the variables QUEUE_DEFAULT, QUEUE_HPSS, and QUEUE_FCST will be used to specify queues/QOS's only. --- ush/config_defaults.sh | 81 +++++++++++++++++++++++----------- ush/generate_FV3LAM_wflow.sh | 38 ++++++++++++++-- ush/setup.sh | 67 +++++++++++++++++++++++++--- ush/templates/FV3LAM_wflow.xml | 9 ++-- 4 files changed, 154 insertions(+), 41 deletions(-) diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 87f25e4ae..30f6bd32c 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -52,35 +52,62 @@ RUN_ENVIR="nco" # machine. # # QUEUE_DEFAULT: -# The default queue to which workflow tasks are submitted. If a task -# does not have a specific variable that specifies the queue to which it -# will be submitted (e.g. QUEUE_HPSS, QUEUE_FCST; see below), it will be -# submitted to the queue specified by this variable. If this is not set -# or is set to an empty string, it will be (re)set to a machine-dependent -# value. +# The default queue or QOS (if using the slurm job scheduler, where QOS +# is Quality of Service) to which workflow tasks are submitted. If a +# task does not have a specific variable that specifies the queue to which +# it will be submitted (e.g. QUEUE_HPSS, QUEUE_FCST; see below), it will +# be submitted to the queue specified by this variable. If this is not +# set or is set to an empty string, it will be (re)set to a machine- +# dependent value. # -# QUEUE_DEFAULT_TAG: -# The rocoto xml tag to use for specifying the default queue. For most -# platforms this should be "queue" +## QUEUE_DEFAULT_TAG: +## The rocoto xml tag to use for specifying the default queue. For most +## platforms this should be "queue" # # QUEUE_HPSS: -# The queue to which the tasks that get or create links to external model -# files [which are needed to generate initial conditions (ICs) and lateral -# boundary conditions (LBCs)] are submitted. If this is not set or is -# set to an empty string, it will be (re)set to a machine-dependent value. +# The queue or QOS to which the tasks that get or create links to external +# model files [which are needed to generate initial conditions (ICs) and +# lateral boundary conditions (LBCs)] are submitted. If this is not set +# or is set to an empty string, it will be (re)set to a machine-dependent +# value. # -# QUEUE_HPSS_TAG: -# The rocoto xml tag to use for specifying the HPSS queue. For slurm-based -# platforms this is typically "partition", for others it may be "queue" +## QUEUE_HPSS_TAG: +## The rocoto xml tag to use for specifying the HPSS queue. For slurm-based +## platforms this is typically "partition", for others it may be "queue" # # QUEUE_FCST: -# The queue to which the task that runs a forecast is submitted. If this -# is not set or set to an empty string, it will be (re)set to a machine- -# dependent value. +# The queue or QOS to which the task that runs a forecast is submitted. +# If this is not set or set to an empty string, it will be (re)set to a +# machine-dependent value. +# +## QUEUE_FCST_TAG: +## The rocoto xml tag to use for specifying the fcst queue. For most +## platforms this should be "queue" +# +# PARTITION_DEFAULT: +# If using the slurm job scheduler (i.e. if SCHED is set to "slurm"), +# the default partition to which to submit workflow tasks. If a task +# does not have a specific variable that specifies the partition to which +# it will be submitted (e.g. PARTITION_HPSS, PARTITION_FCST; see below), +# it will be submitted to the partition specified by this variable. If +# this is not set or is set to an empty string, it will be (re)set to a +# machine-dependent value. This is not used if SCHED is not set to +# "slurm". +# +# PARTITION_HPSS: +# If using the slurm job scheduler (i.e. if SCHED is set to "slurm"), +# the partition to which the tasks that get or create links to external +# model files [which are needed to generate initial conditions (ICs) and +# lateral boundary conditions (LBCs)] are submitted. If this is not set +# or is set to an empty string, it will be (re)set to a machine-dependent +# value. This is not used if SCHED is not set to "slurm". # -# QUEUE_FCST_TAG: -# The rocoto xml tag to use for specifying the fcst queue. For most -# platforms this should be "queue" +# PARTITION_FCST: +# If using the slurm job scheduler (i.e. if SCHED is set to "slurm"), +# the partition to which the task that runs forecasts is submitted. If +# this is not set or set to an empty string, it will be (re)set to a +# machine-dependent value. This is not used if SCHED is not set to +# "slurm". # # mach_doc_end # @@ -90,11 +117,15 @@ MACHINE="BIG_COMPUTER" ACCOUNT="project_name" SCHED="" QUEUE_DEFAULT="" -QUEUE_DEFAULT_TAG="queue" +#QUEUE_DEFAULT_TAG="queue" QUEUE_HPSS="" -QUEUE_HPSS_TAG="partition" +#QUEUE_HPSS_TAG="partition" QUEUE_FCST="" -QUEUE_FCST_TAG="queue" +#QUEUE_FCST_TAG="queue" + +PARTITION_DEFAULT="" +PARTITION_HPSS="" +PARTITION_FCST="" # #----------------------------------------------------------------------- # diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 0244cd7cd..c400cfd76 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -159,6 +159,37 @@ if [ "${DO_ENSEMBLE}" = "TRUE" ]; then slash_ensmem_subdir="/mem#${ensmem_indx_name}#" fi +# +# To see if the following is enough (i.e. to see whether the if-statements +# below are really needed), have to run with the following lines uncommented +# on systems that don't use slurm. +# The following seem to work ok on orion, i.e. passing an empty partition +# setting in the rocoto xml, like this: +# This causes rocoto/slurm to submit to the default partition (called "orion"). +# +#partition_default_spec="${PARTITION_DEFAULT}" +#partition_hpss_spec="${PARTITION_HPSS}" +#partition_fcst_spec="${PARTITION_FCST}" + +partition_default_spec="\"\"" +if [ ! -z "${PARTITION_DEFAULT}" ]; then + partition_default_spec="${PARTITION_DEFAULT}" +fi + +partition_hpss_spec="\"\"" +if [ ! -z "${PARTITION_HPSS}" ]; then + partition_hpss_spec="${PARTITION_HPSS}" +fi + +partition_fcst_spec="\"\"" +if [ ! -z "${PARTITION_FCST}" ]; then + partition_fcst_spec="${PARTITION_FCST}" +fi + +# 'queue_default_tag': ${QUEUE_DEFAULT_TAG} +# 'queue_hpss_tag': ${QUEUE_HPSS_TAG} +# 'queue_fcst_tag': ${QUEUE_FCST_TAG} + settings="\ # # Parameters needed by the job scheduler. @@ -166,11 +197,11 @@ settings="\ 'account': $ACCOUNT 'sched': $SCHED 'queue_default': ${QUEUE_DEFAULT} - 'queue_default_tag': ${QUEUE_DEFAULT_TAG} 'queue_hpss': ${QUEUE_HPSS} - 'queue_hpss_tag': ${QUEUE_HPSS_TAG} 'queue_fcst': ${QUEUE_FCST} - 'queue_fcst_tag': ${QUEUE_FCST_TAG} + 'partition_default_spec': ${partition_default_spec} + 'partition_hpss_spec': ${partition_hpss_spec} + 'partition_fcst_spec': ${partition_fcst_spec} 'machine': ${MACHINE} # # Workflow task names. @@ -201,7 +232,6 @@ settings="\ # 'ncores_run_fcst': ${PE_MEMBER01} 'native_run_fcst': --cpus-per-task 4 --exclusive - 'partition_run_fcst': sjet,vjet,kjet,xjet # # Number of logical processes per node for each task. If running without # threading, this is equal to the number of MPI processes per node. diff --git a/ush/setup.sh b/ush/setup.sh index 51ce38c6f..7938597be 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -344,7 +344,7 @@ case $MACHINE in SCHED="lsfcray" QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} - QUEUE_HPSS_TAG="queue" # lsfcray does not support "partition" tag +# QUEUE_HPSS_TAG="queue" # lsfcray does not support "partition" tag QUEUE_FCST=${QUEUE_FCST:-"dev"} ;; @@ -353,31 +353,52 @@ case $MACHINE in SCHED="lsf" QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} - QUEUE_HPSS_TAG="queue" # lsf does not support "partition" tag +# QUEUE_HPSS_TAG="queue" # lsf does not support "partition" tag QUEUE_FCST=${QUEUE_FCST:-"dev"} ;; "HERA") - NCORES_PER_NODE=24 # Should this be 40? Need to check heradocs. + NCORES_PER_NODE=40 SCHED="${SCHED:-slurm}" + + PARTITION_DEFAULT=${PARTITION_DEFAULT:-"hera"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} + + PARTITION_HPSS=${PARTITION_HPSS:-"service"} QUEUE_HPSS=${QUEUE_HPSS:-"service"} +# QUEUE_HPSS_TAG="queue" + + PARTITION_FCST=${PARTITION_FCST:-"hera"} QUEUE_FCST=${QUEUE_FCST:-"batch"} ;; "ORION") NCORES_PER_NODE=40 SCHED="${SCHED:-slurm}" + +# PARTITION_DEFAULT=${PARTITION_DEFAULT:-"orion"} + PARTITION_DEFAULT=${PARTITION_DEFAULT:-""} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} - QUEUE_HPSS=${QUEUE_HPSS:-"service"} + + PARTITION_HPSS=${PARTITION_HPSS:-"service"} + QUEUE_HPSS=${QUEUE_HPSS:-"batch"} +# QUEUE_HPSS_TAG="queue" + + PARTITION_FCST=${PARTITION_FCST:-"orion"} QUEUE_FCST=${QUEUE_FCST:-"batch"} ;; "JET") NCORES_PER_NODE=24 SCHED="${SCHED:-slurm}" + + PARTITION_DEFAULT=${PARTITION_DEFAULT:-"sjet,vjet,kjet,xjet"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} + + PARTITION_HPSS=${PARTITION_HPSS:-"service"} QUEUE_HPSS=${QUEUE_HPSS:-"service"} + + PARTITION_FCST=${PARTITION_FCST:-"sjet,vjet,kjet,xjet"} QUEUE_FCST=${QUEUE_FCST:-"batch"} ;; @@ -394,7 +415,7 @@ case $MACHINE in SCHED="${SCHED:-pbspro}" QUEUE_DEFAULT=${QUEUE_DEFAULT:-"regular"} QUEUE_HPSS=${QUEUE_HPSS:-"regular"} - QUEUE_HPSS_TAG="queue" # pbspro does not support "partition" tag +# QUEUE_HPSS_TAG="queue" # pbspro does not support "partition" tag QUEUE_FCST=${QUEUE_FCST:-"regular"} ;; @@ -407,6 +428,40 @@ case $MACHINE in ;; esac + + +# Is the following needed??? +# +#if [ "$SCHED" = "slurm" ]; then +# +# if [ -z "${PARTITION_DEFAULT}" ]; then +# print_err_msg_exit "\ +#When SCHED is set to \"slurm\", the variable specifying the default +#slurm partition to which to submit tasks (PARTITION_DEFAULT) cannot be +#empty: +# SCHED = \"$SCHED\" +# PARTITION_DEFAULT = \"${PARTITION_DEFAULT}\"" +# fi +# +# if [ -z "${PARTITION_HPSS}" ]; then +# print_err_msg_exit "\ +#When SCHED is set to \"slurm\", the variable specifying the slurm +#partition to which to submit the tasks that fetch/copy external model +#files (PARTITION_HPSS) cannot be empty: +# SCHED = \"$SCHED\" +# PARTITION_HPSS = \"${PARTITION_HPSS}\"" +# fi +# +# if [ -z "${PARTITION_FCST}" ]; then +# print_err_msg_exit "\ +#When SCHED is set to \"slurm\", the variable specifying the slurm +#partition to which to submit the forecast task (PARTITION_FCST) cannot +#be empty: +# SCHED = \"$SCHED\" +# PARTITION_DEFAULT = \"${PARTITION_FCST}\"" +# fi +# +#fi # #----------------------------------------------------------------------- # @@ -749,7 +804,7 @@ case $MACHINE in "ORION") FIXgsm=${FIXgsm:-"/work/noaa/fv3-cam/emc.campara/fix_fv3cam/fix_am"} - TOPO_DIR=${TOPO_DIR:-"/work/noaa/gsd-fv3-dev/gsketefia/UFS/fix_orog"} + TOPO_DIR=${TOPO_DIR:-"/work/noaa/fv3-cam/emc.campara/fix_fv3cam/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/noaa/gsd-fv3-dev/gsketefia/UFS/climo_fields_netcdf"} ;; diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index 4fbc7ae05..c0343f3aa 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -63,9 +63,9 @@ tasks other than GET_EXTRN_ICS_TN, GET_EXTRN_LBCS_TN, and RUN_FCST_TN; the "HPSS" type is used for the GET_EXTRN_ICS_TN and GET_EXTRN_LBCS_TN tasks; and the "FCST" type is used for the RUN_FCST_TN task. --> -&ACCOUNT;<{{ queue_default_tag }}>&QUEUE_DEFAULT;"> -&ACCOUNT;<{{ queue_hpss_tag }}>&QUEUE_HPSS;"> -&ACCOUNT;<{{ queue_fcst_tag }}>&QUEUE_FCST;"> +&ACCOUNT;&QUEUE_DEFAULT;{{ partition_default_spec }}"> +&ACCOUNT;&QUEUE_HPSS;{{ partition_hpss_spec }}"> +&ACCOUNT;&QUEUE_FCST;{{ partition_fcst_spec }}"> ]> @@ -343,9 +343,6 @@ MODULES_RUN_TASK_FP script. {% if machine in ["JET", "HERA"] %} {{ ncores_run_fcst }} {{ native_run_fcst }} - {% if machine == "JET" %} - {{ partition_run_fcst }} - {% endif %} {% else %} {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }} &NCORES_PER_NODE; From 172500eb316699dab408d4fe575d66385c699867 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Tue, 13 Oct 2020 17:19:27 -0500 Subject: [PATCH 14/27] Remove USE_CCPP variable in preparation for merge with develop branch. --- scripts/exregional_run_fcst.sh | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 5dd04f437..8481877e5 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -444,16 +444,11 @@ else ln_vrfy -sf ${relative_or_null} ${FV3_NML_FP} ${run_dir} fi -if [ "${USE_CCPP}" = "TRUE" ]; then - - ln_vrfy -sf ${relative_or_null} ${CCPP_PHYS_SUITE_FP} ${run_dir} - - if [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then - ln_vrfy -sf ${relative_or_null} $EXPTDIR/CCN_ACTIVATE.BIN ${run_dir} - fi - +ln_vrfy -sf ${relative_or_null} ${CCPP_PHYS_SUITE_FP} ${run_dir} +if [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ] || \ + [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then + ln_vrfy -sf ${relative_or_null} $EXPTDIR/CCN_ACTIVATE.BIN ${run_dir} fi # #----------------------------------------------------------------------- From 73714fb81f6486e4cbea4bf49b856395267e49c1 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Tue, 13 Oct 2020 21:51:48 -0500 Subject: [PATCH 15/27] On Orion, set the default value of PARTITION_DEFAULT to "orion" instead of a null string (which was used for testing). --- ush/setup.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/setup.sh b/ush/setup.sh index 5ddc167df..ecd1de52a 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -397,8 +397,7 @@ case $MACHINE in NCORES_PER_NODE=40 SCHED="${SCHED:-slurm}" -# PARTITION_DEFAULT=${PARTITION_DEFAULT:-"orion"} - PARTITION_DEFAULT=${PARTITION_DEFAULT:-""} + PARTITION_DEFAULT=${PARTITION_DEFAULT:-"orion"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} PARTITION_HPSS=${PARTITION_HPSS:-"service"} From 960e5ff9fe26159a9330807d4dbb1bdbc8ec95ce Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 14 Oct 2020 13:51:51 -0500 Subject: [PATCH 16/27] Manual merging with develop branch to make later conflict resolution via mergetool easier. --- scripts/exregional_make_grid.sh | 14 +++++--- scripts/exregional_make_orog.sh | 48 ++++++++++++++++------------ scripts/exregional_make_sfc_climo.sh | 15 ++++++--- ush/setup.sh | 10 ++---- 4 files changed, 48 insertions(+), 39 deletions(-) diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 7ea5f499c..8f025bbc5 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -99,23 +99,27 @@ export OMP_STACKSIZE=2048m # case $MACHINE in - "WCOSS_C" | "WCOSS") + "WCOSS_CRAY") { save_shell_opts; set +x; } > /dev/null 2>&1 - . $MODULESHOME/init/sh module load PrgEnv-intel cfp-intel-sandybridge/1.1.0 module list - { restore_shell_opts; } > /dev/null 2>&1 - export NODES=1 export APRUN="aprun -n 1 -N 1 -j 1 -d 1 -cc depth" export KMP_AFFINITY=disabled - ulimit -s unlimited ulimit -a ;; + "WCOSS_DELL_P3") + { save_shell_opts; set +x; } > /dev/null 2>&1 + module list + { restore_shell_opts; } > /dev/null 2>&1 + export APRUN="mpirun" + ulimit -s unlimited + ;; + "HERA") APRUN="time" ;; diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 045da170f..2b8c07dfe 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -98,21 +98,23 @@ export OMP_STACKSIZE=2048m # case $MACHINE in - "WCOSS_C" | "WCOSS") + "WCOSS_CRAY") { save_shell_opts; set +x; } > /dev/null 2>&1 - . $MODULESHOME/init/sh module load PrgEnv-intel cfp-intel-sandybridge/1.1.0 module list - { restore_shell_opts; } > /dev/null 2>&1 - export NODES=1 export APRUN="aprun -n 1 -N 1 -j 1 -d 1 -cc depth" export KMP_AFFINITY=disabled + ulimit -s unlimited + ulimit -a + ;; + "WCOSS_DELL_P3") ulimit -s unlimited ulimit -a + APRUN="mpirun" ;; "HERA") @@ -287,21 +289,18 @@ cat "${input_redirect_fn}" print_info_msg "$VERBOSE" " Starting orography file generation..." - case $MACHINE in - -"WCOSS_C" | "WCOSS") + "WCOSS_CRAY") # # On WCOSS and WCOSS_C, use cfp to run multiple tiles simulatneously for # the orography. For now, we have only one tile in the regional case, # but in the future we will have more. First, create an input file for # cfp. # - ufs_utils_ushdir="${UFS_UTILS_DIR}/ush" - res="0" # What should this be set to??? - - printf "%s\n" "\ + ufs_utils_ushdir="${UFS_UTILS_DIR}/ush" + res="0" # What should this be set to??? + printf "%s\n" "\ ${ufs_utils_ushdir}/${orog_gen_scr} \ $res \ ${TILE_RGNL} \ @@ -310,21 +309,28 @@ ${raw_dir} \ ${UFS_UTILS_DIR} \ ${TOPO_DIR} \ ${tmp_dir}" \ - >> ${tmp_dir}/orog.file1 - - aprun -j 1 -n 4 -N 4 -d 6 -cc depth cfp ${tmp_dir}/orog.file1 - rm_vrfy ${tmp_dir}/orog.file1 - ;; - + >> ${tmp_dir}/orog.file1 + aprun -j 1 -n 4 -N 4 -d 6 -cc depth cfp ${tmp_dir}/orog.file1 + rm_vrfy ${tmp_dir}/orog.file1 + ;; -"CHEYENNE" | "HERA" | "JET" | "ODIN" | "STAMPEDE" | "ORION") - $APRUN "${exec_fp}" < "${input_redirect_fn}" || \ - print_err_msg_exit "\ + "WCOSS_DELL_P3") + ufs_utils_ushdir="${UFS_UTILS_DIR}/ush" + res="0" # What should this be set to??? + "${exec_fp}" < "${input_redirect_fn}" || \ + print_err_msg_exit "\ Call to executable (exec_fp) that generates the raw orography file returned with nonzero exit code: exec_fp = \"${exec_fp}\"" - ;; + ;; + "CHEYENNE" | "HERA" | "ORION" | "JET" | "ODIN" | "STAMPEDE") + $APRUN "${exec_fp}" < "${input_redirect_fn}" || \ + print_err_msg_exit "\ +Call to executable (exec_fp) that generates the raw orography file returned +with nonzero exit code: + exec_fp = \"${exec_fp}\"" + ;; esac # diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index 0e4067f0f..31f2c877f 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -128,14 +128,19 @@ EOF # case $MACHINE in - "WCOSS_C") -# This could be wrong. Just a guess since I don't have access to this machine. + "WCOSS_CRAY") APRUN=${APRUN:-"aprun -j 1 -n 6 -N 6"} ;; - "WCOSS") -# This could be wrong. Just a guess since I don't have access to this machine. - APRUN=${APRUN:-"aprun -j 1 -n 6 -N 6"} + "WCOSS_DELL_P3") +# Specify computational resources. + export NODES=2 + export ntasks=48 + export ptile=24 + export threads=1 + export MP_LABELIO=yes + export OMP_NUM_THREADS=$threads + APRUN="mpirun" ;; "HERA") diff --git a/ush/setup.sh b/ush/setup.sh index ecd1de52a..2099c4fd7 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -776,19 +776,13 @@ case $MACHINE in "WCOSS_CRAY") FIXgsm=${FIXgsm:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} TOPO_DIR=${TOPO_DIR:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-""} + SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix/fix_sfc_climo"} ;; "WCOSS_DELL_P3") FIXgsm=${FIXgsm:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} TOPO_DIR=${TOPO_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-""} - ;; - - "DELL") - FIXgsm=${FIXgsm:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_am"} - TOPO_DIR=${TOPO_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_orog"} - SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-""} + SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix/fix_sfc_climo"} ;; "HERA") From bbe8d58f4bf8c2f72b14e7e2ed435e33470c01ea Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 15 Oct 2020 18:16:52 -0600 Subject: [PATCH 17/27] Bug fixes. --- ush/setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/setup.sh b/ush/setup.sh index da6a88cfa..531333b12 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -386,7 +386,7 @@ case $MACHINE in QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} PARTITION_HPSS=${PARTITION_HPSS:-"service"} - QUEUE_HPSS=${QUEUE_HPSS:-"service"} + QUEUE_HPSS=${QUEUE_HPSS:-"batch"} # QUEUE_HPSS_TAG="queue" PARTITION_FCST=${PARTITION_FCST:-"hera"} @@ -416,7 +416,7 @@ case $MACHINE in QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} PARTITION_HPSS=${PARTITION_HPSS:-"service"} - QUEUE_HPSS=${QUEUE_HPSS:-"service"} + QUEUE_HPSS=${QUEUE_HPSS:-"batch"} PARTITION_FCST=${PARTITION_FCST:-"sjet,vjet,kjet,xjet"} QUEUE_FCST=${QUEUE_FCST:-"batch"} From b71dd13929c17fc0790b95ec6cf113efe0278a18 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 16 Oct 2020 18:02:00 -0500 Subject: [PATCH 18/27] Add orion stanzas to the ex-scripts for the make_ics and make_lbcs tasks. This was probably inadvertantly changed during a conflict resolution with another PR. --- scripts/exregional_make_ics.sh | 5 +++++ scripts/exregional_make_lbcs.sh | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 7463de5fb..e684657b5 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -93,6 +93,11 @@ case "$MACHINE" in APRUN="srun" ;; + "ORION") + ulimit -s unlimited + APRUN="srun" + ;; + "JET") ulimit -s unlimited APRUN="srun" diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 7967c68dd..61dd0ebff 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -93,6 +93,11 @@ case "$MACHINE" in APRUN="srun" ;; + "ORION") + ulimit -s unlimited + APRUN="srun" + ;; + "JET") ulimit -s unlimited APRUN="srun" From 30dfc8e9efb52ebaa03162127bd5e94890cd81c5 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 16 Oct 2020 19:18:53 -0500 Subject: [PATCH 19/27] Clean up comments; remove commented out code. --- scripts/exregional_make_grid.sh | 2 -- scripts/exregional_make_orog.sh | 2 -- ush/config_defaults.sh | 64 +++++++++++++-------------------- ush/generate_FV3LAM_wflow.sh | 22 ++---------- ush/setup.sh | 48 ------------------------- 5 files changed, 27 insertions(+), 111 deletions(-) diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 8f025bbc5..a7921b5bb 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -82,8 +82,6 @@ print_input_args valid_args # # The orography code runs with threads. On Cray, the code is optimized # for six threads. Do not change. -# Note that OMP_NUM_THREADS and OMP_STACKSIZE only affect the threaded <== I don't think this is true. Remove?? -# executions on Cray; they don't affect executions on theia. # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 2b8c07dfe..bcdeb2bee 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -78,8 +78,6 @@ print_input_args valid_args # # The orography code runs with threads. On Cray, the code is optimized # for six threads. Do not change. -# Note that OMP_NUM_THREADS and OMP_STACKSIZE only affect the threaded <== I don't think this is true. Remove?? -# executions on Cray; they don't affect executions on theia. # #----------------------------------------------------------------------- # diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index eb3553464..251d6f061 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -51,39 +51,6 @@ RUN_ENVIR="nco" # order for the experiment generation script to set it depending on the # machine. # -# QUEUE_DEFAULT: -# The default queue or QOS (if using the slurm job scheduler, where QOS -# is Quality of Service) to which workflow tasks are submitted. If a -# task does not have a specific variable that specifies the queue to which -# it will be submitted (e.g. QUEUE_HPSS, QUEUE_FCST; see below), it will -# be submitted to the queue specified by this variable. If this is not -# set or is set to an empty string, it will be (re)set to a machine- -# dependent value. -# -## QUEUE_DEFAULT_TAG: -## The rocoto xml tag to use for specifying the default queue. For most -## platforms this should be "queue" -# -# QUEUE_HPSS: -# The queue or QOS to which the tasks that get or create links to external -# model files [which are needed to generate initial conditions (ICs) and -# lateral boundary conditions (LBCs)] are submitted. If this is not set -# or is set to an empty string, it will be (re)set to a machine-dependent -# value. -# -## QUEUE_HPSS_TAG: -## The rocoto xml tag to use for specifying the HPSS queue. For slurm-based -## platforms this is typically "partition", for others it may be "queue" -# -# QUEUE_FCST: -# The queue or QOS to which the task that runs a forecast is submitted. -# If this is not set or set to an empty string, it will be (re)set to a -# machine-dependent value. -# -## QUEUE_FCST_TAG: -## The rocoto xml tag to use for specifying the fcst queue. For most -## platforms this should be "queue" -# # PARTITION_DEFAULT: # If using the slurm job scheduler (i.e. if SCHED is set to "slurm"), # the default partition to which to submit workflow tasks. If a task @@ -94,6 +61,15 @@ RUN_ENVIR="nco" # machine-dependent value. This is not used if SCHED is not set to # "slurm". # +# QUEUE_DEFAULT: +# The default queue or QOS (if using the slurm job scheduler, where QOS +# is Quality of Service) to which workflow tasks are submitted. If a +# task does not have a specific variable that specifies the queue to which +# it will be submitted (e.g. QUEUE_HPSS, QUEUE_FCST; see below), it will +# be submitted to the queue specified by this variable. If this is not +# set or is set to an empty string, it will be (re)set to a machine- +# dependent value. +# # PARTITION_HPSS: # If using the slurm job scheduler (i.e. if SCHED is set to "slurm"), # the partition to which the tasks that get or create links to external @@ -102,6 +78,13 @@ RUN_ENVIR="nco" # or is set to an empty string, it will be (re)set to a machine-dependent # value. This is not used if SCHED is not set to "slurm". # +# QUEUE_HPSS: +# The queue or QOS to which the tasks that get or create links to external +# model files [which are needed to generate initial conditions (ICs) and +# lateral boundary conditions (LBCs)] are submitted. If this is not set +# or is set to an empty string, it will be (re)set to a machine-dependent +# value. +# # PARTITION_FCST: # If using the slurm job scheduler (i.e. if SCHED is set to "slurm"), # the partition to which the task that runs forecasts is submitted. If @@ -109,6 +92,11 @@ RUN_ENVIR="nco" # machine-dependent value. This is not used if SCHED is not set to # "slurm". # +# QUEUE_FCST: +# The queue or QOS to which the task that runs a forecast is submitted. +# If this is not set or set to an empty string, it will be (re)set to a +# machine-dependent value. +# # mach_doc_end # #----------------------------------------------------------------------- @@ -116,16 +104,12 @@ RUN_ENVIR="nco" MACHINE="BIG_COMPUTER" ACCOUNT="project_name" SCHED="" -QUEUE_DEFAULT="" -#QUEUE_DEFAULT_TAG="queue" -QUEUE_HPSS="" -#QUEUE_HPSS_TAG="partition" -QUEUE_FCST="" -#QUEUE_FCST_TAG="queue" - PARTITION_DEFAULT="" +QUEUE_DEFAULT="" PARTITION_HPSS="" +QUEUE_HPSS="" PARTITION_FCST="" +QUEUE_FCST="" # #----------------------------------------------------------------------- # diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 071bdc4fb..09682b3a0 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -159,18 +159,6 @@ if [ "${DO_ENSEMBLE}" = "TRUE" ]; then slash_ensmem_subdir="/mem#${ensmem_indx_name}#" fi -# -# To see if the following is enough (i.e. to see whether the if-statements -# below are really needed), have to run with the following lines uncommented -# on systems that don't use slurm. -# The following seem to work ok on orion, i.e. passing an empty partition -# setting in the rocoto xml, like this: -# This causes rocoto/slurm to submit to the default partition (called "orion"). -# -#partition_default_spec="${PARTITION_DEFAULT}" -#partition_hpss_spec="${PARTITION_HPSS}" -#partition_fcst_spec="${PARTITION_FCST}" - partition_default_spec="\"\"" if [ ! -z "${PARTITION_DEFAULT}" ]; then partition_default_spec="${PARTITION_DEFAULT}" @@ -186,22 +174,18 @@ if [ ! -z "${PARTITION_FCST}" ]; then partition_fcst_spec="${PARTITION_FCST}" fi -# 'queue_default_tag': ${QUEUE_DEFAULT_TAG} -# 'queue_hpss_tag': ${QUEUE_HPSS_TAG} -# 'queue_fcst_tag': ${QUEUE_FCST_TAG} - settings="\ # # Parameters needed by the job scheduler. # 'account': $ACCOUNT 'sched': $SCHED - 'queue_default': ${QUEUE_DEFAULT} - 'queue_hpss': ${QUEUE_HPSS} - 'queue_fcst': ${QUEUE_FCST} 'partition_default_spec': ${partition_default_spec} + 'queue_default': ${QUEUE_DEFAULT} 'partition_hpss_spec': ${partition_hpss_spec} + 'queue_hpss': ${QUEUE_HPSS} 'partition_fcst_spec': ${partition_fcst_spec} + 'queue_fcst': ${QUEUE_FCST} 'machine': ${MACHINE} # # Workflow task names. diff --git a/ush/setup.sh b/ush/setup.sh index 89121f899..024a14aef 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -366,7 +366,6 @@ case $MACHINE in SCHED="lsfcray" QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} -# QUEUE_HPSS_TAG="queue" # lsfcray does not support "partition" tag QUEUE_FCST=${QUEUE_FCST:-"dev"} ;; @@ -375,21 +374,16 @@ case $MACHINE in SCHED="lsf" QUEUE_DEFAULT=${QUEUE_DEFAULT:-"dev"} QUEUE_HPSS=${QUEUE_HPSS:-"dev_transfer"} -# QUEUE_HPSS_TAG="queue" # lsf does not support "partition" tag QUEUE_FCST=${QUEUE_FCST:-"dev"} ;; "HERA") NCORES_PER_NODE=40 SCHED="${SCHED:-slurm}" - PARTITION_DEFAULT=${PARTITION_DEFAULT:-"hera"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} - PARTITION_HPSS=${PARTITION_HPSS:-"service"} QUEUE_HPSS=${QUEUE_HPSS:-"batch"} -# QUEUE_HPSS_TAG="queue" - PARTITION_FCST=${PARTITION_FCST:-"hera"} QUEUE_FCST=${QUEUE_FCST:-"batch"} ;; @@ -397,14 +391,10 @@ case $MACHINE in "ORION") NCORES_PER_NODE=40 SCHED="${SCHED:-slurm}" - PARTITION_DEFAULT=${PARTITION_DEFAULT:-"orion"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} - PARTITION_HPSS=${PARTITION_HPSS:-"service"} QUEUE_HPSS=${QUEUE_HPSS:-"batch"} -# QUEUE_HPSS_TAG="queue" - PARTITION_FCST=${PARTITION_FCST:-"orion"} QUEUE_FCST=${QUEUE_FCST:-"batch"} ;; @@ -412,13 +402,10 @@ case $MACHINE in "JET") NCORES_PER_NODE=24 SCHED="${SCHED:-slurm}" - PARTITION_DEFAULT=${PARTITION_DEFAULT:-"sjet,vjet,kjet,xjet"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} - PARTITION_HPSS=${PARTITION_HPSS:-"service"} QUEUE_HPSS=${QUEUE_HPSS:-"batch"} - PARTITION_FCST=${PARTITION_FCST:-"sjet,vjet,kjet,xjet"} QUEUE_FCST=${QUEUE_FCST:-"batch"} ;; @@ -436,7 +423,6 @@ case $MACHINE in SCHED="${SCHED:-pbspro}" QUEUE_DEFAULT=${QUEUE_DEFAULT:-"regular"} QUEUE_HPSS=${QUEUE_HPSS:-"regular"} -# QUEUE_HPSS_TAG="queue" # pbspro does not support "partition" tag QUEUE_FCST=${QUEUE_FCST:-"regular"} ;; @@ -449,40 +435,6 @@ case $MACHINE in ;; esac - - -# Is the following needed??? -# -#if [ "$SCHED" = "slurm" ]; then -# -# if [ -z "${PARTITION_DEFAULT}" ]; then -# print_err_msg_exit "\ -#When SCHED is set to \"slurm\", the variable specifying the default -#slurm partition to which to submit tasks (PARTITION_DEFAULT) cannot be -#empty: -# SCHED = \"$SCHED\" -# PARTITION_DEFAULT = \"${PARTITION_DEFAULT}\"" -# fi -# -# if [ -z "${PARTITION_HPSS}" ]; then -# print_err_msg_exit "\ -#When SCHED is set to \"slurm\", the variable specifying the slurm -#partition to which to submit the tasks that fetch/copy external model -#files (PARTITION_HPSS) cannot be empty: -# SCHED = \"$SCHED\" -# PARTITION_HPSS = \"${PARTITION_HPSS}\"" -# fi -# -# if [ -z "${PARTITION_FCST}" ]; then -# print_err_msg_exit "\ -#When SCHED is set to \"slurm\", the variable specifying the slurm -#partition to which to submit the forecast task (PARTITION_FCST) cannot -#be empty: -# SCHED = \"$SCHED\" -# PARTITION_DEFAULT = \"${PARTITION_FCST}\"" -# fi -# -#fi # #----------------------------------------------------------------------- # From 26b4089c906398a9a1560886d9eafe4e31cf90a3 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Tue, 20 Oct 2020 10:18:19 -0600 Subject: [PATCH 20/27] Move code involving xml from the generate script to the rocoto XML template. --- ush/generate_FV3LAM_wflow.sh | 38 +++++++++++++++++++--------------- ush/templates/FV3LAM_wflow.xml | 19 ++++++++++++++--- 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 071bdc4fb..00d2ce800 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -171,25 +171,29 @@ fi #partition_hpss_spec="${PARTITION_HPSS}" #partition_fcst_spec="${PARTITION_FCST}" -partition_default_spec="\"\"" -if [ ! -z "${PARTITION_DEFAULT}" ]; then - partition_default_spec="${PARTITION_DEFAULT}" -fi - -partition_hpss_spec="\"\"" -if [ ! -z "${PARTITION_HPSS}" ]; then - partition_hpss_spec="${PARTITION_HPSS}" -fi - -partition_fcst_spec="\"\"" -if [ ! -z "${PARTITION_FCST}" ]; then - partition_fcst_spec="${PARTITION_FCST}" -fi +#partition_default_spec="\"\"" +#if [ ! -z "${PARTITION_DEFAULT}" ]; then +# partition_default_spec="${PARTITION_DEFAULT}" +#fi +# +#partition_hpss_spec="\"\"" +#if [ ! -z "${PARTITION_HPSS}" ]; then +# partition_hpss_spec="${PARTITION_HPSS}" +#fi +# +#partition_fcst_spec="\"\"" +#if [ ! -z "${PARTITION_FCST}" ]; then +# partition_fcst_spec="${PARTITION_FCST}" +#fi # 'queue_default_tag': ${QUEUE_DEFAULT_TAG} # 'queue_hpss_tag': ${QUEUE_HPSS_TAG} # 'queue_fcst_tag': ${QUEUE_FCST_TAG} +# 'partition_default_spec': ${partition_default_spec} +# 'partition_hpss_spec': ${partition_hpss_spec} +# 'partition_fcst_spec': ${partition_fcst_spec} + settings="\ # # Parameters needed by the job scheduler. @@ -199,9 +203,9 @@ settings="\ 'queue_default': ${QUEUE_DEFAULT} 'queue_hpss': ${QUEUE_HPSS} 'queue_fcst': ${QUEUE_FCST} - 'partition_default_spec': ${partition_default_spec} - 'partition_hpss_spec': ${partition_hpss_spec} - 'partition_fcst_spec': ${partition_fcst_spec} + 'partition_default': ${PARTITION_DEFAULT} + 'partition_hpss': ${PARTITION_HPSS} + 'partition_fcst': ${PARTITION_FCST} 'machine': ${MACHINE} # # Workflow task names. diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index c0343f3aa..7667e7730 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -63,9 +63,22 @@ tasks other than GET_EXTRN_ICS_TN, GET_EXTRN_LBCS_TN, and RUN_FCST_TN; the "HPSS" type is used for the GET_EXTRN_ICS_TN and GET_EXTRN_LBCS_TN tasks; and the "FCST" type is used for the RUN_FCST_TN task. --> -&ACCOUNT;&QUEUE_DEFAULT;{{ partition_default_spec }}"> -&ACCOUNT;&QUEUE_HPSS;{{ partition_hpss_spec }}"> -&ACCOUNT;&QUEUE_FCST;{{ partition_fcst_spec }}"> + +{%- if (partition_default != "") %} +&ACCOUNT;&QUEUE_DEFAULT;{{ partition_default }}"> +{%- else %} +&ACCOUNT;&QUEUE_DEFAULT;"> +{%- endif %} +{%- if (partition_hpss != "") %} +&ACCOUNT;&QUEUE_HPSS;{{ partition_hpss }}"> +{%- else %} +&ACCOUNT;&QUEUE_HPSS;"> +{%- endif %} +{%- if (partition_fcst != "") %} +&ACCOUNT;&QUEUE_FCST;{{ partition_fcst }}"> +{%- else %} +&ACCOUNT;&QUEUE_FCST;"> +{%- endif %} ]> From 12bd2e2925609f62ac34c3631ed1b0cab9d50190 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Tue, 20 Oct 2020 10:35:14 -0600 Subject: [PATCH 21/27] Delete unused/commented out lines. --- ush/generate_FV3LAM_wflow.sh | 41 +++--------------------------------- 1 file changed, 3 insertions(+), 38 deletions(-) diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 00d2ce800..4104f4f3f 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -159,53 +159,18 @@ if [ "${DO_ENSEMBLE}" = "TRUE" ]; then slash_ensmem_subdir="/mem#${ensmem_indx_name}#" fi -# -# To see if the following is enough (i.e. to see whether the if-statements -# below are really needed), have to run with the following lines uncommented -# on systems that don't use slurm. -# The following seem to work ok on orion, i.e. passing an empty partition -# setting in the rocoto xml, like this: -# This causes rocoto/slurm to submit to the default partition (called "orion"). -# -#partition_default_spec="${PARTITION_DEFAULT}" -#partition_hpss_spec="${PARTITION_HPSS}" -#partition_fcst_spec="${PARTITION_FCST}" - -#partition_default_spec="\"\"" -#if [ ! -z "${PARTITION_DEFAULT}" ]; then -# partition_default_spec="${PARTITION_DEFAULT}" -#fi -# -#partition_hpss_spec="\"\"" -#if [ ! -z "${PARTITION_HPSS}" ]; then -# partition_hpss_spec="${PARTITION_HPSS}" -#fi -# -#partition_fcst_spec="\"\"" -#if [ ! -z "${PARTITION_FCST}" ]; then -# partition_fcst_spec="${PARTITION_FCST}" -#fi - -# 'queue_default_tag': ${QUEUE_DEFAULT_TAG} -# 'queue_hpss_tag': ${QUEUE_HPSS_TAG} -# 'queue_fcst_tag': ${QUEUE_FCST_TAG} - -# 'partition_default_spec': ${partition_default_spec} -# 'partition_hpss_spec': ${partition_hpss_spec} -# 'partition_fcst_spec': ${partition_fcst_spec} - settings="\ # # Parameters needed by the job scheduler. # 'account': $ACCOUNT 'sched': $SCHED - 'queue_default': ${QUEUE_DEFAULT} - 'queue_hpss': ${QUEUE_HPSS} - 'queue_fcst': ${QUEUE_FCST} 'partition_default': ${PARTITION_DEFAULT} + 'queue_default': ${QUEUE_DEFAULT} 'partition_hpss': ${PARTITION_HPSS} + 'queue_hpss': ${QUEUE_HPSS} 'partition_fcst': ${PARTITION_FCST} + 'queue_fcst': ${QUEUE_FCST} 'machine': ${MACHINE} # # Workflow task names. From bf652cffb18910b6d2ce084839cac9d5e56f61d0 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 21 Oct 2020 13:37:00 -0600 Subject: [PATCH 22/27] Changes to try to get the PR running on Odin and Stampede (not yet complete). * Remove the QUEUE_... statements in config.community.sh because they are overriding the default values for the machine in setup.sh. * Add new lines for PARTITION_... in the case-statement in setup.sh that sets the default values of PARTITION_... and QUEUE_.... For now, set the default values to blanks. These need to be properly specified, but I don't know the names of the partitions (nor QOS's) on these two machines. Probably Yunheng will need to fill these out and test. --- ush/config.community.sh | 4 ---- ush/setup.sh | 6 ++++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ush/config.community.sh b/ush/config.community.sh index 1b5b9018a..e63716cf9 100644 --- a/ush/config.community.sh +++ b/ush/config.community.sh @@ -2,10 +2,6 @@ MACHINE="hera" ACCOUNT="an_account" EXPT_SUBDIR="test_community" -QUEUE_DEFAULT="batch" -QUEUE_HPSS="service" -QUEUE_FCST="batch" - VERBOSE="TRUE" RUN_ENVIR="community" diff --git a/ush/setup.sh b/ush/setup.sh index 4b9727b64..d33a73d16 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -413,8 +413,11 @@ case $MACHINE in "ODIN") NCORES_PER_NODE=24 SCHED="${SCHED:-slurm}" + PARTITION_DEFAULT=${PARTITION_DEFAULT:-""} QUEUE_DEFAULT=${QUEUE_DEFAULT:-""} + PARTITION_HPSS=${PARTITION_HPSS:-""} QUEUE_HPSS=${QUEUE_HPSS:-""} + PARTITION_FCST=${PARTITION_FCST:-""} QUEUE_FCST=${QUEUE_FCST:-""} ;; @@ -429,8 +432,11 @@ case $MACHINE in "STAMPEDE") NCORES_PER_NODE=68 SCHED="slurm" + PARTITION_DEFAULT=${PARTITION_DEFAULT:-""} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"} + PARTITION_HPSS=${PARTITION_HPSS:-""} QUEUE_HPSS=${QUEUE_HPSS:-"development"} + PARTITION_FCST=${PARTITION_FCST:-""} QUEUE_FCST=${QUEUE_FCST:-"normal"} ;; From 4fcf764ba5b7e7e4151e72bbdc94243f26f467df Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 21 Oct 2020 15:52:16 -0500 Subject: [PATCH 23/27] Fix in run_post.sh wrapper script to use 3-digit forecast hour instead of 2. --- ush/wrappers/run_post.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/wrappers/run_post.sh b/ush/wrappers/run_post.sh index 174cc62b5..1c8f3d4df 100755 --- a/ush/wrappers/run_post.sh +++ b/ush/wrappers/run_post.sh @@ -10,6 +10,6 @@ export ENSMEM_INDX="" num_fcst_hrs=${FCST_LEN_HRS} for (( i=0; i<=$((num_fcst_hrs)); i++ )); do - export fhr=`printf "%02i" ${i}` + export fhr=`printf "%03i" ${i}` ${JOBSDIR}/JREGIONAL_RUN_POST done From 11fa204dc2e1985c746372d159960df00a602039 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 21 Oct 2020 17:31:49 -0500 Subject: [PATCH 24/27] Set the PARTITION_... variables on Odin and Stampede as specified by Yunheng Wang. Still need to set QUEUE_... for these machines. --- ush/setup.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ush/setup.sh b/ush/setup.sh index d33a73d16..ab2b20c52 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -413,11 +413,11 @@ case $MACHINE in "ODIN") NCORES_PER_NODE=24 SCHED="${SCHED:-slurm}" - PARTITION_DEFAULT=${PARTITION_DEFAULT:-""} + PARTITION_DEFAULT=${PARTITION_DEFAULT:-"workq"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-""} - PARTITION_HPSS=${PARTITION_HPSS:-""} + PARTITION_HPSS=${PARTITION_HPSS:-"workq"} QUEUE_HPSS=${QUEUE_HPSS:-""} - PARTITION_FCST=${PARTITION_FCST:-""} + PARTITION_FCST=${PARTITION_FCST:-"workq"} QUEUE_FCST=${QUEUE_FCST:-""} ;; @@ -432,11 +432,11 @@ case $MACHINE in "STAMPEDE") NCORES_PER_NODE=68 SCHED="slurm" - PARTITION_DEFAULT=${PARTITION_DEFAULT:-""} + PARTITION_DEFAULT=${PARTITION_DEFAULT:-"normal"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"} - PARTITION_HPSS=${PARTITION_HPSS:-""} + PARTITION_HPSS=${PARTITION_HPSS:-"normal"} QUEUE_HPSS=${QUEUE_HPSS:-"development"} - PARTITION_FCST=${PARTITION_FCST:-""} + PARTITION_FCST=${PARTITION_FCST:-"normal"} QUEUE_FCST=${QUEUE_FCST:-"normal"} ;; From b487e27a8e157b4500294d635b3f0223ad050ab4 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 21 Oct 2020 19:31:56 -0500 Subject: [PATCH 25/27] Set the QUEUE_... variables for Odin and Stampede as specified by Yunheng Wang. --- ush/setup.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/setup.sh b/ush/setup.sh index ab2b20c52..f9d0c1789 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -414,11 +414,11 @@ case $MACHINE in NCORES_PER_NODE=24 SCHED="${SCHED:-slurm}" PARTITION_DEFAULT=${PARTITION_DEFAULT:-"workq"} - QUEUE_DEFAULT=${QUEUE_DEFAULT:-""} + QUEUE_DEFAULT=${QUEUE_DEFAULT:-"workq"} PARTITION_HPSS=${PARTITION_HPSS:-"workq"} - QUEUE_HPSS=${QUEUE_HPSS:-""} + QUEUE_HPSS=${QUEUE_HPSS:-"workq"} PARTITION_FCST=${PARTITION_FCST:-"workq"} - QUEUE_FCST=${QUEUE_FCST:-""} + QUEUE_FCST=${QUEUE_FCST:-"workq"} ;; "CHEYENNE") @@ -435,7 +435,7 @@ case $MACHINE in PARTITION_DEFAULT=${PARTITION_DEFAULT:-"normal"} QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"} PARTITION_HPSS=${PARTITION_HPSS:-"normal"} - QUEUE_HPSS=${QUEUE_HPSS:-"development"} + QUEUE_HPSS=${QUEUE_HPSS:-"normal"} PARTITION_FCST=${PARTITION_FCST:-"normal"} QUEUE_FCST=${QUEUE_FCST:-"normal"} ;; From db938415881e08a83293b1aeb467e76ac71d49e4 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 21 Oct 2020 22:25:51 -0500 Subject: [PATCH 26/27] Clean up indentation of error messages (they shouldn't be indented). --- ush/load_modules_run_task.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index e8654e807..b469e5d77 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -275,23 +275,23 @@ Call to \"module use\" command failed." # if [ ${use_default_modulefile} -eq 0 ]; then - module use "${modules_dir}" || print_err_msg_exit "\ - Call to \"module use\" command failed." + module use -a "${modules_dir}" || print_err_msg_exit "\ +Call to \"module use\" command failed." - module load ${modulefile_name} || print_err_msg_exit "\ - Loading of module file (modulefile_name; in directory specified by mod- - ules_dir) for the specified task (task_name) failed: - task_name = \"${task_name}\" - modulefile_name = \"${modulefile_name}\" - modules_dir = \"${modules_dir}\"" + module load "${modulefile_name}" || print_err_msg_exit "\ +Loading of module file (modulefile_name; in directory specified by mod- +ules_dir) for the specified task (task_name) failed: + task_name = \"${task_name}\" + modulefile_name = \"${modulefile_name}\" + modules_dir = \"${modules_dir}\"" else # using default modulefile - module load ${default_modulefile_name} || print_err_msg_exit "\ - Loading of default module file failed: - task_name = \"${task_name}\" - default_modulefile_name = \"${default_modulefile_name}\" - default_modules_dir = \"${default_modules_dir}\"" + module load "${default_modulefile_name}" || print_err_msg_exit "\ +Loading of default module file failed: + task_name = \"${task_name}\" + default_modulefile_name = \"${default_modulefile_name}\" + default_modules_dir = \"${default_modules_dir}\"" fi From 5d53c5c72979478d1581be40f7acf29f4a7baa0a Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 23 Oct 2020 16:04:49 -0500 Subject: [PATCH 27/27] Change the module files to use the contrib module for regional_workflow set up by Christina Holt. --- modulefiles/tasks/orion/make_grid.local | 12 +++++------- modulefiles/tasks/orion/make_ics.local | 12 +++++------- modulefiles/tasks/orion/make_lbcs.local | 12 +++++------- modulefiles/tasks/orion/run_fcst.local | 12 +++++------- 4 files changed, 20 insertions(+), 28 deletions(-) diff --git a/modulefiles/tasks/orion/make_grid.local b/modulefiles/tasks/orion/make_grid.local index 7b0776c62..e7b20cb66 100644 --- a/modulefiles/tasks/orion/make_grid.local +++ b/modulefiles/tasks/orion/make_grid.local @@ -1,7 +1,5 @@ - -#module use -a /contrib/miniconda3/modulefiles -#module load miniconda3 -#if [module-info mode load] { -# system "conda activate regional_workflow" -#} -module load python/3.7 +module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles +module load miniconda3 +if [module-info mode load] { + system "conda activate regional_workflow" +} diff --git a/modulefiles/tasks/orion/make_ics.local b/modulefiles/tasks/orion/make_ics.local index 7b0776c62..e7b20cb66 100644 --- a/modulefiles/tasks/orion/make_ics.local +++ b/modulefiles/tasks/orion/make_ics.local @@ -1,7 +1,5 @@ - -#module use -a /contrib/miniconda3/modulefiles -#module load miniconda3 -#if [module-info mode load] { -# system "conda activate regional_workflow" -#} -module load python/3.7 +module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles +module load miniconda3 +if [module-info mode load] { + system "conda activate regional_workflow" +} diff --git a/modulefiles/tasks/orion/make_lbcs.local b/modulefiles/tasks/orion/make_lbcs.local index 7b0776c62..e7b20cb66 100644 --- a/modulefiles/tasks/orion/make_lbcs.local +++ b/modulefiles/tasks/orion/make_lbcs.local @@ -1,7 +1,5 @@ - -#module use -a /contrib/miniconda3/modulefiles -#module load miniconda3 -#if [module-info mode load] { -# system "conda activate regional_workflow" -#} -module load python/3.7 +module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles +module load miniconda3 +if [module-info mode load] { + system "conda activate regional_workflow" +} diff --git a/modulefiles/tasks/orion/run_fcst.local b/modulefiles/tasks/orion/run_fcst.local index 2b539c8f0..e7b20cb66 100644 --- a/modulefiles/tasks/orion/run_fcst.local +++ b/modulefiles/tasks/orion/run_fcst.local @@ -1,7 +1,5 @@ - -#module use -a /contrib/miniconda3/modulefiles -#module load miniconda3 -#if [module-info mode load] { -# system "conda activate regional_workflow" -#} -#module load python/3.7 +module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles +module load miniconda3 +if [module-info mode load] { + system "conda activate regional_workflow" +}