From f3a8c3a31155a73bbb57d6a8281fa9f3d36fa0cc Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Thu, 29 Sep 2022 14:26:24 +0000 Subject: [PATCH 01/11] Make machine files yaml. --- modulefiles/wflow_noaacloud | 5 +- scripts/exregional_make_grid.sh | 1 - scripts/exregional_make_ics.sh | 1 - scripts/exregional_make_lbcs.sh | 1 - scripts/exregional_make_orog.sh | 1 - scripts/exregional_make_sfc_climo.sh | 1 - scripts/exregional_run_fcst.sh | 1 - scripts/exregional_run_post.sh | 1 - tests/WE2E/run_WE2E_tests.sh | 15 +---- ush/config_defaults.yaml | 39 +++++++----- ush/launch_FV3LAM_wflow.sh | 1 - ush/load_modules_run_task.sh | 1 - ush/machine/cheyenne.sh | 74 ---------------------- ush/machine/cheyenne.yaml | 31 +++++++++ ush/machine/gaea.sh | 78 ----------------------- ush/machine/gaea.yaml | 30 +++++++++ ush/machine/hera.sh | 75 ---------------------- ush/machine/hera.yaml | 36 +++++++++++ ush/machine/jet.sh | 95 ---------------------------- ush/machine/jet.yaml | 43 +++++++++++++ ush/machine/linux.sh | 67 -------------------- ush/machine/linux.yaml | 19 ++++++ ush/machine/macos.sh | 67 -------------------- ush/machine/macos.yaml | 19 ++++++ ush/machine/noaacloud.sh | 74 ---------------------- ush/machine/noaacloud.yaml | 31 +++++++++ ush/machine/odin.sh | 80 ----------------------- ush/machine/odin.yaml | 31 +++++++++ ush/machine/orion.sh | 74 ---------------------- ush/machine/orion.yaml | 36 +++++++++++ ush/machine/singularity.sh | 58 ----------------- ush/machine/singularity.yaml | 28 ++++++++ ush/machine/stampede.sh | 81 ------------------------ ush/machine/stampede.yaml | 31 +++++++++ ush/python_utils/config_parser.py | 7 +- ush/setup.py | 81 ++++++++++++------------ 36 files changed, 409 insertions(+), 905 deletions(-) delete mode 100644 ush/machine/cheyenne.sh create mode 100644 ush/machine/cheyenne.yaml delete mode 100755 ush/machine/gaea.sh create mode 100644 ush/machine/gaea.yaml delete mode 100644 ush/machine/hera.sh create mode 100644 ush/machine/hera.yaml delete mode 100644 ush/machine/jet.sh create mode 100644 ush/machine/jet.yaml delete mode 100644 ush/machine/linux.sh create mode 100644 ush/machine/linux.yaml delete mode 100644 ush/machine/macos.sh create mode 100644 ush/machine/macos.yaml delete mode 100755 ush/machine/noaacloud.sh create mode 100644 ush/machine/noaacloud.yaml delete mode 100644 ush/machine/odin.sh create mode 100644 ush/machine/odin.yaml delete mode 100644 ush/machine/orion.sh create mode 100644 ush/machine/orion.yaml delete mode 100644 ush/machine/singularity.sh create mode 100644 ush/machine/singularity.yaml delete mode 100644 ush/machine/stampede.sh create mode 100644 ush/machine/stampede.yaml diff --git a/modulefiles/wflow_noaacloud b/modulefiles/wflow_noaacloud index 0335013240..764982cc55 100644 --- a/modulefiles/wflow_noaacloud +++ b/modulefiles/wflow_noaacloud @@ -13,9 +13,12 @@ module load rocoto module use /contrib/GST/miniconda3/modulefiles module load miniconda3/4.10.3 +setenv PROJ_LIB "/contrib/GST/miniconda/envs/regional_workflow/share/proj" +setenv OPT "/contrib/EPIC/hpc-modules" +append-path PATH "/contrib/GST/miniconda/envs/regional_workflow/bin" + if { [module-info mode load] } { puts stderr "Please do the following to activate conda: > conda activate regional_workflow" } - diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 2592d1cfdd..f1c3f1a8b8 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -60,7 +60,6 @@ This is the ex-script for the task that generates grid files. # #----------------------------------------------------------------------- # -. ${MACHINE_FILE} eval ${PRE_TASK_CMDS} if [ -z "${RUN_CMD_SERIAL:-}" ] ; then diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index c6d7b3a674..ab78f4bb78 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -63,7 +63,6 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_ICS} # #----------------------------------------------------------------------- # -. ${MACHINE_FILE} eval ${PRE_TASK_CMDS} nprocs=$(( NNODES_MAKE_ICS*PPN_MAKE_ICS )) diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index c60c11221e..581ba76b5f 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -63,7 +63,6 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_LBCS} # #----------------------------------------------------------------------- # -. ${MACHINE_FILE} eval ${PRE_TASK_CMDS} nprocs=$(( NNODES_MAKE_LBCS*PPN_MAKE_LBCS )) diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index d10693dece..f743748ecd 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -65,7 +65,6 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_OROG} # #----------------------------------------------------------------------- # -. ${MACHINE_FILE} eval ${PRE_TASK_CMDS} if [ -z "${RUN_CMD_SERIAL:-}" ] ; then diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index 48d3b13bc1..d32b39260a 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -106,7 +106,6 @@ EOF # #----------------------------------------------------------------------- # -. ${MACHINE_FILE} eval ${PRE_TASK_CMDS} nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO )) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 2a72f34043..1eb8e3e78f 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -62,7 +62,6 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_RUN_FCST} # #----------------------------------------------------------------------- # -. ${MACHINE_FILE} eval ${PRE_TASK_CMDS} nprocs=$(( NNODES_RUN_FCST*PPN_RUN_FCST )) diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index da1a117545..2c18754c09 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -62,7 +62,6 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_RUN_POST} # #----------------------------------------------------------------------- # -. ${MACHINE_FILE} eval ${PRE_TASK_CMDS} nprocs=$(( NNODES_RUN_POST*PPN_RUN_POST )) diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 31df790145..4e4135d2b3 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -720,6 +720,8 @@ Please correct and rerun." #----------------------------------------------------------------------- # source_config ${USHdir}/config_defaults.yaml + MACHINE_FILE=${machine_file:-"${USHdir}/machine/${machine,,}.yaml"} + source_config ${MACHINE_FILE} source_config ${test_config_fp} # #----------------------------------------------------------------------- @@ -772,13 +774,6 @@ Please correct and rerun." DEBUG=${debug:-"FALSE"} VERBOSE=${verbose:-"TRUE"} - MACHINE_FILE=${machine_file:-"${USHdir}/machine/${machine,,}.sh"} - - # Set the machine-specific configuration settings by sourcing the - # machine file in the ush directory - - . ${MACHINE_FILE} - expt_config_str=${expt_config_str}"\ # # The machine on which to run, the account to which to charge computational @@ -812,11 +807,7 @@ EXPT_SUBDIR=\"${EXPT_SUBDIR}\" USE_CRON_TO_RELAUNCH=\"${USE_CRON_TO_RELAUNCH}\" CRON_RELAUNCH_INTVL_MNTS=\"${CRON_RELAUNCH_INTVL_MNTS}\" # -# Path to machine configuration file. -# -MACHINE_FILE=\"${MACHINE_FILE}\" -# -# Flag specifying whether to run in verbose mode. +# Flags specifying whether to run in debug and verbose mode. # DEBUG=\"${DEBUG}\" VERBOSE=\"${VERBOSE}\"" diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 73a6d0bfd7..78203fd33d 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -49,17 +49,11 @@ user: # Please see ush/valid_param_vals.sh for a full list of supported # platforms. # - # MACHINE_FILE: - # Path to a configuration file with machine-specific settings. If none - # is provided, setup.sh will attempt to set the path to for a supported - # platform. - # # ACCOUNT: # The account under which to submit jobs to the queue. # #----------------------------------------------------------------------- MACHINE: "BIG_COMPUTER" - MACHINE_FILE: "" ACCOUNT: "project_name" #---------------------------- @@ -149,7 +143,7 @@ platform: # #----------------------------------------------------------------------- # - WORKFLOW_MANAGER: "none" + WORKFLOW_MANAGER: "" NCORES_PER_NODE: "" LMOD_PATH: "" BUILD_MOD_FN: "" @@ -183,9 +177,10 @@ platform: # #----------------------------------------------------------------------- # - RUN_CMD_UTILS: "mpirun -np 1" - RUN_CMD_FCST: "mpirun -np ${PE_MEMBER01}" - RUN_CMD_POST: "mpirun -np 1" + RUN_CMD_SERIAL: "" + RUN_CMD_UTILS: "" + RUN_CMD_FCST: "" + RUN_CMD_POST: "" # #----------------------------------------------------------------------- @@ -288,7 +283,7 @@ platform: # MODEL: "" MET_INSTALL_DIR: "" - MET_BIN_EXEC: "bin" + MET_BIN_EXEC: "" METPLUS_PATH: "" CCPA_OBS_DIR: "" MRMS_OBS_DIR: "" @@ -315,8 +310,24 @@ platform: #----------------------------------------------------------------------- # DOMAIN_PREGEN_BASEDIR: "" + # + #----------------------------------------------------------------------- + # Scritps and commands needed by workflow and tasks + #----------------------------------------------------------------------- + # + ENV_INIT_SCRIPTS_FPS: [] + PRE_TASK_CMDS: "" + # + #----------------------------------------------------------------------- + # Test directories + #----------------------------------------------------------------------- + # + TEST_EXTRN_MDL_SOURCE_BASEDIR: "" + TEST_PREGEN_BASEDIR: "" + TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: "" + TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: "" -#---------------------------- +#----------------------------- # WORKFLOW config parameters #----------------------------- workflow: @@ -1420,10 +1431,6 @@ task_run_fcst: # which the top level routine in the dynamics is called as well as the # frequency with which the physics is called." # - # CPL: parameter for coupling - # (set automatically based on FCST_MODEL in ush/setup.sh) - # (ufs-weather-model:FALSE, fv3gfs_aqm:TRUE) - # # RESTART_INTERVAL: # frequency of the output restart files (unit:hour). # Default=0: restart files are produced at the end of a forecast run diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index e4cf0e932b..02f1de0e5e 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -98,7 +98,6 @@ fi . $USHdir/source_util_funcs.sh . $USHdir/constants.sh . $USHdir/init_env.sh -. ${MACHINE_FILE} # #----------------------------------------------------------------------- # diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index c17610d124..c91650e76a 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -71,7 +71,6 @@ fi # #----------------------------------------------------------------------- # -. ${MACHINE_FILE} env_init_scripts_fps_str="( "$(printf "\"%s\" " "${ENV_INIT_SCRIPTS_FPS[@]}")")" init_env env_init_scripts_fps="${env_init_scripts_fps_str}" # diff --git a/ush/machine/cheyenne.sh b/ush/machine/cheyenne.sh deleted file mode 100644 index 2118c9867b..0000000000 --- a/ush/machine/cheyenne.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash - -function file_location() { - - # Return the default location of external model files on disk - - local external_file_fmt external_model location - - external_model=${1} - external_file_fmt=${2} - - location="" - case ${external_model} in - - "FV3GFS") - location='/glade/p/ral/jntp/UFS_CAM/COMGFS/gfs.${yyyymmdd}/${hh}' - ;; - - esac - echo ${location:-} - -} - -EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ - ${EXTRN_MDL_NAME_ICS} \ - ${FV3GFS_FILE_FMT_ICS})} -EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ - ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_LBCS})} - -# System scripts to source to initialize various commands within workflow -# scripts (e.g. "module"). -if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then - ENV_INIT_SCRIPTS_FPS=( "/etc/profile" ) -fi - -# Commands to run at the start of each workflow task. -PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' - -# Architecture information -WORKFLOW_MANAGER="rocoto" -NCORES_PER_NODE="${NCORES_PER_NODE:-36}" -SCHED=${SCHED:-"pbspro"} -QUEUE_DEFAULT=${QUEUE_DEFAULT:-"regular"} -QUEUE_HPSS=${QUEUE_HPSS:-"regular"} -QUEUE_FCST=${QUEUE_FCST:-"regular"} - -# UFS SRW App specific paths -staged_data_dir="/glade/p/ral/jntp/UFS_SRW_App/develop" -FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} -FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} -FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} -DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} - -# Run commands for executables -RUN_CMD_SERIAL="time" -RUN_CMD_UTILS='mpirun -np $nprocs' -RUN_CMD_FCST='mpirun -np ${PE_MEMBER01}' -RUN_CMD_POST='mpirun -np $nprocs' - -# MET/METplus-Related Paths -MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/glade/p/ral/jntp/MET/MET_releases/10.1.1"} -METPLUS_PATH=${METPLUS_PATH:-"/glade/p/ral/jntp/MET/METplus/METplus-4.1.1"} -CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} -MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} -NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} -MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} - -# Test Data Locations -TEST_COMIN="${staged_data_dir}/COMGFS" -TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" -TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" diff --git a/ush/machine/cheyenne.yaml b/ush/machine/cheyenne.yaml new file mode 100644 index 0000000000..cda2924460 --- /dev/null +++ b/ush/machine/cheyenne.yaml @@ -0,0 +1,31 @@ +platform: + WORKFLOW_MANAGER: rocoto + NCORES_PER_NODE: 36 + SCHED: pbspro + CCPA_OBS_DIR: /glade/p/ral/jntp/UFS_SRW_App/develop/obs_data/ccpa/proc + MRMS_OBS_DIR: /glade/p/ral/jntp/UFS_SRW_App/develop/obs_data/mrms/proc + NDAS_OBS_DIR: /glade/p/ral/jntp/UFS_SRW_App/develop/obs_data/ndas/proc + METPLUS_PATH: /glade/p/ral/jntp/MET/METplus/METplus-4.1.1 + MET_BIN_EXEC: bin + MET_INSTALL_DIR: /glade/p/ral/jntp/MET/MET_releases/10.1.1 + DOMAIN_PREGEN_BASEDIR: /glade/p/ral/jntp/UFS_SRW_App/develop/FV3LAM_pregen + QUEUE_DEFAULT: regular + QUEUE_FCST: regular + QUEUE_HPSS: regular + RUN_CMD_FCST: mpirun -np ${PE_MEMBER01} + RUN_CMD_POST: mpirun -np $nprocs + RUN_CMD_SERIAL: time + RUN_CMD_UTILS: mpirun -np $nprocs + ENV_INIT_SCRIPTS_FPS: + - /etc/profile + PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' + TEST_EXTRN_MDL_SOURCE_BASEDIR: /glade/p/ral/jntp/UFS_SRW_App/develop/input_model_data + TEST_PREGEN_BASEDIR: /glade/p/ral/jntp/UFS_SRW_App/develop/FV3LAM_pregen +task_run_fcst: + FIXaer: /glade/p/ral/jntp/UFS_SRW_App/develop/fix/fix_aer + FIXgsm: /glade/p/ral/jntp/UFS_SRW_App/develop/fix/fix_am + FIXlut: /glade/p/ral/jntp/UFS_SRW_App/develop/fix/fix_lut + TOPO_DIR: /glade/p/ral/jntp/UFS_SRW_App/develop/fix/fix_orog + SFC_CLIMO_INPUT_DIR: /glade/p/ral/jntp/UFS_SRW_App/develop/fix/fix_sfc_climo +data: + FV3GFS: /glade/p/ral/jntp/UFS_CAM/COMGFS/gfs.${yyyymmdd}/${hh} diff --git a/ush/machine/gaea.sh b/ush/machine/gaea.sh deleted file mode 100755 index b8b6f49534..0000000000 --- a/ush/machine/gaea.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -function file_location() { - - # Return the default location of external model files on disk - - local external_file_fmt external_model location - - external_model=${1} - external_file_fmt=${2} - - case ${external_model} in - - "FV3GFS") - location='/lustre/f2/dev/Mark.Potts/EPIC/SRW/model_data/FV3GFS/${yyyymmdd}${hh}' - ;; - - esac - echo ${location:-} -} - -export PROJ_LIB=/lustre/f2/dev/role.epic/contrib/miniconda3/4.12.0/envs/regional_workflow/share/proj -export PATH=${PATH}:/lustre/f2/deve/role.epic/contrib/miniconda3/4.12.0/envs/regional_workflow/bin - -EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ - ${EXTRN_MDL_NAME_ICS} \ - ${FV3GFS_FILE_FMT_ICS})} -EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ - ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_ICS})} - -# System scripts to source to initialize various commands within workflow -# scripts (e.g. "module"). -if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then - ENV_INIT_SCRIPTS_FPS=( "/etc/profile" "/lustre/f2/dev/role.epic/contrib/Lmod_init.sh") -fi - - -# Commands to run at the start of each workflow task. -PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' - -# Architecture information -WORKFLOW_MANAGER="rocoto" -SLURM_NATIVE_CMD="-M c3" -NCORES_PER_NODE=${NCORES_PER_NODE:-32} -SCHED=${SCHED:-"slurm"} -QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"} -QUEUE_HPSS=${QUEUE_DEFAULT:-"normal"} -QUEUE_FCST=${QUEUE_DEFAULT:-"normal"} -WTIME_MAKE_LBCS="00:60:00" - -# UFS SRW App specific paths -staged_data_dir="/lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop" -FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} -FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} -FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} -TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" - -RUN_CMD_SERIAL="time" -#Run Commands currently differ for GNU/openmpi -#RUN_CMD_UTILS='mpirun --mca btl tcp,vader,self -np $nprocs' -#RUN_CMD_FCST='mpirun --mca btl tcp,vader,self -np ${PE_MEMBER01}' -#RUN_CMD_POST='mpirun --mca btl tcp,vader,self -np $nprocs' -RUN_CMD_UTILS='srun --mpi=pmi2 -n $nprocs' -RUN_CMD_FCST='srun --mpi=pmi2 -n ${PE_MEMBER01}' -RUN_CMD_POST='srun --mpi=pmi2 -n $nprocs' - -# MET Installation Locations -MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/usw/met/10.1.2"} -METPLUS_PATH=${METPLUS_PATH:-"/usw/met/METplus/METplus-4.1.3"} -CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} -MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} -NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} -MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} - -# Test Data Locations diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml new file mode 100644 index 0000000000..cf8bcb595c --- /dev/null +++ b/ush/machine/gaea.yaml @@ -0,0 +1,30 @@ +platform: + WORKFLOW_MANAGER: rocoto + NCORES_PER_NODE: 32 + SCHED: slurm + CCPA_OBS_DIR: /lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop/obs_data/ccpa/proc + MRMS_OBS_DIR: /lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop/obs_data/mrms/proc + NDAS_OBS_DIR: /lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop/obs_data/ndas/proc + METPLUS_PATH: /usw/met/METplus/METplus-4.1.3 + MET_BIN_EXEC: bin + MET_INSTALL_DIR: /usw/met/10.1.2 + QUEUE_DEFAULT: normal + QUEUE_FCST: normal + QUEUE_HPSS: normal + RUN_CMD_FCST: srun --mpi=pmi2 -n ${PE_MEMBER01} + RUN_CMD_POST: srun --mpi=pmi2 -n $nprocs + RUN_CMD_SERIAL: time + RUN_CMD_UTILS: srun --mpi=pmi2 -n $nprocs + ENV_INIT_SCRIPTS_FPS: + - /etc/profile + PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' + TEST_EXTRN_MDL_SOURCE_BASEDIR: /lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop/input_model_data + TEST_PREGEN_BASEDIR: /lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop/FV3LAM_pregen +task_run_fcst: + FIXaer: /lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop/fix/fix_aer + FIXgsm: /lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop/fix/fix_am + FIXlut: /lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop/fix/fix_lut + TOPO_DIR: /lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop/fix/fix_orog + SFC_CLIMO_INPUT_DIR: /lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop/fix/fix_sfc_climo +data: + FV3GFS: /lustre/f2/dev/Mark.Potts/EPIC/SRW/model_data/FV3GFS/${yyyymmdd}${hh} diff --git a/ush/machine/hera.sh b/ush/machine/hera.sh deleted file mode 100644 index b34610d44e..0000000000 --- a/ush/machine/hera.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash - -function file_location() { - - # Return the default location of external model files on disk - # Hera does not currently have any files staged on disk. - - local external_file_fmt external_model location - - external_model=${1} - external_file_fmt=${2} - - location="" - echo ${location:-} - -} - -EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ - ${EXTRN_MDL_NAME_ICS} \ - ${FV3GFS_FILE_FMT_ICS})} -EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ - ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_LBCS})} - -EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"hpss aws nomads"} - -# System scripts to source to initialize various commands within workflow -# scripts (e.g. "module"). -if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then - ENV_INIT_SCRIPTS_FPS=( "/etc/profile" ) -fi - -# Commands to run at the start of each workflow task. -PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' - -# Architecture information -WORKFLOW_MANAGER="rocoto" -NCORES_PER_NODE=${NCORES_PER_NODE:-40} -SCHED=${SCHED:-"slurm"} -PARTITION_DEFAULT=${PARTITION_DEFAULT:-"hera"} -QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} -PARTITION_HPSS=${PARTITION_HPSS:-"service"} -QUEUE_HPSS=${QUEUE_HPSS:-"batch"} -PARTITION_FCST=${PARTITION_FCST:-"hera"} -QUEUE_FCST=${QUEUE_FCST:-"batch"} - -# UFS SRW App specific paths -staged_data_dir="/scratch2/BMC/det/UFS_SRW_App/develop" -FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} -FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} -FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} -DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} - -# Run commands for executables -RUN_CMD_SERIAL="time" -RUN_CMD_UTILS="srun" -RUN_CMD_FCST="srun" -RUN_CMD_POST="srun" - -# MET/METplus-Related Paths -MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/contrib/met/10.1.1"} -METPLUS_PATH=${METPLUS_PATH:-"/contrib/METplus/METplus-4.1.1"} -CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} -MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} -NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} -MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} - -# Test Data Locations -TEST_COMIN="${staged_data_dir}/COMGFS" -TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" -TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" -TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" -TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml new file mode 100644 index 0000000000..33a681ba6a --- /dev/null +++ b/ush/machine/hera.yaml @@ -0,0 +1,36 @@ +platform: + WORKFLOW_MANAGER: rocoto + NCORES_PER_NODE: 40 + SCHED: slurm + CCPA_OBS_DIR: /scratch2/BMC/det/UFS_SRW_App/develop/obs_data/ccpa/proc + MRMS_OBS_DIR: /scratch2/BMC/det/UFS_SRW_App/develop/obs_data/mrms/proc + NDAS_OBS_DIR: /scratch2/BMC/det/UFS_SRW_App/develop/obs_data/ndas/proc + METPLUS_PATH: /contrib/METplus/METplus-4.1.1 + MET_BIN_EXEC: bin + MET_INSTALL_DIR: /contrib/met/10.1.1 + DOMAIN_PREGEN_BASEDIR: /scratch2/BMC/det/UFS_SRW_App/develop/FV3LAM_pregen + PARTITION_DEFAULT: hera + QUEUE_DEFAULT: batch + PARTITION_FCST: hera + QUEUE_FCST: batch + PARTITION_HPSS: service + QUEUE_HPSS: batch + RUN_CMD_FCST: srun + RUN_CMD_POST: srun + RUN_CMD_SERIAL: time + RUN_CMD_UTILS: srun + ENV_INIT_SCRIPTS_FPS: + - /etc/profile + PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' + TEST_EXTRN_MDL_SOURCE_BASEDIR: /scratch2/BMC/det/UFS_SRW_App/develop/input_model_data + TEST_PREGEN_BASEDIR: /scratch2/BMC/det/UFS_SRW_App/develop/FV3LAM_pregen +task_get_extrn_ics: + EXTRN_MDL_DATA_STORES: hpss aws nomads +task_get_extrn_lbcs: + EXTRN_MDL_DATA_STORES: hpss aws nomads +task_run_fcst: + FIXaer: /scratch2/BMC/det/UFS_SRW_App/develop/fix/fix_aer + FIXgsm: /scratch2/BMC/det/UFS_SRW_App/develop/fix/fix_am + FIXlut: /scratch2/BMC/det/UFS_SRW_App/develop/fix/fix_lut + TOPO_DIR: /scratch2/BMC/det/UFS_SRW_App/develop/fix/fix_orog + SFC_CLIMO_INPUT_DIR: /scratch2/BMC/det/UFS_SRW_App/develop/fix/fix_sfc_climo diff --git a/ush/machine/jet.sh b/ush/machine/jet.sh deleted file mode 100644 index ced5345de4..0000000000 --- a/ush/machine/jet.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash - -function file_location() { - - # Return the default location of external model files on disk - - local external_file_fmt external_model location - - external_model=${1} - external_file_fmt=${2} - - location="" - case ${external_model} in - - "FV3GFS") - case $external_file_fmt in - "nemsio") - location='/public/data/grids/gfs/nemsio' - ;; - "grib2") - location='/public/data/grids/gfs/0p25deg/grib2' - ;; - "netcdf") - location='/public/data/grids/gfs/anl/netcdf/' - ;; - esac - ;; - "RAP") - location='/public/data/grids/rap/full/wrfprs/grib2' - ;; - "HRRR") - location='/public/data/grids/hrrr/conus/wrfprs/grib2' - ;; - - esac - echo ${location:-} - -} - -EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ - ${EXTRN_MDL_NAME_ICS} \ - ${FV3GFS_FILE_FMT_ICS})} -EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ - ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_LBCS})} - -EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"hpss aws nomads"} - -# System scripts to source to initialize various commands within workflow -# scripts (e.g. "module"). -if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then - ENV_INIT_SCRIPTS_FPS=( "/etc/profile" ) -fi - -# Commands to run at the start of each workflow task. -PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' - -# Architecture information -WORKFLOW_MANAGER="rocoto" -NCORES_PER_NODE=${NCORES_PER_NODE:-24} -SCHED=${SCHED:-"slurm"} -PARTITION_DEFAULT=${PARTITION_DEFAULT:-"sjet,vjet,kjet,xjet"} -QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} -PARTITION_HPSS=${PARTITION_HPSS:-"service"} -QUEUE_HPSS=${QUEUE_HPSS:-"batch"} -PARTITION_FCST=${PARTITION_FCST:-"sjet,vjet,kjet,xjet"} -QUEUE_FCST=${QUEUE_FCST:-"batch"} - -# UFS SRW App specific paths -staged_data_dir="/mnt/lfs4/BMC/wrfruc/UFS_SRW_App/develop" -FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} -FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} -FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} -DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} - -# Run commands for executables -RUN_CMD_SERIAL="time" -RUN_CMD_UTILS="srun" -RUN_CMD_FCST="srun" -RUN_CMD_POST="srun" - -# MET/METplus-Related Paths -MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/contrib/met/10.1.1"} -METPLUS_PATH=${METPLUS_PATH:-"/contrib/met/METplus/METplus-4.1.1"} -CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} -MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} -NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} -MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} - -# Test Data Locations -TEST_COMIN="${staged_data_dir}/COMGFS" -TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" -TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml new file mode 100644 index 0000000000..d5176843d7 --- /dev/null +++ b/ush/machine/jet.yaml @@ -0,0 +1,43 @@ +platform: + WORKFLOW_MANAGER: rocoto + NCORES_PER_NODE: 24 + SCHED: slurm + CCPA_OBS_DIR: /mnt/lfs4/BMC/wrfruc/UFS_SRW_App/develop/obs_data/ccpa/proc + MRMS_OBS_DIR: /mnt/lfs4/BMC/wrfruc/UFS_SRW_App/develop/obs_data/mrms/proc + NDAS_OBS_DIR: /mnt/lfs4/BMC/wrfruc/UFS_SRW_App/develop/obs_data/ndas/proc + METPLUS_PATH: /contrib/met/METplus/METplus-4.1.1 + MET_BIN_EXEC: bin + MET_INSTALL_DIR: /contrib/met/10.1.1 + DOMAIN_PREGEN_BASEDIR: /mnt/lfs4/BMC/wrfruc/UFS_SRW_App/develop/FV3LAM_pregen + PARTITION_DEFAULT: sjet,vjet,kjet,xjet + QUEUE_DEFAULT: batch + PARTITION_FCST: sjet,vjet,kjet,xjet + QUEUE_FCST: batch + PARTITION_HPSS: service + QUEUE_HPSS: batch + RUN_CMD_FCST: srun + RUN_CMD_POST: srun + RUN_CMD_SERIAL: time + RUN_CMD_UTILS: srun + ENV_INIT_SCRIPTS_FPS: + - /etc/profile + PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' + TEST_EXTRN_MDL_SOURCE_BASEDIR: /mnt/lfs4/BMC/wrfruc/UFS_SRW_App/develop/input_model_data + TEST_PREGEN_BASEDIR: /mnt/lfs4/BMC/wrfruc/UFS_SRW_App/develop/FV3LAM_pregen +task_get_extrn_ics: + EXTRN_MDL_DATA_STORES: hpss aws nomads +task_get_extrn_lbcs: + EXTRN_MDL_DATA_STORES: hpss aws nomads +task_run_fcst: + FIXaer: /mnt/lfs4/BMC/wrfruc/UFS_SRW_App/develop/fix/fix_aer + FIXgsm: /mnt/lfs4/BMC/wrfruc/UFS_SRW_App/develop/fix/fix_am + FIXlut: /mnt/lfs4/BMC/wrfruc/UFS_SRW_App/develop/fix/fix_lut + TOPO_DIR: /mnt/lfs4/BMC/wrfruc/UFS_SRW_App/develop/fix/fix_orog + SFC_CLIMO_INPUT_DIR: /mnt/lfs4/BMC/wrfruc/UFS_SRW_App/develop/fix/fix_sfc_climo +data: + FV3GFS: + - nemsio: /public/data/grids/gfs/nemsio + - grib2: /public/data/grids/gfs/0p25deg/grib2 + - netcdf: /public/data/grids/gfs/anl/netcdf + RAP: /public/data/grids/rap/full/wrfprs/grib2 + HRRR: /public/data/grids/hrrr/conus/wrfprs/grib2 diff --git a/ush/machine/linux.sh b/ush/machine/linux.sh deleted file mode 100644 index cdf055eb4b..0000000000 --- a/ush/machine/linux.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -set -x - -function file_location() { - - # Return the default location of external model files on disk - - local external_file_fmt external_model location - - external_model=${1} - external_file_fmt=${2} - - case ${external_model} in - - "FV3GFS") - location='/home/username/DATA/UFS/FV3GFS/' - ;; - *) - print_info_msg"\ - External model \'${external_model}\' does not have a default - location on Linux systems. " - ;; - - esac - echo ${location:-} -} - - -EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ - ${EXTRN_MDL_NAME_ICS} \ - ${FV3GFS_FILE_FMT_ICS})} -EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ - ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_ICS})} - - System scripts to source to initialize various commands within workflow - scripts (e.g. "module"). -if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then - ENV_INIT_SCRIPTS_FPS=( "/etc/profile" ) -fi - - -# Commands to run at the start of each workflow task. -PRE_TASK_CMDS='{ ulimit -a; }' - -# Architecture information -WORKFLOW_MANAGER="none" -NCORES_PER_NODE=${NCORES_PER_NODE:-8} -SCHED=${SCHED:-"none"} - -# UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/home/username/DATA/UFS/fix/fix_am"} -FIXaer=${FIXaer:-"/home/username/DATA/UFS/fix/fix_aer"} -FIXlut=${FIXlut:-"/home/username/DATA/UFS/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/home/username/DATA/UFS/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/home/username/DATA/UFS/fix/fix_sfc_climo"} - -# Run commands for executables -RUN_CMD_SERIAL="time" -#Run Commands currently differ for GNU/openmpi -RUN_CMD_UTILS='mpirun -n 4' -RUN_CMD_FCST='mpirun -n ${PE_MEMBER01} ' -RUN_CMD_POST='mpirun -n 4 ' - -# MET Installation Locations - diff --git a/ush/machine/linux.yaml b/ush/machine/linux.yaml new file mode 100644 index 0000000000..2b5e95d4fc --- /dev/null +++ b/ush/machine/linux.yaml @@ -0,0 +1,19 @@ +platform: + WORKFLOW_MANAGER: none + NCORES_PER_NODE: 8 + SCHED: none + RUN_CMD_FCST: 'mpirun -n ${PE_MEMBER01} ' + RUN_CMD_POST: 'mpirun -n 4 ' + RUN_CMD_SERIAL: time + RUN_CMD_UTILS: mpirun -n 4 + ENV_INIT_SCRIPTS_FPS: + - /etc/profile + PRE_TASK_CMDS: '{ ulimit -a; }' +task_run_fcst: + FIXaer: /home/username/DATA/UFS/fix/fix_aer + FIXgsm: /home/username/DATA/UFS/fix/fix_am + FIXlut: /home/username/DATA/UFS/fix/fix_lut + TOPO_DIR: /home/username/DATA/UFS/fix/fix_orog + SFC_CLIMO_INPUT_DIR: /home/username/DATA/UFS/fix/fix_sfc_climo +data: + FV3GFS: /home/username/DATA/UFS/FV3GFS diff --git a/ush/machine/macos.sh b/ush/machine/macos.sh deleted file mode 100644 index 42d70c5684..0000000000 --- a/ush/machine/macos.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -set -x - -function file_location() { - - # Return the default location of external model files on disk - - local external_file_fmt external_model location - - external_model=${1} - external_file_fmt=${2} - - case ${external_model} in - - "FV3GFS") - location='/Users/username/DATA/UFS/FV3GFS/' - ;; - *) - print_info_msg"\ - External model \'${external_model}\' does not have a default - location on MacOSX. " - ;; - - esac - echo ${location:-} -} - - -EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ - ${EXTRN_MDL_NAME_ICS} \ - ${FV3GFS_FILE_FMT_ICS})} -EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ - ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_ICS})} - - System scripts to source to initialize various commands within workflow - scripts (e.g. "module"). -if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then - ENV_INIT_SCRIPTS_FPS=( "/etc/profile" ) -fi - - -# Commands to run at the start of each workflow task. -PRE_TASK_CMDS='{ ulimit -a; }' - -# Architecture information -WORKFLOW_MANAGER="none" -NCORES_PER_NODE=${NCORES_PER_NODE:-8} -SCHED=${SCHED:-"none"} - -# UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/Users/username/DATA/UFS/fix/fix_am"} -FIXaer=${FIXaer:-"/Users/username/DATA/UFS/fix/fix_aer"} -FIXlut=${FIXlut:-"/Users/username/DATA/UFS/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/Users/username/DATA/UFS/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/Users/username/DATA/UFS/fix/fix_sfc_climo"} - -# Run commands for executables -RUN_CMD_SERIAL="time" -#Run Commands currently differ for GNU/openmpi -RUN_CMD_UTILS='mpirun -n 4' -RUN_CMD_FCST='mpirun -n ${PE_MEMBER01} ' -RUN_CMD_POST='mpirun -n 4 ' - -# MET Installation Locations - diff --git a/ush/machine/macos.yaml b/ush/machine/macos.yaml new file mode 100644 index 0000000000..c8fbd0c992 --- /dev/null +++ b/ush/machine/macos.yaml @@ -0,0 +1,19 @@ +platform: + WORKFLOW_MANAGER: none + NCORES_PER_NODE: 8 + SCHED: none + RUN_CMD_FCST: 'mpirun -n ${PE_MEMBER01} ' + RUN_CMD_POST: 'mpirun -n 4 ' + RUN_CMD_SERIAL: time + RUN_CMD_UTILS: mpirun -n 4 + ENV_INIT_SCRIPTS_FPS: + - /etc/profile + PRE_TASK_CMDS: '{ ulimit -a; }' +task_run_fcst: + FIXaer: /Users/username/DATA/UFS/fix/fix_aer + FIXgsm: /Users/username/DATA/UFS/fix/fix_am + FIXlut: /Users/username/DATA/UFS/fix/fix_lut + TOPO_DIR: /Users/username/DATA/UFS/fix/fix_orog + SFC_CLIMO_INPUT_DIR: /Users/username/DATA/UFS/fix/fix_sfc_climo +data: + FV3GFS: /Users/username/DATA/UFS/FV3GFS diff --git a/ush/machine/noaacloud.sh b/ush/machine/noaacloud.sh deleted file mode 100755 index 62a698da8a..0000000000 --- a/ush/machine/noaacloud.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash - -set -x - -function file_location() { - - # Return the default location of external model files on disk - - local external_file_fmt external_model location - - external_model=${1} - external_file_fmt=${2} - - case ${external_model} in - - "FV3GFS") - location='/contrib/GST/model_data/FV3GFS/${yyyymmdd}${hh}' - ;; - - esac - echo ${location:-} -} -export PROJ_LIB=/contrib/GST/miniconda/envs/regional_workflow/share/proj -export OPT=/contrib/EPIC/hpc-modules -export PATH=${PATH}:/contrib/GST/miniconda/envs/regional_workflow/bin - -EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ - ${EXTRN_MDL_NAME_ICS} \ - ${FV3GFS_FILE_FMT_ICS})} -EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ - ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_ICS})} - -EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"aws nomads"} - -# System scripts to source to initialize various commands within workflow -# scripts (e.g. "module"). -if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then - ENV_INIT_SCRIPTS_FPS=( "/etc/profile" ) -fi - - -# Commands to run at the start of each workflow task. -PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' - -# Architecture information -WORKFLOW_MANAGER="rocoto" -NCORES_PER_NODE=${NCORES_PER_NODE:-36} -SCHED=${SCHED:-"slurm"} - -# UFS SRW App specific paths -staged_data_dir="/contrib/EPIC/UFS_SRW_App/develop" -FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} -FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} -FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} -TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" - -RUN_CMD_SERIAL="time" -#Run Commands currently differ for GNU/openmpi -#RUN_CMD_UTILS='mpirun --mca btl tcp,vader,self -np $nprocs' -#RUN_CMD_FCST='mpirun --mca btl tcp,vader,self -np ${PE_MEMBER01}' -#RUN_CMD_POST='mpirun --mca btl tcp,vader,self -np $nprocs' -RUN_CMD_UTILS='mpiexec -np $nprocs' -RUN_CMD_FCST='mpiexec -np ${PE_MEMBER01}' -RUN_CMD_POST='mpiexec -np $nprocs' - -export build_mod_fn="wflow_noaacloud" -BUILD_MOD_FN="wflow_noaacloud" - -# MET Installation Locations -# MET Plus is not yet supported on noaacloud -. /contrib/EPIC/.bash_conda diff --git a/ush/machine/noaacloud.yaml b/ush/machine/noaacloud.yaml new file mode 100644 index 0000000000..019f58426b --- /dev/null +++ b/ush/machine/noaacloud.yaml @@ -0,0 +1,31 @@ +platform: + WORKFLOW_MANAGER: rocoto + NCORES_PER_NODE: 36 + SCHED: slurm + CCPA_OBS_DIR: /contrib/EPIC/UFS_SRW_App/develop/obs_data/ccpa/proc + MRMS_OBS_DIR: /contrib/EPIC/UFS_SRW_App/develop/obs_data/mrms/proc + NDAS_OBS_DIR: /contrib/EPIC/UFS_SRW_App/develop/obs_data/ndas/proc + METPLUS_PATH: /contrib/EPIC/spack-stack/envs/ufs-srw-dev.intel/install/intel/2021.3.0/metplus-4.1.0-t4mkmbg + MET_BIN_EXEC: bin + MET_INSTALL_DIR: /contrib/EPIC/spack-stack/envs/ufs-srw-dev.intel/install/intel/2021.3.0/met-10.1.0-mqwqjxt + RUN_CMD_FCST: mpiexec -np ${PE_MEMBER01} + RUN_CMD_POST: mpiexec -np $nprocs + RUN_CMD_SERIAL: time + RUN_CMD_UTILS: mpiexec -np $nprocs + ENV_INIT_SCRIPTS_FPS: + - /etc/profile + PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' + BUILD_MOD_FN: wflow_noaacloud + TEST_EXTRN_MDL_SOURCE_BASEDIR: /contrib/EPIC/UFS_SRW_App/develop/input_model_data +task_get_extrn_ics: + EXTRN_MDL_DATA_STORES: aws nomads +task_get_extrn_lbcs: + EXTRN_MDL_DATA_STORES: aws nomads +task_run_fcst: + FIXaer: /contrib/EPIC/UFS_SRW_App/develop/fix/fix_aer + FIXgsm: /contrib/EPIC/UFS_SRW_App/develop/fix/fix_am + FIXlut: /contrib/EPIC/UFS_SRW_App/develop/fix/fix_lut + TOPO_DIR: /contrib/EPIC/UFS_SRW_App/develop/fix/fix_orog + SFC_CLIMO_INPUT_DIR: /contrib/EPIC/UFS_SRW_App/develop/fix/fix_sfc_climo +data: + FV3GFS: /contrib/GST/model_data/FV3GFS/${yyyymmdd}${hh} diff --git a/ush/machine/odin.sh b/ush/machine/odin.sh deleted file mode 100644 index 58d360a633..0000000000 --- a/ush/machine/odin.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -function file_location() { - - # Return the default location of external model files on disk - - local external_file_fmt external_model location - - external_model=${1} - external_file_fmt=${2} - - staged_data_dir="/scratch/ywang/UFS_SRW_App/develop" - - location="" - case ${external_model} in - - "GSMGFS") - location="${staged_data_dir}/input_model_data/GFS" - ;; - "FV3GFS") - location="${staged_data_dir}/input_model_data/FV3GFS" - ;; - "HRRR") - location="${staged_data_dir}/input_model_data/HRRR" - ;; - "RAP") - location="${staged_data_dir}/input_model_data/RAP" - ;; - "NAM") - location="${staged_data_dir}/input_model_data/NAM" - ;; - esac - echo ${location:-} - -} - -EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ - ${EXTRN_MDL_NAME_ICS} \ - ${FV3GFS_FILE_FMT_ICS})} -EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ - ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_LBCS})} - -# System scripts to source to initialize various commands within workflow -# scripts (e.g. "module"). -if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then - ENV_INIT_SCRIPTS_FPS=() -fi - -# Commands to run at the start of each workflow task. -PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' - -# Architecture information -WORKFLOW_MANAGER="rocoto" -NCORES_PER_NODE=${NCORES_PER_NODE:-24} -SCHED=${SCHED:-"slurm"} -PARTITION_DEFAULT=${PARTITION_DEFAULT:-"workq"} -QUEUE_DEFAULT=${QUEUE_DEFAULT:-"workq"} -PARTITION_HPSS=${PARTITION_HPSS:-"workq"} -QUEUE_HPSS=${QUEUE_HPSS:-"workq"} -PARTITION_FCST=${PARTITION_FCST:-"workq"} -QUEUE_FCST=${QUEUE_FCST:-"workq"} - -# UFS SRW App specific paths -FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} -FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} -FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} -DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} - -# Run commands for executables -RUN_CMD_SERIAL="srun -n 1" -RUN_CMD_UTILS='srun -n $nprocs' -RUN_CMD_FCST='srun -n ${PE_MEMBER01}' -RUN_CMD_POST="srun -n 1" - -# Test Data Locations -TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" -TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" diff --git a/ush/machine/odin.yaml b/ush/machine/odin.yaml new file mode 100644 index 0000000000..f3809a950f --- /dev/null +++ b/ush/machine/odin.yaml @@ -0,0 +1,31 @@ +platform: + WORKFLOW_MANAGER: rocoto + NCORES_PER_NODE: 24 + SCHED: slurm + DOMAIN_PREGEN_BASEDIR: /FV3LAM_pregen + PARTITION_DEFAULT: workq + QUEUE_DEFAULT: workq + PARTITION_FCST: workq + QUEUE_FCST: workq + PARTITION_HPSS: workq + QUEUE_HPSS: workq + RUN_CMD_FCST: srun -n ${PE_MEMBER01} + RUN_CMD_POST: srun -n 1 + RUN_CMD_SERIAL: srun -n 1 + RUN_CMD_UTILS: srun -n $nprocs + ENV_INIT_SCRIPTS_FPS: [] + PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' + TEST_EXTRN_MDL_SOURCE_BASEDIR: /scratch/ywang/UFS_SRW_App/develop/input_model_data + TEST_PREGEN_BASEDIR: /scratch/ywang/UFS_SRW_App/develop/FV3LAM_pregen +task_run_fcst: + FIXaer: /scratch/ywang/UFS_SRW_App/develop/fix/fix_aer + FIXgsm: /scratch/ywang/UFS_SRW_App/develop/fix/fix_am + FIXlut: /scratch/ywang/UFS_SRW_App/develop/fix/fix_lut + TOPO_DIR: /scratch/ywang/UFS_SRW_App/develop/fix/fix_orog + SFC_CLIMO_INPUT_DIR: /scratch/ywang/UFS_SRW_App/develop/fix/fix_sfc_climo +data: + GSMGFS: /scratch/ywang/UFS_SRW_App/develop/input_model_data/GFS + FV3GFS: /scratch/ywang/UFS_SRW_App/develop/input_model_data/FV3GFS + HRRR: /scratch/ywang/UFS_SRW_App/develop/input_model_data/HRRR + RAP: /scratch/ywang/UFS_SRW_App/develop/input_model_data/RAP + NAM: /scratch/ywang/UFS_SRW_App/develop/input_model_data/NAM diff --git a/ush/machine/orion.sh b/ush/machine/orion.sh deleted file mode 100644 index fe01323b75..0000000000 --- a/ush/machine/orion.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash - -function file_location() { - - # Return the default location of external model files on disk - # Orion does not currently have any files staged on disk. - - local external_file_fmt external_model location - - external_model=${1} - external_file_fmt=${2} - - location="" - echo ${location:-} - -} - -EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ - ${EXTRN_MDL_NAME_ICS} \ - ${FV3GFS_FILE_FMT_ICS})} -EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ - ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_LBCS})} - -EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"aws nomads"} - -# System scripts to source to initialize various commands within workflow -# scripts (e.g. "module"). -if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then - ENV_INIT_SCRIPTS_FPS=( "/etc/profile" ) -fi - -# Commands to run at the start of each workflow task. -PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' - -# Architecture information -WORKFLOW_MANAGER="rocoto" -NCORES_PER_NODE=${NCORES_PER_NODE:-40} -SCHED=${SCHED:-"slurm"} -PARTITION_DEFAULT=${PARTITION_DEFAULT:-"orion"} -QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} -PARTITION_HPSS=${PARTITION_HPSS:-"service"} -QUEUE_HPSS=${QUEUE_HPSS:-"batch"} -PARTITION_FCST=${PARTITION_FCST:-"orion"} -QUEUE_FCST=${QUEUE_FCST:-"batch"} - -# UFS SRW App specific paths -staged_data_dir="/work/noaa/fv3-cam/UFS_SRW_App/develop" -FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} -FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} -FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} -DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} - -# Run commands for executables -RUN_CMD_SERIAL="time" -RUN_CMD_UTILS="srun" -RUN_CMD_FCST='srun -n ${PE_MEMBER01}' -RUN_CMD_POST="srun" - -# MET/METplus-Related Paths -MET_INSTALL_DIR=${MET_INSTALL_DIR:-"/apps/contrib/MET/10.1.1"} -METPLUS_PATH=${METPLUS_PATH:-"/apps/contrib/MET/METplus/METplus-4.1.1"} -CCPA_OBS_DIR=${CCPA_OBS_DIR:-"${staged_data_dir}/obs_data/ccpa/proc"} -MRMS_OBS_DIR=${MRMS_OBS_DIR:-"${staged_data_dir}/obs_data/mrms/proc"} -NDAS_OBS_DIR=${NDAS_OBS_DIR:-"${staged_data_dir}/obs_data/ndas/proc"} -MET_BIN_EXEC=${MET_BIN_EXEC:-"bin"} - -# Test Data Locations -TEST_COMIN="${staged_data_dir}/COMGFS" -TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" -TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" - diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml new file mode 100644 index 0000000000..4a75f3f1c2 --- /dev/null +++ b/ush/machine/orion.yaml @@ -0,0 +1,36 @@ +platform: + WORKFLOW_MANAGER: rocoto + NCORES_PER_NODE: 40 + SCHED: slurm + CCPA_OBS_DIR: /work/noaa/fv3-cam/UFS_SRW_App/develop/obs_data/ccpa/proc + MRMS_OBS_DIR: /work/noaa/fv3-cam/UFS_SRW_App/develop/obs_data/mrms/proc + NDAS_OBS_DIR: /work/noaa/fv3-cam/UFS_SRW_App/develop/obs_data/ndas/proc + METPLUS_PATH: /apps/contrib/MET/METplus/METplus-4.1.1 + MET_BIN_EXEC: bin + MET_INSTALL_DIR: /apps/contrib/MET/10.1.1 + DOMAIN_PREGEN_BASEDIR: /work/noaa/fv3-cam/UFS_SRW_App/develop/FV3LAM_pregen + PARTITION_DEFAULT: orion + QUEUE_DEFAULT: batch + PARTITION_FCST: orion + QUEUE_FCST: batch + PARTITION_HPSS: service + QUEUE_HPSS: batch + RUN_CMD_FCST: srun -n ${PE_MEMBER01} + RUN_CMD_POST: srun + RUN_CMD_SERIAL: time + RUN_CMD_UTILS: srun + ENV_INIT_SCRIPTS_FPS: + - /etc/profile + PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' + TEST_EXTRN_MDL_SOURCE_BASEDIR: /work/noaa/fv3-cam/UFS_SRW_App/develop/input_model_data + TEST_PREGEN_BASEDIR: /work/noaa/fv3-cam/UFS_SRW_App/develop/FV3LAM_pregen +task_get_extrn_ics: + EXTRN_MDL_DATA_STORES: aws nomads +task_get_extrn_lbcs: + EXTRN_MDL_DATA_STORES: aws nomads +task_run_fcst: + FIXaer: /work/noaa/fv3-cam/UFS_SRW_App/develop/fix/fix_aer + FIXgsm: /work/noaa/fv3-cam/UFS_SRW_App/develop/fix/fix_am + FIXlut: /work/noaa/fv3-cam/UFS_SRW_App/develop/fix/fix_lut + TOPO_DIR: /work/noaa/fv3-cam/UFS_SRW_App/develop/fix/fix_orog + SFC_CLIMO_INPUT_DIR: /work/noaa/fv3-cam/UFS_SRW_App/develop/fix/fix_sfc_climo diff --git a/ush/machine/singularity.sh b/ush/machine/singularity.sh deleted file mode 100644 index 14f840800b..0000000000 --- a/ush/machine/singularity.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -function file_location() { - - # Return the default location of external model files on disk - - local external_file_fmt external_model location - - external_model=${1} - external_file_fmt=${2} - - location="" - echo ${location:-} - -} - -EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ - ${EXTRN_MDL_NAME_ICS} \ - ${FV3GFS_FILE_FMT_ICS})} -EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ - ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_LBCS})} - -EXTRN_MDL_DATA_STORES=${EXTRN_MDL_DATA_STORES:-"aws nomads"} - -# System scripts to source to initialize various commands within workflow -# scripts (e.g. "module"). -if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then - ENV_INIT_SCRIPTS_FPS=( "/usr/share/lmod/6.6/init/profile" ) -fi - -# Commands to run at the start of each workflow task. -PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' - -# Architecture information -WORKFLOW_MANAGER="rocoto" -NCORES_PER_NODE=${NCORES_PER_NODE:-40} -SCHED=${SCHED:-"slurm"} -PARTITION_DEFAULT=${PARTITION_DEFAULT:-""} -QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} -PARTITION_HPSS=${PARTITION_HPSS:-"service"} -QUEUE_HPSS=${QUEUE_HPSS:-"batch"} -PARTITION_FCST=${PARTITION_FCST:-""} -QUEUE_FCST=${QUEUE_FCST:-"batch"} - -# UFS SRW App specific paths -FIXgsm=${FIXgsm:-"/contrib/global/glopara/fix/fix_am"} -FIXaer=${FIXaer:-"/contrib/global/glopara/fix/fix_aer"} -FIXlut=${FIXlut:-"/contrib/global/glopara/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"/contrib/global/glopara/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/contrib/global/glopara/fix/fix_sfc_climo"} -DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"/needs/to/be/specified"} - -# Run commands for executables -RUN_CMD_SERIAL="time" -RUN_CMD_UTILS='mpirun -np $nprocs --oversubscribe' -RUN_CMD_FCST='mpirun -n ${PE_MEMBER01} --oversubscribe' -RUN_CMD_POST='mpirun -np $nprocs --oversubscribe' diff --git a/ush/machine/singularity.yaml b/ush/machine/singularity.yaml new file mode 100644 index 0000000000..886962cccd --- /dev/null +++ b/ush/machine/singularity.yaml @@ -0,0 +1,28 @@ +platform: + WORKFLOW_MANAGER: rocoto + NCORES_PER_NODE: 40 + SCHED: slurm + DOMAIN_PREGEN_BASEDIR: /needs/to/be/specified + PARTITION_DEFAULT: null + QUEUE_DEFAULT: batch + PARTITION_FCST: null + QUEUE_FCST: batch + PARTITION_HPSS: service + QUEUE_HPSS: batch + RUN_CMD_FCST: mpirun -n ${PE_MEMBER01} --oversubscribe + RUN_CMD_POST: mpirun -np $nprocs --oversubscribe + RUN_CMD_SERIAL: time + RUN_CMD_UTILS: mpirun -np $nprocs --oversubscribe + ENV_INIT_SCRIPTS_FPS: + - /usr/share/lmod/6.6/init/profile + PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' +task_get_extrn_ics: + EXTRN_MDL_DATA_STORES: aws nomads +task_get_extrn_lbcs: + EXTRN_MDL_DATA_STORES: aws nomads +task_run_fcst: + FIXaer: /contrib/global/glopara/fix/fix_aer + FIXgsm: /contrib/global/glopara/fix/fix_am + FIXlut: /contrib/global/glopara/fix/fix_lut + TOPO_DIR: /contrib/global/glopara/fix/fix_orog + SFC_CLIMO_INPUT_DIR: /contrib/global/glopara/fix/fix_sfc_climo diff --git a/ush/machine/stampede.sh b/ush/machine/stampede.sh deleted file mode 100644 index 3f879ea22f..0000000000 --- a/ush/machine/stampede.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -function file_location() { - - # Return the default location of external model files on disk - - local external_file_fmt external_model location - - external_model=${1} - external_file_fmt=${2} - - staged_data_dir="/work2/00315/tg455890/stampede2/UFS_SRW_App/develop" - - location="" - case ${external_model} in - - "GSMGFS") - location="${staged_data_dir}/input_model_data/GFS" - ;; - "FV3GFS") - location="${staged_data_dir}/input_model_data/FV3GFS" - ;; - "HRRR") - location="${staged_data_dir}/input_model_data/HRRR" - ;; - "RAP") - location="${staged_data_dir}/input_model_data/RAP" - ;; - "NAM") - location="${staged_data_dir}/input_model_data/NAM" - ;; - esac - echo ${location:-} - -} - -EXTRN_MDL_SYSBASEDIR_ICS=${EXTRN_MDL_SYSBASEDIR_ICS:-$(file_location \ - ${EXTRN_MDL_NAME_ICS} \ - ${FV3GFS_FILE_FMT_ICS})} -EXTRN_MDL_SYSBASEDIR_LBCS=${EXTRN_MDL_SYSBASEDIR_LBCS:-$(file_location \ - ${EXTRN_MDL_NAME_LBCS} \ - ${FV3GFS_FILE_FMT_LBCS})} - -# System scripts to source to initialize various commands within workflow -# scripts (e.g. "module"). -if [ -z ${ENV_INIT_SCRIPTS_FPS:-""} ]; then - ENV_INIT_SCRIPTS_FPS=() -fi - -# Commands to run at the start of each workflow task. -PRE_TASK_CMDS='{ ulimit -s unlimited; ulimit -a; }' - -# Architecture information -WORKFLOW_MANAGER="rocoto" -NCORES_PER_NODE="${NCORES_PER_NODE:-68}" -SCHED=${SCHED:-"slurm"} -PARTITION_DEFAULT=${PARTITION_DEFAULT:-"normal"} -QUEUE_DEFAULT=${QUEUE_DEFAULT:-"normal"} -PARTITION_HPSS=${PARTITION_HPSS:-"normal"} -QUEUE_HPSS=${QUEUE_HPSS:-"normal"} -PARTITION_FCST=${PARTITION_FCST:-"normal"} -QUEUE_FCST=${QUEUE_FCST:-"normal"} - -# UFS SRW App specific paths -staged_data_dir="/work2/00315/tg455890/stampede2/UFS_SRW_App/develop" -FIXgsm=${FIXgsm:-"${staged_data_dir}/fix/fix_am"} -FIXaer=${FIXaer:-"${staged_data_dir}/fix/fix_aer"} -FIXlut=${FIXlut:-"${staged_data_dir}/fix/fix_lut"} -TOPO_DIR=${TOPO_DIR:-"${staged_data_dir}/fix/fix_orog"} -SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"${staged_data_dir}/fix/fix_sfc_climo"} -DOMAIN_PREGEN_BASEDIR=${DOMAIN_PREGEN_BASEDIR:-"${staged_data_dir}/FV3LAM_pregen"} - -# Run commands for executables -RUN_CMD_SERIAL="time" -RUN_CMD_UTILS='ibrun -np $nprocs' -RUN_CMD_FCST='ibrun -np $nprocs' -RUN_CMD_POST='ibrun -np $nprocs' - -# Test Data Locations -TEST_PREGEN_BASEDIR="${staged_data_dir}/FV3LAM_pregen" -TEST_EXTRN_MDL_SOURCE_BASEDIR="${staged_data_dir}/input_model_data" diff --git a/ush/machine/stampede.yaml b/ush/machine/stampede.yaml new file mode 100644 index 0000000000..d0a3f55ba8 --- /dev/null +++ b/ush/machine/stampede.yaml @@ -0,0 +1,31 @@ +platform: + WORKFLOW_MANAGER: rocoto + NCORES_PER_NODE: 68 + SCHED: slurm + DOMAIN_PREGEN_BASEDIR: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop/FV3LAM_pregen + PARTITION_DEFAULT: normal + QUEUE_DEFAULT: normal + PARTITION_FCST: normal + QUEUE_FCST: normal + PARTITION_HPSS: normal + QUEUE_HPSS: normal + RUN_CMD_FCST: ibrun -np $nprocs + RUN_CMD_POST: ibrun -np $nprocs + RUN_CMD_SERIAL: time + RUN_CMD_UTILS: ibrun -np $nprocs + ENV_INIT_SCRIPTS_FPS: [] + PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' + TEST_EXTRN_MDL_SOURCE_BASEDIR: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop/input_model_data + TEST_PREGEN_BASEDIR: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop/FV3LAM_pregen +task_run_fcst: + FIXaer: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop/fix/fix_aer + FIXgsm: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop/fix/fix_am + FIXlut: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop/fix/fix_lut + TOPO_DIR: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop/fix/fix_orog + SFC_CLIMO_INPUT_DIR: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop/fix/fix_sfc_climo +data: + GSMGFS: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop/input_model_data/GFS + FV3GFS: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop/input_model_data/FV3GFS + HRRR: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop/input_model_data/HRRR + RAP: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop/input_model_data/RAP + NAM: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop/input_model_data/NAM diff --git a/ush/python_utils/config_parser.py b/ush/python_utils/config_parser.py index 7095d7a956..b89ab2b5a7 100644 --- a/ush/python_utils/config_parser.py +++ b/ush/python_utils/config_parser.py @@ -369,7 +369,7 @@ def structure_dict(dict_o, dict_t): return struct_dict -def update_dict(dict_o, dict_t): +def update_dict(dict_o, dict_t, provide_default=False): """Update a dictionary with another Args: @@ -380,9 +380,10 @@ def update_dict(dict_o, dict_t): """ for k, v in dict_t.items(): if isinstance(v, dict): - update_dict(dict_o, v) + update_dict(dict_o, v, provide_default) elif k in dict_o.keys(): - dict_t[k] = dict_o[k] + if (not provide_default) or (dict_t[k] is None) or (len(dict_t[k]) == 0): + dict_t[k] = dict_o[k] def check_structure_dict(dict_o, dict_t): diff --git a/ush/setup.py b/ush/setup.py index 3e6138d372..61e1d5dd90 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -108,6 +108,45 @@ def setup(): cfg_u = flatten_dict(cfg_u) import_vars(dictionary=cfg_u) update_dict(cfg_u, cfg_d) + # + # ----------------------------------------------------------------------- + # Source machine specific file + # ----------------------------------------------------------------------- + # + global MACHINE + MACHINE = uppercase(MACHINE) + MACHINE_FILE = os.path.join(USHdir, "machine", f"{lowercase(MACHINE)}.yaml") + machine_cfg = load_config_file(MACHINE_FILE) + + # ics and lbcs + def get_location(xcs,fmt): + if ("data" in machine_cfg) and (xcs in machine_cfg["data"]): + v = machine_cfg["data"][xcs] + if not isinstance(v,dict): + return v + else: + return v[fmt] + else: + return "" + + global EXTRN_MDL_SYSBASEDIR_ICS, EXTRN_MDL_SYSBASEDIR_LBCS + EXTRN_MDL_SYSBASEDIR_ICS = EXTRN_MDL_SYSBASEDIR_ICS or \ + get_location(EXTRN_MDL_NAME_ICS, FV3GFS_FILE_FMT_ICS) + EXTRN_MDL_SYSBASEDIR_LBCS = EXTRN_MDL_SYSBASEDIR_LBCS or \ + get_location(EXTRN_MDL_NAME_LBCS, FV3GFS_FILE_FMT_LBCS) + + # remove the data key and provide machine specific default values for cfg_d + if "data" in machine_cfg: + machine_cfg.pop("data") + machine_cfg.update({ + "EXTRN_MDL_SYSBASEDIR_ICS": EXTRN_MDL_SYSBASEDIR_ICS, + "EXTRN_MDL_SYSBASEDIR_LBCS": EXTRN_MDL_SYSBASEDIR_LBCS, + }) + machine_cfg = flatten_dict(machine_cfg) + update_dict(machine_cfg, cfg_d, True) + + # import cfg_d again + import_vars(dictionary=flatten_dict(cfg_d)) # # ----------------------------------------------------------------------- @@ -344,18 +383,11 @@ def setup(): # # ----------------------------------------------------------------------- # - global MACHINE, MACHINE_FILE global FIXgsm, FIXaer, FIXlut, TOPO_DIR, SFC_CLIMO_INPUT_DIR, DOMAIN_PREGEN_BASEDIR global RELATIVE_LINK_FLAG, WORKFLOW_MANAGER, NCORES_PER_NODE, SCHED, QUEUE_DEFAULT global QUEUE_HPSS, QUEUE_FCST, PARTITION_DEFAULT, PARTITION_HPSS, PARTITION_FCST - MACHINE = uppercase(MACHINE) RELATIVE_LINK_FLAG = "--relative" - MACHINE_FILE = MACHINE_FILE or os.path.join( - USHdir, "machine", f"{lowercase(MACHINE)}.sh" - ) - machine_cfg = load_shell_config(MACHINE_FILE) - import_vars(dictionary=machine_cfg) if not NCORES_PER_NODE: print_err_msg_exit( @@ -412,7 +444,7 @@ def setup(): # # ----------------------------------------------------------------------- # - if WORKFLOW_MANAGER != "none": + if WORKFLOW_MANAGER is not None: if not ACCOUNT: print_err_msg_exit( f''' @@ -1885,39 +1917,6 @@ def setup(): # # ----------------------------------------------------------------------- # - # If USE_USER_STAGED_EXTRN_FILES is set to \"FALSE\", this is the system - # directory in which the workflow scripts will look for the files generated - # by the external model specified in EXTRN_MDL_NAME_ICS. These files will - # be used to generate the input initial condition and surface files for - # the FV3-LAM. - # - # ----------------------------------------------------------------------- - # - "EXTRN_MDL_SYSBASEDIR_ICS": EXTRN_MDL_SYSBASEDIR_ICS, - # - # ----------------------------------------------------------------------- - # - # If USE_USER_STAGED_EXTRN_FILES is set to \"FALSE\", this is the system - # directory in which the workflow scripts will look for the files generated - # by the external model specified in EXTRN_MDL_NAME_LBCS. These files - # will be used to generate the input lateral boundary condition files for - # the FV3-LAM. - # - # ----------------------------------------------------------------------- - # - "EXTRN_MDL_SYSBASEDIR_LBCS": EXTRN_MDL_SYSBASEDIR_LBCS, - # - # ----------------------------------------------------------------------- - # - # Shift back in time (in units of hours) of the starting time of the ex- - # ternal model specified in EXTRN_MDL_NAME_LBCS. - # - # ----------------------------------------------------------------------- - # - "EXTRN_MDL_LBCS_OFFSET_HRS": EXTRN_MDL_LBCS_OFFSET_HRS, - # - # ----------------------------------------------------------------------- - # # Boundary condition update times (in units of forecast hours). Note that # LBC_SPEC_FCST_HRS is an array, even if it has only one element. # From f1511650f35a49972e64207edb63e51dae04fe68 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sun, 2 Oct 2022 15:37:29 +0000 Subject: [PATCH 02/11] Remove redundant SR_WX dir --- docs/UsersGuide/source/WE2Etests.rst | 12 ++++++------ test/README.md | 8 ++++---- tests/WE2E/run_WE2E_tests.sh | 14 +++++++------- ush/launch_FV3LAM_wflow.sh | 2 +- ush/load_modules_run_task.sh | 4 ++-- ush/setup.py | 13 ++++--------- 6 files changed, 24 insertions(+), 29 deletions(-) diff --git a/docs/UsersGuide/source/WE2Etests.rst b/docs/UsersGuide/source/WE2Etests.rst index 93b075781f..e189668c52 100644 --- a/docs/UsersGuide/source/WE2Etests.rst +++ b/docs/UsersGuide/source/WE2Etests.rst @@ -70,12 +70,12 @@ The examples below demonstrate several common ways that ``run_WE2E_tests.sh`` ca ./run_WE2E_tests.sh tests_file="my_tests.txt" machine="hera" account="rtrr" - This will create the experiment subdirectories for the two sample WE2E tests in the directory ``${SR_WX_APP_TOP_DIR}/../expt_dirs``, where ``SR_WX_APP_TOP_DIR`` is the top-level directory for the ufs-srweather-app repository (usually set to something like ``/path/to/ufs-srweather-app``). Thus, the following two experiment directories will be created: + This will create the experiment subdirectories for the two sample WE2E tests in the directory ``${HOMEdir}/../expt_dirs``, where ``HOMEdir`` is the top-level directory for the ufs-srweather-app repository (usually set to something like ``/path/to/ufs-srweather-app``). Thus, the following two experiment directories will be created: .. code-block:: - ${SR_WX_APP_TOP_DIR}/../expt_dirs/custom_ESGgrid - ${SR_WX_APP_TOP_DIR}/../expt_dirs/grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 + ${HOMEdir}/../expt_dirs/custom_ESGgrid + ${HOMEdir}/../expt_dirs/grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 In addition, by default, cron jobs will be added to the user's cron table to relaunch the workflows of these experiments every 2 minutes. @@ -95,7 +95,7 @@ The examples below demonstrate several common ways that ``run_WE2E_tests.sh`` ca In this case, the user will have to go into each test's experiment directory and either manually call the ``launch_FV3LAM_wflow.sh`` script or use the Rocoto commands described in :numref:`Chapter %s ` to (re)launch the workflow. Note that if using the Rocoto commands directly, the log file ``log.launch_FV3LAM_wflow`` will not be created; in this case, the status of the workflow can be checked using the ``rocotostat`` command (see :numref:`Section %s ` or :numref:`Section %s `). #. To place the experiment subdirectories in a subdirectory named ``test_set_01`` under - ``${SR_WX_APP_TOP_DIR}/../expt_dirs`` (instead of immediately under ``expt_dirs``), use: + ``${HOMEdir}/../expt_dirs`` (instead of immediately under ``expt_dirs``), use: .. code-block:: @@ -105,8 +105,8 @@ The examples below demonstrate several common ways that ``run_WE2E_tests.sh`` ca .. code-block:: - ${SR_WX_APP_TOP_DIR}/../expt_dirs/test_set_01/custom_ESGgrid - ${SR_WX_APP_TOP_DIR}/../expt_dirs/test_set_01/grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 + ${HOMEdir}/../expt_dirs/test_set_01/custom_ESGgrid + ${HOMEdir}/../expt_dirs/test_set_01/grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 This is useful for grouping various sets of tests. diff --git a/test/README.md b/test/README.md index 1512a0a348..da1f0fb648 100644 --- a/test/README.md +++ b/test/README.md @@ -21,9 +21,9 @@ The CMake build is done in the ``build_${compiler}`` directory. The executables for each build are installed under the ``bin_${compiler}`` directory. NOTE: To run the regional workflow using these executables, the ``EXECDIR`` variable in the -``${SR_WX_APP_TOP_DIR}/regional_workflow/ush/setup.sh`` file must be set to the -appropiate directory, for example: ``EXECDIR="${SR_WX_APP_TOP_DIR}/bin_intel/bin"``, -where ``${SR_WX_APP_TOP_DIR}`` is the top-level directory of the cloned ufs-srweather-app repository. +``${HOMEdir}/ush/setup.py`` file must be set to the +appropiate directory, for example: ``EXECDIR="${HOMEdir}/bin_intel/bin"``, +where ``${HOMEdir}`` is the top-level directory of the cloned ufs-srweather-app repository. ## Usage @@ -36,4 +36,4 @@ cd test ./build.sh cheyenne >& build.out & ``` -Check the ``${SR_WX_APP_TOP_DIR}/test/build_test$PID.out`` file for PASS/FAIL. +Check the ``${HOMEdir}/test/build_test$PID.out`` file for PASS/FAIL. diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 4e4135d2b3..9a01e295cb 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -242,14 +242,14 @@ Then: This will create the experiment subdirectories for the two tests in the directory - \${SR_WX_APP_TOP_DIR}/../expt_dirs + \${HOMEdir}/../expt_dirs - where SR_WX_APP_TOP_DIR is the directory in which the ufs-srweather-app + where HOMEdir is the directory in which the ufs-srweather-app repository is cloned. Thus, the following two experiment directories will be created: - \${SR_WX_APP_TOP_DIR}/../expt_dirs/new_ESGgrid - \${SR_WX_APP_TOP_DIR}/../expt_dirs/specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE + \${HOMEdir}/../expt_dirs/new_ESGgrid + \${HOMEdir}/../expt_dirs/specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE In addition, by default, cron jobs will be created in the user's cron table to relaunch the workflows of these experiments every 2 minutes. @@ -268,7 +268,7 @@ Then: 4) To place the experiment subdirectories in a subdirectory named \"test_set_01\" under - \${SR_WX_APP_TOP_DIR}/../expt_dirs + \${HOMEdir}/../expt_dirs (instead of immediately under the latter), use: @@ -276,8 +276,8 @@ Then: In this case, the full paths to the experiment directories will be: - \${SR_WX_APP_TOP_DIR}/../expt_dirs/test_set_01/new_ESGgrid - \${SR_WX_APP_TOP_DIR}/../expt_dirs/test_set_01/specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE + \${HOMEdir}/../expt_dirs/test_set_01/new_ESGgrid + \${HOMEdir}/../expt_dirs/test_set_01/specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE 5) To use a list of tests that is located in diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 02f1de0e5e..575ebc4651 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -147,7 +147,7 @@ expt_name="${EXPT_SUBDIR}" # #----------------------------------------------------------------------- # -module use "${SR_WX_APP_TOP_DIR}/modulefiles" +module use "${HOMEdir}/modulefiles" module load "${WFLOW_MOD_FN}" > /dev/null 2>&1 || print_err_msg_exit "\ Loading of platform-specific module file (WFLOW_MOD_FN) for the workflow task failed: diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index c91650e76a..ede2870429 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -108,8 +108,8 @@ fi # machine=$(echo_lowercase $MACHINE) -source "${SR_WX_APP_TOP_DIR}/etc/lmod-setup.sh" ${machine} -module use "${SR_WX_APP_TOP_DIR}/modulefiles" +source "${HOMEdir}/etc/lmod-setup.sh" ${machine} +module use "${HOMEdir}/modulefiles" module load "${BUILD_MOD_FN}" || print_err_msg_exit "\ Loading of platform- and compiler-specific module file (BUILD_MOD_FN) for the workflow task specified by task_name failed: diff --git a/ush/setup.py b/ush/setup.py index 61e1d5dd90..fbe5bd58e5 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -308,7 +308,7 @@ def get_location(xcs,fmt): # workflow directory. Thus, the workflow directory is the one above the # directory of the current script. # - SR_WX_APP_TOP_DIR = os.path.abspath( + HOMEdir = os.path.abspath( os.path.dirname(__file__) + os.sep + os.pardir ) @@ -325,7 +325,7 @@ def get_location(xcs,fmt): # # ----------------------------------------------------------------------- # - mng_extrns_cfg_fn = os.path.join(SR_WX_APP_TOP_DIR, "Externals.cfg") + mng_extrns_cfg_fn = os.path.join(HOMEdir, "Externals.cfg") try: mng_extrns_cfg_fn = os.readlink(mng_extrns_cfg_fn) except: @@ -333,10 +333,6 @@ def get_location(xcs,fmt): property_name = "local_path" cfg = load_ini_config(mng_extrns_cfg_fn) # - # Get the path to the workflow scripts - # - HOMEdir = SR_WX_APP_TOP_DIR - # # Get the base directory of the FV3 forecast model code. # external_name = FCST_MODEL @@ -348,7 +344,7 @@ def get_location(xcs,fmt): Externals.cfg does not contain "{external_name}".""" ) - UFS_WTHR_MDL_DIR = os.path.join(SR_WX_APP_TOP_DIR, UFS_WTHR_MDL_DIR) + UFS_WTHR_MDL_DIR = os.path.join(HOMEdir, UFS_WTHR_MDL_DIR) if not os.path.exists(UFS_WTHR_MDL_DIR): print_err_msg_exit( f""" @@ -831,7 +827,7 @@ def get_location(xcs,fmt): if (not EXPT_BASEDIR) or (EXPT_BASEDIR[0] != "/"): if not EXPT_BASEDIR: EXPT_BASEDIR = "" - EXPT_BASEDIR = os.path.join(SR_WX_APP_TOP_DIR, "..", "expt_dirs", EXPT_BASEDIR) + EXPT_BASEDIR = os.path.join(HOMEdir, "..", "expt_dirs", EXPT_BASEDIR) try: EXPT_BASEDIR = os.path.realpath(EXPT_BASEDIR) except: @@ -1780,7 +1776,6 @@ def get_location(xcs,fmt): # # ----------------------------------------------------------------------- # - "SR_WX_APP_TOP_DIR": SR_WX_APP_TOP_DIR, "HOMEdir": HOMEdir, "USHdir": USHdir, "SCRIPTSdir": SCRIPTSdir, From 9cbeb9fb79a171a96633875e0f473e786d06ae00 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sun, 2 Oct 2022 16:03:52 +0000 Subject: [PATCH 03/11] Remove some duplicate derived types. --- ush/config_defaults.yaml | 3 ++- ush/setup.py | 31 +++++-------------------------- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 78203fd33d..955b815651 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1359,7 +1359,7 @@ task_make_ics: # USE_FVCOM: false FVCOM_WCSTART: "cold" - FVCOM_DIR: "/user/defined/dir/to/fvcom/data" + FVCOM_DIR: "" FVCOM_FILE: "fvcom.nc" #---------------------------- @@ -1374,6 +1374,7 @@ task_make_lbcs: KMP_AFFINITY_MAKE_LBCS: "scatter" OMP_NUM_THREADS_MAKE_LBCS: 1 OMP_STACKSIZE_MAKE_LBCS: "1024m" + LBC_SPEC_FCST_HRS: [] #---------------------------- # FORECAST config parameters diff --git a/ush/setup.py b/ush/setup.py index fbe5bd58e5..40a289dc59 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -110,7 +110,9 @@ def setup(): update_dict(cfg_u, cfg_d) # # ----------------------------------------------------------------------- + # # Source machine specific file + # # ----------------------------------------------------------------------- # global MACHINE @@ -150,7 +152,9 @@ def get_location(xcs,fmt): # # ----------------------------------------------------------------------- + # # Source constants.sh and save its contents to a variable for later + # # ----------------------------------------------------------------------- # cfg_c = load_config_file(os.path.join(USHdir, CONSTANTS_FN)) @@ -687,6 +691,7 @@ def get_location(xcs,fmt): # # ----------------------------------------------------------------------- # + global LBC_SPEC_FCST_HRS LBC_SPEC_FCST_HRS = [ i for i in range( @@ -1912,15 +1917,6 @@ def get_location(xcs,fmt): # # ----------------------------------------------------------------------- # - # Boundary condition update times (in units of forecast hours). Note that - # LBC_SPEC_FCST_HRS is an array, even if it has only one element. - # - # ----------------------------------------------------------------------- - # - "LBC_SPEC_FCST_HRS": LBC_SPEC_FCST_HRS, - # - # ----------------------------------------------------------------------- - # # The number of cycles for which to make forecasts and the list of # starting dates/hours of these cycles. # @@ -1931,27 +1927,10 @@ def get_location(xcs,fmt): # # ----------------------------------------------------------------------- # - # Parameters that determine whether FVCOM data will be used, and if so, - # their location. - # - # If USE_FVCOM is set to \"TRUE\", then FVCOM data (in the file FVCOM_FILE - # located in the directory FVCOM_DIR) will be used to update the surface - # boundary conditions during the initial conditions generation task - # (MAKE_ICS_TN). - # - # ----------------------------------------------------------------------- - # - "USE_FVCOM": USE_FVCOM, - "FVCOM_DIR": FVCOM_DIR, - "FVCOM_FILE": FVCOM_FILE, - # - # ----------------------------------------------------------------------- - # # Computational parameters. # # ----------------------------------------------------------------------- # - "NCORES_PER_NODE": NCORES_PER_NODE, "PE_MEMBER01": PE_MEMBER01, # # ----------------------------------------------------------------------- From 1e7a2c35722231e58cf91fff3a8fc945715109a7 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sun, 2 Oct 2022 16:56:47 +0000 Subject: [PATCH 04/11] Convert constants to yaml. --- .../WE2E/get_WE2Etest_names_subdirs_descs.sh | 19 +++--- ush/calculate_cost.py | 2 - ush/config_defaults.yaml | 2 +- ush/constants.sh | 65 ------------------- ush/constants.yaml | 59 +++++++++++++++++ ush/launch_FV3LAM_wflow.sh | 1 - ush/set_gridparams_ESGgrid.py | 20 ++++-- ush/setup.py | 4 +- ush/source_util_funcs.sh | 8 --- 9 files changed, 85 insertions(+), 95 deletions(-) delete mode 100644 ush/constants.sh create mode 100644 ush/constants.yaml diff --git a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh index cab45a092f..1222a63a76 100755 --- a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh +++ b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh @@ -256,6 +256,14 @@ function get_WE2Etest_names_subdirs_descs() { # #----------------------------------------------------------------------- # +# Source constant files. +# +#----------------------------------------------------------------------- +# + source_config $USHdir/constants.yaml +# +#----------------------------------------------------------------------- +# # Specify the set of valid argument names for this script or function. # Then process the arguments provided to it on the command line (which # should consist of a set of name-value pairs of the form arg1="value1", @@ -411,14 +419,6 @@ function get_WE2Etest_names_subdirs_descs() { # #----------------------------------------------------------------------- # -# Source files. -# -#----------------------------------------------------------------------- -# - . $USHdir/constants.sh -# -#----------------------------------------------------------------------- -# # Set variables associated with the CSV (comma-separated value) file that # this function may generate. The conditions under which such a file is # generated are described above in the description of this function. @@ -1136,9 +1136,6 @@ configuration files of the primary WE2E tests... params=$(\ PREDEF_GRID_NAME="${PREDEF_GRID_NAME}" \ QUILTING="FALSE" \ - RADIUS_EARTH=${RADIUS_EARTH} \ - DEGS_PER_RADIAN=${DEGS_PER_RADIAN} \ - NH4=${NH4} \ $USHdir/calculate_cost.py -c "${test_configs_basedir}/$subdir/${config_fn}") read dta nxny dta_r nxny_r <<< "${params}" diff --git a/ush/calculate_cost.py b/ush/calculate_cost.py index 1b1ce846b5..67b73ea974 100755 --- a/ush/calculate_cost.py +++ b/ush/calculate_cost.py @@ -106,5 +106,3 @@ def setUp(self): set_env_var("LAYOUT_Y", 36) set_env_var("BLOCKSIZE", 28) set_env_var("QUILTING", False) - set_env_var("RADIUS_EARTH", 6371200.0) - set_env_var("DEGS_PER_RADIAN", 57.2957795131) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 955b815651..a72fe00c90 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -495,7 +495,7 @@ workflow: #----------------------------------------------------------------------- # EXPT_CONFIG_FN: "config.yaml" - CONSTANTS_FN: "constants.sh" + CONSTANTS_FN: "constants.yaml" RGNL_GRID_NML_FN: "regional_grid.nml" diff --git a/ush/constants.sh b/ush/constants.sh deleted file mode 100644 index f5bce884e8..0000000000 --- a/ush/constants.sh +++ /dev/null @@ -1,65 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# Mathematical and physical constants. -# -#----------------------------------------------------------------------- -# - -# Pi. -PI_GEOM="3.14159265358979323846264338327" - -# Degrees per radian. -DEGS_PER_RADIAN=$( bc -l <<< "360.0/(2.0*${PI_GEOM})" ) - -# Radius of the Earth in meters. -RADIUS_EARTH="6371200.0" -# -#----------------------------------------------------------------------- -# -# Valid values that a user may set a boolean variable to (e.g. in the -# SRW App's experiment configuration file). -# -#----------------------------------------------------------------------- -# -valid_vals_BOOLEAN=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -# -#----------------------------------------------------------------------- -# -# Any regional model must be supplied lateral boundary conditions (in -# addition to initial conditions) to be able to perform a forecast. In -# the FV3-LAM model, these boundary conditions (BCs) are supplied using -# a "halo" of grid cells around the regional domain that extend beyond -# the boundary of the domain. The model is formulated such that along -# with files containing these BCs, it needs as input the following files -# (in NetCDF format): -# -# 1) A grid file that includes a halo of 3 cells beyond the boundary of -# the domain. -# 2) A grid file that includes a halo of 4 cells beyond the boundary of -# the domain. -# 3) A (filtered) orography file without a halo, i.e. a halo of width -# 0 cells. -# 4) A (filtered) orography file that includes a halo of 4 cells beyond -# the boundary of the domain. -# -# Note that the regional grid is referred to as "tile 7" in the code. -# We will let: -# -# * NH0 denote the width (in units of number of cells on tile 7) of -# the 0-cell-wide halo, i.e. NH0 = 0; -# -# * NH3 denote the width (in units of number of cells on tile 7) of -# the 3-cell-wide halo, i.e. NH3 = 3; and -# -# * NH4 denote the width (in units of number of cells on tile 7) of -# the 4-cell-wide halo, i.e. NH4 = 4. -# -# We define these variables next. -# -#----------------------------------------------------------------------- -# -NH0=0 -NH3=3 -NH4=4 - diff --git a/ush/constants.yaml b/ush/constants.yaml new file mode 100644 index 0000000000..cd4867a728 --- /dev/null +++ b/ush/constants.yaml @@ -0,0 +1,59 @@ +constants: + # + #----------------------------------------------------------------------- + # + # Mathematical and physical constants. + # + #----------------------------------------------------------------------- + # + PI_GEOM: 3.14159265358979323846264338327 + DEGS_PER_RADIAN: 57.2957795131 + RADIUS_EARTH: 6371200.0 + # + #----------------------------------------------------------------------- + # + # Any regional model must be supplied lateral boundary conditions (in + # addition to initial conditions) to be able to perform a forecast. In + # the FV3-LAM model, these boundary conditions (BCs) are supplied using + # a "halo" of grid cells around the regional domain that extend beyond + # the boundary of the domain. The model is formulated such that along + # with files containing these BCs, it needs as input the following files + # (in NetCDF format): + # + # 1) A grid file that includes a halo of 3 cells beyond the boundary of + # the domain. + # 2) A grid file that includes a halo of 4 cells beyond the boundary of + # the domain. + # 3) A (filtered) orography file without a halo, i.e. a halo of width + # 0 cells. + # 4) A (filtered) orography file that includes a halo of 4 cells beyond + # the boundary of the domain. + # + # Note that the regional grid is referred to as "tile 7" in the code. + # We will let: + # + # * NH0 denote the width (in units of number of cells on tile 7) of + # the 0-cell-wide halo, i.e. NH0 = 0; + # + # * NH3 denote the width (in units of number of cells on tile 7) of + # the 3-cell-wide halo, i.e. NH3 = 3; and + # + # * NH4 denote the width (in units of number of cells on tile 7) of + # the 4-cell-wide halo, i.e. NH4 = 4. + # + # We define these variables next. + # + #----------------------------------------------------------------------- + # + NH0: 0 + NH3: 3 + NH4: 4 + # + #----------------------------------------------------------------------- + # + # Valid values that a user may set a boolean variable to (e.g. in the + # SRW App's experiment configuration file). + # + #----------------------------------------------------------------------- + # + valid_vals_BOOLEAN: [ "TRUE", "true", "YES", "yes", "FALSE", "false", "NO", "no" ] diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 575ebc4651..6f97c0a2e0 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -96,7 +96,6 @@ fi # . $exptdir/var_defns.sh . $USHdir/source_util_funcs.sh -. $USHdir/constants.sh . $USHdir/init_env.sh # #----------------------------------------------------------------------- diff --git a/ush/set_gridparams_ESGgrid.py b/ush/set_gridparams_ESGgrid.py index 7b9350424f..ce8cd02ffe 100644 --- a/ush/set_gridparams_ESGgrid.py +++ b/ush/set_gridparams_ESGgrid.py @@ -1,9 +1,16 @@ #!/usr/bin/env python3 +import os import unittest from datetime import datetime, timedelta -from python_utils import import_vars, set_env_var, print_input_args +from python_utils import ( + import_vars, + set_env_var, + print_input_args, + load_config_file, + flatten_dict, +) def set_gridparams_ESGgrid(lon_ctr, lat_ctr, nx, ny, halo_width, delx, dely, pazi): @@ -25,9 +32,12 @@ def set_gridparams_ESGgrid(lon_ctr, lat_ctr, nx, ny, halo_width, delx, dely, paz print_input_args(locals()) - # get needed environment variables + # get constants IMPORTS = ["RADIUS_EARTH", "DEGS_PER_RADIAN"] - import_vars(env_vars=IMPORTS) + USHdir = os.path.dirname(os.path.abspath(__file__)) + constants_cfg = load_config_file(os.path.join(USHdir,"constants.yaml")) + import_vars(dictionary=flatten_dict(constants_cfg), env_vars=IMPORTS) + # # ----------------------------------------------------------------------- # @@ -119,5 +129,5 @@ def test_set_gridparams_ESGgrid(self): ) def setUp(self): - set_env_var("RADIUS_EARTH", 6371200.0) - set_env_var("DEGS_PER_RADIAN", 57.2957795131) + set_env_var("DEBUG", False) + set_env_var("VERBOSE", False) diff --git a/ush/setup.py b/ush/setup.py index 40a289dc59..b33c73aaac 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -158,7 +158,7 @@ def get_location(xcs,fmt): # ----------------------------------------------------------------------- # cfg_c = load_config_file(os.path.join(USHdir, CONSTANTS_FN)) - import_vars(dictionary=cfg_c) + import_vars(dictionary=flatten_dict(cfg_c)) # # ----------------------------------------------------------------------- @@ -1746,7 +1746,7 @@ def get_location(xcs,fmt): update_dict(globals(), cfg_d) # constants section - cfg_d["constants"] = cfg_c + cfg_d.update(cfg_c) # grid params cfg_d["grid_params"] = grid_params diff --git a/ush/source_util_funcs.sh b/ush/source_util_funcs.sh index 546e5df541..1157664c06 100644 --- a/ush/source_util_funcs.sh +++ b/ush/source_util_funcs.sh @@ -35,14 +35,6 @@ function source_util_funcs() { # #----------------------------------------------------------------------- # -# Source the file containing workflow constants. -# -#----------------------------------------------------------------------- -# - . ${USHdir}/constants.sh -# -#----------------------------------------------------------------------- -# # Source the file containing functions to save and restore shell op- # tions. # From 003241f14c87b19ed07bf362d3fa588299af9d05 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 3 Oct 2022 18:40:27 +0000 Subject: [PATCH 05/11] Bug fix GFDL grid. --- ush/set_gridparams_GFDLgrid.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ush/set_gridparams_GFDLgrid.py b/ush/set_gridparams_GFDLgrid.py index f7999591d7..47253a1c73 100644 --- a/ush/set_gridparams_GFDLgrid.py +++ b/ush/set_gridparams_GFDLgrid.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import os import unittest from python_utils import ( @@ -8,6 +9,8 @@ print_input_args, print_info_msg, print_err_msg_exit, + load_config_file, + flatten_dict, ) @@ -56,8 +59,12 @@ def set_gridparams_GFDLgrid( print_input_args(locals()) # get needed environment variables - IMPORTS = ["VERBOSE", "RUN_ENVIR", "NH4"] + IMPORTS = ["VERBOSE", "RUN_ENVIR"] import_vars(env_vars=IMPORTS) + IMPORTS = ["NH4"] + USHdir = os.path.dirname(os.path.abspath(__file__)) + constants_cfg = load_config_file(os.path.join(USHdir,"constants.yaml")) + import_vars(dictionary=flatten_dict(constants_cfg), env_vars=IMPORTS) # # ----------------------------------------------------------------------- From de724cd2ce69fda673286d5f17a5418cd0164873 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 3 Oct 2022 21:24:20 +0000 Subject: [PATCH 06/11] Bug fix machine lower/upper case. --- ush/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/setup.py b/ush/setup.py index b33c73aaac..0bb8c0ad09 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -116,7 +116,6 @@ def setup(): # ----------------------------------------------------------------------- # global MACHINE - MACHINE = uppercase(MACHINE) MACHINE_FILE = os.path.join(USHdir, "machine", f"{lowercase(MACHINE)}.yaml") machine_cfg = load_config_file(MACHINE_FILE) @@ -149,6 +148,7 @@ def get_location(xcs,fmt): # import cfg_d again import_vars(dictionary=flatten_dict(cfg_d)) + MACHINE = uppercase(MACHINE) # # ----------------------------------------------------------------------- From a39a7c7ed3b36aad311b56c2b44f1c0d2fd77271 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 3 Oct 2022 22:36:05 +0000 Subject: [PATCH 07/11] Fix unittest to capture exit code. --- ush/config.nco.yaml | 2 -- ush/generate_FV3LAM_wflow.py | 12 ++++++++++-- ush/setup.py | 10 ++++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/ush/config.nco.yaml b/ush/config.nco.yaml index b91195eabf..013c765d7d 100644 --- a/ush/config.nco.yaml +++ b/ush/config.nco.yaml @@ -5,8 +5,6 @@ user: RUN_ENVIR: nco MACHINE: hera ACCOUNT: an_account -platform: - DOMAIN_PREGEN_BASEDIR: /scratch2/BMC/det/UFS_SRW_App/develop/FV3LAM_pregen workflow: USE_CRON_TO_RELAUNCH: true CRON_RELAUNCH_INTVL_MNTS: 3 diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 76493c53f1..8bc553bd9b 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -27,6 +27,7 @@ cfg_to_yaml_str, find_pattern_in_str, set_env_var, + get_env_var, ) from setup import setup @@ -1165,18 +1166,25 @@ def run_workflow(): p = Process(target=workflow_func) p.start() p.join() + exit_code = p.exitcode + if exit_code != 0: + sys.exit(exit_code) USHdir = os.path.dirname(os.path.abspath(__file__)) + SED = get_env_var("SED") # community test case - ln_vrfy("-fs", f"{USHdir}/config.community.yaml", f"{USHdir}/config.yaml") + cp_vrfy(f"{USHdir}/config.community.yaml", f"{USHdir}/config.yaml") + run_command(f"""{SED} -i 's/MACHINE: hera/MACHINE: linux/g' {USHdir}/config.yaml""") run_workflow() # nco test case set_env_var("OPSROOT", f"{USHdir}/../../nco_dirs") - ln_vrfy("-fs", f"{USHdir}/config.nco.yaml", f"{USHdir}/config.yaml") + cp_vrfy(f"{USHdir}/config.nco.yaml", f"{USHdir}/config.yaml") + run_command(f"""{SED} -i 's/MACHINE: hera/MACHINE: linux/g' {USHdir}/config.yaml""") run_workflow() def setUp(self): + define_macos_utilities() set_env_var("DEBUG", False) set_env_var("VERBOSE", False) diff --git a/ush/setup.py b/ush/setup.py index 0bb8c0ad09..c2a2cc8446 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -905,8 +905,9 @@ def get_location(xcs,fmt): # Main directory locations if RUN_ENVIR == "nco": - try: OPSROOT - except NameError: OPSROOT = OPSROOT or EXPTDIR + try: OPSROOT = os.path.abspath(f"{EXPT_BASEDIR}{os.sep}..{os.sep}nco_dirs") \ + if OPSROOT is None else OPSROOT + except NameError: OPSROOT = EXPTDIR try: COMROOT except NameError: COMROOT = os.path.join(OPSROOT, "com") try: PACKAGEROOT @@ -1335,6 +1336,11 @@ def get_location(xcs,fmt): # global GRID_DIR, OROG_DIR, SFC_CLIMO_DIR + if DOMAIN_PREGEN_BASEDIR is None: + RUN_TASK_MAKE_GRID = True + RUN_TASK_MAKE_OROG = True + RUN_TASK_MAKE_SFC_CLIMO = True + # # If RUN_TASK_MAKE_GRID is set to False, the workflow will look for # the pregenerated grid files in GRID_DIR. In this case, make sure that From 6f6a1933e07f91eb59204cb1fa2318b68aafeddb Mon Sep 17 00:00:00 2001 From: "Daniel.Abdi" Date: Tue, 4 Oct 2022 02:09:42 +0000 Subject: [PATCH 08/11] Gaea lmod setup fix with tcsh. --- etc/lmod-setup.csh | 9 +++++---- etc/lmod-setup.sh | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/etc/lmod-setup.csh b/etc/lmod-setup.csh index 5941128813..75b821a338 100644 --- a/etc/lmod-setup.csh +++ b/etc/lmod-setup.csh @@ -27,10 +27,11 @@ else if ( "$L_MACHINE" == singularity ) then module purge else if ( "$L_MACHINE" == gaea ) then - export LMOD_SYSTEM_DEFAULT_MODULES=modules/3.2.11.4 - export BASH_ENV=/lustre/f2/dev/role.epic/contrib/apps/lmod/lmod/init/profile - source $BASH_ENV - module --initial_load restore + set ENV="/lustre/f2/dev/role.epic/contrib/apps/lmod/lmod/init/csh" + source $ENV + + setenv LMOD_SYSTEM_DEFAULT_MODULES "modules/3.2.11.4" + module --initial_load --no_redirect restore else if ( "$L_MACHINE" == odin ) then module unload modules diff --git a/etc/lmod-setup.sh b/etc/lmod-setup.sh index 3b87b2893f..9dcc03ae45 100644 --- a/etc/lmod-setup.sh +++ b/etc/lmod-setup.sh @@ -28,11 +28,11 @@ elif [ "$L_MACHINE" = singularity ]; then module purge elif [ "$L_MACHINE" = gaea ]; then - export LMOD_SYSTEM_DEFAULT_MODULES=modules/3.2.11.4 - export BASH_ENV=/lustre/f2/dev/role.epic/contrib/apps/lmod/lmod/init/profile + export BASH_ENV="/lustre/f2/dev/role.epic/contrib/apps/lmod/lmod/init/bash" source $BASH_ENV - module --initial_load restore + export LMOD_SYSTEM_DEFAULT_MODULES="modules/3.2.11.4" + module --initial_load --no_redirect restore elif [ "$L_MACHINE" = odin ]; then module unload modules unset -f module From 01dc89fff78d81da59dd35ea5746f5c898bb0362 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 4 Oct 2022 02:37:10 +0000 Subject: [PATCH 09/11] Add missing gaea commands. --- ush/machine/gaea.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml index cf8bcb595c..b3a69887ae 100644 --- a/ush/machine/gaea.yaml +++ b/ush/machine/gaea.yaml @@ -15,11 +15,14 @@ platform: RUN_CMD_POST: srun --mpi=pmi2 -n $nprocs RUN_CMD_SERIAL: time RUN_CMD_UTILS: srun --mpi=pmi2 -n $nprocs + SLURM_NATIVE_CMD: -M c3 ENV_INIT_SCRIPTS_FPS: - /etc/profile PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop/input_model_data TEST_PREGEN_BASEDIR: /lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop/FV3LAM_pregen +task_make_lbcs: + WTIME_MAKE_LBCS: 00:60:00 task_run_fcst: FIXaer: /lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop/fix/fix_aer FIXgsm: /lustre/f2/pdata/ncep_shared/UFS_SRW_App/develop/fix/fix_am From 1f2277665a3a4478c669dd16f9f736ca1dd01b4a Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 4 Oct 2022 03:37:44 +0000 Subject: [PATCH 10/11] Remove obsolete module-setup scripts. --- modulefiles/module-setup.csh.inc | 63 ------------------------- modulefiles/module-setup.sh.inc | 81 -------------------------------- 2 files changed, 144 deletions(-) delete mode 100644 modulefiles/module-setup.csh.inc delete mode 100644 modulefiles/module-setup.sh.inc diff --git a/modulefiles/module-setup.csh.inc b/modulefiles/module-setup.csh.inc deleted file mode 100644 index d7434b7260..0000000000 --- a/modulefiles/module-setup.csh.inc +++ /dev/null @@ -1,63 +0,0 @@ -set __ms_shell=csh - -eval "if ( -d / ) set __ms_shell=tcsh" - -if ( { test -d /lfs4 } ) then - if ( ! { module help >& /dev/null } ) then - source /apps/lmod/lmod/init/$__ms_shell - endif - module purge -else if ( { test -d /scratch3 } ) then - # We are on NOAA Theia - if ( ! { module help >& /dev/null } ) then - source /apps/lmod/lmod/init/$__ms_shell - endif - module purge -else if ( { test -d /glade } ) then - # We are on NCAR Yellowstone - if ( ! { module help >& /dev/null } ) then - source /usr/share/Modules/init/$__ms_shell - endif - module purge -else if ( { test -d /lustre -a -d /ncrc } ) then - # We are on GAEA. - if ( ! { module help >& /dev/null } ) then - # We cannot simply load the module command. The GAEA - # /etc/csh.login modifies a number of module-related variables - # before loading the module command. Without those variables, - # the module command fails. Hence we actually have to source - # /etc/csh.login here. - source /etc/csh.login - # initialize Lmod/8.7.12 using a wrapper script - source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh - set __ms_source_etc_csh_login=yes - else - set __ms_source_etc_csh_login=no - endif - module purge - unsetenv _LMFILES_ - unsetenv _LMFILES_000 - unsetenv _LMFILES_001 - unsetenv LOADEDMODULES - module load modules - if ( { test -d /opt/cray/ari/modulefiles } ) then - module use -a /opt/cray/ari/modulefiles - endif - if ( { test -d /opt/cray/pe/ari/modulefiles } ) then - module use -a /opt/cray/pe/ari/modulefiles - endif - if ( { test -d /opt/cray/pe/craype/default/modulefiles } ) then - module use -a /opt/cray/pe/craype/default/modulefiles - endif - setenv NCEPLIBS /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib - if ( { test -d /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib } ) then - module use $NCEPLIBS/modulefiles - endif - if ( "$__ms_source_etc_csh_login" == yes ) then - source /etc/csh.login - unset __ms_source_etc_csh_login - endif -else - # Workaround for csh limitation. Use sh to print to stderr. - sh -c 'echo WARNING: UNKNOWN PLATFORM 1>&2' -endif diff --git a/modulefiles/module-setup.sh.inc b/modulefiles/module-setup.sh.inc deleted file mode 100644 index b412594178..0000000000 --- a/modulefiles/module-setup.sh.inc +++ /dev/null @@ -1,81 +0,0 @@ -# Create a test function for sh vs. bash detection. The name is -# randomly generated to reduce the chances of name collision. -__ms_function_name="setup__test_function__$$" -eval "$__ms_function_name() { /bin/true ; }" - -# Determine which shell we are using -__ms_ksh_test=$( eval '__text="text" ; if [[ $__text =~ ^(t).* ]] ; then printf "%s" ${.sh.match[1]} ; fi' 2> /dev/null | cat ) -__ms_bash_test=$( eval 'if ( set | grep '$__ms_function_name' | grep -v name > /dev/null 2>&1 ) ; then echo t ; fi ' 2> /dev/null | cat ) - -if [[ ! -z "$__ms_ksh_test" ]] ; then - __ms_shell=ksh -elif [[ ! -z "$__ms_bash_test" ]] ; then - __ms_shell=bash -else - # Not bash or ksh, so assume sh. - __ms_shell=sh -fi - -if [[ -d /lfs4 ]] ; then - # We are on NOAA Jet - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /apps/lmod/lmod/init/$__ms_shell - fi - module purge -elif [[ -d /scratch3 ]] ; then - # We are on NOAA Theia - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /apps/lmod/lmod/init/$__ms_shell - fi - module purge -elif [[ -d /lustre && -d /ncrc ]] ; then - # We are on GAEA. - if ( ! eval module help > /dev/null 2>&1 ) ; then - # We cannot simply load the module command. The GAEA - # /etc/profile modifies a number of module-related variables - # before loading the module command. Without those variables, - # the module command fails. Hence we actually have to source - # /etc/profile here. - source /etc/profile - # initialize Lmod/8.7.12 using a wrapper script - source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh - __ms_source_etc_profile=yes - else - __ms_source_etc_profile=no - fi - module purge -# clean up after purge - unset _LMFILES_ - unset _LMFILES_000 - unset _LMFILES_001 - unset LOADEDMODULES - module load modules - if [[ -d /opt/cray/ari/modulefiles ]] ; then - module use -a /opt/cray/ari/modulefiles - fi - if [[ -d /opt/cray/pe/ari/modulefiles ]] ; then - module use -a /opt/cray/pe/ari/modulefiles - fi - if [[ -d /opt/cray/pe/craype/default/modulefiles ]] ; then - module use -a /opt/cray/pe/craype/default/modulefiles - fi - if [[ -s /etc/opt/cray/pe/admin-pe/site-config ]] ; then - source /etc/opt/cray/pe/admin-pe/site-config - fi - export NCEPLIBS=/lustre/f1/pdata/ncep_shared/NCEPLIBS/lib - if [[ -d "$NCEPLIBS" ]] ; then - module use $NCEPLIBS/modulefiles - fi - if [[ "$__ms_source_etc_profile" == yes ]] ; then - source /etc/profile - unset __ms_source_etc_profile - fi -else - echo WARNING: UNKNOWN PLATFORM 1>&2 -fi - -unset __ms_shell -unset __ms_ksh_test -unset __ms_bash_test -unset $__ms_function_name -unset __ms_function_name From db574d815bbac98eb9a8adb47556c1685ee140ab Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 4 Oct 2022 04:21:40 +0000 Subject: [PATCH 11/11] Fix linux modulefile name. --- etc/lmod-setup.csh | 6 ++++++ etc/lmod-setup.sh | 6 ++++++ modulefiles/{build_linux_compiler => build_linux_gnu} | 0 3 files changed, 12 insertions(+) rename modulefiles/{build_linux_compiler => build_linux_gnu} (100%) diff --git a/etc/lmod-setup.csh b/etc/lmod-setup.csh index 75b821a338..e78a1ce59c 100644 --- a/etc/lmod-setup.csh +++ b/etc/lmod-setup.csh @@ -20,6 +20,12 @@ if ( "$L_MACHINE" == macos ) then module purge +else if ( "$L_MACHINE" == linux ) then + setenv ENV "/usr/share/lmod/lmod/init/csh" + source $ENV + + module purge + else if ( "$L_MACHINE" == singularity ) then set ENV="/usr/share/lmod/lmod/init/csh" source $ENV diff --git a/etc/lmod-setup.sh b/etc/lmod-setup.sh index 9dcc03ae45..909a324bd9 100644 --- a/etc/lmod-setup.sh +++ b/etc/lmod-setup.sh @@ -21,6 +21,12 @@ if [ "$L_MACHINE" = macos ]; then module purge +elif [ "$L_MACHINE" = linux ]; then + export BASH_ENV="/usr/share/share/lmod/init/bash" + source $BASH_ENV + + module purge + elif [ "$L_MACHINE" = singularity ]; then export BASH_ENV="/usr/share/lmod/lmod/init/bash" source $BASH_ENV diff --git a/modulefiles/build_linux_compiler b/modulefiles/build_linux_gnu similarity index 100% rename from modulefiles/build_linux_compiler rename to modulefiles/build_linux_gnu