From 06dc25468ea71a7bd4a094a53600dd89100233e9 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 13 Dec 2022 11:59:55 -0500 Subject: [PATCH 1/6] add missing jobids in arch.sh earc.sh and gldas.sh --- .../{JENKFGDAS_ARCHIVE => JGDAS_ENKF_ARCHIVE} | 58 +++++++++++-------- jobs/JGLOBAL_ARCHIVE | 57 ++++++++++-------- jobs/rocoto/arch.sh | 3 + jobs/rocoto/earc.sh | 5 +- jobs/rocoto/gldas.sh | 3 + 5 files changed, 77 insertions(+), 49 deletions(-) rename jobs/{JENKFGDAS_ARCHIVE => JGDAS_ENKF_ARCHIVE} (82%) diff --git a/jobs/JENKFGDAS_ARCHIVE b/jobs/JGDAS_ENKF_ARCHIVE similarity index 82% rename from jobs/JENKFGDAS_ARCHIVE rename to jobs/JGDAS_ENKF_ARCHIVE index 305dbe005b7..def922079a1 100755 --- a/jobs/JENKFGDAS_ARCHIVE +++ b/jobs/JGDAS_ENKF_ARCHIVE @@ -2,31 +2,12 @@ source "${HOMEgfs}/ush/preamble.sh" -############################# -# Source relevant config files -############################# -export EXPDIR=${EXPDIR:-${HOMEgfs}/parm/config} -configs="base earc" -for config in ${configs}; do - . "${EXPDIR}"/config."${config}" - status=$? - [[ ${status} -ne 0 ]] && exit "${status}" -done - -########################################## -# Source machine runtime environment -########################################## -. ${HOMEgfs}/env/${machine}.env earc -status=$? -[[ ${status} -ne 0 ]] && exit "${status}" - ############################################## # Obtain unique process id (pid) and make temp directory ############################################## -export pid=${pid:-$$} -export DATA=${DATA:-${DATAROOT}/${jobid:?}} -mkdir -p "${DATA}" -cd "${DATA}" +export DATA=${DATA:-${DATAROOT}/${jobid}} +mkdir -p ${DATA} +cd ${DATA} ############################################## @@ -40,10 +21,31 @@ setpdy.sh ############################################## # Determine Job Output Name on System ############################################## +export pid=${pid:-$$} export pgmout="OUTPUT.${pid}" export pgmerr=errfile +############################# +# Source relevant config files +############################# +export EXPDIR=${EXPDIR:-${HOMEgfs}/parm/config} +configs="base earc" +for config in ${configs}; do + . ${EXPDIR}/config.${config} + status=$? + [[ ${status} -ne 0 ]] && exit "${status}" +done + + +########################################## +# Source machine runtime environment +########################################## +. ${HOMEgfs}/env/${machine}.env earc +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + + ############################################## # Set variables used in the script ############################################## @@ -61,7 +63,17 @@ status=$? ############################################################### -echo "ENDED NORMALLY." +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [ -e "${pgmout}" ] ; then + cat ${pgmout} +fi + ########################################## # Remove the Temporary working directory diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index 97d8e4be356..aae78a0d55a 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -2,31 +2,10 @@ source "${HOMEgfs}/ush/preamble.sh" -export RUN_ENVIR=${RUN_ENVIR:-"nco"} - -############################################# -# Source relevant config files -############################################# -export EXPDIR=${EXPDIR:-${HOMEgfs}/parm/config} -configs="base arch" -for config in ${configs}; do - . ${EXPDIR}/config.${config} - status=$? - [[ ${status} -ne 0 ]] && exit "${status}" -done - -########################################## -# Source machine runtime environment -########################################## -. "${HOMEgfs}"/env/"${machine}".env arch -status=$? -[[ ${status} -ne 0 ]] && exit "${status}" - ############################################## # Obtain unique process id (pid) and make temp directory ############################################## -export pid=${pid:-$$} -export DATA=${DATA:-${DATAROOT}/${jobid:?}} +export DATA=${DATA:-${DATAROOT}/${jobid}} mkdir -p "${DATA}" cd "${DATA}" @@ -42,16 +21,36 @@ setpdy.sh ############################################## # Determine Job Output Name on System ############################################## +export pid=${pid:-$$} export pgmout="OUTPUT.${pid}" export pgmerr=errfile +############################################# +# Source relevant config files +############################################# +export EXPDIR=${EXPDIR:-${HOMEgfs}/parm/config} +configs="base arch" +for config in ${configs}; do + . ${EXPDIR}/config.${config} + status=$? + [[ ${status} -ne 0 ]] && exit "${status}" +done + + +########################################## +# Source machine runtime environment +########################################## +. "${HOMEgfs}/env/${machine}.env" arch +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + ############################################## # Set variables used in the script ############################################## export CDATE=${CDATE:-${PDY}${cyc}} export CDUMP=${CDUMP:-${RUN:-"gfs"}} -export COMPONENT=${COMPONENT:-atmos} + ############################################################### # Run archive script @@ -61,9 +60,17 @@ ${GLOBALARCHIVESH:-${SCRgfs}/exglobal_archive.sh} status=$? [[ ${status} -ne 0 ]] && exit "${status}" -############################################################### +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [ -e "${pgmout}" ] ; then + cat ${pgmout} +fi -echo "ENDED NORMALLY." ########################################## # Remove the Temporary working directory diff --git a/jobs/rocoto/arch.sh b/jobs/rocoto/arch.sh index 378b900b196..2f62d8b3547 100755 --- a/jobs/rocoto/arch.sh +++ b/jobs/rocoto/arch.sh @@ -8,6 +8,9 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" +export job="arch" +export jobid="${job}.$$" + ############################################################### # Execute the JJOB "${HOMEgfs}"/jobs/JGLOBAL_ARCHIVE diff --git a/jobs/rocoto/earc.sh b/jobs/rocoto/earc.sh index 9e3614377d7..98cebdf3d3b 100755 --- a/jobs/rocoto/earc.sh +++ b/jobs/rocoto/earc.sh @@ -8,9 +8,12 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" +export job="earc" +export jobid="${job}.$$" + ############################################################### # Execute the JJOB -"${HOMEgfs}"/jobs/JENKFGDAS_ARCHIVE +${HOMEgfs}/jobs/JGDAS_ENKF_ARCHIVE status=$? diff --git a/jobs/rocoto/gldas.sh b/jobs/rocoto/gldas.sh index db16dd883f1..8d8bb903bb0 100755 --- a/jobs/rocoto/gldas.sh +++ b/jobs/rocoto/gldas.sh @@ -8,6 +8,9 @@ source "$HOMEgfs/ush/preamble.sh" status=$? [[ $status -ne 0 ]] && exit $status +export job="gldas" +export jobid="${job}.$$" + ############################################################### # Execute the JJOB. GLDAS only runs once per day. From 123c59d07d1b77fc18239cc141918f820a30bd80 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 13 Dec 2022 13:26:02 -0500 Subject: [PATCH 2/6] fix shellcheck errors --- jobs/JGDAS_ENKF_ARCHIVE | 15 +++++++-------- jobs/JGLOBAL_ARCHIVE | 4 ++-- scripts/exgdas_enkf_earc.sh | 14 +++++++------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/jobs/JGDAS_ENKF_ARCHIVE b/jobs/JGDAS_ENKF_ARCHIVE index def922079a1..67a31c1da62 100755 --- a/jobs/JGDAS_ENKF_ARCHIVE +++ b/jobs/JGDAS_ENKF_ARCHIVE @@ -6,8 +6,8 @@ source "${HOMEgfs}/ush/preamble.sh" # Obtain unique process id (pid) and make temp directory ############################################## export DATA=${DATA:-${DATAROOT}/${jobid}} -mkdir -p ${DATA} -cd ${DATA} +mkdir -p "${DATA}" +cd "${DATA}" ############################################## @@ -32,7 +32,7 @@ export pgmerr=errfile export EXPDIR=${EXPDIR:-${HOMEgfs}/parm/config} configs="base earc" for config in ${configs}; do - . ${EXPDIR}/config.${config} + . "${EXPDIR}/config.${config}" status=$? [[ ${status} -ne 0 ]] && exit "${status}" done @@ -41,7 +41,7 @@ done ########################################## # Source machine runtime environment ########################################## -. ${HOMEgfs}/env/${machine}.env earc +. "${HOMEgfs}/env/${machine}.env" earc status=$? [[ ${status} -ne 0 ]] && exit "${status}" @@ -51,13 +51,12 @@ status=$? ############################################## export CDATE=${CDATE:-${PDY}${cyc}} export CDUMP=${CDUMP:-${RUN:-"gdas"}} -export COMPONENT="atmos" ############################################################### # Run archive script ############################################################### -${SCRgfs}/exgdas_enkf_earc.sh +"${SCRgfs}/exgdas_enkf_earc.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" @@ -70,8 +69,8 @@ status=$? ############################################## # Final processing ############################################## -if [ -e "${pgmout}" ] ; then - cat ${pgmout} +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" fi diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index aae78a0d55a..19f6c5937e4 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -67,8 +67,8 @@ status=$? ############################################## # Final processing ############################################## -if [ -e "${pgmout}" ] ; then - cat ${pgmout} +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" fi diff --git a/scripts/exgdas_enkf_earc.sh b/scripts/exgdas_enkf_earc.sh index ce0d4166636..79629460344 100755 --- a/scripts/exgdas_enkf_earc.sh +++ b/scripts/exgdas_enkf_earc.sh @@ -7,7 +7,7 @@ source "${HOMEgfs}/ush/preamble.sh" ############################################## export n=$((10#${ENSGRP})) export CDUMP_ENKF=$(echo "${EUPD_CYC:-"gdas"}" | tr a-z A-Z) -export ARCH_LIST="${ROTDIR}/enkf${CDUMP}.${PDY}/${cyc}/${COMPONENT}/earc${ENSGRP}" +export ARCH_LIST="${ROTDIR}/enkf${CDUMP}.${PDY}/${cyc}/atmos/earc${ENSGRP}" # ICS are restarts and always lag INC by $assim_freq hours. EARCINC_CYC=${ARCH_CYC} @@ -122,13 +122,13 @@ if [ "${ENSGRP}" -eq 0 ]; then [[ ! -d ${ARCDIR} ]] && mkdir -p "${ARCDIR}" cd "${ARCDIR}" - nb_copy "${ROTDIR}"/enkf"${CDUMP}"."${PDY}"/"${cyc}"/"${COMPONENT}"/"${CDUMP}".t"${cyc}"z.enkfstat enkfstat."${CDUMP}"."${CDATE}" - nb_copy "${ROTDIR}"/enkf"${CDUMP}"."${PDY}"/"${cyc}"/"${COMPONENT}"/"${CDUMP}".t"${cyc}"z.gsistat.ensmean gsistat."${CDUMP}"."${CDATE}".ensmean + nb_copy "${ROTDIR}/enkf${CDUMP}.${PDY}/${cyc}/atmos/${CDUMP}.t${cyc}z.enkfstat" "enkfstat.${CDUMP}.${CDATE}" + nb_copy "${ROTDIR}/enkf${CDUMP}.${PDY}/${cyc}/atmos/${CDUMP}.t${cyc}z.gsistat.ensmean" "gsistat.${CDUMP}.${CDATE}.ensmean" if [ "${CDUMP_ENKF}" != "GDAS" ]; then - nb_copy "${ROTDIR}"/enkfgfs."${PDY}"/"${cyc}"/"${COMPONENT}"/"${CDUMP}".t"${cyc}"z.enkfstat enkfstat.gfs."${CDATE}" - nb_copy "${ROTDIR}"/enkfgfs."${PDY}"/"${cyc}"/"${COMPONENT}"/"${CDUMP}".t"${cyc}"z.gsistat.ensmean gsistat.gfs."${CDATE}".ensmean - fi + nb_copy "${ROTDIR}/enkfgfs.${PDY}/${cyc}/atmos/${CDUMP}.t${cyc}z.enkfstat" "enkfstat.gfs.${CDATE}" + nb_copy "${ROTDIR}/enkfgfs.${PDY}/${cyc}/atmos/${CDUMP}.t${cyc}z.gsistat.ensmean" "gsistat.gfs.${CDATE}.ensmean" + fi fi @@ -152,7 +152,7 @@ if [ "${ENSGRP}" -eq 0 ]; then # Loop over GDAS and GFS EnKF directories separately. clist="gdas gfs" for ctype in ${clist}; do - COMIN_ENS="${ROTDIR}/enkf${ctype}.${gPDY}/${gcyc}/${COMPONENT}" + COMIN_ENS="${ROTDIR}/enkf${ctype}.${gPDY}/${gcyc}/atmos" if [ -d "${COMIN_ENS}" ]; then rocotolog="${EXPDIR}/logs/${GDATE}.log" if [ -f "${rocotolog}" ]; then From 9a6d94bac2e18c65530b59a9bfafe7e9291a65f4 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 13 Dec 2022 13:27:39 -0500 Subject: [PATCH 3/6] fix shellcheck errors --- jobs/rocoto/earc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/rocoto/earc.sh b/jobs/rocoto/earc.sh index 98cebdf3d3b..c4c73416988 100755 --- a/jobs/rocoto/earc.sh +++ b/jobs/rocoto/earc.sh @@ -4,7 +4,7 @@ source "${HOMEgfs}/ush/preamble.sh" ############################################################### # Source FV3GFS workflow modules -. "${HOMEgfs}"/ush/load_fv3gfs_modules.sh +. "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" @@ -13,7 +13,7 @@ export jobid="${job}.$$" ############################################################### # Execute the JJOB -${HOMEgfs}/jobs/JGDAS_ENKF_ARCHIVE +"${HOMEgfs}/jobs/JGDAS_ENKF_ARCHIVE" status=$? From 8ffea058576cbeb770a13e3daf750855d67462b5 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 13 Dec 2022 13:54:58 -0500 Subject: [PATCH 4/6] fix moreshellcheck errors --- jobs/JGDAS_ENKF_ARCHIVE | 2 +- jobs/JGLOBAL_ARCHIVE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/JGDAS_ENKF_ARCHIVE b/jobs/JGDAS_ENKF_ARCHIVE index 67a31c1da62..56c88e0e3d6 100755 --- a/jobs/JGDAS_ENKF_ARCHIVE +++ b/jobs/JGDAS_ENKF_ARCHIVE @@ -7,7 +7,7 @@ source "${HOMEgfs}/ush/preamble.sh" ############################################## export DATA=${DATA:-${DATAROOT}/${jobid}} mkdir -p "${DATA}" -cd "${DATA}" +cd "${DATA}" || (echo "${DATA} does not exist. ABORT!"; exit 1) ############################################## diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index 19f6c5937e4..f4de6b44e4c 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -7,7 +7,7 @@ source "${HOMEgfs}/ush/preamble.sh" ############################################## export DATA=${DATA:-${DATAROOT}/${jobid}} mkdir -p "${DATA}" -cd "${DATA}" +cd "${DATA}" || (echo "${DATA} does not exist. ABORT!"; exit 1) ############################################## From e6d309a676f36b3755bd0ee691022c31b5de5b28 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 13 Dec 2022 13:59:20 -0500 Subject: [PATCH 5/6] add jobid in jedi jobs --- jobs/rocoto/aeroanlfinal.sh | 3 +++ jobs/rocoto/aeroanlinit.sh | 3 +++ jobs/rocoto/aeroanlrun.sh | 3 +++ jobs/rocoto/ocnanalpost.sh | 3 +++ jobs/rocoto/ocnanalprep.sh | 3 +++ jobs/rocoto/ocnanalrun.sh | 3 +++ 6 files changed, 18 insertions(+) diff --git a/jobs/rocoto/aeroanlfinal.sh b/jobs/rocoto/aeroanlfinal.sh index 1906cc1e7fb..3056959ac8a 100755 --- a/jobs/rocoto/aeroanlfinal.sh +++ b/jobs/rocoto/aeroanlfinal.sh @@ -8,5 +8,8 @@ source "$HOMEgfs/ush/preamble.sh" status=$? [[ $status -ne 0 ]] && exit $status +export job="aeroanlfinal" +export jobid="${job}.$$" + ############################################################### echo "Do nothing for now" diff --git a/jobs/rocoto/aeroanlinit.sh b/jobs/rocoto/aeroanlinit.sh index 1906cc1e7fb..a3d02383694 100755 --- a/jobs/rocoto/aeroanlinit.sh +++ b/jobs/rocoto/aeroanlinit.sh @@ -8,5 +8,8 @@ source "$HOMEgfs/ush/preamble.sh" status=$? [[ $status -ne 0 ]] && exit $status +export job="aeroanlinit" +export jobid="${job}.$$" + ############################################################### echo "Do nothing for now" diff --git a/jobs/rocoto/aeroanlrun.sh b/jobs/rocoto/aeroanlrun.sh index 1906cc1e7fb..5dc731a94c2 100755 --- a/jobs/rocoto/aeroanlrun.sh +++ b/jobs/rocoto/aeroanlrun.sh @@ -8,5 +8,8 @@ source "$HOMEgfs/ush/preamble.sh" status=$? [[ $status -ne 0 ]] && exit $status +export job="aeroanlrun" +export jobid="${job}.$$" + ############################################################### echo "Do nothing for now" diff --git a/jobs/rocoto/ocnanalpost.sh b/jobs/rocoto/ocnanalpost.sh index bad01c93f22..5dead927bd3 100755 --- a/jobs/rocoto/ocnanalpost.sh +++ b/jobs/rocoto/ocnanalpost.sh @@ -8,6 +8,9 @@ module purge module use "${HOMEgfs}/sorc/gdas.cd/modulefiles" module load GDAS/"${machine,,}" +export job="ocnanalpost" +export jobid="${job}.$$" + ############################################################### # Execute the JJOB "${HOMEgfs}"/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST diff --git a/jobs/rocoto/ocnanalprep.sh b/jobs/rocoto/ocnanalprep.sh index 7cf14a66ce7..0771872629f 100755 --- a/jobs/rocoto/ocnanalprep.sh +++ b/jobs/rocoto/ocnanalprep.sh @@ -8,6 +8,9 @@ module purge module use "${HOMEgfs}/sorc/gdas.cd/modulefiles" module load "GDAS/${machine,,}" +export job="ocnanalprep" +export jobid="${job}.$$" + ############################################################### # Execute the JJOB "${HOMEgfs}"/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP diff --git a/jobs/rocoto/ocnanalrun.sh b/jobs/rocoto/ocnanalrun.sh index 460169a55f6..cd57d13c440 100755 --- a/jobs/rocoto/ocnanalrun.sh +++ b/jobs/rocoto/ocnanalrun.sh @@ -8,6 +8,9 @@ module purge module use "${HOMEgfs}/sorc/gdas.cd/modulefiles" module load GDAS/"${machine,,}" +export job="ocnanalrun" +export jobid="${job}.$$" + ############################################################### # Execute the JJOB "${HOMEgfs}"/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN From f96d7f971b36287a7573cff0784d9957e53132e8 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Tue, 13 Dec 2022 14:04:29 -0500 Subject: [PATCH 6/6] fix more shellcheck errors --- jobs/JGDAS_ENKF_ARCHIVE | 2 +- jobs/JGLOBAL_ARCHIVE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/JGDAS_ENKF_ARCHIVE b/jobs/JGDAS_ENKF_ARCHIVE index 56c88e0e3d6..1988bd76b61 100755 --- a/jobs/JGDAS_ENKF_ARCHIVE +++ b/jobs/JGDAS_ENKF_ARCHIVE @@ -77,7 +77,7 @@ fi ########################################## # Remove the Temporary working directory ########################################## -cd "${DATAROOT}" +cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1) [[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" exit 0 diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index f4de6b44e4c..21b20e1273c 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -75,7 +75,7 @@ fi ########################################## # Remove the Temporary working directory ########################################## -cd "${DATAROOT}" +cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1) [[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" exit 0