From 1af53c339b56579ac08d738c240606037009e31b Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Wed, 5 Apr 2023 11:35:41 -0400 Subject: [PATCH 01/41] Change the fix link based on g-w On branch feature/v13_atmos_prep modified: rocoto/parm/setbase modified: sorc/link_gefs.sh new file: versions/fix.ver Refs: #87 --- rocoto/parm/setbase | 2 +- sorc/link_gefs.sh | 229 ++++++++++++++++++++++++-------------------- versions/fix.ver | 24 +++++ 3 files changed, 152 insertions(+), 103 deletions(-) create mode 100644 versions/fix.ver diff --git a/rocoto/parm/setbase b/rocoto/parm/setbase index 1e0cf5472..6832c889f 100755 --- a/rocoto/parm/setbase +++ b/rocoto/parm/setbase @@ -49,7 +49,7 @@ elif [[ -d /apps/prod ]]; then echo "Using settings for WCOSS2" export machine='WCOSS2' export JOBPHASE= - export HOMEDIR=${HOMEDIR:-/lfs/h2/emc/ens/noscrub/emc.ens/common/git/gfs/gw_gefs_v12.3.3_1a7f074_Nov07_2022} + export HOMEDIR=${HOMEDIR:-/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS_v13/gw_walter} export HOMEtrak=/lfs/h2/emc/ens/noscrub/emc.ens/common/git/ens_tracker/ens_tracker.v2.1.2 export archsyndir=/lfs/h2/emc/ens/noscrub/emc.ens/common/git/ens_tracker/TCvitalData/syndat export HOMEdata=/lfs/h2/emc/ens/noscrub/emc.ens/common/canned_data diff --git a/sorc/link_gefs.sh b/sorc/link_gefs.sh index e4c3e4d5b..6bdbf725b 100755 --- a/sorc/link_gefs.sh +++ b/sorc/link_gefs.sh @@ -5,13 +5,11 @@ set -ex while getopts e:m: option do - case "${option}" - in - e) RUN_ENVIR=${OPTARG};; - m) machine=${OPTARG};; - - esac - + case "${option}" + in + e) RUN_ENVIR=${OPTARG};; + m) machine=${OPTARG};; + esac done RUN_ENVIR=${RUN_ENVIR:-nco} @@ -28,158 +26,185 @@ pwd=$(pwd -P) #------------------------------ #--model fix fields #------------------------------ -if [ $machine = "hera" ]; then +case "${machine}" in + "wcoss2") + FIX_DIR="/lfs/h2/emc/ens/save/emc.ens/FIX/gefs/fix_nco_gefsv12.3" + FIX_DIR_FV3="/lfs/h2/emc/global/noscrub/emc.global/FIX/fix" + ;; + "hera") FIX_DIR="/scratch2/NCEPDEV/ensemble/noscrub/common/FIX/gefs/fix_nco_gefsv12.3" FIX_DIR_FV3="/scratch1/NCEPDEV/global/glopara/fix" -elif [ $machine == "wcoss2" ]; then - FIX_DIR="/lfs/h2/emc/ens/save/emc.ens/FIX/gefs/fix_nco_gefsv12.3" - FIX_DIR_FV3="/lfs/h2/emc/global/save/emc.global/FIX/fix_nco_gfsv15" -fi + ;; + *) + echo "FATAL: Unknown target machine ${machine}, couldn't set FIX_DIR" + exit 1 + ;; +esac # Delete Fix folder and relink/recopy it cd ${pwd}/../fix for dir in fix_gefs fix_wave fix_emission; do - if [[ -d $dir ]]; then - echo "Fix folder exists, deleting it..." - rm -rf $dir - fi - $LINK $FIX_DIR/$dir $dir + if [[ -d $dir ]]; then + echo "Fix folder exists, deleting it..." + rm -rf $dir + fi + $LINK $FIX_DIR/$dir $dir done cd ${pwd} if [[ -d global-workflow.fd ]] ; then - cd ${pwd}/../fix - - for gw_dir in fix_am fix_fv3_gmted2010/C384 fix_chem; do - if [[ -d $gw_dir ]]; then rm -Rf $gw_dir; fi - mkdir -p $(dirname $gw_dir) - $LINK $FIX_DIR_FV3/$gw_dir $gw_dir - done - - # product - sFolder=product - if [[ -d $sFolder ]]; then - rm -rf $sFolder - fi - $LINK ${pwd}/../sorc/global-workflow.fd/fix/${sFolder} ${sFolder} + # Source fix version file + source "${pwd}/../versions/fix.ver" + + cd ${pwd}/../fix - cd ${pwd} + for gw_dir in am aer lut orog chem ugwd wave; do + if [[ -d $gw_dir ]]; then + rm -rf $gw_dir + fi + #mkdir -p $(dirname $gw_dir) + fix_ver="${gw_dir}_ver" + ${LINK} "${FIX_DIR_FV3}/${gw_dir}/${!fix_ver}" "${gw_dir}" + done + + # product + sFolder=product + if [[ -d $sFolder ]]; then + rm -rf $sFolder + fi + $LINK ${pwd}/../sorc/global-workflow.fd/fix/${sFolder} ${sFolder} + + cd ${pwd} fi # copy/link exec files cd $pwd if [[ -d global-workflow.fd ]] ; then - sPath=../sorc/global-workflow.fd/sorc/fv3gfs.fd/WW3/model/exe - for sFile in ${sPath}/ww3_* - do - echo $sFile - done - $LINK ${sPath}/ww3_* ../exec/ + sPath=../sorc/global-workflow.fd/exec + for sFile in ${sPath}/ww3_* + do + echo $sFile + done + $LINK ${sPath}/ww3_* ../exec/ - sPath=../sorc/global-workflow.fd/sorc/fv3gfs.fd/NEMS/exe - $LINK ${sPath}/global_fv3gfs.* ../exec/ + sPath=../sorc/global-workflow.fd/sorc/ufs_model.fd/tests + $LINK ${sPath}/ufs_model.x ../exec/ - sPath=../sorc/global-workflow.fd/sorc/gfs_post.fd/exec - $LINK ${sPath}/ncep_post ../exec/gfs_ncep_post + sPath=../sorc/global-workflow.fd/sorc/ufs_model.fd/FV3/upp/exec + $LINK ${sPath}/upp.x ../exec/ - sPath=../sorc/global-workflow.fd/sorc/gsi.fd/exec - $LINK ${sPath}/getsigensmeanp_smooth.x ../exec/ - $LINK ${sPath}/getsfcensmeanp.x ../exec/ + sPath=../sorc/global-workflow.fd/sorc/gsi_utils.fd/install/bin + $LINK ${sPath}/getsigensmeanp_smooth.x ../exec/ + $LINK ${sPath}/getsfcensmeanp.x ../exec/ - sPath=../sorc/global-workflow.fd/exec - $LINK ${sPath}/gfs_bufr ../exec/ - $LINK ${sPath}/tocsbufr ../exec/ + sPath=../sorc/global-workflow.fd/sorc/gfs_utils.fd/install/bin + $LINK ${sPath}/gfs_bufr.x ../exec/ + $LINK ${sPath}/tocsbufr.x ../exec/ - # chem_prep_emissions - sPath=../sorc/global-workflow.fd/sorc/gsd_prep_chem.fd/workflow/emc-global/exec - $LINK ${sPath}/prep_chem_sources_RADM_FV3_SIMPLE.exe ../exec/ + # chem_prep_emissions + #sPath=../sorc/global-workflow.fd/sorc/gsd_prep_chem.fd/workflow/emc-global/exec + #$LINK ${sPath}/prep_chem_sources_RADM_FV3_SIMPLE.exe ../exec/ fi # Copy/Link parm files cd $pwd if [[ -d global-workflow.fd ]] ; then - if [[ -d ../parm/parm_fv3diag ]]; then - rm -rf ../parm/parm_fv3diag + if [[ -d ../parm/post ]]; then + rm -rf ../parm/post + fi + $LINK ../sorc/global-workflow.fd/sorc/ufs_model.fd/FV3/upp/parm ../parm/post + + for fn in parm_fv3diag product config; do + echo ${fn} + if [[ -d ../parm/${fn} ]]; then + rm -rf ../parm/${fn} fi - $LINK ../sorc/global-workflow.fd/parm/parm_fv3diag ../parm/ - - if [[ -d ../parm/post ]]; then - rm -rf ../parm/post - fi - $LINK ../sorc/global-workflow.fd/sorc/gfs_post.fd/parm ../parm/post - - if [[ -d ../parm/product ]]; then - rm -rf ../parm/product - fi - $LINK ../sorc/global-workflow.fd/parm/product ../parm/ + ${LINK} ../sorc/global-workflow.fd/parm/${fn} ../parm/ + done fi + # Copy/Link ush files cd $pwd if [[ -d global-workflow.fd ]] ; then - $LINK ../sorc/global-workflow.fd/sorc/ufs_utils.fd/ush/chgres_cube.sh ../ush/ - - $LINK ../sorc/global-workflow.fd/sorc/gfs_post.fd/ush/gfs_nceppost.sh ../ush/ - + $LINK ../sorc/global-workflow.fd/sorc/ufs_utils.fd/ush/chgres_cube.sh ../ush/ + #$LINK ../sorc/global-workflow.fd/sorc/gfs_post.fd/ush/gfs_nceppost.sh ../ush/ fi # For Forecast cd $pwd if [[ -d global-workflow.fd ]] ; then - sFile=exglobal_fcst_nemsfv3gfs.sh - if [[ -e ../scripts/$sFile ]]; then - if [[ -L ../scripts/$sFile ]]; then - rm ../scripts/$sFile - $LINK ../sorc/global-workflow.fd/scripts/$sFile ../scripts/ - fi + for sFile in scripts/exglobal_forecast.sh \ + ush/preamble.sh \ + ush/cplvalidate.sh \ + ush/forecast_predet.sh \ + ush/forecast_det.sh \ + ush/forecast_postdet.sh \ + ush/nems_configure.sh \ + ush/parsing_model_configure_FV3.sh \ + ush/parsing_model_configure_DATM.sh \ + ush/parsing_namelists_FV3.sh \ + ush/parsing_namelists_WW3.sh \ + ush/parsing_namelists_MOM6.sh \ + ush/parsing_namelists_CICE.sh \ + sorc/ufs_model.fd + do + if [[ -e ../$sFile ]]; then + if [[ -L ../$sFile ]]; then + rm ../$sFile + $LINK ../sorc/global-workflow.fd/$sFile ../$sFile + fi else - $LINK ../sorc/global-workflow.fd/scripts/$sFile ../scripts/ + $LINK ../sorc/global-workflow.fd/$sFile ../$sFile fi + done fi # For wave echo $pwd cd $pwd if [[ -d global-workflow.fd ]]; then - lScripts="exwave_init.sh exwave_nawips.sh exwave_post_sbs.sh exwave_prep.sh exwave_stat.sh" - for sFile in $lScripts; do - $LINK ../sorc/global-workflow.fd/scripts/$sFile ../scripts/ - done - - lUsh="wave_ens_bull.sh wave_ens_stat.sh wave_grib2_sbs.sh wave_grid_interp_sbs.sh wave_grid_moddef.sh wave_outp_spec.sh wave_prnc_cur.sh wave_prnc_ice.sh wave_tar.sh" - for sFile in $lUsh; do - $LINK ../sorc/global-workflow.fd/ush/$sFile ../ush/ - done - - if [[ -e ../env ]]; then - if [[ -L ../env ]]; then - rm ../env - $LINK ${pwd}/global-workflow.fd/env ../ - fi - else - $LINK ${pwd}/global-workflow.fd/env ../ + lScripts="exgfs_wave_init.sh exgfs_wave_nawips.sh exgfs_wave_post_gridded_sbs.sh exgfs_wave_prep.sh exgfs_wave_prdgen_bulls.sh exgfs_wave_prdgen_gridded.sh" #exwave_stat.sh" + for sFile in $lScripts; do + $LINK ../sorc/global-workflow.fd/scripts/$sFile ../scripts/ + done + + lUsh=`ls ../sorc/global-workflow.fd/ush/wave_*` #"wave_ens_bull.sh wave_ens_stat.sh wave_grib2_sbs.sh wave_grid_interp_sbs.sh wave_grid_moddef.sh wave_outp_spec.sh wave_prnc_cur.sh wave_prnc_ice.sh wave_tar.sh" + for sFile in $lUsh; do + sFile=$(basename ${sFile}) + echo $sFile + $LINK ../sorc/global-workflow.fd/ush/$sFile ../ush/ + done + + if [[ -e ../env ]]; then + if [[ -L ../env ]]; then + rm ../env + $LINK ${pwd}/global-workflow.fd/env ../ fi + else + $LINK ${pwd}/global-workflow.fd/env ../ + fi fi # for CHEM if [[ -d global-workflow.fd ]]; then - # for chem_prep_emissions - $LINK ../sorc/global-workflow.fd/sorc/gsd_prep_chem.fd/workflow/emc-global/scripts/exglobal_prep_chem.sh ../scripts/ - $LINK ../sorc/global-workflow.fd/sorc/gsd_prep_chem.fd/workflow/emc-global/parm/prep_chem_sources.inp.IN ../parm/ + # for chem_prep_emissions + #$LINK ../sorc/global-workflow.fd/sorc/gsd_prep_chem.fd/workflow/emc-global/scripts/exglobal_prep_chem.sh ../scripts/ + #$LINK ../sorc/global-workflow.fd/sorc/gsd_prep_chem.fd/workflow/emc-global/parm/prep_chem_sources.inp.IN ../parm/ - # for init_aerosol - $LINK ../sorc/global-workflow.fd/ush/merge_fv3_chem_tile.py ../ush/ + # for init_aerosol + $LINK ../sorc/global-workflow.fd/ush/merge_fv3_aerosol_tile.py ../ush/ fi -# for atmos_prep for GFSv16 +# for atmos_prep for GFSv17 if [[ -d global-workflow.fd ]]; then - $LINK ../sorc/global-workflow.fd/sorc/ufs_utils.fd/exec/chgres_cube ../exec/ - $LINK ../sorc/global-workflow.fd/exec/chgres_recenter_ncio.exe ../exec/ - $LINK ../sorc/global-workflow.fd/sorc/gsi.fd/exec/calc_increment_ens_ncio.x ../exec/ -fi + $LINK ../sorc/global-workflow.fd/sorc/ufs_utils.fd/exec/chgres_cube ../exec/ + #$LINK ../sorc/global-workflow.fd/exec/chgres_recenter_ncio.exe ../exec/ + #$LINK ../sorc/global-workflow.fd/sorc/gsi.fd/exec/calc_increment_ens_ncio.x ../exec/ +fi exit 0 diff --git a/versions/fix.ver b/versions/fix.ver new file mode 100644 index 000000000..256d8efc5 --- /dev/null +++ b/versions/fix.ver @@ -0,0 +1,24 @@ +#!/bin/bash +# Fix file subfolder versions + +export aer_ver=20220805 +export am_ver=20220805 +export chem_ver=20220805 +export cice_ver=20220805 +export cpl_ver=20220805 +export datm_ver=20220805 +export gdas_bump_ver=20220805 +export gdas_crtm_ver=20220805 +export gdas_fv3jedi_ver=20220805 +export gdas_gsibec_ver=20221031 +export gldas_ver=20220920 +export glwu_ver=20220805 +export gsi_ver=20221128 +export lut_ver=20220805 +export mom6_ver=20220805 +export orog_ver=20220805 +export reg2grb2_ver=20220805 +export sfc_climo_ver=20220805 +export ugwd_ver=20220805 +export verif_ver=20220805 +export wave_ver=20220805 From a2f62eca3f3e91fc8ff060c1dd582f7c0e61bd02 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Wed, 5 Apr 2023 13:40:54 -0400 Subject: [PATCH 02/41] Add some changes for atmos_prep in the pre develop On branch feature/v13_atmos_prep modified: jobs/JGEFS_ATMOS_PREP modified: rocoto/bin/wcoss2/atmos_prep.sh modified: rocoto/bin/wcoss2/common.sh modified: rocoto/py/GEFS_XML_For_Tasks.py modified: scripts/exgefs_atmos_prep.sh modified: ush/gefs_atmos_prep.sh Refs: #87 --- jobs/JGEFS_ATMOS_PREP | 31 ++-- rocoto/bin/wcoss2/atmos_prep.sh | 3 +- rocoto/bin/wcoss2/common.sh | 11 +- rocoto/py/GEFS_XML_For_Tasks.py | 7 +- scripts/exgefs_atmos_prep.sh | 11 +- ush/gefs_atmos_prep.sh | 259 ++++++++++++++++++-------------- 6 files changed, 179 insertions(+), 143 deletions(-) diff --git a/jobs/JGEFS_ATMOS_PREP b/jobs/JGEFS_ATMOS_PREP index 9dde779c1..60fe1bc9b 100755 --- a/jobs/JGEFS_ATMOS_PREP +++ b/jobs/JGEFS_ATMOS_PREP @@ -4,8 +4,8 @@ echo "$(date -u) begin $(basename $BASH_SOURCE)" set -xa if [[ ${STRICT:-NO} == "YES" ]]; then - # Turn on strict bash error checking - set -eu + # Turn on strict bash error checking + set -eu fi export PS4='$SECONDS + $(basename $(basename $BASH_SOURCE))[$LINENO] ' @@ -68,24 +68,25 @@ export FIXgefs=${FIXgefs:-$HOMEgefs/fix/fix_gefs} ############################################## ver=${ver:-$(echo ${gefs_ver}|cut -c1-5)} export COMPONENT="atmos" -export COMOUT=${COMOUT:-$(compath.py -o $NET/${ver})/${RUN}.${PDY}/$cyc/$COMPONENT} -export GESOUT=${GESOUT:-$(compath.py -o $NET/${ver})/${RUN}.${PDY}/$cyc/nwges} + ############################# # Source relevant config files ############################# configs="gefs gefs_atmos_prep" config_path=$PARMgefs for config in $configs; do - . $config_path/$config.parm - export err=$? - if [[ $err != 0 ]]; then - echo "FATAL ERROR in $(basename $BASH_SOURCE): Error while loading parm file $config_path/$config.parm" - exit $err - fi + . $config_path/$config.parm + export err=$? + if [[ $err != 0 ]]; then + echo "FATAL ERROR in $(basename $BASH_SOURCE): Error while loading parm file $config_path/$config.parm" + exit $err + fi done -export COMINgfs=${COMINgfs:-$(compath.py ${envir}/com/gfs/${gfs_ver})/gfs.${PDY}/${cyc}/atmos} -export COMINenkf=${COMINenkf:-$(compath.py ${envir}/com/gfs/${gfs_ver})/enkfgdas.${pdyp}/${cycp}/atmos} +export COMOUT=${COMOUT:-$(compath.py -o $NET/${ver})/${RUN}.${PDY}/${cyc}} + +export COMINgfs=${COMINgfs:-$(compath.py ${envir}/com/gfs/${gfs_ver})/gfs.${PDY}/${cyc}} +export COMINenkfgfs=${COMINenkfgfs:-$(compath.py ${envir}/com/gfs/${gfs_ver})/enkfgfs.${PDY}/${cyc}} NCP=${NCP:-"/bin/cp -p"} @@ -97,10 +98,10 @@ env | sort $HOMEgefs/scripts/exgefs_atmos_prep.sh export err=$? if [[ $err == 0 ]]; then - echo "$job completed normally!" + echo "$job completed normally!" else - echo "FATAL ERROR in $(basename $BASH_SOURCE): $job failed!" - exit $err + echo "FATAL ERROR in $(basename $BASH_SOURCE): $job failed!" + exit $err fi ############################################################# diff --git a/rocoto/bin/wcoss2/atmos_prep.sh b/rocoto/bin/wcoss2/atmos_prep.sh index 0a2930c92..18a5f1ae8 100755 --- a/rocoto/bin/wcoss2/atmos_prep.sh +++ b/rocoto/bin/wcoss2/atmos_prep.sh @@ -1,4 +1,4 @@ -#!/bin/ksh -l +#! /usr/bin/env bash set -x ulimit -s unlimited @@ -21,6 +21,7 @@ module load prod_util/$prod_util_ver module load prod_envir/$prod_envir_ver module load hdf5/$hdf5_ver module load netcdf/$netcdf_ver +module load libjpeg/$libjpeg_ver module load cfp/$cfp_ver diff --git a/rocoto/bin/wcoss2/common.sh b/rocoto/bin/wcoss2/common.sh index 3da732d9c..0c0026014 100644 --- a/rocoto/bin/wcoss2/common.sh +++ b/rocoto/bin/wcoss2/common.sh @@ -36,7 +36,16 @@ if [[ $RocotoGen == 0 ]]; then #export COMPATH=$HOMEdata/canned/com/gfs:$HOMEdata/canned/com/cfs:$HOMEdata/canned/com/nawips:$HOMEdata/canned/com/ecmwf:$HOMEdata/canned/com/nam:${WORKDIR}/$envir/com/${NET} #export DCOMROOT=${HOMEdata}/canned/dcom # For prod data - export COMPATH=${WORKDIR}/$envir/com/${NET} + if [[ $CASEHR == "C48" ]]; then + export COMPATH=${WORKDIR}/$envir/com/${NET}:/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS_v13/z_DATA/lfs_C96_C48_NOIAU_anal/h1/ops/prod/com/gfs + #export COMPATH=${WORKDIR}/$envir/com/${NET}:/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS_v13/z_DATA/lfs_C96_C48_NOIAU_anal_20230316/h1/ops/prod/com/gfs + elif [[ $CASEHR == "C384" ]]; then + export COMPATH=${WORKDIR}/$envir/com/${NET}:/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS_v13/z_DATA/lfs_C768_C384_NOIAU_anal/h1/ops/prod/com/gfs + #export COMPATH=${WORKDIR}/$envir/com/${NET}:/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS_v13/z_DATA/lfs_C768_C384_NOIAU_anal_20230223_updated_dev/h1/ops/prod/com/gfs + else + echo "Wrong CASEHR .." + exit -1 + fi elif [[ $RocotoGen == 1 ]]; then export HOMEtrak=/lfs/h2/emc/ens/noscrub/emc.ens/common/git/ens_tracker/ens_tracker.v2.1.2 diff --git a/rocoto/py/GEFS_XML_For_Tasks.py b/rocoto/py/GEFS_XML_For_Tasks.py index 5853a7b93..b12d87930 100755 --- a/rocoto/py/GEFS_XML_For_Tasks.py +++ b/rocoto/py/GEFS_XML_For_Tasks.py @@ -47,9 +47,10 @@ def config_tasknames(dicBase): dicBase[sTaskName.upper()] = "atmos_prep" # ---init_recenter - iTaskName_Num += 1 - sTaskName = "taskname_{0}".format(iTaskName_Num) - dicBase[sTaskName.upper()] = "init_recenter" + # Since maybe there is no need to use init_recenter, so comment these lines + # iTaskName_Num += 1 + # sTaskName = "taskname_{0}".format(iTaskName_Num) + # dicBase[sTaskName.upper()] = "init_recenter" elif dicBase['RUN_INIT'] == "FV3_WARM": # ---init_recenter diff --git a/scripts/exgefs_atmos_prep.sh b/scripts/exgefs_atmos_prep.sh index 0648d6960..a85819a5a 100755 --- a/scripts/exgefs_atmos_prep.sh +++ b/scripts/exgefs_atmos_prep.sh @@ -1,23 +1,20 @@ -#! /bin/bash +#! /usr/bin/env bash echo "$(date -u) begin $(basename $BASH_SOURCE)" set -xa if [[ ${STRICT:-NO} == "YES" ]]; then - # Turn on strict bash error checking - set -eu + # Turn on strict bash error checking + set -eu fi export HOMEgfs=${HOMEgfs:-${HOMEgefs}} export HOMEufs=${HOMEufs:-${HOMEgfs}} export USHgfs=$HOMEgfs/ush export FIXgfs=$HOMEgfs/fix -export FIXfv3=${FIXfv3:-$FIXgfs/fix_fv3_gmted2010} -export FIXam=${FIXam:-$FIXgfs/fix_am} -export VCOORD_FILE=${VCOORD_FILE:-$FIXam/global_hyblev.l${LEVS}.txt} +export CDUMP=$RUNMEM mem=$(echo $RUNMEM|cut -c3-5) -sfc_mem=${sfc_mem:-"c00"} cd $DATA diff --git a/ush/gefs_atmos_prep.sh b/ush/gefs_atmos_prep.sh index 6c2e08241..5ea31b79d 100755 --- a/ush/gefs_atmos_prep.sh +++ b/ush/gefs_atmos_prep.sh @@ -1,154 +1,181 @@ -#!/bin/bash +#! /usr/bin/env bash echo "$(date -u) begin $(basename $BASH_SOURCE)" export PS4="${PS4}${1}: " set -xa if [[ ${STRICT:-NO} == "YES" ]]; then - # Turn on strict bash error checking - set -eu + # Turn on strict bash error checking + set -eu fi export mem=$1 export nmem=$(echo $mem|cut -c 2-) nmem=${nmem#0} -export INIDIR=$DATA -export OUTDIR=$GESOUT/enkf/$mem -INITDIR=$GESOUT/init/$mem -mkdir -p $INIDIR -mkdir -p $OUTDIR -mkdir -p $INITDIR +#export INIDIR=$DATA +if [[ ${NewCOM} == "YES" ]]; then + export OUTDIR=${COMOUT}/$mem/atmos/INPUT +else + export OUTDIR=${COMOUT}/atmos/init/${mem} +fi -cd $INIDIR +mkdir -p $OUTDIR +cd $DATA + +# Copy from set_fixed_files.sh +#--------------------------------------------------------------------------- +# Set directory names and file names for orog data +# The old and new (support fractional grid) orog data have different file names +#--------------------------------------------------------------------------- +export FRAC_ORO="yes" +if [ "${FRAC_ORO:-"no"}" = "yes" ]; then + if [ ${CASE} == 'C48' ] ; then + OCNRES='500' + elif [ ${CASE} == 'C96' ] ; then + OCNRES='100' + elif [ ${CASE} == 'C192' ] ; then + OCNRES='050' + elif [ ${CASE} == 'C384' ] || [ ${CASE} == 'C768' ] || [ ${CASE} == 'C1152' ]; then + OCNRES='025' + fi + ORO_DIR="${CASE}.mx${OCNRES}_frac" + ORO_NAME="oro_${CASE}.mx${OCNRES}" +else + ORO_DIR="${CASE}" + ORO_NAME="${CASE}_oro_data" +fi if [[ $mem = c00 ]] ;then - # Control intial conditions from current GFS cycle - ATMFILE=$COMINgfs/gfs.t${cyc}z.atmanl.nc - if [[ -f $ATMFILE ]]; then - $NCP $ATMFILE $INIDIR - export ATM_FILES_INPUT="gfs.t${cyc}z.atmanl.nc" - else - msg="FATAL ERROR in $(basename $BASH_SOURCE): GFS atmospheric analysis file $ATMFILE not found!" - echo "$msg" - export err=101 - err_chk || exit $err - fi - export CONVERT_SFC=".true." + # Control intial conditions from current GFS cycle + export ATM_FILES_INPUT="gfs.t${cyc}z.atmanl.nc" + ATMFILE=${COMINgfs}/atmos/${ATM_FILES_INPUT} #gfs.t${cyc}z.atmanl.nc + if [[ -f $ATMFILE ]]; then + $NCP $ATMFILE $DATA + #export ATM_FILES_INPUT="gfs.t${cyc}z.atmanl.nc" + else + msg="FATAL ERROR in $(basename $BASH_SOURCE): GFS atmospheric analysis file $ATMFILE not found!" + echo "$msg" + export err=101 + err_chk || exit $err + fi + export CONVERT_SFC=".true." else - i=0 - success="NO" - (( cmem = nmem + memshift )) - while [[ $success == "NO" && $i < $MAX_ENKF_SEARCHES ]]; do - if (( cmem > 80 )); then - (( cmem = cmem - 80 )) - fi - - memchar="mem"$(printf %03i $cmem) - ATMFILE="$COMINenkf/$memchar/gdas.t${cycp}z.atmf006.nc" - - if [[ -f $ATMFILE ]]; then - $NCP $ATMFILE $INIDIR - export ATM_FILES_INPUT="gdas.t${cycp}z.atmf006.nc" - success="YES" - - else - (( i = i + 1 )) - if [[ $i < $MAX_ENKF_SEARCHES ]]; then - echo "EnKF atmospheric file $ATMFILE not found, trying different member" - (( cmem = cmem + ENKF_SEARCH_LEAP )) - - else - msg="FATAL ERROR in $(basename $BASH_SOURCE): Unable to find EnKF atmospheric file after $MAX_ENKF_SEARCHES attempts" - echo $msg - export err=102 - err_chk || exit $err - fi - fi # [[ -f $ATMFILE ]] - done # [[ success == "NO" && $i < MAX_ENKF_SEARCHES ]] - export CONVERT_SFC=".false." + i=0 + success="NO" + (( cmem = nmem + memshift )) + while [[ $success == "NO" && $i < $MAX_ENKF_SEARCHES ]]; do + if (( cmem > 80 )); then + (( cmem = cmem - 80 )) + fi + + memchar="mem"$(printf %03i $cmem) + export ATM_FILES_INPUT="enkfgfs.t${cyc}z.ratmanl.nc" + ATMFILE="${COMINenkfgfs}/$memchar/atmos/${ATM_FILES_INPUT}" #gfs.t${cycp}z.atmanl.nc" + + if [[ -f $ATMFILE ]]; then + $NCP $ATMFILE $DATA + success="YES" + + else + (( i = i + 1 )) + if [[ $i < $MAX_ENKF_SEARCHES ]]; then + echo "EnKF atmospheric file $ATMFILE not found, trying different member" + (( cmem = cmem + ENKF_SEARCH_LEAP )) + + else + msg="FATAL ERROR in $(basename $BASH_SOURCE): Unable to find EnKF atmospheric file after $MAX_ENKF_SEARCHES attempts" + echo $msg + export err=102 + err_chk || exit $err + fi + fi # [[ -f $ATMFILE ]] + done # [[ success == "NO" && $i < MAX_ENKF_SEARCHES ]] + export CONVERT_SFC=".false." fi if [[ $CONVERT_SFC == ".true." ]]; then - export SFC_FILES_INPUT="gfs.t${cyc}z.sfcanl.nc" - SFCFILE="$COMINgfs/$SFC_FILES_INPUT" - if [[ -f $SFCFILE ]]; then - $NCP $SFCFILE $INIDIR - else - msg="FATAL ERROR in $(basename $BASH_SOURCE): GFS surfce analysis $SFCFILE not found!" - echo $msg - export err=100 - err_chk || exit $err - fi + export SFC_FILES_INPUT="gfs.t${cyc}z.sfcanl.nc" + SFCFILE="${COMINgfs}/atmos/${SFC_FILES_INPUT}" + if [[ -f $SFCFILE ]]; then + $NCP ${SFCFILE} ${DATA} + else + msg="FATAL ERROR in $(basename ${BASH_SOURCE}): GFS surfce analysis ${SFCFILE} not found!" + echo ${msg} + export err=100 + err_chk || exit $err + fi fi -export CRES=$(echo $CASE |cut -c2-5) -export COMIN=$INIDIR +export CRES=$(echo ${CASE} |cut -c2-5) +export COMIN=$DATA export INPUT_TYPE="gaussian_netcdf" -export FIXfv3=$FIXgfs/fix_fv3_gmted2010/C$CRES +export FIXfv3=$FIXgfs/orog/${ORO_DIR} export FIXsfc=$FIXfv3/fix_sfc +export FIXam=${FIXam:-$FIXgfs/am} +export VCOORD_FILE=${VCOORD_FILE:-$FIXam/global_hyblev.l${LEVS}.txt} + +export TRACERS_INPUT="spfh","clwmr","o3mr","icmr","rwmr","snmr","grle" +export TRACERS_TARGET="sphum","liq_wat","o3mr","ice_wat","rainwat","snowwat","graupel" +#export TRACERS_TARGET='"sphum","liq_wat","o3mr","ice_wat","rainwat","snowwat","graupel"' +#export TRACERS_INPUT='"sphum","liq_wat","o3mr","ice_wat","rainwat","snowwat","graupel"' + +OROG_FILES_TARGET_GRID='' +for tile in {1..6} +do + OROG_FILES_TARGET_GRID=${OROG_FILES_TARGET_GRID}"${ORO_NAME}.tile${tile}.nc" + if [[ $tile != 6 ]]; then + OROG_FILES_TARGET_GRID=${OROG_FILES_TARGET_GRID}'","' + fi +done +export OROG_FILES_TARGET_GRID + ############################################################# # Execute the script $USHgfs/chgres_cube.sh export err=$? if [[ $err != 0 ]]; then - echo "FATAL ERROR in $(basename $BASH_SOURCE): chgres_cube failed!" - exit $err + echo "FATAL ERROR in $(basename $BASH_SOURCE): chgres_cube failed!" + exit $err fi ############################################################# -# Move files to the nwges directory -for tile in tile1 tile2 tile3 tile4 tile5 tile6; do - mv ${DATA}/out.atm.${tile}.nc $OUTDIR/gfs_data.${tile}.nc -done -mv ${DATA}/gfs_ctrl.nc $OUTDIR/. - touch ${OUTDIR}/chgres_atm.log # recenter can start now -if [[ $CONVERT_SFC == ".true." ]]; then - # Copy sfc files to the nwges directory for all members - for mem2 in $memberlist; do - INITDIR2=$GESOUT/init/$mem2 - mkdir -p $INITDIR2 - for tile in tile1 tile2 tile3 tile4 tile5 tile6; do - $NCP ${DATA}/out.sfc.${tile}.nc $INITDIR2/sfc_data.${tile}.nc - done - done -fi - -# Copy control file to init -$NCP $OUTDIR/gfs_ctrl.nc $INITDIR - if [[ $SENDCOM == "YES" ]]; then - MODCOM=$(echo ${NET}_${COMPONENT} | tr '[a-z]' '[A-Z]') - DBNTYP=${MODCOM}_INIT - COMDIR=$COMOUT/init/$mem - mkdir -p $COMDIR - $NCP $OUTDIR/gfs_ctrl.nc $COMDIR - if [[ $SENDDBN = YES ]];then - $DBNROOT/bin/dbn_alert MODEL $DBNTYP $job $COMDIR/gfs_ctrl.nc - fi - if [[ $mem == "c00" ]]; then - $NCP $OUTDIR/gfs_data*.nc $COMDIR - if [[ $SENDDBN = YES ]];then - for tile in tile1 tile2 tile3 tile4 tile5 tile6; do - $DBNROOT/bin/dbn_alert MODEL $DBNTYP $job $COMDIR/gfs_data.${tile}.nc - done - fi - fi - if [[ $CONVERT_SFC == ".true." ]]; then - for mem2 in $memberlist; do - COMDIR2=$COMOUT/init/$mem2 - mkdir -p $COMDIR2 - for tile in tile1 tile2 tile3 tile4 tile5 tile6; do - $NCP $GESOUT/init/$mem/sfc_data.${tile}.nc $COMDIR2 - if [[ $SENDDBN = YES ]];then - $DBNROOT/bin/dbn_alert MODEL $DBNTYP $job $COMDIR2/sfc_data.${tile}.nc - fi - done - done - fi + MODCOM=$(echo ${NET}_${COMPONENT} | tr '[a-z]' '[A-Z]') + DBNTYP=${MODCOM}_INIT + COMDIR=${OUTDIR} #$COMOUT/$mem/atmos/INPUT #/init/$mem + mkdir -p $COMDIR + $NCP ${DATA}/gfs_ctrl.nc $COMDIR + if [[ $SENDDBN = YES ]];then + $DBNROOT/bin/dbn_alert MODEL $DBNTYP $job $COMDIR/gfs_ctrl.nc + fi + + for tile in tile1 tile2 tile3 tile4 tile5 tile6; do + $NCP ${DATA}/out.atm.${tile}.nc $OUTDIR/gfs_data.${tile}.nc + if [[ $SENDDBN = YES ]];then + $DBNROOT/bin/dbn_alert MODEL $DBNTYP $job $COMDIR/gfs_data.${tile}.nc + fi + done + + if [[ $CONVERT_SFC == ".true." ]]; then + for mem2 in $memberlist; do + if [[ ${NewCOM} == "YES" ]]; then + COMDIR2=$COMOUT/${mem2}/atmos/INPUT #init/$mem2 + else + COMDIR2=$COMOUT/atmos/init/${mem2} + fi + mkdir -p $COMDIR2 + for tile in tile1 tile2 tile3 tile4 tile5 tile6; do + $NCP ${DATA}/out.sfc.${tile}.nc $COMDIR2/sfc_data.${tile}.nc + if [[ $SENDDBN = YES ]];then + $DBNROOT/bin/dbn_alert MODEL $DBNTYP $job $COMDIR2/sfc_data.${tile}.nc + fi + done + done + fi fi echo "$(date -u) end $(basename $BASH_SOURCE)" From a698abfc8cf29d9441dfec1318ad243f1fdeee25 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Thu, 6 Apr 2023 00:40:33 -0400 Subject: [PATCH 03/41] Update HOMEDIR on HERA On branch feature/v13_atmos_prep modified: rocoto/parm/setbase Refs: #87 --- rocoto/parm/setbase | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocoto/parm/setbase b/rocoto/parm/setbase index 6832c889f..385a21dde 100755 --- a/rocoto/parm/setbase +++ b/rocoto/parm/setbase @@ -29,7 +29,7 @@ if [ -d /scratch2/NCEPDEV ]; then echo "Using settings for Hera" export machine='HERA' - export HOMEDIR=${HOMEDIR:-/scratch2/NCEPDEV/ensemble/noscrub/common/git/global_workflow/gefs_v12.1.0} + export HOMEDIR=${HOMEDIR:-/scratch2/NCEPDEV/ensemble/noscrub/Xianwu.Xue/GEFS_v13/gw_walter} export HOMERFC=/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/ens_tracker export HOMEdata=/scratch2/NCEPDEV/ensemble/noscrub/common From dde793381f26e2910a2ab6e57ee5bae66df8eedc Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Thu, 6 Apr 2023 00:43:25 -0400 Subject: [PATCH 04/41] Add linked folders & files to the .gitignore On branch feature/v13_atmos_prep modified: fix/.gitignore modified: scripts/.gitignore modified: sorc/.gitignore modified: ush/.gitignore Refs: #87 --- fix/.gitignore | 8 ++++++++ scripts/.gitignore | 4 ++-- sorc/.gitignore | 2 +- ush/.gitignore | 14 +++++++++++++- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/fix/.gitignore b/fix/.gitignore index 7d7840a82..fed2309fb 100644 --- a/fix/.gitignore +++ b/fix/.gitignore @@ -1,2 +1,10 @@ +am +chem +orog +aer +lut +ugwd +wave + fix_* product diff --git a/scripts/.gitignore b/scripts/.gitignore index 1dc371977..8a2a76a17 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -1,3 +1,3 @@ exglobal_prep_chem.sh -exglobal_fcst_nemsfv3gfs.sh -exwave_*.sh +exglobal_forecast.sh +exgfs_wave_*.sh diff --git a/sorc/.gitignore b/sorc/.gitignore index 381bc98db..6112934ea 100644 --- a/sorc/.gitignore +++ b/sorc/.gitignore @@ -1,4 +1,4 @@ # Ignore logs directories logs/ global-workflow.fd - +ufs_model.fd diff --git a/ush/.gitignore b/ush/.gitignore index f745c3165..000874909 100644 --- a/ush/.gitignore +++ b/ush/.gitignore @@ -2,5 +2,17 @@ global_chgres.sh global_chgres_driver.sh gfs_nceppost.sh chgres_cube.sh -merge_fv3_chem_tile.py +merge_fv3_aerosol_tile.py wave_*.sh + +preamble.sh +cplvalidate.sh +forecast_predet.sh +forecast_det.sh +forecast_postdet.sh +nems_configure.sh +parsing_model_configure_FV3.sh +parsing_model_configure_DATM.sh + +parsing_namelists_* + From 76f558f6a5de757f95d5bdc6e42acb5a9f0f482c Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Thu, 6 Apr 2023 00:44:32 -0400 Subject: [PATCH 05/41] Remove parm/config from .gitignore On branch feature/v13_atmos_prep modified: sorc/link_gefs.sh Refs: #87 --- sorc/link_gefs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/link_gefs.sh b/sorc/link_gefs.sh index 6bdbf725b..2646e5b8e 100755 --- a/sorc/link_gefs.sh +++ b/sorc/link_gefs.sh @@ -117,7 +117,7 @@ if [[ -d global-workflow.fd ]] ; then fi $LINK ../sorc/global-workflow.fd/sorc/ufs_model.fd/FV3/upp/parm ../parm/post - for fn in parm_fv3diag product config; do + for fn in parm_fv3diag product; do echo ${fn} if [[ -d ../parm/${fn} ]]; then rm -rf ../parm/${fn} From b6af6b678fd74eaa9efcb58a4622c89913475f89 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Thu, 6 Apr 2023 15:51:58 -0400 Subject: [PATCH 06/41] Move some parm files to parm/parm_gefs On branch feature/v13_atmos_prep renamed: parm/gefs.parm -> parm/parm_gefs/gefs.parm renamed: parm/gefs_atmos_prep.parm -> parm/parm_gefs/gefs_atmos_prep.parm Refs: #87 --- parm/{ => parm_gefs}/gefs.parm | 0 parm/{ => parm_gefs}/gefs_atmos_prep.parm | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename parm/{ => parm_gefs}/gefs.parm (100%) rename parm/{ => parm_gefs}/gefs_atmos_prep.parm (100%) diff --git a/parm/gefs.parm b/parm/parm_gefs/gefs.parm similarity index 100% rename from parm/gefs.parm rename to parm/parm_gefs/gefs.parm diff --git a/parm/gefs_atmos_prep.parm b/parm/parm_gefs/gefs_atmos_prep.parm similarity index 100% rename from parm/gefs_atmos_prep.parm rename to parm/parm_gefs/gefs_atmos_prep.parm From 9438ef79ca8e60b1fdcaf326e8d514073cd49a57 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Thu, 6 Apr 2023 15:53:19 -0400 Subject: [PATCH 07/41] Modify parm files for atmos_prep On branch feature/v13_atmos_prep modified: parm/parm_gefs/gefs.parm modified: parm/parm_gefs/gefs_atmos_prep.parm Refs: #87 --- parm/parm_gefs/gefs.parm | 106 ++++++++++++++++++++++------ parm/parm_gefs/gefs_atmos_prep.parm | 16 ++--- 2 files changed, 92 insertions(+), 30 deletions(-) diff --git a/parm/parm_gefs/gefs.parm b/parm/parm_gefs/gefs.parm index b645e8d11..d26ffe90d 100755 --- a/parm/parm_gefs/gefs.parm +++ b/parm/parm_gefs/gefs.parm @@ -1,4 +1,4 @@ -#!/bin/ksh -l +#! /usr/bin/env bash ################################################################### # This parm file defines the control variables for the GEFS model @@ -31,20 +31,20 @@ export fhrp=${fhrp:-06} export gefs_cych=${gefs_cych:-6} case $cyc in - (00) - fhmax=$fhmax00 ;; - (06) - fhmax=$fhmax06 ;; - (12) - fhmax=$fhmax12 ;; - (18) - fhmax=$fhmax18 ;; - (*) - echo - echo UNDEFINED CYCLE cyc=$cyc - echo - echo unmodified fhmax=$fhmax - ;; + 00) + fhmax=$fhmax00;; + 06) + fhmax=$fhmax06;; + 12) + fhmax=$fhmax12;; + 18) + fhmax=$fhmax18;; + *) + echo + echo UNDEFINED CYCLE cyc=$cyc + echo + echo unmodified fhmax=$fhmax + ;; esac memberlist="c00" @@ -52,8 +52,8 @@ memberlist="c00" imem=1 while [[ imem -le $npert ]]; do - memberlist="$memberlist p$(printf %02i $imem)" - (( imem++ )) + memberlist="$memberlist p$(printf %02i $imem)" + (( imem++ )) done export memberlist @@ -96,7 +96,7 @@ export FHOUT_HF_WAV=${FHOUTHF:-3} ####################################### #For the half-month range part of FV3 based forecast export CASEHR=${CASEHR:-C384} -export LEVSHR=${LEVSHR:-64} +export LEVSHR=${LEVSHR:-127} export DELTIM=${DELTIM:-450} export JCAPFV=${JCAPFV:-766} export LATBFV=${LATBFV:-768} @@ -105,7 +105,7 @@ export MTNRSLFV=${MTNRSLFV:-766.1536.768} #For the longer range part of FV3 based forecast export CASELR=${CASELR:-C384} -export LEVSLR=${LEVSLR:-64} +export LEVSLR=${LEVSLR:-127} export DELTIMLR=${DELTIMLR:-450} export JCAPLR=${JCAPLR:-766} export LATBLR=${LATBLR:-768} @@ -113,24 +113,86 @@ export LONBLR=${LONBLR:-1536} export MTNRSLLR=${MTNRSLLR:-766.1536.768} export NTRACLR=${NTRACLR:-3} +export FORECAST_SEGMENT=${FORECAST_SEGMENT:-"hr"} +if [[ ${FORECAST_SEGMENT} == "hr" ]]; then + export LEVS=$((LEVSHR + 1)) +else + export LEVS=$((LEVSLR + 1)) +fi + +CASE=${CASEHR} +if [[ ${FORECAST_SEGMENT} == "lr" ]]; then + CASE=$CASELR +fi + #for new vertical coordinate export IDVC=${IDVC:-2} #export IDVM=32 -export imp_physics=${imp_physics:-"11"} +#################################### +# Directories relative to installation areas +#################################### +export HOMEgefs=${HOMEgefs:-${PACKAGEROOT}/gefs.${gefs_ver}} +export EXECgefs=${EXECgefs:-${HOMEgefs}/exec} +export USHgefs=${USHgefs:-${HOMEgefs}/ush} +#export FIXgefs=${FIXgefs:-${HOMEgefs}/fix/fix_gefs} +export PARMgefs=${PARMgefs:-${HOMEgefs}/parm} + +export HOMEgfs=${HOMEgfs:-${HOMEgefs}} +export PARMgfs=${HOMEgfs}/parm +export EXPDIR=${PARMgfs}/config +#export FIXgfs=${HOMEgfs}/fix +#export USHgfs=${HOMEgfs}/ush +#export UTILgfs=${HOMEgfs}/util +#export EXECgfs=${HOMEgfs}/exec +#export SCRgfs=${HOMEgfs}/scripts + +#export FIXcice=${HOMEgfs}/fix/cice +#export FIXmom=${HOMEgfs}/fix/mom6 +#export FIXreg2grb2=${HOMEgfs}/fix/reg2grb2 + +#################################### +# CONVENIENT utility scripts and other environment parameters +export NCP="/bin/cp -p" +export NMV="/bin/mv" +export NLN="/bin/ln -sf" +export KEEPDATA=${KEEPDATA:-"NO"} + +#################################### +# Specify NET and RUN Name and model +#################################### +export NET=${NET:-gefs} +export envir=${envir:-prod} +export RUN=${RUN:-gefs} + +###################################### +# SENDCOM - Copy Files From TMPDIR to $COMOUT +# SENDDBN - Issue DBNet Client Calls +# SENDECF - Flag Events on ECF +# VERBOSE - Specify Verbose Output +###################################### +export SENDECF=${SENDECF:-"YES"} +export SENDCOM=${SENDCOM:-"YES"} +export SENDDBN_NTC=${SENDDBN_NTC:-"NO"} +export SENDDBN=${SENDDBN:-"YES"} +export DBNROOT=${DBNROOT:-${UTILROOT}/fakedbn} +export VERBOSE=${VERBOSE:-YES} + +# Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL +export imp_physics=${imp_physics:-"8"} ######################################## # This section defines the output flags for ncep_post + prdgen ######################################## export FV3OUT_GRID="gaussian_grid" export GRIBVERSION=grib2 -export REMAP_GRID=${REMAP_GRID:-latlon} #gaussian or latlon for using regrid or regrid_nemsio for remapping +export REMAP_GRID=${REMAP_GRID:-latlon} #gaussian or latlon for using regrid or regrid_nemsio for remapping export gfsfhmaxh=240 export gfsfhoroglist=252 # The following options are fore re-forecast -export save_pgrb2_p5=${save_pgrb2_p5:-NO} +export save_pgrb2_p5=${save_pgrb2_p5:-NO} export save_pgrb2_p25=${save_pgrb2_p25:-NO} # List of contacts to mail when there is mail sent by mail.py for product degredation diff --git a/parm/parm_gefs/gefs_atmos_prep.parm b/parm/parm_gefs/gefs_atmos_prep.parm index 4faac30d7..60253457e 100644 --- a/parm/parm_gefs/gefs_atmos_prep.parm +++ b/parm/parm_gefs/gefs_atmos_prep.parm @@ -1,15 +1,15 @@ -#!/bin/ksh +#! /usr/bin/env bash case $cyc in - 00) memshift=0;; - 06) memshift=20;; - 12) memshift=40;; - 18) memshift=60;; + 00) memshift=0;; + 06) memshift=20;; + 12) memshift=40;; + 18) memshift=60;; esac -export pdycycp=$($NDATE -$fhrp $PDY$cyc) -export pdyp=$(echo $pdycycp|cut -c1-8) -export cycp=$(echo $pdycycp|cut -c9-10) +#export pdycycp=$($NDATE -$fhrp $PDY$cyc) +#export pdyp=$(echo $pdycycp|cut -c1-8) +#export cycp=$(echo $pdycycp|cut -c9-10) export CDATE=$PDY$cyc From 6e3ee10948f410c4522c9c7a3a0f79e34cb137e9 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Thu, 6 Apr 2023 16:48:38 -0400 Subject: [PATCH 08/41] Improve to make atmos_prep work On branch feature/v13_atmos_prep modified: jobs/JGEFS_ATMOS_PREP modified: rocoto/user_full.conf modified: ush/gefs_atmos_prep.sh modified: versions/run.ver Refs: #87 --- jobs/JGEFS_ATMOS_PREP | 2 +- rocoto/user_full.conf | 21 +++++++++++---------- ush/gefs_atmos_prep.sh | 12 ++---------- versions/run.ver | 4 ++-- 4 files changed, 16 insertions(+), 23 deletions(-) diff --git a/jobs/JGEFS_ATMOS_PREP b/jobs/JGEFS_ATMOS_PREP index 60fe1bc9b..1809435b7 100755 --- a/jobs/JGEFS_ATMOS_PREP +++ b/jobs/JGEFS_ATMOS_PREP @@ -73,7 +73,7 @@ export COMPONENT="atmos" # Source relevant config files ############################# configs="gefs gefs_atmos_prep" -config_path=$PARMgefs +config_path=$PARMgefs/parm_gefs for config in $configs; do . $config_path/$config.parm export err=$? diff --git a/rocoto/user_full.conf b/rocoto/user_full.conf index fd4f43cfb..d47cd5928 100755 --- a/rocoto/user_full.conf +++ b/rocoto/user_full.conf @@ -1,8 +1,8 @@ #SOURCEDIR = /gpfs/h2/emc/modeling/save/Dingchen.Hou/GIT/GEFS # -SDATE = 2022072100 -EDATE = 2022072118 -npert = 30 +SDATE = 2022012112 +EDATE = 2022012112 +npert = 2 INCYC = 6 #ACCOUNT = GEN-T2O #CUE2RUN = dev @@ -123,13 +123,14 @@ saveflux = yes savesfcsig = no sigzvd = no # tempororay setting for the low-resolution +CASEHR = C384 #C48 #C384 CASELR = C384 -LEVSLR = 64 -DELTIMLR = 450 -JCAPLR = 766 -LATBLR = 768 -LONBLR = 1536 -MTNRSLLR = 766.1536.768 +#LEVSLR = 127 +#DELTIMLR = 450 +#JCAPLR = 766 +#LATBLR = 768 +#LONBLR = 1536 +#MTNRSLLR = 766.1536.768 #RFHOME = /gpfs/h2/nco/storage/fv3gefs #enslistend = "avg spr" #SSTDIR = $HOMEdata/2tsst @@ -169,7 +170,7 @@ RUN_CLEANUP = NO #taskname = wave_prep #taskname = rf_prep #taskname = prdgen_gfs -#taskname = atmos_prep +taskname = atmos_prep #taskname = init_recenter #taskname = chem_prep_emissions #taskname = chem_init diff --git a/ush/gefs_atmos_prep.sh b/ush/gefs_atmos_prep.sh index 5ea31b79d..b773bbbac 100755 --- a/ush/gefs_atmos_prep.sh +++ b/ush/gefs_atmos_prep.sh @@ -14,11 +14,7 @@ export nmem=$(echo $mem|cut -c 2-) nmem=${nmem#0} #export INIDIR=$DATA -if [[ ${NewCOM} == "YES" ]]; then - export OUTDIR=${COMOUT}/$mem/atmos/INPUT -else - export OUTDIR=${COMOUT}/atmos/init/${mem} -fi +export OUTDIR=${COMOUT}/$mem/atmos/INPUT mkdir -p $OUTDIR cd $DATA @@ -162,11 +158,7 @@ if [[ $SENDCOM == "YES" ]]; then if [[ $CONVERT_SFC == ".true." ]]; then for mem2 in $memberlist; do - if [[ ${NewCOM} == "YES" ]]; then - COMDIR2=$COMOUT/${mem2}/atmos/INPUT #init/$mem2 - else - COMDIR2=$COMOUT/atmos/init/${mem2} - fi + COMDIR2=$COMOUT/${mem2}/atmos/INPUT #init/$mem2 mkdir -p $COMDIR2 for tile in tile1 tile2 tile3 tile4 tile5 tile6; do $NCP ${DATA}/out.sfc.${tile}.nc $COMDIR2/sfc_data.${tile}.nc diff --git a/versions/run.ver b/versions/run.ver index c4db74bb5..197a7b9ba 100755 --- a/versions/run.ver +++ b/versions/run.ver @@ -1,6 +1,6 @@ -export gefs_ver=v12.3.6 +export gefs_ver=v13.0.0 -export gfs_ver=v16.3 +export gfs_ver=v17.0 export cfs_ver=v2.3 export ecmwf_ver=v2.1 export nawips_ver=v0.0 From c258966dfd02cb843ca9b8b8867e39122451d0b8 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Thu, 6 Apr 2023 17:13:35 -0400 Subject: [PATCH 09/41] Use preamble.sh in j-job of atmos_prep On branch feature/v13_atmos_prep modified: jobs/JGEFS_ATMOS_PREP Refs: #87 --- jobs/JGEFS_ATMOS_PREP | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/jobs/JGEFS_ATMOS_PREP b/jobs/JGEFS_ATMOS_PREP index 1809435b7..2179b2b54 100755 --- a/jobs/JGEFS_ATMOS_PREP +++ b/jobs/JGEFS_ATMOS_PREP @@ -1,13 +1,6 @@ -#!/bin/bash +#! /usr/bin/env bash -echo "$(date -u) begin $(basename $BASH_SOURCE)" - -set -xa -if [[ ${STRICT:-NO} == "YES" ]]; then - # Turn on strict bash error checking - set -eu -fi -export PS4='$SECONDS + $(basename $(basename $BASH_SOURCE))[$LINENO] ' +source "${HOMEgfs:-${HOMEgefs}}/ush/preamble.sh" #################################### # Determine Job Output Name on System From 250810510406e5c7d7ae5d59374315c091c78250 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Thu, 6 Apr 2023 19:56:39 -0400 Subject: [PATCH 10/41] Use generate_com to get COM_* for atmos_prep job On branch feature/v13_atmos_prep modified: jobs/JGEFS_ATMOS_PREP new file: parm/config/config.com modified: parm/parm_gefs/gefs.parm modified: parm/parm_gefs/gefs_atmos_prep.parm modified: scripts/exgefs_atmos_prep.sh modified: ush/gefs_atmos_prep.sh Refs: #87 --- jobs/JGEFS_ATMOS_PREP | 1 + parm/config/config.com | 92 +++++++++++++++++++++++++++++ parm/parm_gefs/gefs.parm | 4 ++ parm/parm_gefs/gefs_atmos_prep.parm | 1 + scripts/exgefs_atmos_prep.sh | 14 +++-- ush/gefs_atmos_prep.sh | 25 ++++---- 6 files changed, 120 insertions(+), 17 deletions(-) create mode 100644 parm/config/config.com diff --git a/jobs/JGEFS_ATMOS_PREP b/jobs/JGEFS_ATMOS_PREP index 2179b2b54..15340326f 100755 --- a/jobs/JGEFS_ATMOS_PREP +++ b/jobs/JGEFS_ATMOS_PREP @@ -76,6 +76,7 @@ for config in $configs; do fi done +export ROTDIR=${COMOUT:-$(compath.py -o $NET/${ver})} export COMOUT=${COMOUT:-$(compath.py -o $NET/${ver})/${RUN}.${PDY}/${cyc}} export COMINgfs=${COMINgfs:-$(compath.py ${envir}/com/gfs/${gfs_ver})/gfs.${PDY}/${cyc}} diff --git a/parm/config/config.com b/parm/config/config.com new file mode 100644 index 000000000..2308658fe --- /dev/null +++ b/parm/config/config.com @@ -0,0 +1,92 @@ +# shellcheck shell=bash +echo "BEGIN: config.com" + +# These are just templates. All templates must use single quotations so variable +# expansion does not occur when this file is sourced. Substitution happens later +# during runtime. It is recommended to use the helper function `generate_com()`, +# to do this substitution, which is defined in `ush/preamble.sh`. +# +# Syntax for generate_com(): +# generate_com [-rx] $var1[:$tmpl1] [$var2[:$tmpl2]] [...]] +# +# options: +# -r: Make variable read-only (same as `decalre -r`) +# -x: Mark variable for declare -rx (same as `declare -x`) +# var1, var2, etc: Variable names whose values will be generated from a template +# and declared +# tmpl1, tmpl2, etc: Specify the template to use (default is "${var}_TMPL") +# +# Examples: +# # Current cycle and RUN +# YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_ANALYSIS +# +# # Previous cycle and gdas +# RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \ +# COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL +# +# # Current cycle and COM for first member +# MEMDIR='mem001' YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_HISTORY +# + +# +# If any restart, input, or analysis template is updated, `setup_expt.py.fill_COMROT_cycled()` +# must correspondingly be updated to match. +# + +# Ignore shellcheck warnings about variables not being expanded; this is what we want +# shellcheck disable=SC2016 +if [[ "${RUN_ENVIR:-emc}" == "nco" ]]; then + COM_OBS_TMPL=$(compath.py "${envir}/obsproc/${obsproc_ver}")'/${RUN}.${YMD}/${HH}/atmos' + COM_RTOFS_TMPL=$(compath.py "${envir}/${WAVECUR_DID}/${rtofs_ver}") +else + COM_OBS_TMPL='${ROTDIR}/${RUN}.${YMD}/${HH}/obs' + COM_RTOFS_TMPL='${DMPDIR}' +fi +declare -rx COM_OBS_TMPL COM_RTOFS_TMPL +declare -rx COM_OBSDMP_TMPL='${DMPDIR}/${DUMP}${DUMP_SUFFIX}.${YMD}/${HH}/atmos' + +COM_BASE='${ROTDIR}/${RUN}.${YMD}/${HH}/${MEMDIR}' + +declare -rx COM_TOP_TMPL='${ROTDIR}/${RUN}.${YMD}/${HH}' + +declare -rx COM_ATMOS_INPUT_TMPL=${COM_BASE}'/model_data/atmos/input' +declare -rx COM_ATMOS_RESTART_TMPL=${COM_BASE}'/model_data/atmos/restart' +declare -rx COM_ATMOS_ANALYSIS_TMPL=${COM_BASE}'/analysis/atmos' +declare -rx COM_ATMOS_HISTORY_TMPL=${COM_BASE}'/model_data/atmos/history' +declare -rx COM_ATMOS_MASTER_TMPL=${COM_BASE}'/model_data/atmos/master' +declare -rx COM_ATMOS_GRIB_TMPL=${COM_BASE}'/products/atmos/grib/${RES}' +declare -rx COM_ATMOS_BUFR_TMPL=${COM_BASE}'/products/atmos/bufr' +declare -rx COM_ATMOS_GEMPAK_TMPL=${COM_BASE}'/products/atmos/gempak' +declare -rx COM_ATMOS_GENESIS_TMPL=${COM_BASE}'/products/atmos/cyclone/genesis_vital' +declare -rx COM_ATMOS_TRACK_TMPL=${COM_BASE}'/products/atmos/cyclone/tracks' +declare -rx COM_ATMOS_GOES_TMPL=${COM_BASE}'/products/atmos/goes_sim' +declare -rx COM_ATMOS_IMAGERY_TMPL=${COM_BASE}'/products/atmos/imagery' +declare -rx COM_ATMOS_MINMON_TMPL=${COM_BASE}'/products/atmos/minmon' +declare -rx COM_ATMOS_WAFS_TMPL=${COM_BASE}'/products/atmos/wafs' +declare -rx COM_ATMOS_WMO_TMPL=${COM_BASE}'/products/atmos/wmo' + +declare -rx COM_WAVE_RESTART_TMPL=${COM_BASE}'/model_data/wave/restart' +declare -rx COM_WAVE_PREP_TMPL=${COM_BASE}'/model_data/wave/prep' +declare -rx COM_WAVE_HISTORY_TMPL=${COM_BASE}'/model_data/wave/history' +declare -rx COM_WAVE_GRID_TMPL=${COM_BASE}'/products/wave/gridded' +declare -rx COM_WAVE_STATION_TMPL=${COM_BASE}'/products/wave/station' +declare -rx COM_WAVE_GEMPAK_TMPL=${COM_BASE}'/products/wave/gempak' +declare -rx COM_WAVE_WMO_TMPL=${COM_BASE}'/products/wave/wmo' + +declare -rx COM_OCEAN_HISTORY_TMPL=${COM_BASE}'/model_data/ocean/history' +declare -rx COM_OCEAN_RESTART_TMPL=${COM_BASE}'/model_data/ocean/restart' +declare -rx COM_OCEAN_INPUT_TMPL=${COM_BASE}'/model_data/ocean/input' +declare -rx COM_OCEAN_ANALYSIS_TMPL=${COM_BASE}'/analysis/ocean' +declare -rx COM_OCEAN_2D_TMPL=${COM_BASE}'/products/ocean/2D' +declare -rx COM_OCEAN_3D_TMPL=${COM_BASE}'/products/ocean/3D' +declare -rx COM_OCEAN_DAILY_TMPL=${COM_BASE}'/products/ocean/daily' +declare -rx COM_OCEAN_XSECT_TMPL=${COM_BASE}'/products/ocean/xsect' +declare -rx COM_OCEAN_GRIB_TMPL=${COM_BASE}'/products/ocean/grib/${RES}' + +declare -rx COM_ICE_INPUT_TMPL=${COM_BASE}'/model_data/ice/input' +declare -rx COM_ICE_HISTORY_TMPL=${COM_BASE}'/model_data/ice/history' +declare -rx COM_ICE_RESTART_TMPL=${COM_BASE}'/model_data/ice/restart' + +declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model_data/chem/history' + +declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model_data/med/restart' diff --git a/parm/parm_gefs/gefs.parm b/parm/parm_gefs/gefs.parm index d26ffe90d..304509acd 100755 --- a/parm/parm_gefs/gefs.parm +++ b/parm/parm_gefs/gefs.parm @@ -178,6 +178,10 @@ export SENDDBN=${SENDDBN:-"YES"} export DBNROOT=${DBNROOT:-${UTILROOT}/fakedbn} export VERBOSE=${VERBOSE:-YES} +# Get all the COM path templates +source "${PARMgefs}/config/config.com" + + # Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL export imp_physics=${imp_physics:-"8"} diff --git a/parm/parm_gefs/gefs_atmos_prep.parm b/parm/parm_gefs/gefs_atmos_prep.parm index 60253457e..453410d8a 100644 --- a/parm/parm_gefs/gefs_atmos_prep.parm +++ b/parm/parm_gefs/gefs_atmos_prep.parm @@ -6,6 +6,7 @@ case $cyc in 12) memshift=40;; 18) memshift=60;; esac +export memshift #export pdycycp=$($NDATE -$fhrp $PDY$cyc) #export pdyp=$(echo $pdycycp|cut -c1-8) diff --git a/scripts/exgefs_atmos_prep.sh b/scripts/exgefs_atmos_prep.sh index a85819a5a..f88f48236 100755 --- a/scripts/exgefs_atmos_prep.sh +++ b/scripts/exgefs_atmos_prep.sh @@ -1,12 +1,14 @@ #! /usr/bin/env bash -echo "$(date -u) begin $(basename $BASH_SOURCE)" +source "${HOMEgfs:-${HOMEgefs}}/ush/preamble.sh" -set -xa -if [[ ${STRICT:-NO} == "YES" ]]; then - # Turn on strict bash error checking - set -eu -fi +#echo "$(date -u) begin $(basename $BASH_SOURCE)" + +#set -xa +#if [[ ${STRICT:-NO} == "YES" ]]; then +# # Turn on strict bash error checking +# set -eu +#fi export HOMEgfs=${HOMEgfs:-${HOMEgefs}} export HOMEufs=${HOMEufs:-${HOMEgfs}} diff --git a/ush/gefs_atmos_prep.sh b/ush/gefs_atmos_prep.sh index b773bbbac..74f80bb94 100755 --- a/ush/gefs_atmos_prep.sh +++ b/ush/gefs_atmos_prep.sh @@ -1,20 +1,24 @@ #! /usr/bin/env bash -echo "$(date -u) begin $(basename $BASH_SOURCE)" -export PS4="${PS4}${1}: " +source "${HOMEgfs:-${HOMEgefs}}/ush/preamble.sh" -set -xa -if [[ ${STRICT:-NO} == "YES" ]]; then - # Turn on strict bash error checking - set -eu -fi +#echo "$(date -u) begin $(basename $BASH_SOURCE)" +#export PS4="${PS4}${1}: " + +#set -xa +#if [[ ${STRICT:-NO} == "YES" ]]; then +# # Turn on strict bash error checking +# set -eu +#fi export mem=$1 export nmem=$(echo $mem|cut -c 2-) nmem=${nmem#0} +YMD=${PDY} HH=${cyc} MEMDIR=${mem} generate_com -x COM_ATMOS_INPUT + #export INIDIR=$DATA -export OUTDIR=${COMOUT}/$mem/atmos/INPUT +export OUTDIR=${COM_ATMOS_INPUT} # ${COMOUT}/$mem/atmos/INPUT mkdir -p $OUTDIR cd $DATA @@ -114,8 +118,6 @@ export VCOORD_FILE=${VCOORD_FILE:-$FIXam/global_hyblev.l${LEVS}.txt} export TRACERS_INPUT="spfh","clwmr","o3mr","icmr","rwmr","snmr","grle" export TRACERS_TARGET="sphum","liq_wat","o3mr","ice_wat","rainwat","snowwat","graupel" -#export TRACERS_TARGET='"sphum","liq_wat","o3mr","ice_wat","rainwat","snowwat","graupel"' -#export TRACERS_INPUT='"sphum","liq_wat","o3mr","ice_wat","rainwat","snowwat","graupel"' OROG_FILES_TARGET_GRID='' for tile in {1..6} @@ -158,7 +160,8 @@ if [[ $SENDCOM == "YES" ]]; then if [[ $CONVERT_SFC == ".true." ]]; then for mem2 in $memberlist; do - COMDIR2=$COMOUT/${mem2}/atmos/INPUT #init/$mem2 + YMD=${PDY} HH=${cyc} MEMDIR=${mem2} generate_com -x COM_ATMOS_INPUT + COMDIR2=${COM_ATMOS_INPUT} #${echo "${!COM_ATMOS_INPUT_TMPL}"} # $(echo "${!COM_ATMOS_INPUT_TMPL}" | envsubst) #${COM_ATMOS_INPUT} #$COMOUT/${mem2}/atmos/INPUT #init/$mem2 mkdir -p $COMDIR2 for tile in tile1 tile2 tile3 tile4 tile5 tile6; do $NCP ${DATA}/out.sfc.${tile}.nc $COMDIR2/sfc_data.${tile}.nc From ed4a5d634d119ea17ca7d6d591e06b264933235a Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 10 Apr 2023 11:40:45 -0400 Subject: [PATCH 11/41] Modify to port atmos_prep to hera On branch feature/v13_atmos_prep modified: rocoto/bin/hera/atmos_prep.sh modified: rocoto/bin/hera/common.sh Refs: #87 --- rocoto/bin/hera/atmos_prep.sh | 28 ++++++++++++++++++++++++---- rocoto/bin/hera/common.sh | 16 +++++++++++----- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/rocoto/bin/hera/atmos_prep.sh b/rocoto/bin/hera/atmos_prep.sh index e11360ddc..bb6eabb3e 100755 --- a/rocoto/bin/hera/atmos_prep.sh +++ b/rocoto/bin/hera/atmos_prep.sh @@ -3,12 +3,32 @@ # EXPORT list here set -x -export IOBUF_PARAMS= - - +#export IOBUF_PARAMS= ulimit -s unlimited ulimit -a +# module_ver.h +. $GEFS_ROCOTO/dev/versions/run_hera.ver + +# Load modules +. /apps/lmod/lmod/init/ksh +module list +module purge + +module load EnvVars/$EnvVars_ver +module load intel/$ips_ver +module load impi/$impi_ver +module load prod_util/$prod_util_ver +module load prod_envir/$prod_envir_ver +module load hdf5_parallel/$HDF5_parallel_ver +module load netcdf_parallel/$NetCDF_parallel_ver + +module list + +# For Development +. $GEFS_ROCOTO/bin/hera/common.sh + + export MP_SHARED_MEMORY=yes export MEMORY_AFFINITY=core:2 @@ -17,7 +37,7 @@ export MEMORY_AFFINITY=core:2 #export OMP_NUM_THREADS=2 #export taskspernode=12 -export FORECAST_SEGMENT=hr +#export FORECAST_SEGMENT=hr # Export List (( OMP_NUM_THREADS_CH = 40 / GEFS_PPN )) diff --git a/rocoto/bin/hera/common.sh b/rocoto/bin/hera/common.sh index fb8009b68..8ba3408bd 100755 --- a/rocoto/bin/hera/common.sh +++ b/rocoto/bin/hera/common.sh @@ -31,17 +31,23 @@ export NTASKS=$SLURM_NTASKS export outid="LL$job" export jobid="${outid}.o${pid}" export pgmout="OUTPUT.${pid}" +export LSB_JOBID=${pid} #/\ Only for Hera -export gefsmpexec="srun -n $NTASKS" -export gefsmpexec_mpmd="srun -n $NTASKS /scratch2/NCEPDEV/ensemble/noscrub/common/soft/mpiserial.cd/mpiserial" +export gefsmpexec="srun -n $total_tasks" +export gefsmpexec_mpmd="srun -n $total_tasks /scratch2/NCEPDEV/ensemble/noscrub/common/soft/mpiserial.cd/mpiserial" export wavempexec="srun -n" export wave_mpmd="/scratch2/NCEPDEV/ensemble/noscrub/common/soft/mpiserial.cd/mpiserial" # -m" #export errchk="eval if [[ \$err != 0 ]]; then exit \$err; fi" -export APRUNC="srun -n 1" -export APRUN_RECENT="srun -n $NTASKS" +#export APRUNC="srun -n 1" +#export APRUN_RECENT="srun -n $NTASKS" export APRUN_CHGRES="srun -n 1" -export aprun_gec00="srun -n 1" +#export aprun_gec00="srun -n 1" export APRUN_CALCINC="srun -n 1" + +. $GEFS_ROCOTO/parm/setbase +. $GEFS_ROCOTO/parm/gefs_config +. $GEFS_ROCOTO/parm/gefs_dev.parm + From 203f6399ef09ab6d235aa18a8b6d6d7c1a26653c Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 10 Apr 2023 11:47:10 -0400 Subject: [PATCH 12/41] Change the default value of CASEHR to C48 On branch feature/v13_atmos_prep modified: rocoto/user_full.conf Refs: #87 --- rocoto/user_full.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocoto/user_full.conf b/rocoto/user_full.conf index d47cd5928..4b4b878f0 100755 --- a/rocoto/user_full.conf +++ b/rocoto/user_full.conf @@ -123,7 +123,7 @@ saveflux = yes savesfcsig = no sigzvd = no # tempororay setting for the low-resolution -CASEHR = C384 #C48 #C384 +CASEHR = C48 #C384 #C48 #C384 CASELR = C384 #LEVSLR = 127 #DELTIMLR = 450 From c5673d9c7377627acd84b5208e7501ca30d43ed6 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 10 Apr 2023 11:58:51 -0400 Subject: [PATCH 13/41] Fix a minor bug for ROTDIR On branch feature/v13_atmos_prep modified: jobs/JGEFS_ATMOS_PREP Refs: #87 --- jobs/JGEFS_ATMOS_PREP | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/JGEFS_ATMOS_PREP b/jobs/JGEFS_ATMOS_PREP index 15340326f..78d8949d4 100755 --- a/jobs/JGEFS_ATMOS_PREP +++ b/jobs/JGEFS_ATMOS_PREP @@ -76,7 +76,7 @@ for config in $configs; do fi done -export ROTDIR=${COMOUT:-$(compath.py -o $NET/${ver})} +export ROTDIR=${ROTDIR:-$(compath.py -o $NET/${ver})} export COMOUT=${COMOUT:-$(compath.py -o $NET/${ver})/${RUN}.${PDY}/${cyc}} export COMINgfs=${COMINgfs:-$(compath.py ${envir}/com/gfs/${gfs_ver})/gfs.${PDY}/${cyc}} From 6a6392e4bb70b159a5aba5de4e88748e480d3bfe Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 10 Apr 2023 12:19:25 -0400 Subject: [PATCH 14/41] Modify for running atmos_prep on hera On branch feature/v13_atmos_prep modified: rocoto/dev/versions/run_hera.ver modified: rocoto/parm/gefs_config Refs: #87 --- rocoto/dev/versions/run_hera.ver | 4 ++- rocoto/parm/gefs_config | 54 ++++++++++++++++++++++++-------- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/rocoto/dev/versions/run_hera.ver b/rocoto/dev/versions/run_hera.ver index 5eafe5b21..36297c05e 100755 --- a/rocoto/dev/versions/run_hera.ver +++ b/rocoto/dev/versions/run_hera.ver @@ -1,4 +1,6 @@ -export gefs_ver=v12.3.1 +export gefs_ver=v13.0.0 +export gfs_ver=v17.0 +export cfs_ver=v2.3 export hpc_ver=1.2.0 diff --git a/rocoto/parm/gefs_config b/rocoto/parm/gefs_config index 82846d56f..a53f8de8a 100644 --- a/rocoto/parm/gefs_config +++ b/rocoto/parm/gefs_config @@ -21,13 +21,16 @@ if [[ $machine == "WCOSS2" ]]; then export COMROOT=${WORKDIR}/$envir/com export DATAROOT=$baseoutput/tmp else - export COMROOT=$baseoutput/com + export COMROOT=$baseoutput/dev/com export GESROOT=$baseoutput/nwges - export DATAROOT=$baseoutput/tmpnwprd - export DCOMROOT=${DCOMROOT:-/dcom} - export PCOMROOT=$baseoutput/pcom/$envir + export DATAROOT=$baseoutput/tmp + #export DCOMROOT=${DCOMROOT:-/dcom} + #export PCOMROOT=$baseoutput/pcom/$envir + + #export GESROOTp1=$GESROOTp1 - export GESROOTp1=$GESROOTp1 + #export GESOUT=$baseoutput/com/ + #export COMOUT=$baseoutput/com fi export archsyndir=${archsyndir:-$COMROOTp1/arch/prod/syndat} @@ -49,6 +52,10 @@ fi GFSArchivedData=yes #GFSArchivedData=no +if [[ $WHERE_AM_I == "hera" ]]; then + GFSArchivedData=yes + UseHOMEdata=yes +fi #Please check with parm/gefs_init.parm to make sure the next line matching fhrp=6 @@ -62,15 +69,36 @@ else if [[ $GFSArchivedData == "yes" ]]; then # HOMEdata, COMINgfs_base, COMINenkf_base and COMINcfs_base are defined in setbase, pointing to the NCO gfs.v15 parallel # However, if you use gfs/enkf data from archived data, you may speficfy them in this "if" block and make GFSArchivedData=yes - UseHOMEdata=no + UseHOMEdata=${UseHOMEdata:-no} if [[ $UseHOMEdata == "yes" ]]; then - HOMEdata=/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS/HOMEdata/canned/com - export COMINenkf_base=$HOMEdata/fv3init/enkf.gdas. - export COMINgfs_base=$HOMEdata/fv3init/gfs. - export COMINcfs_base=$HOMEdata/cfs/cfs. - export COMIN_WAV_ICE=$HOMEdata/sea_ice - export COMINgfs=${COMINgfs:-${COMINgfs_base}${PDY}/$cyc} - export COMINenkf=${COMINenkf:-${COMINenkf_base}${pdyp}/$cycp} + if [[ $WHERE_AM_I == "hera" ]]; then + source dev/versions/run_hera.ver + export HOMEdata=/scratch2/NCEPDEV/ensemble/noscrub/Xianwu.Xue/GEFS_v13/z_DATA/lfs_C96_C48_NOIAU_anal_20230114/h1/ops/prod/com + export COMINgfs_base=${HOMEdata}/gfs/${gfs_ver} + #export COMINenkf_base=${HOMEdata}/com/gfs/v16.2/enkfgdas. + export COMINgfs=${COMINgfs:-${COMINgfs_base}/gfs.${PDY}/${cyc}} + #export COMINenkf=${COMINenkf:-${COMINenkf_base}${pdyp}/$cycp/atmos} + export COMINenkfgfs=${COMINenkfgfs:-${COMINgfs_base}/enkfgfs.${PDY}/${cyc}} + + export COMINcfs_base=${HOMEdata}/com/cfs/v2.3/cfs. + export DCOMROOT=$HOMEdata/dcom + + # For gempak_meta + export COMINsgfs=${HOMEdata}/com/gfs/v16.2 #${COMINsgfs:-$(compath.py gfs/prod)} + export COMINnawips="NONE" #${COMINnawips:-$(compath.py nawips/prod)} + export COMINecmwf="NONE" #${COMINecmwf:-$(compath.py ecmwf/prod/ecmwf)} + export COMINukmet="NONE" #${COMINukmet:-$(compath.py nawips/prod/ukmet)} + export COMINnam="NONE" #${COMINnam:-$(compath.py nam/prod)} + + else + HOMEdata=/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS/HOMEdata/canned/com + export COMINenkf_base=$HOMEdata/fv3init/enkf.gdas. + export COMINgfs_base=$HOMEdata/fv3init/gfs. + export COMINcfs_base=$HOMEdata/cfs/cfs. + export COMIN_WAV_ICE=$HOMEdata/sea_ice + export COMINgfs=${COMINgfs:-${COMINgfs_base}${PDY}/$cyc} + export COMINenkf=${COMINenkf:-${COMINenkf_base}${pdyp}/$cycp} + fi else HOMEdata=/gpfs/h2/emc/modeling/noscrub/Xianwu.Xue/gw_for_acorn/HOMEdata export COMPATH=$HOMEdata/com/gfs/prod From b2c62ba4468368011950cf4f5fd17d1526b4abda Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 10 Apr 2023 22:37:54 -0400 Subject: [PATCH 15/41] Optimize pyGEFS On branch feature/v13_atmos_prep modified: rocoto/py/GEFS_Crontab.py modified: rocoto/py/GEFS_Parm.py modified: rocoto/py/GEFS_UserConfig.py modified: rocoto/py/GEFS_XML.py modified: rocoto/py/GEFS_XML_For_Tasks.py modified: rocoto/py/add_crontab.py modified: rocoto/py/run_pyGEFS.py Refs: #87 --- rocoto/py/GEFS_Crontab.py | 17 ++++---- rocoto/py/GEFS_Parm.py | 15 ++----- rocoto/py/GEFS_UserConfig.py | 73 ++++----------------------------- rocoto/py/GEFS_XML.py | 39 ++++++++++-------- rocoto/py/GEFS_XML_For_Tasks.py | 28 ++++--------- rocoto/py/add_crontab.py | 2 +- rocoto/py/run_pyGEFS.py | 3 -- 7 files changed, 48 insertions(+), 129 deletions(-) diff --git a/rocoto/py/GEFS_Crontab.py b/rocoto/py/GEFS_Crontab.py index 08f9db2b5..85f407d6b 100755 --- a/rocoto/py/GEFS_Crontab.py +++ b/rocoto/py/GEFS_Crontab.py @@ -20,7 +20,7 @@ def create_crontab(dicBase, OnlyForTest=False, cronint=5): ########################################################## # No point creating a crontab if rocotorun is not available. - import sys + import sys, os from distutils.spawn import find_executable if OnlyForTest: @@ -47,20 +47,19 @@ def create_crontab(dicBase, OnlyForTest=False, cronint=5): system = dicBase["WHERE_AM_I"] - rocotorun_args = rocotoruncmd + ' -d ' + sRocotoPath + '/' + sDB_FileName \ - + ' -w ' + sRocotoPath + '/' + sXML_FileName - if system == 'wins' or system == 'hera': - crontab_string += crontab_usage - crontab_string += crontab_time + rocotorun_args - elif system == 'wcoss2': + sXML_File = os.path.join(sRocotoPath, sXML_FileName) + sDB_File = os.path.join(sRocotoPath, sDB_FileName) + rocotorun_args = f"{rocotoruncmd} -d f{sDB_File} -w f{sXML_File}" + if system in ['wcoss2', 'hera', 'wins']: crontab_string += crontab_usage crontab_string += crontab_time + rocotorun_args else: - print("CRITICAL ERROR: auto-crontab file generation for %s still needs to be implemented" % system) + print(f"CRITICAL ERROR: auto-crontab file generation for {system} still needs to be implemented") sys.exit(-502) crontab_string += "\n" - crontab_file = open(sRocotoPath + "/" + sCrontab_FileName, 'w') + sFile = os.path.join(sRocotoPath, sCrontab_FileName) + crontab_file = open(sFile, 'w') crontab_file.write(crontab_string) crontab_file.close() diff --git a/rocoto/py/GEFS_Parm.py b/rocoto/py/GEFS_Parm.py index 93e50107f..df1809831 100755 --- a/rocoto/py/GEFS_Parm.py +++ b/rocoto/py/GEFS_Parm.py @@ -70,12 +70,9 @@ def read_dicParm(sConfig): # ======================================================= def get_and_merge_default_dicParm(dicParm, WHERE_AM_I): import os, sys - sSep = "/" - if sys.platform == 'win32': - sSep = r'\\' # To get the WHERE_AM_I from dicParm or identify it using default methode - sDefaultConfig_File = sys.path[0] + sSep + "user_{0}.conf".format(WHERE_AM_I) + sDefaultConfig_File = os.path.join(sys.path[0], f"user_{WHERE_AM_I}.conf") if os.path.exists(sDefaultConfig_File): #print("----Getting default parameters' value ...") @@ -168,10 +165,6 @@ def create_gets_dev_parm(dicBase, listBaseParm): import sys import os - sSep = "/" - if sys.platform == 'win32': - sSep = r'\\' - strings = [] strings.append('#!/bin/ksh\n') @@ -201,12 +194,12 @@ def create_gets_dev_parm(dicBase, listBaseParm): sPath = dicBase["GEFS_ROCOTO"] - sPath += sSep + "parm" + sPath = os.path.join(sPath, "parm") if not os.path.exists(sPath): os.mkdir(sPath) - sgefs_dev_parm_File = sPath + sSep + "gefs_dev.parm" + sgefs_dev_parm_File = os.path.join(sPath, "gefs_dev.parm") fh = open(sgefs_dev_parm_File, 'w') fh.write(strings) @@ -219,8 +212,6 @@ def create_gets_dev_parm(dicBase, listBaseParm): fh.write('\n# cpu geometry\n') elif sVarName.upper() == 'gfssource'.upper(): fh.write('\n# for test, NOT USED FOR PRODUCTION gfssource = dev, para, prod\n') - elif sVarName.upper() == 'gfssource'.upper(): - fh.write('\n# for test, NOT USED FOR PRODUCTION gfssource = dev, para, prod\n') elif sVarName.upper() == 'makepgrba'.upper(): fh.write('\n# set all the following "make" and "save" flags to "yes" to simulate production\n') diff --git a/rocoto/py/GEFS_UserConfig.py b/rocoto/py/GEFS_UserConfig.py index f5fab5154..0679a755b 100755 --- a/rocoto/py/GEFS_UserConfig.py +++ b/rocoto/py/GEFS_UserConfig.py @@ -1,13 +1,12 @@ # ======================================================= def get_and_merge_default_config(dicBase): import os, sys - sSep = "/" # To get the WHERE_AM_I from dicBase or identify it using default methode get_WHERE_AM_I(dicBase) WHERE_AM_I = dicBase["WHERE_AM_I"] print("----*You are working on machine: {0}".format(WHERE_AM_I)) - sDefaultConfig_File = sys.path[0] + sSep + "user_{0}.conf".format(WHERE_AM_I) + sDefaultConfig_File =os.path.join(sys.path[0], f"user_{WHERE_AM_I}.conf") if os.path.exists(sDefaultConfig_File): print("----Default User Configure file was found! Reading ...") @@ -18,62 +17,15 @@ def get_and_merge_default_config(dicBase): if sDic not in dicBase: dicBase[sDic] = dicBase_Default[sDic] - -# ======================================================= -def get_config_file(OnlyForTest=False): - import os, sys - - sSep = "/" - - sRocoto_WS = os.getcwd() - sConfig = "" # "user_conf" - if OnlyForTest: - - sRocoto_WS = os.getcwd() + sSep + '..' - - # sConfig = "user.conf" - sConfig = sRocoto_WS + sSep + "user_full.conf" - - if not os.path.exists(sConfig): - sRocoto_WS = os.getcwd() - sConfig = sRocoto_WS + sSep + "user_full.conf" - else: - - if len(sys.argv) == 2: - sConfig = sys.argv[1] - else: - sConfig = "user.conf" - - if not os.path.exists(sConfig): - sConfig = ".." + sSep + "user.conf" - - if not os.path.exists(sConfig): - sConfig = "user_full.conf" - - if not os.path.exists(sConfig): - sConfig = ".." + sSep + "user_full.conf" - - if not os.path.exists(sConfig): - print("Please check whether you have config file in your rocoto path!") - sys.exit(-5) - else: - sRocoto_WS = os.getcwd() + sSep + ".." - - sRocoto_WS = os.path.abspath(sRocoto_WS) - return sConfig, sRocoto_WS - - # ======================================================= def get_config_file2(sConfigFile="user_full.conf"): import os, sys - sSep = "/" - sRocoto_WS = os.getcwd() - sConfig = sRocoto_WS + sSep + sConfigFile + sConfig = os.path.join(sRocoto_WS, sConfigFile) if not os.path.exists(sConfig): - sRocoto_WS = os.getcwd() + sSep + ".." - sConfig = sRocoto_WS + sSep + sConfigFile + sRocoto_WS = os.path.join(os.getcwd(), "..") + sConfig = os.path.join(sRocoto_WS, sConfigFile) if not os.path.exists(sConfig): print("Please check whether you have config file in your rocoto path!") sys.exit(-5) @@ -81,7 +33,6 @@ def get_config_file2(sConfigFile="user_full.conf"): sRocoto_WS = os.path.abspath(sRocoto_WS) return sConfig, sRocoto_WS - # ======================================================= def read_config(sConfig): # read config file @@ -135,8 +86,6 @@ def read_config(sConfig): def create_folders(dicBase): import os, sys - sSep = "/" - EXPID = dicBase['EXPID'] WORKDIR = str(dicBase['WORKDIR']).replace("&EXPID;", EXPID) WHERE_AM_I = dicBase["WHERE_AM_I".upper()] @@ -144,17 +93,11 @@ def create_folders(dicBase): if not os.path.exists(WORKDIR): os.makedirs(WORKDIR) - if WHERE_AM_I.lower() == "wcoss2": - sPath = WORKDIR + sSep + 'tmp' - else: - sPath = WORKDIR + sSep + 'tmpnwprd' + sPath = os.path.join(WORKDIR, "tmp") if not os.path.exists(sPath): os.makedirs(sPath) - if WHERE_AM_I.lower() == "wcoss2": - sPath = WORKDIR + '{0}dev{0}output{0}'.format(sSep) - else: - sPath = WORKDIR + '{0}com{0}output{0}dev{0}'.format(sSep) + sPath = os.path.join(WORKDIR, "dev", "output") if not os.path.exists(sPath): os.makedirs(sPath) @@ -164,7 +107,7 @@ def create_folders(dicBase): day = datetime.timedelta(days=1) while date1 <= date2: - sPath1 = sPath + date1.strftime('%Y%m%d') + sPath1 = os.path.join(sPath, date1.strftime('%Y%m%d')) if not os.path.exists(sPath1): #print(sPath1) os.makedirs(sPath1) @@ -177,8 +120,6 @@ def get_WHERE_AM_I(dicBase): import os if sVarName not in dicBase: - sCPath = os.getcwd() - if os.path.exists('/scratch1/NCEPDEV'): dicBase[sVarName] = 'hera' elif os.path.exists('/apps/prod'): diff --git a/rocoto/py/GEFS_XML.py b/rocoto/py/GEFS_XML.py index db3b0fa02..ccdd96f28 100755 --- a/rocoto/py/GEFS_XML.py +++ b/rocoto/py/GEFS_XML.py @@ -3,7 +3,6 @@ def assign_default_for_xml_def(dicBase, sRocoto_WS=""): import sys import os - sSep = "/" # == get_MEMLIST(dicBase) # == @@ -48,9 +47,9 @@ def assign_default_for_xml_def(dicBase, sRocoto_WS=""): sRocoto_Path = dicBase[sVarName_2] sVarValue = os.path.basename(os.path.abspath(sRocoto_Path)) if sVarValue.startswith("rocoto"): - sVarValue = os.path.basename(os.path.abspath(sRocoto_Path + sSep + "..")) + sVarValue = os.path.basename(os.path.abspath(os.path.join(sRocoto_Path, ".."))) if sVarValue == "nwdev": - sVarValue = os.path.basename(os.path.abspath(sRocoto_Path + sSep + ".." + sSep + "..")) + sVarValue = os.path.basename(os.path.abspath(os.path.join(sRocoto_Path, "..", ".."))) dicBase[sVarName] = sVarValue else: @@ -62,28 +61,30 @@ def assign_default_for_xml_def(dicBase, sRocoto_WS=""): sVarName = "SOURCEDIR".upper() sVarValue = "" if sVarName not in dicBase: - sVarValue = os.path.abspath(sRocoto_WS + sSep + "..") - if not (os.path.exists(sVarValue + sSep + "parm") and os.path.exists(sVarValue + sSep + "sorc")): + sVarValue = os.path.abspath(os.path.join(sRocoto_WS, "..")) + if not (os.path.exists(os.path.join(sVarValue, "parm")) and os.path.exists(os.path.join(sVarValue, "sorc"))): print('!!! It seems that your GEFS SOURE is not in the same path with ROCOTO, therefore, you must assign a avalue for "SOURCEDIR" in the user configure file!!!') exit(-5) else: sVarValue = replace_First_Last(dicBase, sVarName) - if not (os.path.exists(sVarValue + sSep + "parm") and os.path.exists(sVarValue + sSep + "sorc")): - sPathTem = sVarValue + sSep + dicBase["EXPID"] + if not (os.path.exists(os.path.join(sVarValue, "parm")) and os.path.exists(os.path.join(sVarValue, "sorc"))): + sPathTem = os.path.join(sVarValue, dicBase["EXPID"]) if os.path.exists(sPathTem): - if os.path.exists(sPathTem + sSep + "parm") and os.path.exists(sPathTem + sSep + "sorc"): + if os.path.exists(os.path.join(sPathTem,"parm")) and os.path.exists(os.path.join(sPathTem, "sorc")): sVarValue = sPathTem else: - sPathTem = sVarValue + sSep + dicBase["EXPID"] + sSep + "nwdev" + sPathTem = os.path.join(sVarValue, dicBase["EXPID"], "nwdev") if os.path.exists(sPathTem): - if os.path.exists(sPathTem + sSep + "parm") and os.path.exists(sPathTem + sSep + "sorc"): + if os.path.exists(os.path.join(sPathTem, "parm")) and os.path.exists(os.path.join(sPathTem, "sorc")): sVarValue = sPathTem else: print("Please check your SOURCEDIR - {0}".format(sVarValue)) exit(-6) else: - sPathTem = sVarValue + sSep + "nwdev" - if os.path.exists(sPathTem) and os.path.exists(sPathTem + sSep + "parm") and os.path.exists(sPathTem + sSep + "sorc"): + sPathTem = os.path.join(sVarValue, "nwdev") + if os.path.exists(sPathTem) \ + and os.path.exists(os.path.join(sPathTem, "parm")) \ + and os.path.exists(os.path.join(sPathTem, "sorc")): sVarValue = sPathTem # sVarValue += "/&EXPID;/nwdev" @@ -254,6 +255,7 @@ def get_preamble(): # ======================================================= def get_definitions(dicBase): + import os ''' Create entities related to the experiment ''' @@ -277,18 +279,19 @@ def get_definitions(dicBase): GenTaskEnt = get_GenTaskEnt(dicBase) if GenTaskEnt: - import sys - sSep = "/" - + sPath_task = os.path.join(dicBase['GEFS_ROCOTO'], "tasks") # ----------------------------------------------------------------------------------------------- strings.append('\t\n') - strings.append('\t\n'.format(dicBase['GEFS_ROCOTO'], sSep)) - strings.append('\t\n'.format(dicBase['GEFS_ROCOTO'], sSep)) + sFile = os.path.join(sPath_task, "env_vars.ent") + strings.append(f'\t\n') + sFile = os.path.join(sPath_task, "date_vars.ent") + strings.append(f'\t\n') strings.append('\n') # ----------------------------------------------------------------------------------------------- strings.append('\t\n') - strings.append('\t\n'.format(dicBase['GEFS_ROCOTO'], sSep)) + sFile = os.path.join(sPath_task, "all.ent") + strings.append(f'\t\n') @@ -704,16 +700,16 @@ def write_to_all_ent(GenTaskEnt, dicBase): fh.close() # ---- - sPath = dicBase["GEFS_ROCOTO"] + sSep + "tasks" + sPath = os.path.join(dicBase["GEFS_ROCOTO"], "tasks") # create date_vars.ent - sFile = sPath + sSep + "date_vars.ent" + sFile = os.path.join(sPath, "date_vars.ent") fh = open(sFile, 'w') strings = get_DATE_VARS("") fh.write(strings) fh.flush() fh.close() # create env_vars.ent - sFile = sPath + sSep + "env_vars.ent" + sFile = os.path.join(sPath, "env_vars.ent") fh = open(sFile, 'w') strings = get_ENV_VARS("") fh.write(strings) @@ -726,21 +722,16 @@ def write_to_ent(taskname, dicBase, GenTaskEnt=False): import sys import os - sSep = "/" - if sys.platform == 'win32': - sSep = r'\\' - strings = create_metatask_task(dicBase, taskname=taskname, sPre="", GenTaskEnt=GenTaskEnt) strings = ''.join(strings) sPath = dicBase["GEFS_ROCOTO"] - sPath += sSep + "tasks" - + sPath += os.path.join(sPath, "tasks") if not os.path.exists(sPath): os.mkdir(sPath) - sFile = sPath + sSep + "{0}.ent".format(taskname) + sFile = os.path.join(sPath, f"{taskname}.ent") fh = open(sFile, 'w') @@ -1434,11 +1425,8 @@ def get_metatask_names(taskname=""): def get_jobname(taskname): import os import sys - sSep = "/" - if sys.platform == 'win32': - sSep = r'\\' - sDefaultJobID_File = sys.path[0] + sSep + "job_id.conf" + sDefaultJobID_File = os.path.join(sys.path[0], "job_id.conf") jobname_short = "--" if os.path.exists(sDefaultJobID_File): # print("---Default Job-ID Configure file was found! Reading ...") diff --git a/rocoto/py/add_crontab.py b/rocoto/py/add_crontab.py index 68d1e0a7a..ce3be5304 100755 --- a/rocoto/py/add_crontab.py +++ b/rocoto/py/add_crontab.py @@ -13,7 +13,7 @@ def main(): # print(os.environ['HOME']) sHomeDIR = os.environ['HOME'] - sMyCrontab = sHomeDIR + "/cron/mycrontab" + sMyCrontab = os.path.join(sHomeDIR, "cron", "mycrontab") ss = Add_Cron_To_myCrontab(sMyCrontab, sCronLine) # print(ss) diff --git a/rocoto/py/run_pyGEFS.py b/rocoto/py/run_pyGEFS.py index 76e66ae91..b6ba57194 100755 --- a/rocoto/py/run_pyGEFS.py +++ b/rocoto/py/run_pyGEFS.py @@ -14,9 +14,6 @@ def main(): import os, sys - sSep = "/" - if sys.platform == 'win32': - sSep = r'\\' import argparse parser = argparse.ArgumentParser(description='pyGEFS: Python-Based Workflow Management of NCEP Global Ensemble Forecast System!') From d293b096fa1a98d11ec530f6f904c2fc6b4f5578 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Tue, 11 Apr 2023 01:21:13 -0400 Subject: [PATCH 16/41] Change ksh to bash for link_gefs.sh On branch feature/v13_atmos_prep modified: sorc/link_gefs.sh Refs: #87 --- sorc/link_gefs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/link_gefs.sh b/sorc/link_gefs.sh index 2646e5b8e..f66daca9d 100755 --- a/sorc/link_gefs.sh +++ b/sorc/link_gefs.sh @@ -1,4 +1,4 @@ -#!/bin/ksh +#!/bin/bash set -ex #--make symbolic links for EMC installation and hardcopies for NCO delivery From 6436d788842d992f5573e510d1c0ad82300ce71e Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Tue, 11 Apr 2023 01:28:59 -0400 Subject: [PATCH 17/41] Fix some bugs in pyGEFS On branch feature/v13_atmos_prep modified: rocoto/py/GEFS_XML.py modified: rocoto/py/GEFS_XML_For_Tasks.py modified: rocoto/py/user_hera.conf Refs: #87 --- rocoto/py/GEFS_XML.py | 2 +- rocoto/py/GEFS_XML_For_Tasks.py | 7 ++----- rocoto/py/user_hera.conf | 8 ++++---- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/rocoto/py/GEFS_XML.py b/rocoto/py/GEFS_XML.py index ccdd96f28..408a16fe5 100755 --- a/rocoto/py/GEFS_XML.py +++ b/rocoto/py/GEFS_XML.py @@ -291,7 +291,7 @@ def get_definitions(dicBase): # ----------------------------------------------------------------------------------------------- strings.append('\t\n') sFile = os.path.join(sPath_task, "all.ent") - strings.append(f'\t\n') strings.append('\t%TASKS;\n') strings.append('\n') diff --git a/rocoto/py/GEFS_XML_For_Tasks.py b/rocoto/py/GEFS_XML_For_Tasks.py index a1ea394ee..c4247568c 100755 --- a/rocoto/py/GEFS_XML_For_Tasks.py +++ b/rocoto/py/GEFS_XML_For_Tasks.py @@ -444,10 +444,7 @@ def create_metatask_task(dicBase, taskname="atmos_prep", sPre="\t", GenTaskEnt=F strings += (create_envar(name="SUBJOB", value=taskname.replace("ensavg_nemsio_", ""), sPre=sPre_2)) # Add command - sPRE = "&PRE; " - if WHERE_AM_I.upper() in ["WCOSS2".upper()]: - sPRE = "" - + sPRE = "" if taskname in ['keep_init', 'copy_init', 'keep_data_atm', 'archive_atm', 'cleanup_atm', 'keep_data_wave', 'archive_wave', 'cleanup_wave', 'keep_data_chem', 'archive_chem', 'cleanup_chem']: if WHERE_AM_I.upper() in ["WCOSS2".upper()]: strings += sPre_2 + '{1}&BIN;/{0}.sh\n'.format(taskname, sPRE) @@ -727,7 +724,7 @@ def write_to_ent(taskname, dicBase, GenTaskEnt=False): strings = ''.join(strings) sPath = dicBase["GEFS_ROCOTO"] - sPath += os.path.join(sPath, "tasks") + sPath = os.path.join(sPath, "tasks") if not os.path.exists(sPath): os.mkdir(sPath) diff --git a/rocoto/py/user_hera.conf b/rocoto/py/user_hera.conf index 9baad552f..295a54ce5 100755 --- a/rocoto/py/user_hera.conf +++ b/rocoto/py/user_hera.conf @@ -213,7 +213,7 @@ extractvars_dep = ## atmos_prep*************************************************** atmos_prep_walltime = 00:30:00 atmos_prep_nodes = 1 -atmos_prep_ppn = 24 +atmos_prep_ppn = 36 atmos_prep_tpp = 1 #atmos_prep_memory = 50000M atmos_prep_join = &LOG_DIR;/@Y@m@d/gefs_#member#_atmos_prep_@H.%j @@ -574,9 +574,9 @@ chem_post_cleanup_dep = auto BIN = &GEFS_ROCOTO;/bin/&WHERE_AM_I; PRE = &GEFS_ROCOTO;/bin/gefs_pre_job.sh WORKFLOW_LOG_DIR = &GEFS_ROCOTO;/logs -LOG_DIR = &WORKDIR;/com/output/dev -tmpnwprd = &WORKDIR;/tmpnwprd -DATA_DIR = &WORKDIR;/com/gefs/dev +LOG_DIR = &WORKDIR;/dev/output +tmpnwprd = &WORKDIR;/tmp +DATA_DIR = &WORKDIR;/dev/com/gefs/v12.3 #------- XML = gefs.xml DB = gefs.db From d14acdbdc33bbe9b7a527ecdbd401780bcb9a070 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Tue, 11 Apr 2023 01:40:35 -0400 Subject: [PATCH 18/41] Improve pyGEFS to run atmos_prep on Hera On branch feature/v13_atmos_prep modified: rocoto/bin/hera/atmos_prep.sh modified: rocoto/bin/hera/common.sh modified: rocoto/compile_install_all.sh modified: rocoto/dev/versions/run_hera.ver modified: rocoto/py/GEFS_XML.py Refs: #87 --- rocoto/bin/hera/atmos_prep.sh | 22 ++++++++++++++-------- rocoto/bin/hera/common.sh | 4 +++- rocoto/compile_install_all.sh | 4 ++-- rocoto/dev/versions/run_hera.ver | 2 +- rocoto/py/GEFS_XML.py | 2 +- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/rocoto/bin/hera/atmos_prep.sh b/rocoto/bin/hera/atmos_prep.sh index bb6eabb3e..d2cb11506 100755 --- a/rocoto/bin/hera/atmos_prep.sh +++ b/rocoto/bin/hera/atmos_prep.sh @@ -11,17 +11,19 @@ ulimit -a . $GEFS_ROCOTO/dev/versions/run_hera.ver # Load modules -. /apps/lmod/lmod/init/ksh module list module purge -module load EnvVars/$EnvVars_ver -module load intel/$ips_ver -module load impi/$impi_ver -module load prod_util/$prod_util_ver -module load prod_envir/$prod_envir_ver -module load hdf5_parallel/$HDF5_parallel_ver -module load netcdf_parallel/$NetCDF_parallel_ver +module use -a /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack + +module load hpc/$hpc_ver + +module load hpc-intel/${intel_ver} +module load hpc-impi/${impi_ver} +module load grib_util/${grib_util_ver} +module load prod_util/${prod_util_ver} + +module load netcdf/${netcdf_ver} module list @@ -43,5 +45,9 @@ export MEMORY_AFFINITY=core:2 (( OMP_NUM_THREADS_CH = 40 / GEFS_PPN )) export OMP_NUM_THREADS_CH +ver=${ver:-$(echo ${gefs_ver}|cut -c1-5)} +export COMOUT=${COMOUT:-${COMROOT}/gefs/$ver/${RUN}.${PDY}/$cyc} +export ROTDIR=${COMROOT}/gefs/${ver} + # CALL executable job script here $SOURCEDIR/jobs/JGEFS_ATMOS_PREP diff --git a/rocoto/bin/hera/common.sh b/rocoto/bin/hera/common.sh index 8ba3408bd..1162707da 100755 --- a/rocoto/bin/hera/common.sh +++ b/rocoto/bin/hera/common.sh @@ -28,10 +28,11 @@ export taskspernode=${GEFS_PPN} export NTASKS=$SLURM_NTASKS #\/ Only on Hera +export pid=${pid:-${SLURM_JOBID}} export outid="LL$job" export jobid="${outid}.o${pid}" export pgmout="OUTPUT.${pid}" -export LSB_JOBID=${pid} +export PBS_JOBID=${pid} #/\ Only for Hera export gefsmpexec="srun -n $total_tasks" @@ -47,6 +48,7 @@ export APRUN_CHGRES="srun -n 1" #export aprun_gec00="srun -n 1" export APRUN_CALCINC="srun -n 1" + . $GEFS_ROCOTO/parm/setbase . $GEFS_ROCOTO/parm/gefs_config . $GEFS_ROCOTO/parm/gefs_dev.parm diff --git a/rocoto/compile_install_all.sh b/rocoto/compile_install_all.sh index 4d36f54f3..8a5afe7c5 100755 --- a/rocoto/compile_install_all.sh +++ b/rocoto/compile_install_all.sh @@ -174,9 +174,9 @@ if [ $RunRocoto = "yes" ]; then cd $sWS if [ $machine = "hera" ]; then module load intel/18.0.5.274 - module load rocoto/1.3.1 + module load rocoto/1.3.3 module load contrib - module load intelpython/3.6.8 + module load anaconda/latest elif [ $machine = "wcoss2" ]; then diff --git a/rocoto/dev/versions/run_hera.ver b/rocoto/dev/versions/run_hera.ver index 36297c05e..f89e6cd91 100755 --- a/rocoto/dev/versions/run_hera.ver +++ b/rocoto/dev/versions/run_hera.ver @@ -10,7 +10,7 @@ export impi_ver=2022.1.2 export prod_util_ver=1.2.2 export grib_util_ver=1.2.4 -export netcdf_ver=4.7.0 +export netcdf_ver=4.7.4 # -- For FV3 diff --git a/rocoto/py/GEFS_XML.py b/rocoto/py/GEFS_XML.py index 408a16fe5..6a6ac1b97 100755 --- a/rocoto/py/GEFS_XML.py +++ b/rocoto/py/GEFS_XML.py @@ -261,7 +261,7 @@ def get_definitions(dicBase): ''' lstEntity = ["MEMLIST", "CYCLE_THROTTLE", "TASK_THROTTLE", "SDATE", "EDATE", \ - "INCYC", "WHERE_AM_I", "GEFS_ROCOTO", "BIN", "PRE", \ + "INCYC", "WHERE_AM_I", "GEFS_ROCOTO", "BIN", \ "WORKFLOW_LOG_DIR", "LOG_DIR", "tmpnwprd", "DATA_DIR", "EXPID", \ "PSLOT", "SOURCEDIR", "WORKDIR", "KEEP_DIR", "INIT_DIR", \ "HPSS_DIR", "DIRS_TO_KEEP", "DIRS_TO_ARCHIVE", "DIRS_TO_KEEP_WAVE", "DIRS_TO_ARCHIVE_WAVE", \ From cee21ecbb5c156d0fac63bac3418b431d7e9189c Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Wed, 12 Apr 2023 12:01:06 -0400 Subject: [PATCH 19/41] Optimize and clean up some scripts On branch feature/v13_atmos_prep modified: jobs/JGEFS_ATMOS_PREP modified: parm/parm_gefs/gefs.parm modified: rocoto/bin/hera/atmos_prep.sh modified: rocoto/parm/gefs_config modified: rocoto/parm/setbase modified: rocoto/py/GEFS_XML_For_Tasks.py Refs: #87 --- jobs/JGEFS_ATMOS_PREP | 36 +++++------ parm/parm_gefs/gefs.parm | 2 +- rocoto/bin/hera/atmos_prep.sh | 4 +- rocoto/parm/gefs_config | 110 ++++++++------------------------ rocoto/parm/setbase | 49 +++++++------- rocoto/py/GEFS_XML_For_Tasks.py | 2 +- 6 files changed, 71 insertions(+), 132 deletions(-) diff --git a/jobs/JGEFS_ATMOS_PREP b/jobs/JGEFS_ATMOS_PREP index 78d8949d4..7fbc38912 100755 --- a/jobs/JGEFS_ATMOS_PREP +++ b/jobs/JGEFS_ATMOS_PREP @@ -1,11 +1,11 @@ #! /usr/bin/env bash -source "${HOMEgfs:-${HOMEgefs}}/ush/preamble.sh" +source "${HOMEgefs:-${HOMEgfs}}/ush/preamble.sh" #################################### # Determine Job Output Name on System #################################### -export jobid=${jobid:-"$job.$PBS_JOBID"} +export jobid=${jobid:-"${job}.${PBS_JOBID}"} export pgmout="OUTPUT.${PBS_JOBID}" export pgmerr=errfile @@ -20,7 +20,7 @@ cd $DATA # Set up compute resources ###################################### export total_tasks=${total_tasks:-36} # Must be divisible number of tiles (6) -export APRUN=${gefsmpexec:-"mpiexec -n $total_tasks"} +export APRUN=${gefsmpexec:-"mpiexec -n ${total_tasks}"} #################################### # Specify NET and RUN Name and model @@ -51,10 +51,10 @@ source PDY # Specify Execution Areas #################################### export HOMEgefs=${HOMEgefs:-${PACKAGEROOT}/gefs.${gefs_ver}} -export PARMgefs=${PARMgefs:-$HOMEgefs/parm} -export EXECgefs=${EXECgefs:-$HOMEgefs/exec} -export USHgefs=${USHgefs:-$HOMEgefs/ush} -export FIXgefs=${FIXgefs:-$HOMEgefs/fix/fix_gefs} +export PARMgefs=${PARMgefs:-${HOMEgefs}/parm} +export EXECgefs=${EXECgefs:-${HOMEgefs}/exec} +export USHgefs=${USHgefs:-${HOMEgefs}/ush} +export FIXgefs=${FIXgefs:-${HOMEgefs}/fix/fix_gefs} ############################################## # Define COM directories @@ -68,21 +68,19 @@ export COMPONENT="atmos" configs="gefs gefs_atmos_prep" config_path=$PARMgefs/parm_gefs for config in $configs; do - . $config_path/$config.parm + . ${config_path}/${config}.parm export err=$? if [[ $err != 0 ]]; then - echo "FATAL ERROR in $(basename $BASH_SOURCE): Error while loading parm file $config_path/$config.parm" - exit $err + echo "FATAL ERROR in $(basename $BASH_SOURCE): Error while loading parm file ${config_path}/${config}.parm" + exit ${err} fi done -export ROTDIR=${ROTDIR:-$(compath.py -o $NET/${ver})} -export COMOUT=${COMOUT:-$(compath.py -o $NET/${ver})/${RUN}.${PDY}/${cyc}} - export COMINgfs=${COMINgfs:-$(compath.py ${envir}/com/gfs/${gfs_ver})/gfs.${PDY}/${cyc}} export COMINenkfgfs=${COMINenkfgfs:-$(compath.py ${envir}/com/gfs/${gfs_ver})/enkfgfs.${PDY}/${cyc}} -NCP=${NCP:-"/bin/cp -p"} +export ROTDIR=${ROTDIR:-$(compath.py -o $NET/${ver})} +#export COMOUT=${COMOUT:-$(compath.py -o $NET/${ver})/${RUN}.${PDY}/${cyc}} echo "Environment before calling script" env | sort @@ -91,10 +89,10 @@ env | sort # Execute the script $HOMEgefs/scripts/exgefs_atmos_prep.sh export err=$? -if [[ $err == 0 ]]; then - echo "$job completed normally!" +if [[ ${err} == 0 ]]; then + echo "${job} completed normally!" else - echo "FATAL ERROR in $(basename $BASH_SOURCE): $job failed!" + echo "FATAL ERROR in $(basename $BASH_SOURCE): ${job} failed!" exit $err fi ############################################################# @@ -102,8 +100,8 @@ fi ############################## # Remove the Temporary working directory ############################## -cd $DATAROOT -if [ ${KEEPDATA:-NO} = NO ] ; then rm -rf $DATA ; fi +cd ${DATAROOT} +if [ ${KEEPDATA:-NO} = NO ] ; then rm -rf ${DATA} ; fi echo "$(date -u) end $(basename $BASH_SOURCE)" diff --git a/parm/parm_gefs/gefs.parm b/parm/parm_gefs/gefs.parm index 304509acd..27784f03a 100755 --- a/parm/parm_gefs/gefs.parm +++ b/parm/parm_gefs/gefs.parm @@ -163,6 +163,7 @@ export KEEPDATA=${KEEPDATA:-"NO"} #################################### export NET=${NET:-gefs} export envir=${envir:-prod} +export RUN_ENVIR=${RUN_ENVIR:-nco} export RUN=${RUN:-gefs} ###################################### @@ -181,7 +182,6 @@ export VERBOSE=${VERBOSE:-YES} # Get all the COM path templates source "${PARMgefs}/config/config.com" - # Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL export imp_physics=${imp_physics:-"8"} diff --git a/rocoto/bin/hera/atmos_prep.sh b/rocoto/bin/hera/atmos_prep.sh index d2cb11506..4540089b8 100755 --- a/rocoto/bin/hera/atmos_prep.sh +++ b/rocoto/bin/hera/atmos_prep.sh @@ -16,7 +16,7 @@ module purge module use -a /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack -module load hpc/$hpc_ver +module load hpc/${hpc_ver} module load hpc-intel/${intel_ver} module load hpc-impi/${impi_ver} @@ -50,4 +50,4 @@ export COMOUT=${COMOUT:-${COMROOT}/gefs/$ver/${RUN}.${PDY}/$cyc} export ROTDIR=${COMROOT}/gefs/${ver} # CALL executable job script here -$SOURCEDIR/jobs/JGEFS_ATMOS_PREP +${SOURCEDIR}/jobs/JGEFS_ATMOS_PREP diff --git a/rocoto/parm/gefs_config b/rocoto/parm/gefs_config index a53f8de8a..090f60d53 100644 --- a/rocoto/parm/gefs_config +++ b/rocoto/parm/gefs_config @@ -1,9 +1,11 @@ +#! /usr/bin/env bash + ####################################################### # This file defines the various input varibles for Development Tests of GEFS ####################################################### # # Define DEVELOPMENT Enviorenmenat variables related to the TYPE of test -export RUN_ENVIR=${RUN_ENVIR:-dev} +export RUN_ENVIR=${RUN_ENVIR:-emc} # emc, nco based on g-w export envir=${envir:-dev} export NET=${NET:-gefs} export RUN=${RUN:-gefs} @@ -17,21 +19,8 @@ export KEEPDATA=${KEEPDATA:-YES} # Enviorenmenat variables related to Development work place and output files # -if [[ $machine == "WCOSS2" ]]; then - export COMROOT=${WORKDIR}/$envir/com - export DATAROOT=$baseoutput/tmp -else - export COMROOT=$baseoutput/dev/com - export GESROOT=$baseoutput/nwges - export DATAROOT=$baseoutput/tmp - #export DCOMROOT=${DCOMROOT:-/dcom} - #export PCOMROOT=$baseoutput/pcom/$envir - - #export GESROOTp1=$GESROOTp1 - - #export GESOUT=$baseoutput/com/ - #export COMOUT=$baseoutput/com -fi +export COMROOT=${WORKDIR}/${envir}/com +export DATAROOT=$baseoutput/tmp export archsyndir=${archsyndir:-$COMROOTp1/arch/prod/syndat} export DBNROOT= @@ -44,17 +33,10 @@ export HOMEwave=${HOMEwave:-$HOMEgfs} export HOMEpost=${HOMEpost:-$HOMEgfs} if [[ ! -d $SOURCEDIR/sorc/global-workflow.fd ]] ; then - export EXECwave=${HOMEgefs}/exec + export EXECwave=${HOMEgefs}/exec - export FCSTEXECDIR=${FCSTEXECDIR:-$HOMEgfs/sorc/fv3gfs.fd/NEMS/exe} - export HOMEchem=${HOMEchem:-$HOMEgfs/sorc/gsd_prep_chem.fd/workflow/emc-global} -fi - -GFSArchivedData=yes -#GFSArchivedData=no -if [[ $WHERE_AM_I == "hera" ]]; then - GFSArchivedData=yes - UseHOMEdata=yes + export FCSTEXECDIR=${FCSTEXECDIR:-$HOMEgfs/sorc/fv3gfs.fd/NEMS/exe} + export HOMEchem=${HOMEchem:-$HOMEgfs/sorc/gsd_prep_chem.fd/workflow/emc-global} fi #Please check with parm/gefs_init.parm to make sure the next line matching @@ -63,65 +45,25 @@ export pdycycp=$($NDATE -$fhrp $PDY$cyc) export pdyp=$(echo $pdycycp|cut -c1-8) export cycp=$(echo $pdycycp|cut -c9-10) -if [[ $machine == "WCOSS2" ]]; then - echo "This is on WCOSS2!" -else - if [[ $GFSArchivedData == "yes" ]]; then - # HOMEdata, COMINgfs_base, COMINenkf_base and COMINcfs_base are defined in setbase, pointing to the NCO gfs.v15 parallel - # However, if you use gfs/enkf data from archived data, you may speficfy them in this "if" block and make GFSArchivedData=yes - UseHOMEdata=${UseHOMEdata:-no} - if [[ $UseHOMEdata == "yes" ]]; then - if [[ $WHERE_AM_I == "hera" ]]; then - source dev/versions/run_hera.ver - export HOMEdata=/scratch2/NCEPDEV/ensemble/noscrub/Xianwu.Xue/GEFS_v13/z_DATA/lfs_C96_C48_NOIAU_anal_20230114/h1/ops/prod/com - export COMINgfs_base=${HOMEdata}/gfs/${gfs_ver} - #export COMINenkf_base=${HOMEdata}/com/gfs/v16.2/enkfgdas. - export COMINgfs=${COMINgfs:-${COMINgfs_base}/gfs.${PDY}/${cyc}} - #export COMINenkf=${COMINenkf:-${COMINenkf_base}${pdyp}/$cycp/atmos} - export COMINenkfgfs=${COMINenkfgfs:-${COMINgfs_base}/enkfgfs.${PDY}/${cyc}} - - export COMINcfs_base=${HOMEdata}/com/cfs/v2.3/cfs. - export DCOMROOT=$HOMEdata/dcom +# HOMEdata, COMINgfs_base, COMINenkf_base and COMINcfs_base are defined in setbase, pointing to the NCO gfs.v15 parallel +# However, if you use gfs/enkf data from archived data, you may speficfy them in this "if" block and make GFSArchivedData=yes +if [[ $WHERE_AM_I == "hera" ]]; then + source dev/versions/run_hera.ver + export HOMEdata=/scratch2/NCEPDEV/ensemble/noscrub/Xianwu.Xue/GEFS_v13/z_DATA/lfs_C96_C48_NOIAU_anal_20230114/h1/ops/prod/com + export COMINgfs_base=${HOMEdata}/gfs/${gfs_ver} + export COMINenkfgfs=${COMINenkfgfs:-${COMINgfs_base}/enkfgfs.${PDY}/${cyc}} - # For gempak_meta - export COMINsgfs=${HOMEdata}/com/gfs/v16.2 #${COMINsgfs:-$(compath.py gfs/prod)} - export COMINnawips="NONE" #${COMINnawips:-$(compath.py nawips/prod)} - export COMINecmwf="NONE" #${COMINecmwf:-$(compath.py ecmwf/prod/ecmwf)} - export COMINukmet="NONE" #${COMINukmet:-$(compath.py nawips/prod/ukmet)} - export COMINnam="NONE" #${COMINnam:-$(compath.py nam/prod)} + export COMINcfs_base=${HOMEdata}/com/cfs/v2.3/cfs. + export DCOMROOT=$HOMEdata/dcom - else - HOMEdata=/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS/HOMEdata/canned/com - export COMINenkf_base=$HOMEdata/fv3init/enkf.gdas. - export COMINgfs_base=$HOMEdata/fv3init/gfs. - export COMINcfs_base=$HOMEdata/cfs/cfs. - export COMIN_WAV_ICE=$HOMEdata/sea_ice - export COMINgfs=${COMINgfs:-${COMINgfs_base}${PDY}/$cyc} - export COMINenkf=${COMINenkf:-${COMINenkf_base}${pdyp}/$cycp} - fi - else - HOMEdata=/gpfs/h2/emc/modeling/noscrub/Xianwu.Xue/gw_for_acorn/HOMEdata - export COMPATH=$HOMEdata/com/gfs/prod - export COMINgfs_base=$(compath.py gfs/prod)/gfs. - export COMINenkf_base=$(compath.py gfs/prod)/enkfgdas. - export COMPATH=$HOMEdata/com/cfs/prod - export COMINcfs_base=$(compath.py cfs/prod)/cfs/cfs. - export COMINgfs=${COMINgfs:-${COMINgfs_base}${PDY}/$cyc/atmos} - export COMINenkf=${COMINenkf:-${COMINenkf_base}${pdyp}/$cycp/atmos} - export DCOMROOT=$HOMEdata/dcom - fi - else - export COMINgfs_base=/gpfs/h2/ptmp/emc.glopara/ROTDIRS/v16rt2/gfs/para/gfs. - #export COMINgfs_base=$(compath.py gfs/prod)/gfs. - export COMINenkf_base=/gpfs/h2/ptmp/emc.glopara/ROTDIRS/v16rt2/gfs/para/enkfgdas. - #export COMINenkf_base=$(compath.py gfs/prod)/enkfgdas. - export COMINcfs_base=$(compath.py cfs/prod)/cfs/cfs. - export COMIN_WAV_ICE=$(compath.py gfs/prod)/gfs.${PDY}/${cyc}/atmos - export COMINgfs=${COMINgfs:-${COMINgfs_base}${PDY}/$cyc/atmos} - export COMINenkf=${COMINenkf:-${COMINenkf_base}${pdyp}/$cycp/atmos} - fi # [[ $GFSArchivedData == "yes" ]] + # For gempak_meta + export COMINsgfs=${COMINgfs_base} + export COMINnawips="NONE" #${COMINnawips:-$(compath.py nawips/prod)} + export COMINecmwf="NONE" #${COMINecmwf:-$(compath.py ecmwf/prod/ecmwf)} + export COMINukmet="NONE" #${COMINukmet:-$(compath.py nawips/prod/ukmet)} + export COMINnam="NONE" #${COMINnam:-$(compath.py nam/prod)} - export COMIN_WAV_ICE=$COMINgfs - export COMINenkf=${COMINenkf:-${COMINenkf_base}} - export COMINcfs=${COMINcfs:-${COMINcfs_base}} + export COMIN_WAV_ICE=$COMINgfs + export COMINenkf=${COMINenkf:-${COMINenkf_base}} + export COMINcfs=${COMINcfs:-${COMINcfs_base}} fi diff --git a/rocoto/parm/setbase b/rocoto/parm/setbase index 385a21dde..6dc0e7968 100755 --- a/rocoto/parm/setbase +++ b/rocoto/parm/setbase @@ -1,4 +1,4 @@ -#!/bin/ksh +#! /usr/bin/env bash # # set up base variables # execute with dot @@ -22,37 +22,36 @@ export STRICT="YES" # For virtical structure if [[ -d $SOURCEDIR/sorc/global-workflow.fd ]] ; then - export HOMEDIR=$SOURCEDIR + export HOMEDIR=$SOURCEDIR fi if [ -d /scratch2/NCEPDEV ]; then - echo "Using settings for Hera" - export machine='HERA' - - export HOMEDIR=${HOMEDIR:-/scratch2/NCEPDEV/ensemble/noscrub/Xianwu.Xue/GEFS_v13/gw_walter} - export HOMERFC=/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/ens_tracker - export HOMEdata=/scratch2/NCEPDEV/ensemble/noscrub/common + echo "Using settings for Hera" + export machine='HERA' - export COMROOThps=${HOMEdata} - export COMROOTp1=/scratch2/NCEPDEV/rstprod/com - export PATH=$PATH:/scratch2/NCEPDEV/nwprod/NCEPLIBS/util/prod_util.v1.1.0/ush + export HOMEDIR=${HOMEDIR:-/scratch2/NCEPDEV/ensemble/noscrub/Xianwu.Xue/GEFS_v13/gw_walter} + export HOMERFC=/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/ens_tracker + export HOMEdata=/scratch2/NCEPDEV/ensemble/noscrub/common - export GBBEPX_DATA_DIR_TODAY=$HOMEdata/GBBEPx - export GBBEPX_DATA_DIR_YESTERDAY=$HOMEdata/GBBEPx - export gbbepx_pattern='$GBBEPX_DATA_DIR/${local_name}.${day}.FV3.${CASE}Grid.${tiledir}.bin' # single-quotes required - export gbbepx_days='$PDYm1' # single-quotes required - export gbbepx_list="GBBEPx.bc GBBEPx.oc GBBEPx.so2 GBBEPx.pm25 meanFRP" - export inout_list="BBURN3-bb,ebu_pm_10 SO4-bb,ebu_sulf plume,plumestuff GBBEPx.bc,ebu_bc GBBEPx.oc,ebu_oc GBBEPx.so2,ebu_so2 GBBEPx.pm25,ebu_pm_25 meanFRP,plumefrp" + export COMROOThps=${HOMEdata} + export COMROOTp1=/scratch2/NCEPDEV/rstprod/com + export PATH=$PATH:/scratch2/NCEPDEV/nwprod/NCEPLIBS/util/prod_util.v1.1.0/ush + + export GBBEPX_DATA_DIR_TODAY=$HOMEdata/GBBEPx + export GBBEPX_DATA_DIR_YESTERDAY=$HOMEdata/GBBEPx + export gbbepx_pattern='$GBBEPX_DATA_DIR/${local_name}.${day}.FV3.${CASE}Grid.${tiledir}.bin' # single-quotes required + export gbbepx_days='$PDYm1' # single-quotes required + export gbbepx_list="GBBEPx.bc GBBEPx.oc GBBEPx.so2 GBBEPx.pm25 meanFRP" + export inout_list="BBURN3-bb,ebu_pm_10 SO4-bb,ebu_sulf plume,plumestuff GBBEPx.bc,ebu_bc GBBEPx.oc,ebu_oc GBBEPx.so2,ebu_so2 GBBEPx.pm25,ebu_pm_25 meanFRP,plumefrp" elif [[ -d /apps/prod ]]; then - # We are on NOAA WCOSS2 - echo "Using settings for WCOSS2" - export machine='WCOSS2' - export JOBPHASE= - export HOMEDIR=${HOMEDIR:-/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS_v13/gw_walter} - export HOMEtrak=/lfs/h2/emc/ens/noscrub/emc.ens/common/git/ens_tracker/ens_tracker.v2.1.2 - export archsyndir=/lfs/h2/emc/ens/noscrub/emc.ens/common/git/ens_tracker/TCvitalData/syndat - export HOMEdata=/lfs/h2/emc/ens/noscrub/emc.ens/common/canned_data + # We are on NOAA WCOSS2 + echo "Using settings for WCOSS2" + export machine='WCOSS2' + export HOMEDIR=${HOMEDIR:-/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS_v13/gw_walter} + export HOMEtrak=/lfs/h2/emc/ens/noscrub/emc.ens/common/git/ens_tracker/ens_tracker.v2.1.2 + export archsyndir=/lfs/h2/emc/ens/noscrub/emc.ens/common/git/ens_tracker/TCvitalData/syndat + export HOMEdata=/lfs/h2/emc/ens/noscrub/emc.ens/common/canned_data fi diff --git a/rocoto/py/GEFS_XML_For_Tasks.py b/rocoto/py/GEFS_XML_For_Tasks.py index c4247568c..7d10d7681 100755 --- a/rocoto/py/GEFS_XML_For_Tasks.py +++ b/rocoto/py/GEFS_XML_For_Tasks.py @@ -1507,7 +1507,7 @@ def get_DATE_VARS(sPre="\t\t"): def get_ENV_VARS(sPre="\t\t"): dicENV_VARS = {} dicENV_VARS['envir'] = 'dev' - dicENV_VARS['RUN_ENVIR'] = 'dev' + dicENV_VARS['RUN_ENVIR'] = 'emc' dicENV_VARS['WHERE_AM_I'] = '&WHERE_AM_I;' dicENV_VARS['GEFS_ROCOTO'] = '&GEFS_ROCOTO;' dicENV_VARS['WORKDIR'] = '&WORKDIR;' From 008adf836ea0b78ce3c2eaf1ffc797bd6a8e3c71 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Wed, 12 Apr 2023 14:29:22 -0400 Subject: [PATCH 20/41] Add COMINgfs back On branch feature/v13_atmos_prep modified: rocoto/parm/gefs_config Refs: #87 --- rocoto/parm/gefs_config | 1 + 1 file changed, 1 insertion(+) diff --git a/rocoto/parm/gefs_config b/rocoto/parm/gefs_config index 090f60d53..257d703b8 100644 --- a/rocoto/parm/gefs_config +++ b/rocoto/parm/gefs_config @@ -51,6 +51,7 @@ if [[ $WHERE_AM_I == "hera" ]]; then source dev/versions/run_hera.ver export HOMEdata=/scratch2/NCEPDEV/ensemble/noscrub/Xianwu.Xue/GEFS_v13/z_DATA/lfs_C96_C48_NOIAU_anal_20230114/h1/ops/prod/com export COMINgfs_base=${HOMEdata}/gfs/${gfs_ver} + export COMINgfs=${COMINgfs:-${COMINgfs_base}/gfs.${PDY}/${cyc}} export COMINenkfgfs=${COMINenkfgfs:-${COMINgfs_base}/enkfgfs.${PDY}/${cyc}} export COMINcfs_base=${HOMEdata}/com/cfs/v2.3/cfs. From cdac12e4481048e8a1445bc0e32eb72c5c42e644 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Wed, 12 Apr 2023 14:32:52 -0400 Subject: [PATCH 21/41] Delete unused files On branch feature/v13_atmos_prep deleted: rocoto/bin/gefs_load_modules.inc deleted: rocoto/bin/gefs_pre_job.sh Refs: #87 --- rocoto/bin/gefs_load_modules.inc | 31 ------------------------------- rocoto/bin/gefs_pre_job.sh | 26 -------------------------- 2 files changed, 57 deletions(-) delete mode 100755 rocoto/bin/gefs_load_modules.inc delete mode 100755 rocoto/bin/gefs_pre_job.sh diff --git a/rocoto/bin/gefs_load_modules.inc b/rocoto/bin/gefs_load_modules.inc deleted file mode 100755 index 93d84d583..000000000 --- a/rocoto/bin/gefs_load_modules.inc +++ /dev/null @@ -1,31 +0,0 @@ -# This is a KSH include file that should be sourced at the top of -# every job. It initializes the environment for various clusters so -# that the rest of the GEFS or Rocoto can run properly. - -if [[ "$WHERE_AM_I" == hera ]]; then - echo "Loading modules for hera" - . /apps/lmod/lmod/init/ksh - module purge - module load contrib - module load hpss - module load intel/18.0.5.274 - module load impi/2018.0.4 - module load hdf5parallel/1.10.5 - module load netcdf/4.7.0 - module load nco/4.7.0 - module load hdf5/1.10.5 - module load intelpython/3.6.8 - - module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles - module load prod_util/1.1.0 - module load g2tmpl/1.5.1 - module load grib_util/1.1.1 - - module load gempak/7.4.2 - - export PATH=$PATH:. - - # Temporary setting to test common settings - . $GEFS_ROCOTO/bin/hera/common.sh - -fi diff --git a/rocoto/bin/gefs_pre_job.sh b/rocoto/bin/gefs_pre_job.sh deleted file mode 100755 index 898caa131..000000000 --- a/rocoto/bin/gefs_pre_job.sh +++ /dev/null @@ -1,26 +0,0 @@ -#! /bin/ksh -set -x -e -date -# . $GEFS_ROCOTO/bin/gefs_load_modules.inc -export expid=${EXPID} -export SOURCEDIR=${SOURCEDIR} -export WORKDIR=${WORKDIR} - -if [[ "$WHERE_AM_I" == hera ]]; then - # Temporary setting to test common settings - #. $GEFS_ROCOTO/bin/hera/common.sh - #. $SOURCEDIR/versions/gefs_hera.ver - . $GEFS_ROCOTO/bin/gefs_load_modules.inc - . $GEFS_ROCOTO/parm/setbase - . $GEFS_ROCOTO/parm/gefs_config - . $GEFS_ROCOTO/parm/gefs_dev.parm - -fi - -# to test and will be deleted after finalized all tasks - -#. $GEFS_ROCOTO/parm/setbase -#. $GEFS_ROCOTO/parm/gefs_config -#. $GEFS_ROCOTO/parm/gefs_dev.parm - -"$@" From 781c433730a08ba172ceefa2d9bdc570e968b872 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Wed, 12 Apr 2023 21:33:39 -0400 Subject: [PATCH 22/41] Optimize compile_install_all.sh On branch feature/v13_atmos_prep modified: rocoto/compile_install_all.sh Refs: #87 --- rocoto/compile_install_all.sh | 303 ++++++++++++++++------------------ 1 file changed, 146 insertions(+), 157 deletions(-) diff --git a/rocoto/compile_install_all.sh b/rocoto/compile_install_all.sh index 8a5afe7c5..8f16a9f13 100755 --- a/rocoto/compile_install_all.sh +++ b/rocoto/compile_install_all.sh @@ -1,24 +1,24 @@ #!/bin/bash -set -eu #x +set -eux sWS=$(pwd) echo $sWS while getopts c:a:r:m:f:b:e:s:l:o: option do - case "${option}" - in - c) CompileCode=${OPTARG};; - a) CleanAll=${OPTARG};; - r) RunRocoto=${OPTARG};; - m) machine=${OPTARG};; - f) userConfigFile=${OPTARG};; - b) AddCrontabToMyCrontab=${OPTARG};; - e) RunEnvir=${OPTARG};; - s) Structure=${OPTARG};; - l) Link=${OPTARG};; - o) Operation=${OPTARG};; - esac + case "${option}" + in + c) CompileCode=${OPTARG};; + a) CleanAll=${OPTARG};; + r) RunRocoto=${OPTARG};; + m) machine=${OPTARG};; + f) userConfigFile=${OPTARG};; + b) AddCrontabToMyCrontab=${OPTARG};; + e) RunEnvir=${OPTARG};; + s) Structure=${OPTARG};; + l) Link=${OPTARG};; + o) Operation=${OPTARG};; + esac done CompileCode=${CompileCode:-no} @@ -34,14 +34,14 @@ Link=${Link:-no} Operation=${Operation:-no} # ecflow, rocoto, lsf if [ $machine = "nomachine" ]; then - if [ -d /scratch1/NCEPDEV ]; then - machine=hera - elif [[ -d /apps/prod ]]; then # WCOSS2 - machine=wcoss2 - else - echo "This is not supported by this script!" - exit 55 - fi + if [ -d /scratch1/NCEPDEV ]; then + machine=hera + elif [[ -d /apps/prod ]]; then # WCOSS2 + machine=wcoss2 + else + echo "This is not supported by this script!" + exit 55 + fi fi echo $CompileCode @@ -54,177 +54,166 @@ echo ${Structure} echo ${Link} if [ $CompileCode = "yes" ]; then - Link=yes + Link=yes fi if [ $CompileCode = "yes" ]; then - cd $sWS/../sorc - - if [[ $Structure == "dev" ]]; then - echo "...working on it ..." - if [[ $machine == "hera" ]]; then - sHeader='/scratch2/NCEPDEV/ensemble' - elif [[ $machine == "wcoss2" ]]; then - sHeader='/lfs/h' - fi - sHOMEDIR=$(grep 'export HOMEDIR=${HOMEDIR:-'${sHeader} -r ${sWS}/parm/setbase | sed 's/export HOMEDIR=${HOMEDIR:-//g'| sed 's/}//g') - echo $sHOMEDIR - - if [[ -L global-workflow.fd ]] ; then - rm global-workflow.fd - elif [[ -d global-workflow.fd ]] ; then - rm -rf global-workflow.fd - fi - ln -sf $sHOMEDIR global-workflow.fd - - elif [[ $Structure == "prod" ]]; then - # Checkout the global-workflow if needed - if [[ -d global-workflow.fd ]] ; then - rm -rf global-workflow.fd - fi - ./checkout.sh + cd $sWS/../sorc + + if [[ $Structure == "dev" ]]; then + echo "...working on it ..." + if [[ $machine == "hera" ]]; then + sHeader='/scratch2/NCEPDEV/ensemble' + elif [[ $machine == "wcoss2" ]]; then + sHeader='/lfs/h' + fi + sHOMEDIR=$(grep 'export HOMEDIR=${HOMEDIR:-'${sHeader} -r ${sWS}/parm/setbase | sed 's/export HOMEDIR=${HOMEDIR:-//g'| sed 's/}//g') + echo $sHOMEDIR + + if [[ -L global-workflow.fd ]] ; then + rm global-workflow.fd + elif [[ -d global-workflow.fd ]] ; then + rm -rf global-workflow.fd fi + ln -sf $sHOMEDIR global-workflow.fd - ## Build the code and install - ./build_all.sh + elif [[ $Structure == "prod" ]]; then + # Checkout the global-workflow if needed + if [[ -d global-workflow.fd ]] ; then + rm -rf global-workflow.fd + fi + ./checkout.sh + fi + + ## Build the code and install + ./build_all.sh fi # for Link if [ $Link = "yes" ]; then - cd $sWS/../sorc - if [ $machine = "hera" ]; then - ./link_gefs.sh -e emc -m hera - elif [ $machine = "wcoss2" ]; then - ./link_gefs.sh -e $RunEnvir -m wcoss2 - fi + cd $sWS/../sorc + if [ $machine = "hera" ]; then + ./link_gefs.sh -e emc -m hera + elif [ $machine = "wcoss2" ]; then + ./link_gefs.sh -e $RunEnvir -m wcoss2 + fi fi # for cleanning if [ $CleanAll = "yes" ]; then - echo "Cleaning ..." - - rm -rf gefs.xml - rm -rf cron_rocoto - rm -rf tasks + echo "Cleaning ..." - cd $sWS/../sorc + rm -rf gefs.xml + rm -rf cron_rocoto + rm -rf tasks - rm -rf logs + cd $sWS/../sorc - if [[ -L global-workflow.fd ]] ; then - rm global-workflow.fd - elif [[ -d global-workflow.fd ]] ; then - rm -rf global-workflow.fd - fi - - for dir in global_ensadd.fd global_enspqpf.fd gefs_ensstat.fd global_ensppf.fd ; do - if [ -f $dir ]; then - cd $dir - make clean - cd .. - fi - done - - for dir in global_enscvprcp.fd global_enspvrfy.fd global_enssrbias.fd global_enscqpf.fd global_enscvt24h.fd global_ensrfmat.fd ; do - if [ -f $dir ]; then - cd $dir - make clean - cd .. - fi - done - - for dir in ../util/sorc/overenstr.grib.fd; do - if [ -f $dir ]; then - cd $dir - make clean - cd ../../../sorc - fi - done - - cd ${sWS}/../sorc - rm -rf ../exec - rm -rf ../util/exec - rm -rf ../fix/fix_* - rm -rf ../fix/product - - # Clean the new links + rm -rf logs + + if [[ -L global-workflow.fd ]] ; then + rm global-workflow.fd + elif [[ -d global-workflow.fd ]] ; then rm -rf global-workflow.fd - rm -rf ../parm/parm_fv3diag - rm -rf ../parm/post - rm -rf ../parm/product - rm -f ../ush/gfs_nceppost.sh - rm -f ../ush/global_chgres.sh - rm -f ../ush/global_chgres_driver.sh - rm -f ../ush/wave_* - rm -f ../ush/merge_fv3_chem_tile.py - - rm -f ../parm/prep_chem_sources.inp.IN - rm -f ../scripts/exglobal_prep_chem.sh - rm -f ../scripts/exwave_* - rm -f ../scripts/exglobal_fcst_nemsfv3gfs.sh - rm -rf ../env + fi + + for dir in global_ensadd.fd global_enspqpf.fd gefs_ensstat.fd global_ensppf.fd; do + if [ -f $dir ]; then + cd $dir + make clean + cd ${sWS}/../sorc + fi + done + + for dir in global_enscvprcp.fd global_enspvrfy.fd global_enssrbias.fd global_enscqpf.fd global_enscvt24h.fd global_ensrfmat.fd; do + if [ -f $dir ]; then + cd $dir + make clean + cd ${sWS}/../sorc + fi + done + + for dir in ../util/sorc/overenstr.grib.fd; do + if [ -f $dir ]; then + cd $dir + make clean + cd ${sWS}/../sorc + fi + done + + cd ${sWS}/../fix + for gw_dir in product fix_emission fix_gefs fix_wave am aer lut orog chem ugwd wave; do + rm -rf ${gw_dir} + done + + cd ${sWS}/../parm + for dir in parm_fv3diag post product; do + rm -rf ${dir} + done + + cd ${sWS}/../scripts + for file in `ls exgfs*` exglobal_forecast.sh; do + rm -rf ${file} + done + + cd ${sWS}/../ush + for file in preamble.sh cplvalidate.sh `ls forecast_* ` `ls wave_*` merge_fv3_aerosol_tile.py chgres_cube.sh nems_configure.sh `ls parsing_*.sh`; do + rm -rf ${file} + done + + cd ${sWS} + rm -rf ../exec + rm -rf ../util/exec + rm -rf ../sorc/ufs_model.fd + rm -rf ../env fi # for CleanAll # for rocoto if [ $RunRocoto = "yes" ]; then - cd $sWS - if [ $machine = "hera" ]; then - module load intel/18.0.5.274 - module load rocoto/1.3.3 - module load contrib - module load anaconda/latest + cd $sWS + if [ $machine = "hera" ]; then + module load intel/18.0.5.274 + module load rocoto/1.3.3 + module load contrib + module load anaconda/latest - elif [ $machine = "wcoss2" ]; then + elif [ $machine = "wcoss2" ]; then - module purge - module load envvar/1.0 + module purge + module load envvar/1.0 - module load PrgEnv-intel/8.1.0 - module load craype/2.7.10 - module load intel/19.1.3.304 - module load cray-mpich/8.1.9 + module load PrgEnv-intel/8.1.0 + module load craype/2.7.10 + module load intel/19.1.3.304 + module load cray-mpich/8.1.9 - module load python/3.8.6 + module load python/3.8.6 - module use /apps/ops/test/nco/modulefiles/ - module load core/rocoto/1.3.5 + module use /apps/ops/test/nco/modulefiles/ + module load core/rocoto/1.3.5 - fi - ./py/run_pyGEFS.py -r yes -f $userConfigFile - echo "Generated xml and/or ent and updated bin file!" + fi + ./py/run_pyGEFS.py -r yes -f $userConfigFile + echo "Generated xml and/or ent and updated bin file!" fi # For RunRocoto #echo "crontab" # For Crontab if [ $AddCrontabToMyCrontab = "yes" ]; then - cd $sWS - if [ $machine = "hera" ]; then - if [ -f $HOME/cron/mycrontab ]; then - echo "Adding crontab to $HOME/cron/mycrontab!" - else - mkdir $HOME/cron - touch $HOME/cron/mycrontab - fi + cd $sWS + if [ -f $HOME/cron/mycrontab ]; then + echo "Adding crontab to $HOME/cron/mycrontab!" + else + mkdir $HOME/cron + touch $HOME/cron/mycrontab + fi - py/add_crontab.py - crontab $HOME/cron/mycrontab - echo "Added crontab to $HOME/cron/mycrontab!" - - elif [ $machine = "wcoss2" ]; then - if [ -f $HOME/cron/mycrontab ]; then - echo "Adding crontab to $HOME/cron/mycrontab!" - else - mkdir $HOME/cron - touch $HOME/cron/mycrontab - fi - - py/add_crontab.py - crontab $HOME/cron/mycrontab - echo "Added crontab to $HOME/cron/mycrontab!" + py/add_crontab.py + crontab $HOME/cron/mycrontab + echo "Added crontab to $HOME/cron/mycrontab!" - fi fi From 6073dc31c67e70584ef14c4ebeeb9dc0709d0703 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Wed, 12 Apr 2023 22:03:32 -0400 Subject: [PATCH 23/41] Clean up and optimize the compile_install_all.sh On branch feature/v13_atmos_prep modified: rocoto/compile_install_all.sh Refs: #87 --- rocoto/compile_install_all.sh | 149 ++++++++++++++++++---------------- 1 file changed, 78 insertions(+), 71 deletions(-) diff --git a/rocoto/compile_install_all.sh b/rocoto/compile_install_all.sh index 8f16a9f13..4b3084894 100755 --- a/rocoto/compile_install_all.sh +++ b/rocoto/compile_install_all.sh @@ -1,8 +1,8 @@ #!/bin/bash -set -eux +set -eu #x sWS=$(pwd) -echo $sWS +echo ${sWS} while getopts c:a:r:m:f:b:e:s:l:o: option do @@ -33,7 +33,7 @@ Structure=${Structure:-no} # dev (use HOMEDIR to link), prod (clone global-workf Link=${Link:-no} Operation=${Operation:-no} # ecflow, rocoto, lsf -if [ $machine = "nomachine" ]; then +if [ ${machine} = "nomachine" ]; then if [ -d /scratch1/NCEPDEV ]; then machine=hera elif [[ -d /apps/prod ]]; then # WCOSS2 @@ -44,41 +44,41 @@ if [ $machine = "nomachine" ]; then fi fi -echo $CompileCode -echo $CleanAll -echo $RunRocoto -echo $machine -echo $userConfigFile -echo $RunEnvir +echo ${CompileCode} +echo ${CleanAll} +echo ${RunRocoto} +echo ${machine} +echo ${userConfigFile} +echo ${RunEnvir} echo ${Structure} echo ${Link} -if [ $CompileCode = "yes" ]; then +if [ ${CompileCode} = "yes" ]; then Link=yes fi -if [ $CompileCode = "yes" ]; then - cd $sWS/../sorc +if [ ${CompileCode} = "yes" ]; then + cd ${sWS}/../sorc - if [[ $Structure == "dev" ]]; then + if [[ ${Structure} == "dev" ]]; then echo "...working on it ..." - if [[ $machine == "hera" ]]; then + if [[ ${machine} == "hera" ]]; then sHeader='/scratch2/NCEPDEV/ensemble' - elif [[ $machine == "wcoss2" ]]; then + elif [[ ${machine} == "wcoss2" ]]; then sHeader='/lfs/h' fi sHOMEDIR=$(grep 'export HOMEDIR=${HOMEDIR:-'${sHeader} -r ${sWS}/parm/setbase | sed 's/export HOMEDIR=${HOMEDIR:-//g'| sed 's/}//g') - echo $sHOMEDIR + echo ${sHOMEDIR} if [[ -L global-workflow.fd ]] ; then rm global-workflow.fd elif [[ -d global-workflow.fd ]] ; then rm -rf global-workflow.fd fi - ln -sf $sHOMEDIR global-workflow.fd + ln -sf ${sHOMEDIR} global-workflow.fd - elif [[ $Structure == "prod" ]]; then + elif [[ ${Structure} == "prod" ]]; then # Checkout the global-workflow if needed if [[ -d global-workflow.fd ]] ; then rm -rf global-workflow.fd @@ -93,83 +93,90 @@ fi # for Link -if [ $Link = "yes" ]; then - cd $sWS/../sorc - if [ $machine = "hera" ]; then +if [ ${Link} = "yes" ]; then + cd ${sWS}/../sorc + if [ ${machine} = "hera" ]; then ./link_gefs.sh -e emc -m hera - elif [ $machine = "wcoss2" ]; then - ./link_gefs.sh -e $RunEnvir -m wcoss2 + elif [ ${machine} = "wcoss2" ]; then + ./link_gefs.sh -e ${RunEnvir} -m wcoss2 fi fi + # for cleanning -if [ $CleanAll = "yes" ]; then +if [ ${CleanAll} = "yes" ]; then echo "Cleaning ..." + cd ${sWS} + rm -rf ../exec + rm -rf ../util/exec + rm -rf ../env + rm -rf gefs.xml + rm -rf gefs*.db rm -rf cron_rocoto rm -rf tasks - - cd $sWS/../sorc - rm -rf logs + cd ${sWS}/../sorc + rm -rf logs + rm -rf ufs_model.fd if [[ -L global-workflow.fd ]] ; then rm global-workflow.fd elif [[ -d global-workflow.fd ]] ; then rm -rf global-workflow.fd fi - for dir in global_ensadd.fd global_enspqpf.fd gefs_ensstat.fd global_ensppf.fd; do - if [ -f $dir ]; then - cd $dir - make clean - cd ${sWS}/../sorc - fi - done - - for dir in global_enscvprcp.fd global_enspvrfy.fd global_enssrbias.fd global_enscqpf.fd global_enscvt24h.fd global_ensrfmat.fd; do - if [ -f $dir ]; then - cd $dir + for sDir in global_ensadd \ + gefs_ensstat \ + global_ensppf \ + wave_stat \ + gefs_nemsio2nc \ + global_enscqpf \ + global_enscvprcp \ + global_enssrbias \ + global_enscvt24h \ + global_enspqpf \ + global_enspvrfy \ + global_ensrfmat \ + global_enssrbias; do + cd ${sWS}/../sorc + if [ -f ${sDir}.fd ]; then + cd ${sDir}.fd make clean - cd ${sWS}/../sorc fi done - for dir in ../util/sorc/overenstr.grib.fd; do - if [ -f $dir ]; then - cd $dir + cd ${sWS}/../util/sorc + for sDir in overenstr.grib.fd; do + if [ -f ${sDir} ]; then + cd ${sDir} make clean - cd ${sWS}/../sorc fi done cd ${sWS}/../fix - for gw_dir in product fix_emission fix_gefs fix_wave am aer lut orog chem ugwd wave; do - rm -rf ${gw_dir} + for sDir in product \ + fix_emission fix_gefs fix_wave \ + am aer lut orog chem ugwd wave; do + rm -rf ${sDir} done cd ${sWS}/../parm - for dir in parm_fv3diag post product; do - rm -rf ${dir} + for sDir in parm_fv3diag post product; do + rm -rf ${sDir} done cd ${sWS}/../scripts - for file in `ls exgfs*` exglobal_forecast.sh; do - rm -rf ${file} + for sFile in `ls exgfs*` exglobal_forecast.sh; do + rm -rf ${sFile} done cd ${sWS}/../ush - for file in preamble.sh cplvalidate.sh `ls forecast_* ` `ls wave_*` merge_fv3_aerosol_tile.py chgres_cube.sh nems_configure.sh `ls parsing_*.sh`; do - rm -rf ${file} + for sFile in preamble.sh cplvalidate.sh `ls forecast_* ` `ls wave_*` merge_fv3_aerosol_tile.py chgres_cube.sh nems_configure.sh `ls parsing_*.sh`; do + rm -rf ${sFile} done - cd ${sWS} - rm -rf ../exec - rm -rf ../util/exec - rm -rf ../sorc/ufs_model.fd - rm -rf ../env - fi # for CleanAll # for rocoto @@ -186,8 +193,8 @@ if [ $RunRocoto = "yes" ]; then module purge module load envvar/1.0 - module load PrgEnv-intel/8.1.0 - module load craype/2.7.10 + module load PrgEnv-intel/8.3.3 + module load craype/2.7.17 module load intel/19.1.3.304 module load cray-mpich/8.1.9 @@ -197,23 +204,23 @@ if [ $RunRocoto = "yes" ]; then module load core/rocoto/1.3.5 fi - ./py/run_pyGEFS.py -r yes -f $userConfigFile + ./py/run_pyGEFS.py -r yes -f ${userConfigFile} echo "Generated xml and/or ent and updated bin file!" fi # For RunRocoto -#echo "crontab" # For Crontab -if [ $AddCrontabToMyCrontab = "yes" ]; then - cd $sWS - if [ -f $HOME/cron/mycrontab ]; then - echo "Adding crontab to $HOME/cron/mycrontab!" - else - mkdir $HOME/cron - touch $HOME/cron/mycrontab +if [ ${AddCrontabToMyCrontab} = "yes" ]; then + cd ${sWS} + sFile=${HOME}/cron/mycrontab + if [ -f ${sFile} ]; then + echo "Adding crontab to ${sFile}!" + else + mkdir ${HOME}/cron + touch ${sFile} fi - + py/add_crontab.py - crontab $HOME/cron/mycrontab - echo "Added crontab to $HOME/cron/mycrontab!" + crontab ${sFile} + echo "Added crontab to ${sFile}!" fi From 9fb9ace73bb9180197bca8ec213973d90e066888 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Wed, 12 Apr 2023 22:08:20 -0400 Subject: [PATCH 24/41] Fix a bug On branch feature/v13_atmos_prep modified: rocoto/compile_install_all.sh Refs: #87 --- rocoto/compile_install_all.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/rocoto/compile_install_all.sh b/rocoto/compile_install_all.sh index 4b3084894..f1eae543d 100755 --- a/rocoto/compile_install_all.sh +++ b/rocoto/compile_install_all.sh @@ -147,13 +147,16 @@ if [ ${CleanAll} = "yes" ]; then fi done - cd ${sWS}/../util/sorc - for sDir in overenstr.grib.fd; do - if [ -f ${sDir} ]; then - cd ${sDir} - make clean - fi - done + sPath=${sWS}/../util/sorc + if [[ -e ${sPath} ]]; then + cd ${sWS}/../util/sorc + for sDir in overenstr.grib.fd; do + if [ -f ${sDir} ]; then + cd ${sDir} + make clean + fi + done + fi cd ${sWS}/../fix for sDir in product \ From 0ef6d52414807c55edfaa7553004ff2dc80a50d5 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Wed, 12 Apr 2023 22:53:27 -0400 Subject: [PATCH 25/41] Change ksh to bash On branch feature/v13_atmos_prep modified: rocoto/bin/hera/common.sh modified: rocoto/bin/wcoss2/common.sh Refs: #87 --- rocoto/bin/hera/common.sh | 2 +- rocoto/bin/wcoss2/common.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rocoto/bin/hera/common.sh b/rocoto/bin/hera/common.sh index 1162707da..26cdd7003 100755 --- a/rocoto/bin/hera/common.sh +++ b/rocoto/bin/hera/common.sh @@ -1,4 +1,4 @@ -#!/bin/ksh +#!/usr/bin/env bash export envir=${envir:-dev} export RUN_ENVIR=${RUN_ENVIR:-dev} diff --git a/rocoto/bin/wcoss2/common.sh b/rocoto/bin/wcoss2/common.sh index 0c0026014..9f238a13b 100644 --- a/rocoto/bin/wcoss2/common.sh +++ b/rocoto/bin/wcoss2/common.sh @@ -1,4 +1,4 @@ -#!/bin/ksh -l +#!/usr/bin/env bash RocotoGen=${RocotoGen:-0} From f2bc775c38db51b8e2bf2cf361f07472022f5458 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 17 Apr 2023 13:32:14 -0400 Subject: [PATCH 26/41] Modify based on reviewer's comments On branch feature/v13_atmos_prep modified: parm/parm_gefs/gefs_atmos_prep.parm modified: rocoto/bin/wcoss2/atmos_prep.sh modified: rocoto/bin/wcoss2/common.sh modified: rocoto/py/GEFS_Crontab.py modified: rocoto/py/GEFS_Parm.py modified: rocoto/py/GEFS_UserConfig.py modified: rocoto/py/run_pyGEFS.py modified: scripts/exgefs_atmos_prep.sh Refs: #87 --- parm/parm_gefs/gefs_atmos_prep.parm | 4 ---- rocoto/bin/wcoss2/atmos_prep.sh | 2 +- rocoto/bin/wcoss2/common.sh | 23 +++++++++++++---------- rocoto/py/GEFS_Crontab.py | 3 ++- rocoto/py/GEFS_Parm.py | 3 ++- rocoto/py/GEFS_UserConfig.py | 8 +++++--- rocoto/py/run_pyGEFS.py | 3 ++- scripts/exgefs_atmos_prep.sh | 8 -------- 8 files changed, 25 insertions(+), 29 deletions(-) diff --git a/parm/parm_gefs/gefs_atmos_prep.parm b/parm/parm_gefs/gefs_atmos_prep.parm index 453410d8a..a56c9af7f 100644 --- a/parm/parm_gefs/gefs_atmos_prep.parm +++ b/parm/parm_gefs/gefs_atmos_prep.parm @@ -8,10 +8,6 @@ case $cyc in esac export memshift -#export pdycycp=$($NDATE -$fhrp $PDY$cyc) -#export pdyp=$(echo $pdycycp|cut -c1-8) -#export cycp=$(echo $pdycycp|cut -c9-10) - export CDATE=$PDY$cyc export MAX_ENKF_SEARCHES=${MAX_ENKF_SEARCHES:-3} diff --git a/rocoto/bin/wcoss2/atmos_prep.sh b/rocoto/bin/wcoss2/atmos_prep.sh index 18a5f1ae8..5265a215e 100755 --- a/rocoto/bin/wcoss2/atmos_prep.sh +++ b/rocoto/bin/wcoss2/atmos_prep.sh @@ -8,7 +8,7 @@ ulimit -a . $SOURCEDIR/versions/run.ver # Load modules -module purge +module reset module load envvar/$envvar_ver module load PrgEnv-intel/$PrgEnv_intel_ver module load craype/$craype_ver diff --git a/rocoto/bin/wcoss2/common.sh b/rocoto/bin/wcoss2/common.sh index 0c0026014..985047548 100644 --- a/rocoto/bin/wcoss2/common.sh +++ b/rocoto/bin/wcoss2/common.sh @@ -36,16 +36,19 @@ if [[ $RocotoGen == 0 ]]; then #export COMPATH=$HOMEdata/canned/com/gfs:$HOMEdata/canned/com/cfs:$HOMEdata/canned/com/nawips:$HOMEdata/canned/com/ecmwf:$HOMEdata/canned/com/nam:${WORKDIR}/$envir/com/${NET} #export DCOMROOT=${HOMEdata}/canned/dcom # For prod data - if [[ $CASEHR == "C48" ]]; then - export COMPATH=${WORKDIR}/$envir/com/${NET}:/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS_v13/z_DATA/lfs_C96_C48_NOIAU_anal/h1/ops/prod/com/gfs - #export COMPATH=${WORKDIR}/$envir/com/${NET}:/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS_v13/z_DATA/lfs_C96_C48_NOIAU_anal_20230316/h1/ops/prod/com/gfs - elif [[ $CASEHR == "C384" ]]; then - export COMPATH=${WORKDIR}/$envir/com/${NET}:/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS_v13/z_DATA/lfs_C768_C384_NOIAU_anal/h1/ops/prod/com/gfs - #export COMPATH=${WORKDIR}/$envir/com/${NET}:/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS_v13/z_DATA/lfs_C768_C384_NOIAU_anal_20230223_updated_dev/h1/ops/prod/com/gfs - else - echo "Wrong CASEHR .." - exit -1 - fi + case ${CASEHR} in + "C48") + export COMPATH=${WORKDIR}/$envir/com/${NET}:/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS_v13/z_DATA/lfs_C96_C48_NOIAU_anal/h1/ops/prod/com/gfs + #export COMPATH=${WORKDIR}/$envir/com/${NET}:/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS_v13/z_DATA/lfs_C96_C48_NOIAU_anal_20230316/h1/ops/prod/com/gfs + ;; + "C384") + export COMPATH=${WORKDIR}/$envir/com/${NET}:/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS_v13/z_DATA/lfs_C768_C384_NOIAU_anal/h1/ops/prod/com/gfs + #export COMPATH=${WORKDIR}/$envir/com/${NET}:/lfs/h2/emc/ens/noscrub/xianwu.xue/GEFS_v13/z_DATA/lfs_C768_C384_NOIAU_anal_20230223_updated_dev/h1/ops/prod/com/gfs + ;; + *) + echo "Wrong CASEHR .." + exit -1 + esac elif [[ $RocotoGen == 1 ]]; then export HOMEtrak=/lfs/h2/emc/ens/noscrub/emc.ens/common/git/ens_tracker/ens_tracker.v2.1.2 diff --git a/rocoto/py/GEFS_Crontab.py b/rocoto/py/GEFS_Crontab.py index 85f407d6b..5c1891256 100755 --- a/rocoto/py/GEFS_Crontab.py +++ b/rocoto/py/GEFS_Crontab.py @@ -20,7 +20,8 @@ def create_crontab(dicBase, OnlyForTest=False, cronint=5): ########################################################## # No point creating a crontab if rocotorun is not available. - import sys, os + import sys + import os from distutils.spawn import find_executable if OnlyForTest: diff --git a/rocoto/py/GEFS_Parm.py b/rocoto/py/GEFS_Parm.py index df1809831..844f0d53a 100755 --- a/rocoto/py/GEFS_Parm.py +++ b/rocoto/py/GEFS_Parm.py @@ -69,7 +69,8 @@ def read_dicParm(sConfig): # ======================================================= def get_and_merge_default_dicParm(dicParm, WHERE_AM_I): - import os, sys + import os + import sys # To get the WHERE_AM_I from dicParm or identify it using default methode sDefaultConfig_File = os.path.join(sys.path[0], f"user_{WHERE_AM_I}.conf") diff --git a/rocoto/py/GEFS_UserConfig.py b/rocoto/py/GEFS_UserConfig.py index 0679a755b..19c1b4798 100755 --- a/rocoto/py/GEFS_UserConfig.py +++ b/rocoto/py/GEFS_UserConfig.py @@ -1,6 +1,7 @@ # ======================================================= def get_and_merge_default_config(dicBase): - import os, sys + import os + import sys # To get the WHERE_AM_I from dicBase or identify it using default methode get_WHERE_AM_I(dicBase) @@ -19,7 +20,8 @@ def get_and_merge_default_config(dicBase): # ======================================================= def get_config_file2(sConfigFile="user_full.conf"): - import os, sys + import os + import sys sRocoto_WS = os.getcwd() sConfig = os.path.join(sRocoto_WS, sConfigFile) @@ -84,7 +86,7 @@ def read_config(sConfig): # ======================================================= def create_folders(dicBase): - import os, sys + import os EXPID = dicBase['EXPID'] WORKDIR = str(dicBase['WORKDIR']).replace("&EXPID;", EXPID) diff --git a/rocoto/py/run_pyGEFS.py b/rocoto/py/run_pyGEFS.py index b6ba57194..4a34dd277 100755 --- a/rocoto/py/run_pyGEFS.py +++ b/rocoto/py/run_pyGEFS.py @@ -13,7 +13,8 @@ import GEFS_Crontab as gefs_crontab def main(): - import os, sys + import os + import sys import argparse parser = argparse.ArgumentParser(description='pyGEFS: Python-Based Workflow Management of NCEP Global Ensemble Forecast System!') diff --git a/scripts/exgefs_atmos_prep.sh b/scripts/exgefs_atmos_prep.sh index f88f48236..4992917cf 100755 --- a/scripts/exgefs_atmos_prep.sh +++ b/scripts/exgefs_atmos_prep.sh @@ -2,14 +2,6 @@ source "${HOMEgfs:-${HOMEgefs}}/ush/preamble.sh" -#echo "$(date -u) begin $(basename $BASH_SOURCE)" - -#set -xa -#if [[ ${STRICT:-NO} == "YES" ]]; then -# # Turn on strict bash error checking -# set -eu -#fi - export HOMEgfs=${HOMEgfs:-${HOMEgefs}} export HOMEufs=${HOMEufs:-${HOMEgfs}} export USHgfs=$HOMEgfs/ush From 33232ef79c8972fcc9a5b9705f74606286007c30 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 17 Apr 2023 13:34:28 -0400 Subject: [PATCH 27/41] Reorganize link_gefs.sh On branch feature/v13_atmos_prep modified: sorc/link_gefs.sh Refs: #87 --- sorc/link_gefs.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/sorc/link_gefs.sh b/sorc/link_gefs.sh index f66daca9d..b9f92d383 100755 --- a/sorc/link_gefs.sh +++ b/sorc/link_gefs.sh @@ -85,7 +85,7 @@ if [[ -d global-workflow.fd ]] ; then sPath=../sorc/global-workflow.fd/exec for sFile in ${sPath}/ww3_* do - echo $sFile + echo ${sFile} done $LINK ${sPath}/ww3_* ../exec/ @@ -152,13 +152,13 @@ if [[ -d global-workflow.fd ]] ; then ush/parsing_namelists_CICE.sh \ sorc/ufs_model.fd do - if [[ -e ../$sFile ]]; then - if [[ -L ../$sFile ]]; then - rm ../$sFile - $LINK ../sorc/global-workflow.fd/$sFile ../$sFile + if [[ -e ../${sFile} ]]; then + if [[ -L ../${sFile} ]]; then + rm ../${sFile} + $LINK ../sorc/global-workflow.fd/${sFile} ../${sFile} fi else - $LINK ../sorc/global-workflow.fd/$sFile ../$sFile + $LINK ../sorc/global-workflow.fd/${sFile} ../${sFile} fi done fi @@ -172,11 +172,10 @@ if [[ -d global-workflow.fd ]]; then $LINK ../sorc/global-workflow.fd/scripts/$sFile ../scripts/ done - lUsh=`ls ../sorc/global-workflow.fd/ush/wave_*` #"wave_ens_bull.sh wave_ens_stat.sh wave_grib2_sbs.sh wave_grid_interp_sbs.sh wave_grid_moddef.sh wave_outp_spec.sh wave_prnc_cur.sh wave_prnc_ice.sh wave_tar.sh" + lUsh=`ls ../sorc/global-workflow.fd/ush/wave_*` for sFile in $lUsh; do - sFile=$(basename ${sFile}) echo $sFile - $LINK ../sorc/global-workflow.fd/ush/$sFile ../ush/ + $LINK $sFile ../ush/ done if [[ -e ../env ]]; then From c4c48a0957d6a8859b0b09b2810f4127b52c5dea Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 17 Apr 2023 13:52:02 -0400 Subject: [PATCH 28/41] Remove config.com and link it from g-w On branch feature/v13_atmos_prep deleted: parm/config/config.com modified: sorc/link_gefs.sh Refs: #87 --- parm/config/config.com | 92 ------------------------------------------ sorc/link_gefs.sh | 8 ++++ 2 files changed, 8 insertions(+), 92 deletions(-) delete mode 100644 parm/config/config.com diff --git a/parm/config/config.com b/parm/config/config.com deleted file mode 100644 index 2308658fe..000000000 --- a/parm/config/config.com +++ /dev/null @@ -1,92 +0,0 @@ -# shellcheck shell=bash -echo "BEGIN: config.com" - -# These are just templates. All templates must use single quotations so variable -# expansion does not occur when this file is sourced. Substitution happens later -# during runtime. It is recommended to use the helper function `generate_com()`, -# to do this substitution, which is defined in `ush/preamble.sh`. -# -# Syntax for generate_com(): -# generate_com [-rx] $var1[:$tmpl1] [$var2[:$tmpl2]] [...]] -# -# options: -# -r: Make variable read-only (same as `decalre -r`) -# -x: Mark variable for declare -rx (same as `declare -x`) -# var1, var2, etc: Variable names whose values will be generated from a template -# and declared -# tmpl1, tmpl2, etc: Specify the template to use (default is "${var}_TMPL") -# -# Examples: -# # Current cycle and RUN -# YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_ANALYSIS -# -# # Previous cycle and gdas -# RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \ -# COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL -# -# # Current cycle and COM for first member -# MEMDIR='mem001' YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_HISTORY -# - -# -# If any restart, input, or analysis template is updated, `setup_expt.py.fill_COMROT_cycled()` -# must correspondingly be updated to match. -# - -# Ignore shellcheck warnings about variables not being expanded; this is what we want -# shellcheck disable=SC2016 -if [[ "${RUN_ENVIR:-emc}" == "nco" ]]; then - COM_OBS_TMPL=$(compath.py "${envir}/obsproc/${obsproc_ver}")'/${RUN}.${YMD}/${HH}/atmos' - COM_RTOFS_TMPL=$(compath.py "${envir}/${WAVECUR_DID}/${rtofs_ver}") -else - COM_OBS_TMPL='${ROTDIR}/${RUN}.${YMD}/${HH}/obs' - COM_RTOFS_TMPL='${DMPDIR}' -fi -declare -rx COM_OBS_TMPL COM_RTOFS_TMPL -declare -rx COM_OBSDMP_TMPL='${DMPDIR}/${DUMP}${DUMP_SUFFIX}.${YMD}/${HH}/atmos' - -COM_BASE='${ROTDIR}/${RUN}.${YMD}/${HH}/${MEMDIR}' - -declare -rx COM_TOP_TMPL='${ROTDIR}/${RUN}.${YMD}/${HH}' - -declare -rx COM_ATMOS_INPUT_TMPL=${COM_BASE}'/model_data/atmos/input' -declare -rx COM_ATMOS_RESTART_TMPL=${COM_BASE}'/model_data/atmos/restart' -declare -rx COM_ATMOS_ANALYSIS_TMPL=${COM_BASE}'/analysis/atmos' -declare -rx COM_ATMOS_HISTORY_TMPL=${COM_BASE}'/model_data/atmos/history' -declare -rx COM_ATMOS_MASTER_TMPL=${COM_BASE}'/model_data/atmos/master' -declare -rx COM_ATMOS_GRIB_TMPL=${COM_BASE}'/products/atmos/grib/${RES}' -declare -rx COM_ATMOS_BUFR_TMPL=${COM_BASE}'/products/atmos/bufr' -declare -rx COM_ATMOS_GEMPAK_TMPL=${COM_BASE}'/products/atmos/gempak' -declare -rx COM_ATMOS_GENESIS_TMPL=${COM_BASE}'/products/atmos/cyclone/genesis_vital' -declare -rx COM_ATMOS_TRACK_TMPL=${COM_BASE}'/products/atmos/cyclone/tracks' -declare -rx COM_ATMOS_GOES_TMPL=${COM_BASE}'/products/atmos/goes_sim' -declare -rx COM_ATMOS_IMAGERY_TMPL=${COM_BASE}'/products/atmos/imagery' -declare -rx COM_ATMOS_MINMON_TMPL=${COM_BASE}'/products/atmos/minmon' -declare -rx COM_ATMOS_WAFS_TMPL=${COM_BASE}'/products/atmos/wafs' -declare -rx COM_ATMOS_WMO_TMPL=${COM_BASE}'/products/atmos/wmo' - -declare -rx COM_WAVE_RESTART_TMPL=${COM_BASE}'/model_data/wave/restart' -declare -rx COM_WAVE_PREP_TMPL=${COM_BASE}'/model_data/wave/prep' -declare -rx COM_WAVE_HISTORY_TMPL=${COM_BASE}'/model_data/wave/history' -declare -rx COM_WAVE_GRID_TMPL=${COM_BASE}'/products/wave/gridded' -declare -rx COM_WAVE_STATION_TMPL=${COM_BASE}'/products/wave/station' -declare -rx COM_WAVE_GEMPAK_TMPL=${COM_BASE}'/products/wave/gempak' -declare -rx COM_WAVE_WMO_TMPL=${COM_BASE}'/products/wave/wmo' - -declare -rx COM_OCEAN_HISTORY_TMPL=${COM_BASE}'/model_data/ocean/history' -declare -rx COM_OCEAN_RESTART_TMPL=${COM_BASE}'/model_data/ocean/restart' -declare -rx COM_OCEAN_INPUT_TMPL=${COM_BASE}'/model_data/ocean/input' -declare -rx COM_OCEAN_ANALYSIS_TMPL=${COM_BASE}'/analysis/ocean' -declare -rx COM_OCEAN_2D_TMPL=${COM_BASE}'/products/ocean/2D' -declare -rx COM_OCEAN_3D_TMPL=${COM_BASE}'/products/ocean/3D' -declare -rx COM_OCEAN_DAILY_TMPL=${COM_BASE}'/products/ocean/daily' -declare -rx COM_OCEAN_XSECT_TMPL=${COM_BASE}'/products/ocean/xsect' -declare -rx COM_OCEAN_GRIB_TMPL=${COM_BASE}'/products/ocean/grib/${RES}' - -declare -rx COM_ICE_INPUT_TMPL=${COM_BASE}'/model_data/ice/input' -declare -rx COM_ICE_HISTORY_TMPL=${COM_BASE}'/model_data/ice/history' -declare -rx COM_ICE_RESTART_TMPL=${COM_BASE}'/model_data/ice/restart' - -declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model_data/chem/history' - -declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model_data/med/restart' diff --git a/sorc/link_gefs.sh b/sorc/link_gefs.sh index b9f92d383..ddc867c1d 100755 --- a/sorc/link_gefs.sh +++ b/sorc/link_gefs.sh @@ -124,6 +124,14 @@ if [[ -d global-workflow.fd ]] ; then fi ${LINK} ../sorc/global-workflow.fd/parm/${fn} ../parm/ done + + if [[ ! -d ../parm/config ]]; then + mkdir -p ../parm/config + fi + for fn in config.com; do + echo ${fn} + ${LINK} ../sorc/global-workflow.fd/parm/config/${fn} ../parm/config/ + done fi From 0b44b7bb5f6aaf9ca67249ffcae734f74e591ccf Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 17 Apr 2023 13:58:16 -0400 Subject: [PATCH 29/41] Ignore parm/config and fix the link path On branch feature/v13_atmos_prep modified: parm/.gitignore modified: sorc/link_gefs.sh Refs: #87 --- parm/.gitignore | 2 ++ sorc/link_gefs.sh | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/parm/.gitignore b/parm/.gitignore index a10ecd8d4..2b97400e6 100644 --- a/parm/.gitignore +++ b/parm/.gitignore @@ -2,3 +2,5 @@ parm_fv3diag post product prep_chem_sources.inp.IN + +config diff --git a/sorc/link_gefs.sh b/sorc/link_gefs.sh index ddc867c1d..2b2ebd02a 100755 --- a/sorc/link_gefs.sh +++ b/sorc/link_gefs.sh @@ -128,9 +128,10 @@ if [[ -d global-workflow.fd ]] ; then if [[ ! -d ../parm/config ]]; then mkdir -p ../parm/config fi + cd ../parm/config for fn in config.com; do echo ${fn} - ${LINK} ../sorc/global-workflow.fd/parm/config/${fn} ../parm/config/ + ${LINK} ../../sorc/global-workflow.fd/parm/config/${fn} ./ done fi From e94350f4ef9e2109953893f00193191a59cb1789 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 17 Apr 2023 14:06:51 -0400 Subject: [PATCH 30/41] Move all import statements to the top of scripts On branch feature/v13_atmos_prep modified: rocoto/py/GEFS_Crontab.py modified: rocoto/py/GEFS_Parm.py modified: rocoto/py/GEFS_UserConfig.py modified: rocoto/py/GEFS_XML.py modified: rocoto/py/GEFS_XML_For_Tasks.py modified: rocoto/py/add_crontab.py modified: rocoto/py/run_pyGEFS.py Refs: #87 --- rocoto/py/GEFS_Crontab.py | 10 +++++++--- rocoto/py/GEFS_Parm.py | 16 +++++++--------- rocoto/py/GEFS_UserConfig.py | 20 +++++++------------- rocoto/py/GEFS_XML.py | 18 +++++++----------- rocoto/py/GEFS_XML_For_Tasks.py | 19 +++++++------------ rocoto/py/add_crontab.py | 8 +++----- rocoto/py/run_pyGEFS.py | 9 ++------- 7 files changed, 40 insertions(+), 60 deletions(-) diff --git a/rocoto/py/GEFS_Crontab.py b/rocoto/py/GEFS_Crontab.py index 5c1891256..138629202 100755 --- a/rocoto/py/GEFS_Crontab.py +++ b/rocoto/py/GEFS_Crontab.py @@ -1,3 +1,9 @@ +#!/usr/bin/env python3 + +import sys +import os +from distutils.spawn import find_executable + def create_crontab(dicBase, OnlyForTest=False, cronint=5): ''' Create crontab to execute rocotorun every cronint (5) minutes @@ -20,9 +26,7 @@ def create_crontab(dicBase, OnlyForTest=False, cronint=5): ########################################################## # No point creating a crontab if rocotorun is not available. - import sys - import os - from distutils.spawn import find_executable + if OnlyForTest: rocotoruncmd = "/opt/modules/3.2.10.3/init/sh" diff --git a/rocoto/py/GEFS_Parm.py b/rocoto/py/GEFS_Parm.py index 844f0d53a..4e4eb4d51 100755 --- a/rocoto/py/GEFS_Parm.py +++ b/rocoto/py/GEFS_Parm.py @@ -1,3 +1,10 @@ +#!/usr/bin/env python3 + +import os +import sys +from collections import OrderedDict +import GEFS_XML_For_Tasks as gefs_xml_for_tasks + # ======================================================= def create_parm(sConfig, dicBase): # For gets_dev.parm @@ -18,7 +25,6 @@ def get_lstParm(sConfig, dicBase): # ======================================================= def read_dicParm(sConfig): # read config file - from collections import OrderedDict dicBaseParm = OrderedDict() IsParm = False StartParm = "# Start Parm" @@ -69,9 +75,6 @@ def read_dicParm(sConfig): # ======================================================= def get_and_merge_default_dicParm(dicParm, WHERE_AM_I): - import os - import sys - # To get the WHERE_AM_I from dicParm or identify it using default methode sDefaultConfig_File = os.path.join(sys.path[0], f"user_{WHERE_AM_I}.conf") @@ -88,8 +91,6 @@ def get_and_merge_default_dicParm(dicParm, WHERE_AM_I): # ======================================================= def assign_default_for_gets_dev_parm(dicBase, lstBaseParm): - import GEFS_XML_For_Tasks as gefs_xml_for_tasks - # == sVarName = "First" if sVarName not in lstBaseParm: @@ -163,9 +164,6 @@ def create_parm2(sConfig, dicBase): # ======================================================= def create_gets_dev_parm(dicBase, listBaseParm): - import sys - import os - strings = [] strings.append('#!/bin/ksh\n') diff --git a/rocoto/py/GEFS_UserConfig.py b/rocoto/py/GEFS_UserConfig.py index 19c1b4798..0e96b5ecc 100755 --- a/rocoto/py/GEFS_UserConfig.py +++ b/rocoto/py/GEFS_UserConfig.py @@ -1,8 +1,12 @@ +#!/usr/bin/env python3 + +import os +import sys +import datetime +from collections import OrderedDict + # ======================================================= def get_and_merge_default_config(dicBase): - import os - import sys - # To get the WHERE_AM_I from dicBase or identify it using default methode get_WHERE_AM_I(dicBase) WHERE_AM_I = dicBase["WHERE_AM_I"] @@ -20,9 +24,6 @@ def get_and_merge_default_config(dicBase): # ======================================================= def get_config_file2(sConfigFile="user_full.conf"): - import os - import sys - sRocoto_WS = os.getcwd() sConfig = os.path.join(sRocoto_WS, sConfigFile) if not os.path.exists(sConfig): @@ -38,7 +39,6 @@ def get_config_file2(sConfigFile="user_full.conf"): # ======================================================= def read_config(sConfig): # read config file - from collections import OrderedDict dicBase = OrderedDict() iTaskName_Num = 0 with open(sConfig, "r") as f: @@ -86,8 +86,6 @@ def read_config(sConfig): # ======================================================= def create_folders(dicBase): - import os - EXPID = dicBase['EXPID'] WORKDIR = str(dicBase['WORKDIR']).replace("&EXPID;", EXPID) WHERE_AM_I = dicBase["WHERE_AM_I".upper()] @@ -103,7 +101,6 @@ def create_folders(dicBase): if not os.path.exists(sPath): os.makedirs(sPath) - import datetime date1 = datetime.datetime.strptime(dicBase['SDATE'][0:8], "%Y%m%d") date2 = datetime.datetime.strptime(dicBase['EDATE'][0:8], "%Y%m%d") day = datetime.timedelta(days=1) @@ -115,12 +112,9 @@ def create_folders(dicBase): os.makedirs(sPath1) date1 = date1 + day - # ======================================================= def get_WHERE_AM_I(dicBase): sVarName = 'WHERE_AM_I' - import os - if sVarName not in dicBase: if os.path.exists('/scratch1/NCEPDEV'): dicBase[sVarName] = 'hera' diff --git a/rocoto/py/GEFS_XML.py b/rocoto/py/GEFS_XML.py index 6a6ac1b97..c0bbb53ed 100755 --- a/rocoto/py/GEFS_XML.py +++ b/rocoto/py/GEFS_XML.py @@ -1,15 +1,18 @@ +#!/usr/bin/env python3 + +import os +from datetime import datetime +import GEFS_XML_For_Tasks as gefs_xml_for_tasks + + # ======================================================= def assign_default_for_xml_def(dicBase, sRocoto_WS=""): - import sys - import os - # == get_MEMLIST(dicBase) # == sVarName = "First".upper() sVarValue = 'Xianwu' if sVarName not in dicBase: - import os sVarValue = os.environ.get("USER") if sVarValue in ['emc.ens']: sVarValue = os.environ.get("SUDO_USER") @@ -20,7 +23,6 @@ def assign_default_for_xml_def(dicBase, sRocoto_WS=""): sVarName = "Last".upper() sVarValue = 'Xue' if sVarName not in dicBase: - import os sVarValue = os.environ.get("USER") if sVarValue in ['emc.ens']: sVarValue = os.environ.get("SUDO_USER") @@ -188,7 +190,6 @@ def assign_default_for_xml_def(dicBase, sRocoto_WS=""): def replace_First_Last(dicBase, sVarName): # to replace the first and last names in the strValue # Modified on 10/17/2018 to avoid the path has individual "First" or "Last" to be replaced by mistake. If just replace the "First.Last", it will be safer. - import os sUSER = os.environ.get("USER") GroupNames = ['emc.ens'] if sUSER in GroupNames: @@ -231,7 +232,6 @@ def get_preamble(): ''' Generate preamble for XML ''' - from datetime import datetime strings = [] @@ -255,7 +255,6 @@ def get_preamble(): # ======================================================= def get_definitions(dicBase): - import os ''' Create entities related to the experiment ''' @@ -307,12 +306,9 @@ def get_workflow_body(dicBase): Create the workflow body ''' - import datetime as dt StartDate = dt.datetime.strptime(dicBase['SDATE'][0:10], "%Y%m%d%H") EndDate = dt.datetime.strptime(dicBase['EDATE'][0:10], "%Y%m%d%H") - import GEFS_XML_For_Tasks as gefs_xml_for_tasks - GenTaskEnt = get_GenTaskEnt(dicBase) # print("---Config your tasks...") diff --git a/rocoto/py/GEFS_XML_For_Tasks.py b/rocoto/py/GEFS_XML_For_Tasks.py index 7d10d7681..1c3d80001 100755 --- a/rocoto/py/GEFS_XML_For_Tasks.py +++ b/rocoto/py/GEFS_XML_For_Tasks.py @@ -1,3 +1,10 @@ +#!/usr/bin/env python3 + +import os +import sys +import math +import textwrap + # ======================================================= def IsCoupleCHEM(dicBase): if dicBase['RUN_AEROSOL_MEMBER'].upper()[0] == "Y": @@ -668,8 +675,6 @@ def Add_Subjobs_to_dicBase(dicBase, iTaskName_Num, taskname="post_hr", sNSubJobs # ======================================================= def write_to_all_ent(GenTaskEnt, dicBase): if GenTaskEnt: - import os - import sys # sPath = dicBase["GEFS_ROCOTO"] + r"/tasks/" + dicBase["WHERE_AM_I"] + r"/" sPath = dicBase["GEFS_ROCOTO"] @@ -716,9 +721,6 @@ def write_to_all_ent(GenTaskEnt, dicBase): # ======================================================= def write_to_ent(taskname, dicBase, GenTaskEnt=False): - import sys - import os - strings = create_metatask_task(dicBase, taskname=taskname, sPre="", GenTaskEnt=GenTaskEnt) strings = ''.join(strings) @@ -740,8 +742,6 @@ def write_to_ent(taskname, dicBase, GenTaskEnt=False): # ======================================================= def calc_fcst_resources(dicBase, taskname="forecast_hr"): - import math - if taskname == "forecast_hr": layout_x = int(dicBase['layout_x'.upper()]) layout_y = int(dicBase['layout_y'.upper()]) @@ -805,7 +805,6 @@ def calc_fcst_resources(dicBase, taskname="forecast_hr"): # ======================================================= def get_param_of_task(dicBase, taskname): - import textwrap sWalltime = "" sNodes = "" sMemory = "" @@ -1310,7 +1309,6 @@ def get_param_of_task(dicBase, taskname): # ======================================================= def calc_gempak_resources(dicBase): - import math taskname="gempak" ncores_per_node = Get_NCORES_PER_NODE(dicBase) WHERE_AM_I = dicBase['WHERE_AM_I'].upper() @@ -1420,9 +1418,6 @@ def get_metatask_names(taskname=""): # ======================================================= def get_jobname(taskname): - import os - import sys - sDefaultJobID_File = os.path.join(sys.path[0], "job_id.conf") jobname_short = "--" if os.path.exists(sDefaultJobID_File): diff --git a/rocoto/py/add_crontab.py b/rocoto/py/add_crontab.py index ce3be5304..343010dd4 100755 --- a/rocoto/py/add_crontab.py +++ b/rocoto/py/add_crontab.py @@ -1,5 +1,8 @@ #!/usr/bin/env python +import os +import sys + # ======================================================= def main(): sInputCron = "cron_rocoto" @@ -8,8 +11,6 @@ def main(): # print(sCronLine) - import os - # print(os.environ['HOME']) sHomeDIR = os.environ['HOME'] @@ -41,7 +42,6 @@ def Add_Cron_To_myCrontab(sMyCrontab, sCronLine): sFile = open(sMyCrontab, "r") for sLine in sFile: # print(sLine) - # import sys # sys.stdout.write(sLine) if sLine.strip() == sCronLine: @@ -66,8 +66,6 @@ def Add_Cron_To_myCrontab(sMyCrontab, sCronLine): if __name__ == '__main__': - import sys - main() sys.exit(0) diff --git a/rocoto/py/run_pyGEFS.py b/rocoto/py/run_pyGEFS.py index 4a34dd277..476d4a605 100755 --- a/rocoto/py/run_pyGEFS.py +++ b/rocoto/py/run_pyGEFS.py @@ -5,18 +5,15 @@ ## 03/19/2018 first released version by Xianwu Xue ## 03/22/2018 Revised to use the default config file based on WHERE_AM_ID by Xianwu Xue +import sys +import argparse import GEFS_UserConfig as gefs_config import GEFS_XML as gefs_xml import GEFS_XML_For_Tasks as gefs_xml_for_tasks import GEFS_Parm as gefs_parm -#import GEFS_Bin as gefs_bin import GEFS_Crontab as gefs_crontab def main(): - import os - import sys - - import argparse parser = argparse.ArgumentParser(description='pyGEFS: Python-Based Workflow Management of NCEP Global Ensemble Forecast System!') parser.add_argument("-r", "--Rocoto", default="yes", type=str, help="Generate rocoto xml related files! [yes|no]") parser.add_argument("-o", "--Operation", default="no", type=str, help="Generate operation workflow related files! [rocoto|devecf|ecflow|no]") @@ -74,8 +71,6 @@ def main(): if __name__ == '__main__': - import sys - main() print("--Done to generate all files!") From 4b0186a7fbff50c6859a7c726b36e400cde99303 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 17 Apr 2023 14:10:20 -0400 Subject: [PATCH 31/41] Fix a bug by add missing import statement On branch feature/v13_atmos_prep modified: rocoto/py/GEFS_XML.py Refs: #87 --- rocoto/py/GEFS_XML.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rocoto/py/GEFS_XML.py b/rocoto/py/GEFS_XML.py index c0bbb53ed..4a28b053d 100755 --- a/rocoto/py/GEFS_XML.py +++ b/rocoto/py/GEFS_XML.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import os -from datetime import datetime +import datetime as dt import GEFS_XML_For_Tasks as gefs_xml_for_tasks @@ -247,7 +247,7 @@ def get_preamble(): strings.append('\t\tXianwu.Xue@noaa.gov\n') strings.append('\n') strings.append('\tNOTES:\n') - strings.append('\t\tThis workflow was automatically generated at %s\n' % datetime.now()) + strings.append('\t\tThis workflow was automatically generated at %s\n' % dt.datetime.now()) strings.append('\t-->\n') return ''.join(strings) From d7eaa846f13e71337f48b56bc9fae2ef06ffb65a Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 17 Apr 2023 14:17:21 -0400 Subject: [PATCH 32/41] Modify to delete parm/config On branch feature/v13_atmos_prep modified: rocoto/compile_install_all.sh Refs: #87 --- rocoto/compile_install_all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocoto/compile_install_all.sh b/rocoto/compile_install_all.sh index f1eae543d..c0ab6376a 100755 --- a/rocoto/compile_install_all.sh +++ b/rocoto/compile_install_all.sh @@ -166,7 +166,7 @@ if [ ${CleanAll} = "yes" ]; then done cd ${sWS}/../parm - for sDir in parm_fv3diag post product; do + for sDir in parm_fv3diag post product config; do rm -rf ${sDir} done From 643e6bd3b7915ed81e81226ed76501ddc45f38a0 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 17 Apr 2023 14:20:45 -0400 Subject: [PATCH 33/41] Improve to use generate_com On branch feature/v13_atmos_prep modified: ush/gefs_atmos_prep.sh Refs: #87 --- ush/gefs_atmos_prep.sh | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/ush/gefs_atmos_prep.sh b/ush/gefs_atmos_prep.sh index 74f80bb94..1701b161f 100755 --- a/ush/gefs_atmos_prep.sh +++ b/ush/gefs_atmos_prep.sh @@ -2,23 +2,11 @@ source "${HOMEgfs:-${HOMEgefs}}/ush/preamble.sh" -#echo "$(date -u) begin $(basename $BASH_SOURCE)" -#export PS4="${PS4}${1}: " - -#set -xa -#if [[ ${STRICT:-NO} == "YES" ]]; then -# # Turn on strict bash error checking -# set -eu -#fi - export mem=$1 export nmem=$(echo $mem|cut -c 2-) nmem=${nmem#0} -YMD=${PDY} HH=${cyc} MEMDIR=${mem} generate_com -x COM_ATMOS_INPUT - -#export INIDIR=$DATA -export OUTDIR=${COM_ATMOS_INPUT} # ${COMOUT}/$mem/atmos/INPUT +YMD=${PDY} HH=${cyc} MEMDIR=${memchar} generate_com -rx OUTDIR:COM_ATMOS_INPUT_TMPL mkdir -p $OUTDIR cd $DATA @@ -48,11 +36,11 @@ fi if [[ $mem = c00 ]] ;then # Control intial conditions from current GFS cycle - export ATM_FILES_INPUT="gfs.t${cyc}z.atmanl.nc" - ATMFILE=${COMINgfs}/atmos/${ATM_FILES_INPUT} #gfs.t${cyc}z.atmanl.nc + export ATM_FILE_INPUT="gfs.t${cyc}z.atmanl.nc" + ATMFILE=${COMINgfs}/atmos/${ATM_FILE_INPUT} #gfs.t${cyc}z.atmanl.nc if [[ -f $ATMFILE ]]; then $NCP $ATMFILE $DATA - #export ATM_FILES_INPUT="gfs.t${cyc}z.atmanl.nc" + #export ATM_FILE_INPUT="gfs.t${cyc}z.atmanl.nc" else msg="FATAL ERROR in $(basename $BASH_SOURCE): GFS atmospheric analysis file $ATMFILE not found!" echo "$msg" @@ -71,8 +59,10 @@ else fi memchar="mem"$(printf %03i $cmem) - export ATM_FILES_INPUT="enkfgfs.t${cyc}z.ratmanl.nc" - ATMFILE="${COMINenkfgfs}/$memchar/atmos/${ATM_FILES_INPUT}" #gfs.t${cycp}z.atmanl.nc" + export ATM_FILE_INPUT="enkfgfs.t${cyc}z.ratmanl.nc" + ATMFILE="${COMINenkfgfs}/$memchar/atmos/${ATM_FILE_INPUT}" #gfs.t${cycp}z.atmanl.nc" + ATMFILE=${COM_ATMOS_ANALYSIS_GFS}/${ATM_FILE_INPUT} + if [[ -f $ATMFILE ]]; then $NCP $ATMFILE $DATA @@ -97,7 +87,7 @@ fi if [[ $CONVERT_SFC == ".true." ]]; then export SFC_FILES_INPUT="gfs.t${cyc}z.sfcanl.nc" - SFCFILE="${COMINgfs}/atmos/${SFC_FILES_INPUT}" + SFCFILE="${COMINgfs}/atmos/${SFC_FILES_INPUT}" #change to the new COM if [[ -f $SFCFILE ]]; then $NCP ${SFCFILE} ${DATA} else @@ -160,8 +150,7 @@ if [[ $SENDCOM == "YES" ]]; then if [[ $CONVERT_SFC == ".true." ]]; then for mem2 in $memberlist; do - YMD=${PDY} HH=${cyc} MEMDIR=${mem2} generate_com -x COM_ATMOS_INPUT - COMDIR2=${COM_ATMOS_INPUT} #${echo "${!COM_ATMOS_INPUT_TMPL}"} # $(echo "${!COM_ATMOS_INPUT_TMPL}" | envsubst) #${COM_ATMOS_INPUT} #$COMOUT/${mem2}/atmos/INPUT #init/$mem2 + YMD=${PDY} HH=${cyc} MEMDIR=${mem2} generate_com COMDIR2:COM_ATMOS_INPUT_TMPL mkdir -p $COMDIR2 for tile in tile1 tile2 tile3 tile4 tile5 tile6; do $NCP ${DATA}/out.sfc.${tile}.nc $COMDIR2/sfc_data.${tile}.nc From b8535494974b5048ac52907bd181ec0a96b26ff9 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 17 Apr 2023 15:20:46 -0400 Subject: [PATCH 34/41] Use COM_ATMOS_ANALYSIS_TMPL On branch feature/v13_atmos_prep modified: jobs/JGEFS_ATMOS_PREP modified: ush/gefs_atmos_prep.sh Refs: #87 --- jobs/JGEFS_ATMOS_PREP | 1 + ush/gefs_atmos_prep.sh | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/jobs/JGEFS_ATMOS_PREP b/jobs/JGEFS_ATMOS_PREP index 7fbc38912..fae47e159 100755 --- a/jobs/JGEFS_ATMOS_PREP +++ b/jobs/JGEFS_ATMOS_PREP @@ -80,6 +80,7 @@ export COMINgfs=${COMINgfs:-$(compath.py ${envir}/com/gfs/${gfs_ver})/gfs.${PDY} export COMINenkfgfs=${COMINenkfgfs:-$(compath.py ${envir}/com/gfs/${gfs_ver})/enkfgfs.${PDY}/${cyc}} export ROTDIR=${ROTDIR:-$(compath.py -o $NET/${ver})} +export ROTDIR_IN=${ROTDIR_IN:-$(compath.py ${envir}/com/gfs/${gfs_ver})} #export COMOUT=${COMOUT:-$(compath.py -o $NET/${ver})/${RUN}.${PDY}/${cyc}} echo "Environment before calling script" diff --git a/ush/gefs_atmos_prep.sh b/ush/gefs_atmos_prep.sh index 1701b161f..4f22c2d84 100755 --- a/ush/gefs_atmos_prep.sh +++ b/ush/gefs_atmos_prep.sh @@ -6,7 +6,7 @@ export mem=$1 export nmem=$(echo $mem|cut -c 2-) nmem=${nmem#0} -YMD=${PDY} HH=${cyc} MEMDIR=${memchar} generate_com -rx OUTDIR:COM_ATMOS_INPUT_TMPL +YMD=${PDY} HH=${cyc} MEMDIR=${mem} generate_com -rx OUTDIR:COM_ATMOS_INPUT_TMPL mkdir -p $OUTDIR cd $DATA @@ -36,8 +36,9 @@ fi if [[ $mem = c00 ]] ;then # Control intial conditions from current GFS cycle - export ATM_FILE_INPUT="gfs.t${cyc}z.atmanl.nc" - ATMFILE=${COMINgfs}/atmos/${ATM_FILE_INPUT} #gfs.t${cyc}z.atmanl.nc + export ATM_FILES_INPUT="gfs.t${cyc}z.atmanl.nc" + YMD=${PDY} HH=${cyc} RUN=gfs MEMDIR="" ROTDIR=${ROTDIR_IN} generate_com -rx COM_ATMOS_ANALYSIS_GFS:COM_ATMOS_ANALYSIS_TMPL + ATMFILE=${COM_ATMOS_ANALYSIS_GFS}/${ATM_FILES_INPUT} if [[ -f $ATMFILE ]]; then $NCP $ATMFILE $DATA #export ATM_FILE_INPUT="gfs.t${cyc}z.atmanl.nc" @@ -59,9 +60,9 @@ else fi memchar="mem"$(printf %03i $cmem) - export ATM_FILE_INPUT="enkfgfs.t${cyc}z.ratmanl.nc" - ATMFILE="${COMINenkfgfs}/$memchar/atmos/${ATM_FILE_INPUT}" #gfs.t${cycp}z.atmanl.nc" - ATMFILE=${COM_ATMOS_ANALYSIS_GFS}/${ATM_FILE_INPUT} + export ATM_FILES_INPUT="enkfgfs.t${cyc}z.ratmanl.nc" + YMD=${PDY} HH=${cyc} RUN=enkfgfs MEMDIR=${memchar} ROTDIR=${ROTDIR_IN} generate_com -rx COM_ATMOS_ANALYSIS_ENKFGFS:COM_ATMOS_ANALYSIS_TMPL + ATMFILE=${COM_ATMOS_ANALYSIS_ENKFGFS}/${ATM_FILES_INPUT} if [[ -f $ATMFILE ]]; then @@ -134,7 +135,7 @@ touch ${OUTDIR}/chgres_atm.log # recenter can start now if [[ $SENDCOM == "YES" ]]; then MODCOM=$(echo ${NET}_${COMPONENT} | tr '[a-z]' '[A-Z]') DBNTYP=${MODCOM}_INIT - COMDIR=${OUTDIR} #$COMOUT/$mem/atmos/INPUT #/init/$mem + COMDIR=${OUTDIR} mkdir -p $COMDIR $NCP ${DATA}/gfs_ctrl.nc $COMDIR if [[ $SENDDBN = YES ]];then From ea31a906c977ee29ef0000814fc39ab359c97311 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 17 Apr 2023 18:16:23 -0400 Subject: [PATCH 35/41] Change COM_ATMOS_ANALYSIS_GFS for SFCFILE On branch feature/v13_atmos_prep modified: jobs/JGEFS_ATMOS_PREP modified: ush/gefs_atmos_prep.sh Refs: #87 --- jobs/JGEFS_ATMOS_PREP | 4 ---- ush/gefs_atmos_prep.sh | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/jobs/JGEFS_ATMOS_PREP b/jobs/JGEFS_ATMOS_PREP index fae47e159..2f7f293ef 100755 --- a/jobs/JGEFS_ATMOS_PREP +++ b/jobs/JGEFS_ATMOS_PREP @@ -76,12 +76,8 @@ for config in $configs; do fi done -export COMINgfs=${COMINgfs:-$(compath.py ${envir}/com/gfs/${gfs_ver})/gfs.${PDY}/${cyc}} -export COMINenkfgfs=${COMINenkfgfs:-$(compath.py ${envir}/com/gfs/${gfs_ver})/enkfgfs.${PDY}/${cyc}} - export ROTDIR=${ROTDIR:-$(compath.py -o $NET/${ver})} export ROTDIR_IN=${ROTDIR_IN:-$(compath.py ${envir}/com/gfs/${gfs_ver})} -#export COMOUT=${COMOUT:-$(compath.py -o $NET/${ver})/${RUN}.${PDY}/${cyc}} echo "Environment before calling script" env | sort diff --git a/ush/gefs_atmos_prep.sh b/ush/gefs_atmos_prep.sh index 4f22c2d84..c92f37c31 100755 --- a/ush/gefs_atmos_prep.sh +++ b/ush/gefs_atmos_prep.sh @@ -88,7 +88,7 @@ fi if [[ $CONVERT_SFC == ".true." ]]; then export SFC_FILES_INPUT="gfs.t${cyc}z.sfcanl.nc" - SFCFILE="${COMINgfs}/atmos/${SFC_FILES_INPUT}" #change to the new COM + SFCFILE="${COM_ATMOS_ANALYSIS_GFS}/${SFC_FILES_INPUT}" if [[ -f $SFCFILE ]]; then $NCP ${SFCFILE} ${DATA} else From 8abf0cb20facae154f8538e9d5d63a65da04c8de Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 17 Apr 2023 19:07:37 -0400 Subject: [PATCH 36/41] Add OCNRES to gefs.parm On branch feature/v13_atmos_prep modified: parm/parm_gefs/gefs.parm RefsP #87 --- parm/parm_gefs/gefs.parm | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/parm/parm_gefs/gefs.parm b/parm/parm_gefs/gefs.parm index 27784f03a..6a4f55b1a 100755 --- a/parm/parm_gefs/gefs.parm +++ b/parm/parm_gefs/gefs.parm @@ -180,7 +180,18 @@ export DBNROOT=${DBNROOT:-${UTILROOT}/fakedbn} export VERBOSE=${VERBOSE:-YES} # Get all the COM path templates -source "${PARMgefs}/config/config.com" +source "${PARMgefs:-${PARMgfs}}/config/config.com" + +# These are the currently available grid-combinations +case "${CASE}" in + "C48") export OCNRES=500;; + "C96") export OCNRES=100;; + "C192") export OCNRES=050;; + "C384") export OCNRES=025;; + "C768") export OCNRES=025;; + *) export OCNRES=025;; +esac +export ICERES=${OCNRES} # Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL export imp_physics=${imp_physics:-"8"} From 6a8ba0845143431454614ab11d795f56438a4c79 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 17 Apr 2023 19:11:54 -0400 Subject: [PATCH 37/41] Remove OCNRES related lines in ush atmos_prep On branch feature/v13_atmos_prep modified: ush/gefs_atmos_prep.sh Refs: #87 --- ush/gefs_atmos_prep.sh | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/ush/gefs_atmos_prep.sh b/ush/gefs_atmos_prep.sh index c92f37c31..1d2672907 100755 --- a/ush/gefs_atmos_prep.sh +++ b/ush/gefs_atmos_prep.sh @@ -16,23 +16,8 @@ cd $DATA # Set directory names and file names for orog data # The old and new (support fractional grid) orog data have different file names #--------------------------------------------------------------------------- -export FRAC_ORO="yes" -if [ "${FRAC_ORO:-"no"}" = "yes" ]; then - if [ ${CASE} == 'C48' ] ; then - OCNRES='500' - elif [ ${CASE} == 'C96' ] ; then - OCNRES='100' - elif [ ${CASE} == 'C192' ] ; then - OCNRES='050' - elif [ ${CASE} == 'C384' ] || [ ${CASE} == 'C768' ] || [ ${CASE} == 'C1152' ]; then - OCNRES='025' - fi - ORO_DIR="${CASE}.mx${OCNRES}_frac" - ORO_NAME="oro_${CASE}.mx${OCNRES}" -else - ORO_DIR="${CASE}" - ORO_NAME="${CASE}_oro_data" -fi +ORO_DIR="${CASE}.mx${OCNRES}_frac" +ORO_NAME="oro_${CASE}.mx${OCNRES}" if [[ $mem = c00 ]] ;then # Control intial conditions from current GFS cycle From e44a6ac5e2cf2944362f88ca4f47da2baa959afc Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 17 Apr 2023 19:15:49 -0400 Subject: [PATCH 38/41] Change "module purge" to "module reset" On branch feature/v13_atmos_prep modified: rocoto/bin/hera/atmos_prep.sh Refs: #87 --- rocoto/bin/hera/atmos_prep.sh | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/rocoto/bin/hera/atmos_prep.sh b/rocoto/bin/hera/atmos_prep.sh index 4540089b8..95d4c54f2 100755 --- a/rocoto/bin/hera/atmos_prep.sh +++ b/rocoto/bin/hera/atmos_prep.sh @@ -1,9 +1,6 @@ -#!/bin/bash -# +#! /usr/bin/env bash -# EXPORT list here set -x -#export IOBUF_PARAMS= ulimit -s unlimited ulimit -a @@ -11,18 +8,15 @@ ulimit -a . $GEFS_ROCOTO/dev/versions/run_hera.ver # Load modules -module list -module purge +module reset module use -a /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack module load hpc/${hpc_ver} - module load hpc-intel/${intel_ver} module load hpc-impi/${impi_ver} module load grib_util/${grib_util_ver} module load prod_util/${prod_util_ver} - module load netcdf/${netcdf_ver} module list @@ -30,7 +24,6 @@ module list # For Development . $GEFS_ROCOTO/bin/hera/common.sh - export MP_SHARED_MEMORY=yes export MEMORY_AFFINITY=core:2 @@ -45,9 +38,10 @@ export MEMORY_AFFINITY=core:2 (( OMP_NUM_THREADS_CH = 40 / GEFS_PPN )) export OMP_NUM_THREADS_CH -ver=${ver:-$(echo ${gefs_ver}|cut -c1-5)} -export COMOUT=${COMOUT:-${COMROOT}/gefs/$ver/${RUN}.${PDY}/$cyc} +ver=${gefs_ver:0:5} +#ver=${ver%${ver##v+([0-9]).+([0-9])}} export ROTDIR=${COMROOT}/gefs/${ver} +export ROTDIR_IN=${HOMEdata}/gfs/${gfs_ver} # CALL executable job script here ${SOURCEDIR}/jobs/JGEFS_ATMOS_PREP From 7dce077f1376649d3654a4ea596b2048c9e9572b Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 17 Apr 2023 20:31:08 -0400 Subject: [PATCH 39/41] Change ROTDIR_IN to ROTDIR_GFS On branch feature/v13_atmos_prep modified: jobs/JGEFS_ATMOS_PREP modified: rocoto/bin/hera/atmos_prep.sh modified: ush/gefs_atmos_prep.sh Refs: #87 --- jobs/JGEFS_ATMOS_PREP | 2 +- rocoto/bin/hera/atmos_prep.sh | 2 +- ush/gefs_atmos_prep.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jobs/JGEFS_ATMOS_PREP b/jobs/JGEFS_ATMOS_PREP index 2f7f293ef..d1f582fe5 100755 --- a/jobs/JGEFS_ATMOS_PREP +++ b/jobs/JGEFS_ATMOS_PREP @@ -77,7 +77,7 @@ for config in $configs; do done export ROTDIR=${ROTDIR:-$(compath.py -o $NET/${ver})} -export ROTDIR_IN=${ROTDIR_IN:-$(compath.py ${envir}/com/gfs/${gfs_ver})} +export ROTDIR_GFS=${ROTDIR_GFS:-$(compath.py ${envir}/com/gfs/${gfs_ver})} echo "Environment before calling script" env | sort diff --git a/rocoto/bin/hera/atmos_prep.sh b/rocoto/bin/hera/atmos_prep.sh index 95d4c54f2..3ca96044e 100755 --- a/rocoto/bin/hera/atmos_prep.sh +++ b/rocoto/bin/hera/atmos_prep.sh @@ -41,7 +41,7 @@ export OMP_NUM_THREADS_CH ver=${gefs_ver:0:5} #ver=${ver%${ver##v+([0-9]).+([0-9])}} export ROTDIR=${COMROOT}/gefs/${ver} -export ROTDIR_IN=${HOMEdata}/gfs/${gfs_ver} +export ROTDIR_GFS=${HOMEdata}/gfs/${gfs_ver} # CALL executable job script here ${SOURCEDIR}/jobs/JGEFS_ATMOS_PREP diff --git a/ush/gefs_atmos_prep.sh b/ush/gefs_atmos_prep.sh index 1d2672907..1b4beb0cc 100755 --- a/ush/gefs_atmos_prep.sh +++ b/ush/gefs_atmos_prep.sh @@ -22,7 +22,7 @@ ORO_NAME="oro_${CASE}.mx${OCNRES}" if [[ $mem = c00 ]] ;then # Control intial conditions from current GFS cycle export ATM_FILES_INPUT="gfs.t${cyc}z.atmanl.nc" - YMD=${PDY} HH=${cyc} RUN=gfs MEMDIR="" ROTDIR=${ROTDIR_IN} generate_com -rx COM_ATMOS_ANALYSIS_GFS:COM_ATMOS_ANALYSIS_TMPL + YMD=${PDY} HH=${cyc} RUN=gfs MEMDIR="" ROTDIR=${ROTDIR_GFS} generate_com -rx COM_ATMOS_ANALYSIS_GFS:COM_ATMOS_ANALYSIS_TMPL ATMFILE=${COM_ATMOS_ANALYSIS_GFS}/${ATM_FILES_INPUT} if [[ -f $ATMFILE ]]; then $NCP $ATMFILE $DATA @@ -46,7 +46,7 @@ else memchar="mem"$(printf %03i $cmem) export ATM_FILES_INPUT="enkfgfs.t${cyc}z.ratmanl.nc" - YMD=${PDY} HH=${cyc} RUN=enkfgfs MEMDIR=${memchar} ROTDIR=${ROTDIR_IN} generate_com -rx COM_ATMOS_ANALYSIS_ENKFGFS:COM_ATMOS_ANALYSIS_TMPL + YMD=${PDY} HH=${cyc} RUN=enkfgfs MEMDIR=${memchar} ROTDIR=${ROTDIR_GFS} generate_com -rx COM_ATMOS_ANALYSIS_ENKFGFS:COM_ATMOS_ANALYSIS_TMPL ATMFILE=${COM_ATMOS_ANALYSIS_ENKFGFS}/${ATM_FILES_INPUT} From 5a89c12eaa80a0a89997292e509298d2f60311d3 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Mon, 17 Apr 2023 21:20:00 -0400 Subject: [PATCH 40/41] Remove fix.ver and use fix.ver in g-w On branch feature/v13_atmos_prep modified: sorc/link_gefs.sh deleted: versions/fix.ver Refs: #87 --- sorc/link_gefs.sh | 2 +- versions/fix.ver | 24 ------------------------ 2 files changed, 1 insertion(+), 25 deletions(-) delete mode 100644 versions/fix.ver diff --git a/sorc/link_gefs.sh b/sorc/link_gefs.sh index 2b2ebd02a..e4f32b69e 100755 --- a/sorc/link_gefs.sh +++ b/sorc/link_gefs.sh @@ -55,7 +55,7 @@ cd ${pwd} if [[ -d global-workflow.fd ]] ; then # Source fix version file - source "${pwd}/../versions/fix.ver" + source "${pwd}/global-workflow.fd/versions/fix.ver" cd ${pwd}/../fix diff --git a/versions/fix.ver b/versions/fix.ver deleted file mode 100644 index 256d8efc5..000000000 --- a/versions/fix.ver +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Fix file subfolder versions - -export aer_ver=20220805 -export am_ver=20220805 -export chem_ver=20220805 -export cice_ver=20220805 -export cpl_ver=20220805 -export datm_ver=20220805 -export gdas_bump_ver=20220805 -export gdas_crtm_ver=20220805 -export gdas_fv3jedi_ver=20220805 -export gdas_gsibec_ver=20221031 -export gldas_ver=20220920 -export glwu_ver=20220805 -export gsi_ver=20221128 -export lut_ver=20220805 -export mom6_ver=20220805 -export orog_ver=20220805 -export reg2grb2_ver=20220805 -export sfc_climo_ver=20220805 -export ugwd_ver=20220805 -export verif_ver=20220805 -export wave_ver=20220805 From e07b3ada5a5e51cd42b251fb449b7ce48c75b495 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Tue, 18 Apr 2023 21:42:19 -0400 Subject: [PATCH 41/41] Use shopt to get ver on hera On branch feature/v13_atmos_prep modified: rocoto/bin/hera/atmos_prep.sh Refs: #87 --- rocoto/bin/hera/atmos_prep.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rocoto/bin/hera/atmos_prep.sh b/rocoto/bin/hera/atmos_prep.sh index 3ca96044e..8e442d1c0 100755 --- a/rocoto/bin/hera/atmos_prep.sh +++ b/rocoto/bin/hera/atmos_prep.sh @@ -38,8 +38,10 @@ export MEMORY_AFFINITY=core:2 (( OMP_NUM_THREADS_CH = 40 / GEFS_PPN )) export OMP_NUM_THREADS_CH -ver=${gefs_ver:0:5} -#ver=${ver%${ver##v+([0-9]).+([0-9])}} +shopt -s extglob +ver=${gefs_ver%${gefs_ver##v+([0-9]).+([0-9])}} +shopt -u extglob + export ROTDIR=${COMROOT}/gefs/${ver} export ROTDIR_GFS=${HOMEdata}/gfs/${gfs_ver}