diff --git a/.gitmodules b/.gitmodules index c80d24c03aa..fda1a5c187c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "sorc/ufs_model.fd"] path = sorc/ufs_model.fd - url = https://github.com/ufs-community/ufs-weather-model + url = https://github.com/AliS-Noaa/ufs-weather-model [submodule "sorc/wxflow"] path = sorc/wxflow url = https://github.com/NOAA-EMC/wxflow diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index df35f0228d9..60ed9b2676f 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -209,8 +209,8 @@ case ${step} in # The wavepost*pnt* jobs are I/O heavy and do not scale well to large nodes. # Limit the number of tasks/node to 40. "wavepostbndpnt") - walltime="03:00:00" - ntasks=240 + walltime="00:30:00" + ntasks=1 threads_per_task=1 tasks_per_node=$(( max_tasks_per_node / threads_per_task )) export is_exclusive=True @@ -222,8 +222,8 @@ case ${step} in ;; "wavepostbndpntbll") - walltime="01:00:00" - ntasks=448 + walltime="00:10:00" + ntasks=2 threads_per_task=1 tasks_per_node=$(( max_tasks_per_node / threads_per_task )) export is_exclusive=True @@ -235,8 +235,8 @@ case ${step} in ;; "wavepostpnt") - walltime="04:00:00" - ntasks=200 + walltime="00:35:00" + ntasks=3 threads_per_task=1 tasks_per_node=$(( max_tasks_per_node / threads_per_task )) export is_exclusive=True diff --git a/parm/wave/ww3_outp_bull.inp.tmpl b/parm/wave/ww3_outp_bull.inp.tmpl index e7741b876c4..b9d5339a0a3 100755 --- a/parm/wave/ww3_outp_bull.inp.tmpl +++ b/parm/wave/ww3_outp_bull.inp.tmpl @@ -1,6 +1,6 @@ $ WAVEWATCH III Point output post-processing $ ------------------------------------------ - TIME DT 999 + TIME DT 999 1 PREFIX $ POINT -1 diff --git a/parm/wave/ww3_outp_spec.inp.tmpl b/parm/wave/ww3_outp_spec.inp.tmpl index 2490b7a8d7e..82865a93014 100755 --- a/parm/wave/ww3_outp_spec.inp.tmpl +++ b/parm/wave/ww3_outp_spec.inp.tmpl @@ -1,6 +1,6 @@ $ WAVEWATCH III Point output post-processing $ ------------------------------------------ - TIME DT 999 + TIME DT 999 1 PREFIX $ POINT -1 diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index bad848c5888..cb5a19ea359 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -11,9 +11,6 @@ # It executes several scripts forpreparing and creating output data # as follows: # -# wave_outp_spec.sh : generates spectral data for output locations -# wave_outp_bull.sh : generates bulletins for output locations -# wave_outp_cat.sh : cats the by hour into the single output file # wave_tar.sh : tars the spectral and bulletin multiple files # # Script history log: @@ -21,6 +18,7 @@ # 2020-06-10 J-Henrique Alves: Porting to R&D machine Hera # 2020-07-30 Jessica Meixner: Points only - no gridded data # 2020-09-29 Jessica Meixner: optimized by changing loop structures +# 2025-02-12 Ali Salimi-Tarazou: Changed to be compatible with the new optimized ww3_outp # # COM inputs: # - ${COMIN_WAVE_PREP}/${RUN}wave.mod_def.${grdID} @@ -85,9 +83,6 @@ source "${USHgfs}/preamble.sh" mkdir -p ${STA_DIR}/spec mkdir -p ${STA_DIR}/bull mkdir -p ${STA_DIR}/cbull - mkdir -p ${STA_DIR}/specfhr - mkdir -p ${STA_DIR}/bullfhr - mkdir -p ${STA_DIR}/cbullfhr set +x echo ' ' @@ -110,14 +105,6 @@ source "${USHgfs}/preamble.sh" # 1.a Model definition files and output files (set up using poe) -# 1.a.1 Set up the parallel command tasks - - rm -f cmdfile - touch cmdfile - chmod 744 cmdfile - - set_trace - # Copy model definition files iloop=0 for grdID in ${waveuoutpGRD}; do @@ -151,7 +138,7 @@ source "${USHgfs}/preamble.sh" fi done -# 1.c Output locations file +# 1.b Output locations file rm -f buoy.loc @@ -190,7 +177,7 @@ source "${USHgfs}/preamble.sh" DOBLL_WAV='NO' fi -# 1.d Input template files +# 1.c Input template files if [ -f ${PARMgfs}/wave/ww3_outp_spec.inp.tmpl ] then @@ -237,39 +224,55 @@ source "${USHgfs}/preamble.sh" DOBLL_WAV='NO' fi +# 1.d Linking the output files + + ymdh=$(${NDATE} -"${WAVHINDH}" "${PDY}${cyc}") + tstart="${ymdh:0:8} ${ymdh:8:2}0000" + N=$(( ($FHMAX_WAV_PNT - $FHMIN_WAV) * 3600 / $DTPNT_WAV + 1 )) + truntime="${PDY} ${cyc}0000" + + # Loop through forecast hours to link output file + fhr=$FHMIN_WAV + while [ $fhr -le $FHMAX_WAV_PNT ]; do + ymdh=$($NDATE $fhr "${PDY}${cyc}") + YMD=${ymdh:0:8} + HMS="${ymdh:8:2}0000" + pfile="${COMIN_WAVE_HISTORY}/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${YMD}.${HMS}.nc" + if [[ -f "${pfile}" ]]; then + ${NLN} "${pfile}" "./${YMD}.${HMS}.out_pnt.ww3.nc" + else + echo '*************************************************** ' + echo " FATAL ERROR : NO RAW POINT OUTPUT FILE ${YMD}.${HMS}.out_pnt.ww3.nc " + echo '*************************************************** ' + [[ "$LOUD" = YES ]] && set -x + err=7; export err; ${errchk} + exit $err + fi + + FHINCP=$(( DTPNT_WAV / 3600 )) + fhrp=$((fhr+FHINCP)) + fhr=$fhrp # no gridded output, loop with out_pnt stride + done + # 1.e Getting buoy information for points if [ "$DOSPC_WAV" = 'YES' ] || [ "$DOBLL_WAV" = 'YES' ] then ymdh=$(${NDATE} -"${WAVHINDH}" "${PDY}${cyc}") + YMD=${ymdh:0:8} tstart="${ymdh:0:8} ${ymdh:8:2}0000" - dtspec=3600. # default time step (not used here) + N=$(( ($FHMAX_WAV_PNT - $FHMIN_WAV) * 3600 / $DTPNT_WAV + 1 )) sed -e "s/TIME/${tstart}/g" \ - -e "s/DT/${dtspec}/g" \ - -e "s/POINT/1/g" \ + -e "s/DT/${DTPNT_WAV}/g" \ + -e "s/999/$N/g" \ + -e "s/PREFIX/${RUN}wave/g" \ + -e "s/^.*POINT.*/\$ &/g" \ -e "s/ITYPE/0/g" \ -e "s/FORMAT/F/g" \ ww3_outp_spec.inp.tmpl > ww3_outp.inp - - ${NLN} mod_def.$waveuoutpGRD mod_def.ww3 - #export OFFSET_START_HOUR=$( printf "%02d" ${half_assim} ) - hh=$( printf "%02d" $(( cyc + OFFSET_START_HOUR )) ) - HMS="${hh}0000" - if [[ -f "${COMIN_WAVE_HISTORY}/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${PDY}.${HMS}" ]]; then - ${NLN} "${COMIN_WAVE_HISTORY}/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${PDY}.${HMS}" \ - "./out_pnt.${waveuoutpGRD}" - else - echo '*************************************************** ' - echo " FATAL ERROR : NO RAW POINT OUTPUT FILE out_pnt.${waveuoutpGRD}.${PDY}.${HMS} " - echo '*************************************************** ' - echo ' ' - set_trace - echo "${WAV_MOD_TAG} post ${waveuoutpGRD} ${PDY}${cyc} ${cycle} : field output missing." - err=4; export err;${errchk} fi rm -f buoy_tmp.loc buoy_log.ww3 ww3_oup.inp - ${NLN} ./out_pnt.${waveuoutpGRD} ./out_pnt.ww3 ${NLN} ./mod_def.${waveuoutpGRD} ./mod_def.ww3 export pgm="${NET,,}_ww3_outp.x" @@ -300,11 +303,7 @@ source "${USHgfs}/preamble.sh" rm -f buoy_log.dat mv buoy_log.tmp buoy_log.dat - grep -F -f ibp_tags buoy_lst.loc > buoy_tmp1.loc - #sed '$d' buoy_tmp1.loc > buoy_tmp2.loc - awk '{ print $1 }' buoy_tmp1.loc > buoy_lst.txt - Nb=$(wc buoy_tmp1.loc | awk '{ print $1 }') - rm -f buoy_tmp1.loc + Nb=$(wc buoy_log.dat | awk '{ print $1 }') if [ -s buoy_log.dat ] then @@ -324,8 +323,6 @@ source "${USHgfs}/preamble.sh" DOBLL_WAV='NO' fi - fi - # 1.f Data summary set +x @@ -343,259 +340,44 @@ source "${USHgfs}/preamble.sh" # --------------------------------------------------------------------------- # # 2. Make files for processing boundary points # -# 2.a Command file set-up +# 2.a Creating ww3_outp.inp for each job and execute ww3_outp set +x echo ' Making command file for wave post points ' set_trace - - rm -f cmdfile - touch cmdfile - chmod 744 cmdfile - -# 1.a.2 Loop over forecast time to generate post files - fhr=$FHMIN_WAV - # Generated sed-searchable paths - escaped_USHgfs="${USHgfs//\//\\\/}" - while [ $fhr -le $FHMAX_WAV_PNT ]; do - - echo " Creating the wave point scripts at : $(date)" - ymdh=$($NDATE "${fhr}" "${PDY}${cyc}") - YMD=${ymdh:0:8} - HMS="${ymdh:8:2}0000" - YMDHMS=${YMD}${HMS} - FH3=$(printf %03i ${fhr}) - - rm -f tmpcmdfile.${FH3} - touch tmpcmdfile.${FH3} - mkdir output_$YMDHMS - cd output_$YMDHMS - -# Create instances of directories for spec and gridded output - export SPECDATA=${DATA}/output_$YMDHMS - escaped_SPECDATA="${SPECDATA//\//\\\/}" - export BULLDATA=${DATA}/output_$YMDHMS - cp $DATA/mod_def.${waveuoutpGRD} mod_def.${waveuoutpGRD} - - pfile="${COMIN_WAVE_HISTORY}/${WAV_MOD_TAG}.out_pnt.${waveuoutpGRD}.${YMD}.${HMS}" - if [ -f ${pfile} ] - then - ${NLN} ${pfile} ./out_pnt.${waveuoutpGRD} - else - echo " FATAL ERROR : NO RAW POINT OUTPUT FILE out_pnt.$waveuoutpGRD.${YMD}.${HMS} " - echo ' ' - set_trace - err=7; export err;${errchk} - exit $err - fi - - cd $DATA - - if [ "$DOSPC_WAV" = 'YES' ] - then - export dtspec=3600. - # Construct the wave_outp_spec (spec) command to run on each buoy in buoy_lst.txt - sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_spec.sh \1 ${ymdh} spec ${escaped_SPECDATA} > ${escaped_SPECDATA}\/spec_\1.out 2>\&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}" - fi - - if [ "$DOBLL_WAV" = 'YES' ] - then - export dtspec=3600. - # Construct the wave_outp_spec (bull) command to run on each buoy in buoy_lst.txt - sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_spec.sh \1 ${ymdh} bull ${escaped_SPECDATA} > ${escaped_SPECDATA}\/bull_\1.out 2>\&1/" buoy_lst.txt >> "tmpcmdfile.${FH3}" - fi - - split -n l/1/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.01 - split -n l/2/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.02 - split -n l/3/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.03 - split -n l/4/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.04 - split -n l/5/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.05 - split -n l/6/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.06 - split -n l/7/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.07 - split -n l/8/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.08 - split -n l/9/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.09 - split -n l/10/10 tmpcmdfile.$FH3 > cmdfile.${FH3}.10 - - rm tmpcmdfile.$FH3 - chmod 744 cmdfile.${FH3}.01 cmdfile.${FH3}.02 cmdfile.${FH3}.03 cmdfile.${FH3}.04 - chmod 744 cmdfile.${FH3}.05 cmdfile.${FH3}.06 cmdfile.${FH3}.07 cmdfile.${FH3}.08 - chmod 744 cmdfile.${FH3}.09 cmdfile.${FH3}.10 - echo "$DATA/cmdfile.${FH3}.01" >> cmdfile - echo "$DATA/cmdfile.${FH3}.02" >> cmdfile - echo "$DATA/cmdfile.${FH3}.03" >> cmdfile - echo "$DATA/cmdfile.${FH3}.04" >> cmdfile - echo "$DATA/cmdfile.${FH3}.05" >> cmdfile - echo "$DATA/cmdfile.${FH3}.06" >> cmdfile - echo "$DATA/cmdfile.${FH3}.07" >> cmdfile - echo "$DATA/cmdfile.${FH3}.08" >> cmdfile - echo "$DATA/cmdfile.${FH3}.09" >> cmdfile - echo "$DATA/cmdfile.${FH3}.10" >> cmdfile - - - FHINCP=$(( DTPNT_WAV / 3600 )) - fhrp=$((fhr+FHINCP)) - fhr=$fhrp # no gridded output, loop with out_pnt stride - - done - - - if [ ${CFP_MP:-"NO"} = "YES" ]; then - nfile=0 - ifile=0 - iline=1 - ifirst='yes' - nlines=$( wc -l cmdfile | awk '{print $1}' ) - while [ $iline -le $nlines ]; do - line=$( sed -n ''$iline'p' cmdfile ) - if [ -z "$line" ]; then - break - else - if [ "$ifirst" = 'yes' ]; then - echo "#!/bin/sh" > cmdmfile.$nfile - echo "$nfile cmdmfile.$nfile" >> cmdmprog - chmod 744 cmdmfile.$nfile - fi - echo $line >> cmdmfile.$nfile - nfile=$(( nfile + 1 )) - if [ $nfile -eq $NTASKS ]; then - nfile=0 - ifirst='no' - fi - iline=$(( iline + 1 )) - fi - done - fi - - wavenproc=$(wc -l cmdfile | awk '{print $1}') - wavenproc=$(echo $((${wavenproc}<${NTASKS}?${wavenproc}:${NTASKS}))) - - set +x - echo ' ' - echo " Executing the wave point scripts at : $(date)" - echo ' ------------------------------------' - echo ' ' - set_trace - - if [ "$wavenproc" -gt '1' ] - then - if [ ${CFP_MP:-"NO"} = "YES" ]; then - ${wavempexec} -n ${wavenproc} ${wave_mpmd} cmdmprog - else - ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile - fi - exit=$? - else - chmod 744 cmdfile - ./cmdfile - exit=$? - fi - - if [ "$exit" != '0' ] - then - set +x - echo ' ' - echo '*************************************' - echo '*** FATAL ERROR: CMDFILE FAILED ***' - echo '*************************************' - echo ' See Details Below ' - echo ' ' - set_trace - err=8; export err;${errchk} - exit $err - fi - -# 2.b Loop over each buoy to cat the final buoy file for all fhr - - cd $DATA - - echo "Before create cmdfile for cat buoy : $(date)" - rm -f cmdfile.buoy - touch cmdfile.buoy - chmod 744 cmdfile.buoy - CATOUTDIR=${DATA}/pnt_cat_out - escaped_CATOUTDIR="${CATOUTDIR//\//\\\/}" - mkdir -p ${CATOUTDIR} - - if [ "$DOSPC_WAV" = 'YES' ] - then - # Construct wave_outp_cat (spec) call for each buoy in buoy_lst.txt - sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} spec > ${escaped_CATOUTDIR}\/spec_cat_\1.out 2>\&1/" buoy_lst.txt >> cmdfile.buoy - fi - - if [ "$DOBLL_WAV" = 'YES' ] - then - # Construct wave_outp_cat (bull) call for each buoy in buoy_lst.txt - sed "s/^\(.*\)$/${escaped_USHgfs}\/wave_outp_cat.sh \1 ${FHMAX_WAV_PNT} bull > ${escaped_CATOUTDIR}\/bull_cat_\1.out 2>\&1/" buoy_lst.txt >> cmdfile.buoy - fi - - if [ ${CFP_MP:-"NO"} = "YES" ]; then - nfile=0 - ifile=0 - iline=1 - ifirst='yes' - nlines=$( wc -l < cmdfile.buoy) - while [ $iline -le $nlines ]; do - line=$( sed -n ''$iline'p' cmdfile.buoy ) - if [ -z "$line" ]; then - break - else - if [ "$ifirst" = 'yes' ]; then - echo "#!/bin/sh" > cmdfile.buoy.$nfile - echo "$nfile cmdfile.buoy.$nfile" >> cmdmprogbuoy - chmod 744 cmdfile.buoy.$nfile - fi - echo $line >> cmdfile.buoy.$nfile - nfile=$(( nfile + 1 )) - if [ $nfile -eq $NTASKS ]; then - nfile=0 - ifirst='no' - fi - iline=$(( iline + 1 )) - fi - done - fi - - wavenproc=$(wc -l < cmdfile.buoy) - wavenproc=$(echo $((${wavenproc}<${NTASKS}?${wavenproc}:${NTASKS}))) - - set +x - echo ' ' - echo " Executing the boundary point cat script at : $(date)" - echo ' ------------------------------------' - echo ' ' - set_trace - - if [ "$wavenproc" -gt '1' ] - then - if [ ${CFP_MP:-"NO"} = "YES" ]; then - # shellcheck disable=SC2086 - ${wavempexec} -n "${wavenproc}" ${wave_mpmd} cmdmprogbuoy - else - # shellcheck disable=SC2086 - ${wavempexec} "${wavenproc}" ${wave_mpmd} cmdfile.buoy - fi - exit=$? - else - chmod 744 ${fcmdnow} - ./${fcmdnow} - exit=$? + + grep -F -f ibp_tags buoy_log.dat | awk '{ print $2 }' > buoys + grep -F -f buoys buoy_log.ww3 | awk '{ print $1 }' > points + points=$(awk '{print $0 "\\n"}' points | tr -d '\n') + rm buoys + + # Generate the ww3_outp.inp file from the template + if [ "$DOSPC_WAV" = 'YES' ]; then + sed -e "s/TIME/${tstart}/g" \ + -e "s/DT/${DTPNT_WAV}/g" \ + -e "s/999/$N/g" \ + -e "s/PREFIX/${RUN}wave/g" \ + -e "s|POINT|$points|g" \ + -e "s/ITYPE/1/g" \ + -e "s/FORMAT/F/g" \ + ww3_outp_spec.inp.tmpl > ww3_outp.inp + + export pgm="${NET,,}_ww3_outp.x" + "${EXECgfs}/${pgm}" > ww3_outp_spec.log 2>&1 fi - if [ "$exit" != '0' ] - then - set +x - echo ' ' - echo '*************************************' - echo '*** FATAL ERROR: CMDFILE FAILED ***' - echo '*************************************' - echo ' See Details Below ' - echo ' ' - set_trace - err=9; export err;${errchk} - exit $err + if [ "$DOBLL_WAV" = 'YES' ]; then + sed -e "s/TIME/${tstart}/g" \ + -e "s/DT/${DTPNT_WAV}/g" \ + -e "s/999/$N/g" \ + -e "s/PREFIX/${RUN}wave/g" \ + -e "s|POINT|$points|g" \ + -e "s/REFT/$truntime/g" \ + ww3_outp_bull.inp.tmpl > ww3_outp.inp + export pgm="${NET,,}_ww3_outp.x" + "${EXECgfs}/${pgm}" > ww3_outp_bull.log 2>&1 fi - # --------------------------------------------------------------------------- # # 3. Compress point output data into tar files @@ -611,7 +393,7 @@ source "${USHgfs}/preamble.sh" set_trace -# 6.b Spectral data files +# 3.b Execute the taring if [ ${CFP_MP:-"NO"} = "YES" ]; then nm=0; fi diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index ee3d52d3620..647d4f79881 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit ee3d52d3620565dbf7bc4f47bd039fe6423fa5c1 +Subproject commit 647d4f7988199f9b892fd16f416ab060a6715ae3 diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index b6e69d00a4a..274f8666d01 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -468,7 +468,7 @@ WW3_postdet() { fhinc=${FHINCP_WAV} while (( fhr <= FHMAX_WAV )); do vdate=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${fhr} hours" +%Y%m%d.%H0000) - ${NLN} "${COMOUT_WAVE_HISTORY}/${wavprfx}.out_pnt.${waveuoutpGRD}.${vdate}" "${DATA}/${vdate}.out_pnt.ww3" + ${NLN} "${COMOUT_WAVE_HISTORY}/${wavprfx}.out_pnt.${waveuoutpGRD}.${vdate}.nc" "${DATA}/${vdate}.out_pnt.ww3.nc" fhr=$((fhr + fhinc)) done diff --git a/ush/wave_tar.sh b/ush/wave_tar.sh index f82849854ff..a31c8b39600 100755 --- a/ush/wave_tar.sh +++ b/ush/wave_tar.sh @@ -88,8 +88,6 @@ source "${USHgfs}/preamble.sh" exit 2 fi - cd "${STA_DIR}/${filext}" - # --------------------------------------------------------------------------- # # 2. Generate tar file (spectral files are compressed)