Restructure the bufr sounding job #2853
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,11 +18,16 @@ | |
| # 7) 2018-07-18 Guang Ping Lou Generalize this version to other platforms | ||
| # 8) 2019-10-18 Guang Ping Lou Transition to reading in NetCDF model data | ||
| # 9) 2019-12-18 Guang Ping Lou generalizing to reading in NetCDF or nemsio | ||
| # 10) 2024-08-08 Bo Cui Update to handle one forecast at a time | ||
| # For GFSv17 bufr, total number of forecast hours is 141(num_hours=141) | ||
| # it requires 7 nodes & allocate 21 processes per node(num_ppn=21) | ||
| ################################################################ | ||
|
|
||
| source "${USHgfs}/preamble.sh" | ||
|
|
||
| runscript=${USHgfs}/gfs_bufr.sh | ||
|
|
||
| cd "${DATA}" || exit 2 | ||
|
|
||
| ######################################## | ||
|
|
||
|
|
@@ -44,47 +49,114 @@ export NINT3=${FHOUT_GFS:-3} | |
|
|
||
| rm -f -r "${COM_ATMOS_BUFR}" | ||
| mkdir -p "${COM_ATMOS_BUFR}" | ||
|
|
||
| GETDIM="${USHgfs}/getncdimlen" | ||
| LEVS=$(${GETDIM} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atmf000.${atmfm}" pfull) | ||
| declare -x LEVS | ||
|
|
||
| # Initialize an empty list to store the hours | ||
| hour_list=() | ||
|
|
||
| # Generate hours from 0 to NEND1 with interval NINT1 | ||
| for (( hour=0; hour<=NEND1 && hour<=ENDHOUR; hour+=NINT1 )); do | ||
| hour_list+=("$(printf "%03d" "$hour")") | ||
| done | ||
|
|
||
| # Generate hours from NEND1 + NINT3 to ENDHOUR with interval NINT3 | ||
| for (( hour=NEND1+NINT3; hour<=ENDHOUR; hour+=NINT3 )); do | ||
| hour_list+=("$(printf "%03d" "$hour")") | ||
| done | ||
|
|
||
| # Print the hour list | ||
| echo "Hour List:" "${hour_list[@]}" | ||
|
|
||
| # Count the number of elements in the hour_list | ||
| export ntasks="${#hour_list[@]}" | ||
|
|
||
| # Print the total number of hours | ||
| echo "Total number of hours: $ntasks" | ||
|
|
||
| # allocate 21 processes per node | ||
| # don't allocate more processes, or it might have memory issue | ||
| #export tasks_per_node=21 | ||
| export APRUN="mpiexec -np ${ntasks} -ppn ${tasks_per_node} --cpu-bind core cfp " | ||
|
|
||
| if [ -s "${DATA}/poescript_bufr" ]; then | ||
| rm ${DATA}/poescript_bufr | ||
| fi | ||
|
Contributor
There was a problem hiding this comment. We have a utility script, ush/run_mpmh.sh, that handles setting up an MPMD job now. That is the preferred method, as it correctly handles both slurm and pbs/torque. You just need to give it the file with your list of commands as an argument. See the atmos products ex-script for an example.
Contributor
Author
There was a problem hiding this comment. ush/run_mpmd.sh, the mpiexec command misses the setting of the process number per node in bufr job exgfs_atmos.postsnd.sh.. Will there be any update for the run_mpmd.sh in the future?
Contributor
There was a problem hiding this comment. I would try without the ppn setting first to confirm it is actually an issue (ideally the MPMD tasks should be equally distributed across all nodes anyway). If it is still required, an entry should be added to the env script on any machine where it is necessary to update the
Contributor
Author
There was a problem hiding this comment. I tested the bufr job using run_mpmd.sh without setting the ppn parameter, and the job failed. After adding the ppn setting, the bufr job completed successfully. The PBS setting in my jobcard is: Please let me know if I’m wrong.
Contributor
There was a problem hiding this comment. See new review. It should maintain the ppn setting while switching to |
||
|
|
||
| for fhr in "${hour_list[@]}"; do | ||
|
|
||
| if [ ! -s "${DATA}/${fhr}" ]; then mkdir -p ${DATA}/${fhr}; fi | ||
| export FINT=${NINT1} | ||
| ## 1-hourly output before $NEND1, 3-hourly output after | ||
| if [[ $((10#${fhr})) -gt $((10#${NEND1})) ]]; then | ||
| export FINT=${NINT3} | ||
| fi | ||
| if [[ $((10#${fhr})) -eq 0 ]]; then | ||
| export F00FLAG="YES" | ||
| else | ||
| export F00FLAG="NO" | ||
| fi | ||
|
|
||
| # Convert fhr to integer | ||
| fhr_int=$((10#$fhr)) | ||
|
|
||
| # Get previous hour | ||
| if (( fhr_int == STARTHOUR )); then | ||
| fhr_p=${fhr_int} | ||
| else | ||
| fhr_p=$(( fhr_int - FINT )) | ||
| fi | ||
|
|
||
| # Format fhr_p with leading zeros | ||
| fhr_p="$(printf "%03d" "$fhr_p")" | ||
|
|
||
| filename="${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atm.logf${fhr}.${logfm}" | ||
| if [[ -z ${filename} ]]; then | ||
| echo "File ${filename} is required but not found." | ||
| err_exit "FATAL ERROR: logf${fhr} not found." | ||
| else | ||
| echo "${runscript} \"${fhr}\" \"${fhr_p}\" \"${FINT}\" \"${F00FLAG}\" \"${DATA}/${fhr}\"" >> "${DATA}/poescript_bufr" | ||
| fi | ||
| done | ||
|
|
||
| # Run with MPMD | ||
| chmod +x "${DATA}/poescript_bufr" | ||
| $APRUN "${DATA}/poescript_bufr" | ||
| export err=$?; err_chk | ||
|
|
||
| # Run with MPMD | ||
| #chmod +x "${DATA}/poescript_bufr" | ||
| #bash +x "${DATA}/poescript_bufr" > mpmd.out 2>&1 | ||
| #export err=$?; err_chk | ||
|
BoCui-NOAA marked this conversation as resolved.
Outdated
|
||
|
|
||
| cd "${DATA}" || exit 2 | ||
|
|
||
| # Initialize fortnum | ||
| fortnum=20 | ||
|
|
||
| # Loop through each element in the array | ||
| for fhr in "${hour_list[@]}"; do | ||
| # Increment fortnum | ||
| fortnum=$((fortnum + 1)) | ||
| ${NLN} "${DATA}/${fhr}/fort.${fortnum}" "fort.${fortnum}" | ||
| done | ||
|
|
||
| export MAKEBUFR=YES | ||
| export fhr=${ENDHOUR} | ||
| export FINT=${NINT1} | ||
| ## 1-hourly output before $NEND1, 3-hourly output after | ||
| if [[ $((10#${fhr})) -gt $((10#${NEND1})) ]]; then | ||
| export FINT=${NINT3} | ||
| fi | ||
| if [[ $((10#${fhr})) -eq 0 ]]; then | ||
| export F00FLAG="YES" | ||
| else | ||
| export F00FLAG="NO" | ||
| fi | ||
| ${runscript} "${fhr}" "${fhr_p}" "${FINT}" "${F00FLAG}" "${DATA}" | ||
|
Comment on lines
+148
to
+153
Contributor
There was a problem hiding this comment. Is there a reason the last hour can't be done as part of the MPMD?
Contributor
Author
There was a problem hiding this comment. I made some modifications to the code gfs_bufr.fd/gfs_bufr.f. This code handles two tasks, which is controlled by flag makebufr. When makebufr is set to false, the code reads separate forecast and generates temporary file for each forecast hour. When makebufr is set to yes, the code merges all the temporary files and generates the final bufr products. So the last hour forecast can be processes as part of the MPMD. |
||
|
|
||
| ############################################################## | ||
| # Tar and gzip the individual bufr files and send them to /com | ||
| ############################################################## | ||
|
|
@@ -105,7 +177,7 @@ fi | |
| # add appropriate WMO Headers. | ||
| ######################################## | ||
| rm -rf poe_col | ||
| for (( m = 1; m <= NUM_SND_COLLECTIVES; m++ )); do | ||
| echo "sh ${USHgfs}/gfs_sndp.sh ${m} " >> poe_col | ||
| done | ||
|
|
||
|
|
@@ -123,4 +195,5 @@ ${APRUN_POSTSNDCFP} cmdfile | |
| sh "${USHgfs}/gfs_bfr2gpk.sh" | ||
|
|
||
|
|
||
|
|
||
| ############## END OF SCRIPT ####################### | ||
| +3 −5 | src/gfs_bufr.fd/CMakeLists.txt | |
| +3 −1 | src/gfs_bufr.fd/buff.f | |
| +60 −94 | src/gfs_bufr.fd/gfsbufr.f | |
| +100 −294 | src/gfs_bufr.fd/meteorg.f | |
| +457 −0 | src/gfs_bufr.fd/modpr_module.f90 | |
| +2 −1 | src/gfs_bufr.fd/modstuff1.f | |
| +0 −1 | src/gfs_bufr.fd/read_netcdf.f |
Uh oh!
There was an error while loading. Please reload this page.