diff --git a/.gitmodules b/.gitmodules index fee315db63..00cc5440f6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,9 @@ [submodule "FV3"] path = FV3 - url = https://github.com/NOAA-EMC/fv3atm - branch = production/RRFS.v1 + #url = https://github.com/NOAA-EMC/fv3atm + #branch = production/RRFS.v1 + url = https://github.com/DusanJovic-NOAA/fv3atm + branch = rrfs_v1_conus13km_tests [submodule "WW3"] path = WW3 url = https://github.com/NOAA-EMC/WW3 diff --git a/FV3 b/FV3 index 81e6d10dea..03fc30cfe6 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit 81e6d10dea838f03aeffdd70e8e9c9ce7001304d +Subproject commit 03fc30cfe621836ea7d09c0b8dcab220d4d452dc diff --git a/tests/default_vars.sh b/tests/default_vars.sh index 819b28d7d7..6b502ff14c 100755 --- a/tests/default_vars.sh +++ b/tests/default_vars.sh @@ -1402,8 +1402,8 @@ export SHAL_CNV=.false. export DO_SAT_ADJ=.false. export DO_DEEP=.false. export CCPP_SUITE='FV3_HRRR' -export INPES=12 -export JNPES=12 +export INPES=10 +export JNPES=8 export NPX=397 export NPY=233 export NPZ=65 diff --git a/tests/fv3_conf/fv3_qsub.IN_acorn b/tests/fv3_conf/fv3_qsub.IN_acorn index db6b3f1d12..efb7051ee5 100644 --- a/tests/fv3_conf/fv3_qsub.IN_acorn +++ b/tests/fv3_conf/fv3_qsub.IN_acorn @@ -5,15 +5,16 @@ #PBS -N @[JBNME] #PBS -A @[ACCNR] #PBS -q @[QUEUE] -#PBS -l place=vscatter,select=@[NODES]:ncpus=@[TPN]:mpiprocs=@[TPN]:mem=500G +#PBS -l place=vscatter,select=@[NODES]:ncpus=@[NCPUS]:mpiprocs=@[TPN]:ompthreads=@[THRD]:mem=500G #PBS -l place=excl #PBS -l walltime=00:@[WLCLK]:00 set -eux -echo -n " $( date +%s )," > job_timestamp.txt cd $PBS_O_WORKDIR +echo -n " $( date +%s )," > job_timestamp.txt + set +x module use $PWD/modulefiles module load modules.fv3 @@ -30,7 +31,7 @@ export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4 export ESMF_RUNTIME_PROFILE=ON export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY" -mpiexec -n @[TASKS] -ppn @[TPN] -depth @[THRD] ./fv3.exe +mpiexec -n @[TASKS] -ppn @[TPN] --cpu-bind core -depth @[THRD] ./fv3.exe echo "Model ended: " `date` echo -n " $( date +%s )," >> job_timestamp.txt diff --git a/tests/fv3_conf/fv3_qsub.IN_wcoss2 b/tests/fv3_conf/fv3_qsub.IN_wcoss2 index db6b3f1d12..efb7051ee5 100644 --- a/tests/fv3_conf/fv3_qsub.IN_wcoss2 +++ b/tests/fv3_conf/fv3_qsub.IN_wcoss2 @@ -5,15 +5,16 @@ #PBS -N @[JBNME] #PBS -A @[ACCNR] #PBS -q @[QUEUE] -#PBS -l place=vscatter,select=@[NODES]:ncpus=@[TPN]:mpiprocs=@[TPN]:mem=500G +#PBS -l place=vscatter,select=@[NODES]:ncpus=@[NCPUS]:mpiprocs=@[TPN]:ompthreads=@[THRD]:mem=500G #PBS -l place=excl #PBS -l walltime=00:@[WLCLK]:00 set -eux -echo -n " $( date +%s )," > job_timestamp.txt cd $PBS_O_WORKDIR +echo -n " $( date +%s )," > job_timestamp.txt + set +x module use $PWD/modulefiles module load modules.fv3 @@ -30,7 +31,7 @@ export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4 export ESMF_RUNTIME_PROFILE=ON export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY" -mpiexec -n @[TASKS] -ppn @[TPN] -depth @[THRD] ./fv3.exe +mpiexec -n @[TASKS] -ppn @[TPN] --cpu-bind core -depth @[THRD] ./fv3.exe echo "Model ended: " `date` echo -n " $( date +%s )," >> job_timestamp.txt diff --git a/tests/rrfs_for_testing_do_not_merge.conf b/tests/rrfs_for_testing_do_not_merge.conf new file mode 100644 index 0000000000..5a88fd3d1b --- /dev/null +++ b/tests/rrfs_for_testing_do_not_merge.conf @@ -0,0 +1,13 @@ + +COMPILE | rrfs_dyn32_phy32_faster | intel | -DAPP=ATM -DFASTER=ON -DCCPP_SUITES=FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON -DENABLE_PARALLELRESTART=NO -DENABLE_RRFS_WAR=NO | - noaacloud | fv3 | +RUN | conus13km_control | - noaacloud | baseline | +RUN | conus13km_2threads | - noaacloud | | +RUN | conus13km_decomp | - noaacloud | | +RUN | conus13km_restart | - noaacloud | | conus13km_control + +COMPILE | rrfs_dyn32_phy32_debug | intel | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR,FV3_HRRR_gf -D32BIT=ON -DCCPP_32BIT=ON -DDEBUG=ON -DENABLE_PARALLELRESTART=NO -DENABLE_RRFS_WAR=NO | - noaacloud | fv3 | +RUN | conus13km_debug | - noaacloud | baseline | +RUN | conus13km_debug_qr | - derecho noaacloud | | +RUN | conus13km_debug_2threads | - derecho noaacloud | | +RUN | conus13km_radar_tten_debug | - noaacloud | baseline | +RUN | conus13km_debug_decomp | - noaacloud | | diff --git a/tests/rt.conf b/tests/rt.conf index ba5a98a4ae..0c834c4200 100644 --- a/tests/rt.conf +++ b/tests/rt.conf @@ -199,12 +199,9 @@ RUN | hrrr_control_restart_dyn32_phy32 | - noaacloud COMPILE | rrfs_dyn32_phy32_faster | intel | -DAPP=ATM -DFASTER=ON -DCCPP_SUITES=FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON | - noaacloud | fv3 | RUN | conus13km_control | - noaacloud | baseline | -RUN | conus13km_2threads | - noaacloud | | conus13km_control -RUN | conus13km_restart_mismatch | - noaacloud | baseline | conus13km_control - -# Expected to fail: -# RUN | conus13km_restart | - noaacloud | | conus13km_control -# RUN | conus13km_decomp | - noaacloud | | conus13km_control +RUN | conus13km_2threads | - noaacloud | | +RUN | conus13km_restart | - noaacloud | | conus13km_control +RUN | conus13km_decomp | - noaacloud | | COMPILE | rrfs_dyn64_phy32 | intel | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -DCCPP_32BIT=ON | - noaacloud | fv3 | RUN | rap_control_dyn64_phy32 | - noaacloud | baseline | @@ -213,13 +210,10 @@ COMPILE | rrfs_dyn32_phy32_debug | intel | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_H RUN | rap_control_debug_dyn32_phy32 | - noaacloud | baseline | RUN | hrrr_control_debug_dyn32_phy32 | - noaacloud | baseline | RUN | conus13km_debug | - noaacloud | baseline | -RUN | conus13km_debug_qr | - derecho noaacloud | | -RUN | conus13km_debug_2threads | - derecho noaacloud | | +RUN | conus13km_debug_qr | - noaacloud | | +RUN | conus13km_debug_2threads | - noaacloud | | RUN | conus13km_radar_tten_debug | - noaacloud | baseline | -# Expected to fail: -# RUN | conus13km_debug_decomp | - noaacloud | | - COMPILE | rrfs_dyn64_phy32_debug | intel | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -DCCPP_32BIT=ON -DDEBUG=ON | - noaacloud | fv3 | RUN | rap_control_dyn64_phy32_debug | - noaacloud | baseline | @@ -371,12 +365,9 @@ RUN | hrrr_control_decomp_dyn32_phy32 | + hera hercules RUN | rap_restart_dyn32_phy32 | + hera hercules | | rap_control_dyn32_phy32 RUN | hrrr_control_restart_dyn32_phy32 | + hera hercules | | hrrr_control_dyn32_phy32 RUN | conus13km_control | + hera hercules | baseline | -RUN | conus13km_2threads | + hera hercules | | conus13km_control -RUN | conus13km_restart_mismatch | + hera hercules | baseline | conus13km_control - -# Expected to fail: -# RUN | conus13km_restart | + hera hercules | | conus13km_control -# RUN | conus13km_decomp | + hera hercules | | conus13km_control +RUN | conus13km_2threads | + hera hercules | | +RUN | conus13km_decomp | + hera hercules | | +RUN | conus13km_restart | + hera hercules | | conus13km_control COMPILE | atm_dyn64_phy32 | gnu | -DAPP=ATM -DCCPP_32BIT=ON | + hera hercules | fv3 | RUN | rap_control_dyn64_phy32 | + hera hercules | baseline | @@ -387,11 +378,9 @@ RUN | hrrr_control_debug_dyn32_phy32 | + hera hercules RUN | conus13km_debug | + hera hercules | baseline | RUN | conus13km_debug_qr | + hera hercules | | RUN | conus13km_debug_2threads | + hera hercules | | +RUN | conus13km_debug_decomp | + hera hercules | | RUN | conus13km_radar_tten_debug | + hera hercules | baseline | -# Expected to fail: -# RUN | conus13km_debug_decomp | + hera derecho hercules | | - COMPILE | atm_dyn64_phy32_debug | gnu | -DAPP=ATM -DCCPP_32BIT=ON -DDEBUG=ON | + hera hercules | fv3 | RUN | rap_control_dyn64_phy32_debug | + hera hercules | baseline | diff --git a/tests/rt.conf_rrfs b/tests/rt.conf_rrfs index a734d96536..28fa639c9d 100644 --- a/tests/rt.conf_rrfs +++ b/tests/rt.conf_rrfs @@ -86,12 +86,9 @@ RUN | hrrr_control_restart_dyn32_phy32 | - noaacloud COMPILE | rrfs_dyn32_phy32_faster | intel | -DAPP=ATM -DFASTER=ON -DCCPP_SUITES=FV3_HRRR -D32BIT=ON -DCCPP_32BIT=ON -DENABLE_PARALLELRESTART=NO -DENABLE_RRFS_WAR=NO | - noaacloud | fv3 | RUN | conus13km_control | - noaacloud | baseline | -RUN | conus13km_2threads | - noaacloud | | conus13km_control -RUN | conus13km_restart_mismatch | - noaacloud | baseline | conus13km_control - -# Expected to fail: -# RUN | conus13km_restart | - noaacloud | | conus13km_control -# RUN | conus13km_decomp | - noaacloud | | conus13km_control +RUN | conus13km_2threads | - noaacloud | | +RUN | conus13km_decomp | - noaacloud | | +RUN | conus13km_restart | - noaacloud | | conus13km_control COMPILE | rrfs_dyn64_phy32 | intel | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -DCCPP_32BIT=ON -DENABLE_PARALLELRESTART=NO -DENABLE_RRFS_WAR=NO | - noaacloud | fv3 | RUN | rap_control_dyn64_phy32 | - noaacloud | baseline | @@ -102,11 +99,9 @@ RUN | hrrr_control_debug_dyn32_phy32 | - noaacloud RUN | conus13km_debug | - noaacloud | baseline | RUN | conus13km_debug_qr | - derecho noaacloud | | RUN | conus13km_debug_2threads | - derecho noaacloud | | +RUN | conus13km_debug_decomp | - noaacloud | | RUN | conus13km_radar_tten_debug | - noaacloud | baseline | -# Expected to fail: -# RUN | conus13km_debug_decomp | - noaacloud | | - COMPILE | rrfs_dyn64_phy32_debug | intel | -DAPP=ATM -DCCPP_SUITES=FV3_RAP,FV3_HRRR -DCCPP_32BIT=ON -DDEBUG=ON -DENABLE_PARALLELRESTART=NO -DENABLE_RRFS_WAR=NO | - noaacloud | fv3 | RUN | rap_control_dyn64_phy32_debug | - noaacloud | baseline | diff --git a/tests/run_test.sh b/tests/run_test.sh index 2b9e3ebf9d..d0d1f6419b 100755 --- a/tests/run_test.sh +++ b/tests/run_test.sh @@ -312,6 +312,8 @@ fi export PPN export UFS_TASKS +export NCPUS=$(( TPN * THRD )) + if [[ $SCHEDULER = 'pbs' ]]; then if [[ -e $PATHRT/fv3_conf/fv3_qsub.IN_${MACHINE_ID} ]]; then atparse < $PATHRT/fv3_conf/fv3_qsub.IN_${MACHINE_ID} > job_card diff --git a/tests/tests/conus13km_2threads b/tests/tests/conus13km_2threads index 9d8620c42a..c72e25708d 100644 --- a/tests/tests/conus13km_2threads +++ b/tests/tests/conus13km_2threads @@ -17,11 +17,12 @@ export LIST_FILES="sfcf000.nc \ atmf001.nc" export_hrrr_conus13km -export SMONTH=5 export RESTART_INTERVAL=1 export FHMAX=1 export atm_omp_num_threads=2 -export INPES=12 -export JNPES=12 -export WRTTASK_PER_GROUP=6 export QUILTING_RESTART=.false. +export USE_MERRA2=.true. + +if [[ " hera orion hercules jet " =~ " ${MACHINE_ID} " ]] ; then + ZSTANDARD_LEVEL=5 +fi diff --git a/tests/tests/conus13km_debug_2threads b/tests/tests/conus13km_debug_2threads index 9daadc7dd6..a520d787cc 100644 --- a/tests/tests/conus13km_debug_2threads +++ b/tests/tests/conus13km_debug_2threads @@ -20,7 +20,4 @@ export_hrrr_conus13km export RESTART_INTERVAL=1 export FHMAX=1 export atm_omp_num_threads=2 -export INPES=12 -export JNPES=12 -export WRTTASK_PER_GROUP=6 export QUILTING_RESTART=.false. diff --git a/tests/tests/conus13km_debug_decomp b/tests/tests/conus13km_debug_decomp index 07f1436d03..4816af063a 100644 --- a/tests/tests/conus13km_debug_decomp +++ b/tests/tests/conus13km_debug_decomp @@ -19,6 +19,6 @@ export LIST_FILES="sfcf000.nc \ export_hrrr_conus13km export RESTART_INTERVAL=1 export FHMAX=1 -export INPES=16 -export JNPES=9 +export INPES=8 +export JNPES=10 export QUILTING_RESTART=.false. diff --git a/tests/tests/conus13km_debug_qr b/tests/tests/conus13km_debug_qr index 918680aacc..038d0aa576 100644 --- a/tests/tests/conus13km_debug_qr +++ b/tests/tests/conus13km_debug_qr @@ -1,13 +1,13 @@ ############################################################################### # -# HRRR physics on 13km domain, control +# HRRR physics on 13km domain, debug with quilting restart # ############################################################################### # This configuration is supposed to match the rrfs_dev1 parallel, so # do not change the namelist without checking the current parallel. -export TEST_DESCR="HRRR physics on 13km domain, control" +export TEST_DESCR="HRRR physics on 13km domain, debug with quilting restart" export CNTL_DIR=conus13km_debug @@ -26,3 +26,4 @@ export LIST_FILES="sfcf000.nc \ export_hrrr_conus13km export RESTART_INTERVAL=1 export FHMAX=1 +export QUILTING_RESTART=.true. diff --git a/tests/tests/conus13km_decomp b/tests/tests/conus13km_decomp index 70ede9c67c..3674882354 100644 --- a/tests/tests/conus13km_decomp +++ b/tests/tests/conus13km_decomp @@ -20,6 +20,11 @@ export LIST_FILES="sfcf000.nc \ export_hrrr_conus13km export RESTART_INTERVAL=1 -export INPES=16 -export JNPES=9 +export INPES=8 +export JNPES=10 export QUILTING_RESTART=.false. +export USE_MERRA2=.true. + +if [[ " hera orion hercules jet " =~ " ${MACHINE_ID} " ]] ; then + ZSTANDARD_LEVEL=5 +fi diff --git a/tests/tests/conus13km_restart b/tests/tests/conus13km_restart index 4a966591bf..0d8488eb6b 100644 --- a/tests/tests/conus13km_restart +++ b/tests/tests/conus13km_restart @@ -19,3 +19,8 @@ export FHROT=1 export RESTART_FILE_PREFIX="${SYEAR}${SMONTH}${SDAY}.$(printf "%02d" $(( ${SHOUR} + ${FHROT} )))0000" export RRFS_RESTART=YES export QUILTING_RESTART=.false. +export USE_MERRA2=.true. + +if [[ " hera orion hercules jet " =~ " ${MACHINE_ID} " ]] ; then + ZSTANDARD_LEVEL=5 +fi