From 98e8764e2d16f7804d764efcd8ba31e3546a3078 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Wed, 18 Nov 2020 21:22:31 +0000 Subject: [PATCH 1/3] Build on wcoss2 (acorn) --- CMakeLists.txt | 2 ++ cmake/configure_wcoss2.cmake | 3 ++ modulefiles/wcoss2/fv3 | 42 +++++++++++++++++++++++++++ modulefiles/wcoss2/fv3_debug | 42 +++++++++++++++++++++++++++ tests/compile.sh | 3 ++ tests/default_vars.sh | 4 +-- tests/detect_machine.sh | 3 ++ tests/fv3_conf/compile_qsub.IN_wcoss2 | 20 +++++++++++++ tests/fv3_conf/fv3_qsub.IN_wcoss2 | 29 ++++++++++++++++++ tests/rt.sh | 33 +++++++++++++++++++++ 10 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 cmake/configure_wcoss2.cmake create mode 100644 modulefiles/wcoss2/fv3 create mode 100644 modulefiles/wcoss2/fv3_debug create mode 100644 tests/fv3_conf/compile_qsub.IN_wcoss2 create mode 100644 tests/fv3_conf/fv3_qsub.IN_wcoss2 diff --git a/CMakeLists.txt b/CMakeLists.txt index a00e1472cf..cdc12033b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -189,6 +189,8 @@ if(WW3) set(WW3_COMP "hera") elseif(${CMAKE_Platform} STREQUAL "orion.intel") set(WW3_COMP "orion") + elseif(${CMAKE_Platform} STREQUAL "wcoss2") + set(WW3_COMP "wcoss_cray") elseif(${CMAKE_Platform} MATCHES "linux*" OR ${CMAKE_Platform} MATCHES "macosx*") if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU") set(WW3_COMP "gnu") diff --git a/cmake/configure_wcoss2.cmake b/cmake/configure_wcoss2.cmake new file mode 100644 index 0000000000..c8a689be6c --- /dev/null +++ b/cmake/configure_wcoss2.cmake @@ -0,0 +1,3 @@ +set(INLINE_POST ON CACHE BOOL "Enable inline post" FORCE) +set(PARALLEL_NETCDF ON CACHE BOOL "Enable parallel NetCDF" FORCE) +set(DEBUG_LINKMPI OFF CACHE BOOL "Enable linkmpi option when DEBUG mode is on" FORCE) diff --git a/modulefiles/wcoss2/fv3 b/modulefiles/wcoss2/fv3 new file mode 100644 index 0000000000..7ea3d01682 --- /dev/null +++ b/modulefiles/wcoss2/fv3 @@ -0,0 +1,42 @@ +#%Module + +proc ModulesHelp {} { + puts stderr "\tcit - loads modules required for building and running UFS Model on the Dell side of WCOSS" +} + +module-whatis "loads UFS Model prerequisites on Venus and Mars" + +module unload cpe-cray cce +module load cpe-intel intel + +module load cmake/3.17.3 +setenv CMAKE_C_COMPILER cc +setenv CMAKE_CXX_COMPILER CC +setenv CMAKE_Fortran_COMPILER ftn +setenv CMAKE_Platform wcoss2 + +module use /lfs/h2/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack + +module load hpc/1.0.0-beta1 +module load hpc-intel/19.1.1.217 +module load hpc-cray-mpich/8.0.15 + +module load jasper/2.0.22 +module load zlib/1.2.11 +module load png/1.6.35 + +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load pio/2.5.1 +module load esmf/8_1_0_beta_snapshot_27 + +module load bacio/2.4.1 +module load crtm/2.3.0 +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load ip/3.3.3 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load upp/10.0.0 diff --git a/modulefiles/wcoss2/fv3_debug b/modulefiles/wcoss2/fv3_debug new file mode 100644 index 0000000000..7ea3d01682 --- /dev/null +++ b/modulefiles/wcoss2/fv3_debug @@ -0,0 +1,42 @@ +#%Module + +proc ModulesHelp {} { + puts stderr "\tcit - loads modules required for building and running UFS Model on the Dell side of WCOSS" +} + +module-whatis "loads UFS Model prerequisites on Venus and Mars" + +module unload cpe-cray cce +module load cpe-intel intel + +module load cmake/3.17.3 +setenv CMAKE_C_COMPILER cc +setenv CMAKE_CXX_COMPILER CC +setenv CMAKE_Fortran_COMPILER ftn +setenv CMAKE_Platform wcoss2 + +module use /lfs/h2/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack + +module load hpc/1.0.0-beta1 +module load hpc-intel/19.1.1.217 +module load hpc-cray-mpich/8.0.15 + +module load jasper/2.0.22 +module load zlib/1.2.11 +module load png/1.6.35 + +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load pio/2.5.1 +module load esmf/8_1_0_beta_snapshot_27 + +module load bacio/2.4.1 +module load crtm/2.3.0 +module load g2/3.4.1 +module load g2tmpl/1.9.1 +module load ip/3.3.3 +module load nemsio/2.5.2 +module load sp/2.3.3 +module load w3emc/2.7.3 +module load w3nco/2.4.1 +module load upp/10.0.0 diff --git a/tests/compile.sh b/tests/compile.sh index 4300685133..ff3ae10bec 100755 --- a/tests/compile.sh +++ b/tests/compile.sh @@ -56,6 +56,9 @@ set +x if [[ $MACHINE_ID == macosx.* ]] || [[ $MACHINE_ID == linux.* ]]; then source $PATHTR/modulefiles/${MACHINE_ID}/fv3 else + if [[ $MACHINE_ID == wcoss2 ]]; then + source /apps/prod/lmodules/startLmod + fi module use $PATHTR/modulefiles/${MACHINE_ID} modulefile="fv3" if [[ "${MAKE_OPT}" == *"DEBUG=Y"* ]]; then diff --git a/tests/default_vars.sh b/tests/default_vars.sh index cc2342367e..3ddf0e364b 100755 --- a/tests/default_vars.sh +++ b/tests/default_vars.sh @@ -7,14 +7,14 @@ # ############################################################################### -if [ $MACHINE_ID = wcoss_cray ]; then +if [[ $MACHINE_ID = wcoss_cray ]]; then TASKS_dflt=150 ; TPN_dflt=24 ; INPES_dflt=3 ; JNPES_dflt=8 TASKS_thrd=84 ; TPN_thrd=12 ; INPES_thrd=3 ; JNPES_thrd=4 TASKS_stretch=48 ; TPN_stretch=24 ; INPES_stretch=2 ; JNPES_stretch=4 TASKS_strnest=96 ; TPN_strnest=24 ; INPES_strnest=2 ; JNPES_strnest=4 -elif [ $MACHINE_ID = wcoss_dell_p3 ]; then +elif [[ $MACHINE_ID = wcoss_dell_p3 || $MACHINE_ID = wcoss2 ]]; then TASKS_dflt=150 ; TPN_dflt=28 ; INPES_dflt=3 ; JNPES_dflt=8 TASKS_thrd=84 ; TPN_thrd=14 ; INPES_thrd=3 ; JNPES_thrd=4 diff --git a/tests/detect_machine.sh b/tests/detect_machine.sh index 8877530d7b..c4bea08a2e 100755 --- a/tests/detect_machine.sh +++ b/tests/detect_machine.sh @@ -27,6 +27,9 @@ case $(hostname -f) in m72a2.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### mars m72a3.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### mars + alogin01) MACHINE_ID=wcoss2 ;; ### acorn + alogin02) MACHINE_ID=wcoss2 ;; ### acorn + gaea9) MACHINE_ID=gaea ;; ### gaea9 gaea10) MACHINE_ID=gaea ;; ### gaea10 gaea11) MACHINE_ID=gaea ;; ### gaea11 diff --git a/tests/fv3_conf/compile_qsub.IN_wcoss2 b/tests/fv3_conf/compile_qsub.IN_wcoss2 new file mode 100644 index 0000000000..ee2b789758 --- /dev/null +++ b/tests/fv3_conf/compile_qsub.IN_wcoss2 @@ -0,0 +1,20 @@ +#!/bin/bash + +#PBS -o out +#PBS -e err +#PBS -N @[JBNME] +# #PBS -A @[ACCNR] +#PBS -q @[QUEUE] +#PBS -l select=1:ncpus=8:mpiprocs=1 +#PBS -l walltime=00:30:00 + +set -eux + +cd $PBS_O_WORKDIR +export CRAY_CONFIG_DIR=$HOME + +echo "Compile started: " `date` + +@[PATHRT]/compile.sh @[MACHINE_ID] "@[MAKE_OPT]" @[COMPILE_NR] + +echo "Compile ended: " `date` diff --git a/tests/fv3_conf/fv3_qsub.IN_wcoss2 b/tests/fv3_conf/fv3_qsub.IN_wcoss2 new file mode 100644 index 0000000000..d97a91ff25 --- /dev/null +++ b/tests/fv3_conf/fv3_qsub.IN_wcoss2 @@ -0,0 +1,29 @@ +#!/bin/bash + +#PBS -o out +#PBS -e err +#PBS -N @[JBNME] +# #PBS -A @[ACCNR] +#PBS -q @[QUEUE] +#PBS -l place=vscatter,select=@[NODES]:ncpus=@[TPN]:mpiprocs=@[TPN] +#PBS -l walltime=00:@[WLCLK]:00 + +set -eux + +cd $PBS_O_WORKDIR +export CRAY_CONFIG_DIR=$HOME + +module list + +echo "Model started: " `date` + +export MPI_TYPE_DEPTH=20 +export OMP_STACKSIZE=512M +export OMP_NUM_THREADS=@[THRD] +export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4 + +cray aprun -n @[TASKS] ./fv3.exe + +echo "Model ended: " `date` + +exit diff --git a/tests/rt.sh b/tests/rt.sh index 524ea63e17..b3a61224f4 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -167,6 +167,33 @@ elif [[ $MACHINE_ID = wcoss_dell_p3 ]]; then cp fv3_conf/fv3_bsub.IN_wcoss_dell_p3 fv3_conf/fv3_bsub.IN cp fv3_conf/compile_bsub.IN_wcoss_dell_p3 fv3_conf/compile_bsub.IN +elif [[ $MACHINE_ID = wcoss2 ]]; then + + source /apps/prod/lmodules/startLmod + + #module use /usrx/local/dev/emc_rocoto/modulefiles + #module load ruby/2.5.1 rocoto/1.3.0rc2 + #ROCOTORUN=$(which rocotorun) + #ROCOTOSTAT=$(which rocotostat) + #ROCOTOCOMPLETE=$(which rocotocomplete) + #ROCOTO_SCHEDULER=lsf + + #module load ips/18.0.1.163 + #module load ecflow/4.7.1 + #ECFLOW_START=${ECF_ROOT}/intel/bin/ecflow_start.sh + #ECF_PORT=$(grep $USER /usrx/local/sys/ecflow/assigned_ports.txt | awk '{print $2}') + + DISKNM=/lfs/h1_old/ptmp/Dusan.Jovic/RT + QUEUE=workq + COMPILE_QUEUE=workq + PARTITION= + ACCNR=GFS-DEV + STMP=/lfs/h2/emc/stmp + PTMP=/lfs/h2/emc/ptmp + SCHEDULER=pbs + cp fv3_conf/fv3_qsub.IN_wcoss2 fv3_conf/fv3_qsub.IN + cp fv3_conf/compile_qsub.IN_wcoss2 fv3_conf/compile_qsub.IN + elif [[ $MACHINE_ID = gaea.* ]]; then source $PATHTR/NEMS/src/conf/module-setup.sh.inc @@ -487,6 +514,10 @@ if [[ $ROCOTO == true ]]; then QUEUE=dev COMPILE_QUEUE=dev_transfer ROCOTO_SCHEDULER=lsf + elif [[ $MACHINE_ID = wcoss2 ]]; then + QUEUE=workq + COMPILE_QUEUE=workq + ROCOTO_SCHEDULER=pbs elif [[ $MACHINE_ID = hera.* ]]; then QUEUE=batch COMPILE_QUEUE=batch @@ -547,6 +578,8 @@ EOF QUEUE=dev elif [[ $MACHINE_ID = wcoss_dell_p3 ]]; then QUEUE=dev + elif [[ $MACHINE_ID = wcoss2 ]]; then + QUEUE=workq elif [[ $MACHINE_ID = hera.* ]]; then QUEUE=batch elif [[ $MACHINE_ID = orion.* ]]; then From ed381e8c23ea8bb00591dc607789b692bbc40c4b Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Thu, 19 Nov 2020 23:43:54 +0000 Subject: [PATCH 2/3] Use -march=core-avx2 instead of -xCORE-AVX2 on wcoss2 --- cmake/Intel.cmake | 6 ++++-- tests/fv3_conf/fv3_qsub.IN_wcoss2 | 3 +++ tests/rt.sh | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cmake/Intel.cmake b/cmake/Intel.cmake index 6d2eb74537..d070ac2927 100644 --- a/cmake/Intel.cmake +++ b/cmake/Intel.cmake @@ -21,8 +21,10 @@ elseif(DEBUG) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qno-opt-dynamic-align") else() if(AVX2) - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align") + #set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align") + #set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -march=core-avx2 -qno-opt-dynamic-align") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=core-avx2 -qno-opt-dynamic-align") elseif(SIMDMULTIARCH) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align") diff --git a/tests/fv3_conf/fv3_qsub.IN_wcoss2 b/tests/fv3_conf/fv3_qsub.IN_wcoss2 index d97a91ff25..d000b7e493 100644 --- a/tests/fv3_conf/fv3_qsub.IN_wcoss2 +++ b/tests/fv3_conf/fv3_qsub.IN_wcoss2 @@ -13,6 +13,9 @@ set -eux cd $PBS_O_WORKDIR export CRAY_CONFIG_DIR=$HOME +source /apps/prod/lmodules/startLmod +module use $( pwd -P ) +module load modules.fv3 module list echo "Model started: " `date` diff --git a/tests/rt.sh b/tests/rt.sh index b3a61224f4..fa1651864f 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -169,7 +169,7 @@ elif [[ $MACHINE_ID = wcoss_dell_p3 ]]; then elif [[ $MACHINE_ID = wcoss2 ]]; then - source /apps/prod/lmodules/startLmod + source /apps/prod/lmodules/startLmod #module use /usrx/local/dev/emc_rocoto/modulefiles #module load ruby/2.5.1 rocoto/1.3.0rc2 @@ -183,7 +183,7 @@ elif [[ $MACHINE_ID = wcoss2 ]]; then #ECFLOW_START=${ECF_ROOT}/intel/bin/ecflow_start.sh #ECF_PORT=$(grep $USER /usrx/local/sys/ecflow/assigned_ports.txt | awk '{print $2}') - DISKNM=/lfs/h1_old/ptmp/Dusan.Jovic/RT + DISKNM=/lfs/h2/emc/ptmp/Dusan.Jovic/RT QUEUE=workq COMPILE_QUEUE=workq PARTITION= From 7a8460280ea2fb0b577ca205914bfe8427978976 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Thu, 3 Dec 2020 20:05:47 +0000 Subject: [PATCH 3/3] Add rt_acorn.conf. Change /lfs/h2 to /lfs/h1. --- modulefiles/wcoss2/fv3 | 2 +- modulefiles/wcoss2/fv3_debug | 2 +- tests/rt.sh | 6 ++--- tests/rt_acorn.conf | 52 ++++++++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 5 deletions(-) create mode 100644 tests/rt_acorn.conf diff --git a/modulefiles/wcoss2/fv3 b/modulefiles/wcoss2/fv3 index 7ea3d01682..fbb682affc 100644 --- a/modulefiles/wcoss2/fv3 +++ b/modulefiles/wcoss2/fv3 @@ -15,7 +15,7 @@ setenv CMAKE_CXX_COMPILER CC setenv CMAKE_Fortran_COMPILER ftn setenv CMAKE_Platform wcoss2 -module use /lfs/h2/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack +module use /lfs/h1/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack module load hpc/1.0.0-beta1 module load hpc-intel/19.1.1.217 diff --git a/modulefiles/wcoss2/fv3_debug b/modulefiles/wcoss2/fv3_debug index 7ea3d01682..fbb682affc 100644 --- a/modulefiles/wcoss2/fv3_debug +++ b/modulefiles/wcoss2/fv3_debug @@ -15,7 +15,7 @@ setenv CMAKE_CXX_COMPILER CC setenv CMAKE_Fortran_COMPILER ftn setenv CMAKE_Platform wcoss2 -module use /lfs/h2/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack +module use /lfs/h1/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack module load hpc/1.0.0-beta1 module load hpc-intel/19.1.1.217 diff --git a/tests/rt.sh b/tests/rt.sh index b8875c2850..bc7da070c1 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -189,13 +189,13 @@ elif [[ $MACHINE_ID = wcoss2 ]]; then #ECFLOW_START=${ECF_ROOT}/intel/bin/ecflow_start.sh #ECF_PORT=$(grep $USER /usrx/local/sys/ecflow/assigned_ports.txt | awk '{print $2}') - DISKNM=/lfs/h2/emc/ptmp/Dusan.Jovic/RT + DISKNM=/lfs/h1/emc/ptmp/Dusan.Jovic/RT QUEUE=workq COMPILE_QUEUE=workq PARTITION= ACCNR=GFS-DEV - STMP=/lfs/h2/emc/stmp - PTMP=/lfs/h2/emc/ptmp + STMP=/lfs/h1/emc/stmp + PTMP=/lfs/h1/emc/ptmp SCHEDULER=pbs cp fv3_conf/fv3_qsub.IN_wcoss2 fv3_conf/fv3_qsub.IN cp fv3_conf/compile_qsub.IN_wcoss2 fv3_conf/compile_qsub.IN diff --git a/tests/rt_acorn.conf b/tests/rt_acorn.conf new file mode 100644 index 0000000000..a5df552904 --- /dev/null +++ b/tests/rt_acorn.conf @@ -0,0 +1,52 @@ +####################################################################################################################################################################################### +# CCPP PROD tests # +####################################################################################################################################################################################### + +COMPILE | CCPP=Y SUITES=FV3_GFS_2017 | standard | | fv3 | + +RUN | fv3_ccpp_control | standard | | fv3 | +RUN | fv3_ccpp_decomp | standard | | | +RUN | fv3_ccpp_2threads | standard | | | +RUN | fv3_ccpp_restart | standard | | fv3 | +RUN | fv3_ccpp_read_inc | standard | | fv3 | +RUN | fv3_ccpp_wrtGauss_netcdf_esmf | standard | | fv3 | +RUN | fv3_ccpp_wrtGauss_netcdf | standard | | fv3 | +RUN | fv3_ccpp_wrtGlatlon_netcdf | standard | | fv3 | +RUN | fv3_ccpp_wrtGauss_nemsio | standard | | fv3 | +RUN | fv3_ccpp_wrtGauss_nemsio_c192 | standard | | fv3 | +RUN | fv3_ccpp_stochy | standard | | fv3 | +RUN | fv3_ccpp_iau | standard | | fv3 | +RUN | fv3_ccpp_lheatstrg | standard | | fv3 | + +COMPILE | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y | standard | | fv3 | +RUN | fv3_ccpp_regional_control | standard | | fv3 | +RUN | fv3_ccpp_regional_restart | standard | | fv3 | fv3_ccpp_regional_control +RUN | fv3_ccpp_regional_quilt | standard | | fv3 | + +COMPILE | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp | standard | | fv3 | +RUN | fv3_ccpp_gfdlmp | standard | | fv3 | +RUN | fv3_ccpp_gfdlmprad_gwd | standard | | fv3 | +RUN | fv3_ccpp_gfdlmprad_noahmp | standard | | fv3 | + +COMPILE | CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf,FV3_GFS_2017_satmedmfq | standard | | fv3 | +RUN | fv3_ccpp_csawmg | standard | | fv3 | +RUN | fv3_ccpp_satmedmf | standard | | fv3 | +RUN | fv3_ccpp_satmedmfq | standard | | fv3 | + +COMPILE | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson,FV3_RAP,FV3_HRRR,FV3_RRFS_v1beta 32BIT=Y | standard | | fv3 | +RUN | fv3_ccpp_gfdlmp_32bit | standard | | fv3 | +RUN | fv3_ccpp_gfdlmprad_32bit_post | standard | | fv3 | +RUN | fv3_ccpp_cpt | standard | | fv3 | +RUN | fv3_ccpp_gsd | standard | | fv3 | +RUN | fv3_ccpp_thompson | standard | | fv3 | +RUN | fv3_ccpp_thompson_no_aero | standard | | fv3 | +RUN | fv3_ccpp_rrfs_v1beta | standard | | fv3 | + +####################################################################################################################################################################################### +# CPLD tests +####################################################################################################################################################################################### + +COMPILE | CCPP=Y SUITES=FV3_GFS_2017_coupled,FV3_GFS_2017_satmedmf_coupled,FV3_GFS_v15p2_coupled S2S=Y | standard | | fv3 | +RUN | cpld_control | standard | | fv3 | +RUN | cpld_2threads | standard | | | +RUN | cpld_decomp | standard | | |