Skip to content

Port ufs-weather-model to acorn (WCOSS2 TDS) #291

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
DusanJovic-NOAA wants to merge 6 commits into from
Closed

Port ufs-weather-model to acorn (WCOSS2 TDS) #291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
98e8764
Build on wcoss2 (acorn)
DusanJovic-NOAA Nov 18, 2020
ed381e8
Use -march=core-avx2 instead of -xCORE-AVX2 on wcoss2
DusanJovic-NOAA Nov 19, 2020
8e524f6
Merge branch 'develop' into wcoss2
DusanJovic-NOAA Nov 19, 2020
7a84602
Add rt_acorn.conf. Change /lfs/h2 to /lfs/h1.
DusanJovic-NOAA Dec 3, 2020
c628df3
Merge remote-tracking branch 'origin/develop' into wcoss2
DusanJovic-NOAA Dec 4, 2020
7e327db
Merge branch 'develop' into wcoss2
DusanJovic-NOAA Dec 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,8 @@ if(WW3)
set(WW3_COMP "hera")
elseif(${CMAKE_Platform} STREQUAL "orion.intel")
set(WW3_COMP "orion")
elseif(${CMAKE_Platform} STREQUAL "wcoss2")
set(WW3_COMP "wcoss_cray")
elseif(${CMAKE_Platform} MATCHES "linux*" OR ${CMAKE_Platform} MATCHES "macosx*")
if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
set(WW3_COMP "gnu")
Expand Down
6 changes: 4 additions & 2 deletions cmake/Intel.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,10 @@ elseif(DEBUG)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qno-opt-dynamic-align")
else()
if(AVX2)
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
#set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
#set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -march=core-avx2 -qno-opt-dynamic-align")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=core-avx2 -qno-opt-dynamic-align")
elseif(SIMDMULTIARCH)
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align")
Expand Down
3 changes: 3 additions & 0 deletions cmake/configure_wcoss2.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
set(INLINE_POST ON CACHE BOOL "Enable inline post" FORCE)
set(PARALLEL_NETCDF ON CACHE BOOL "Enable parallel NetCDF" FORCE)
set(DEBUG_LINKMPI OFF CACHE BOOL "Enable linkmpi option when DEBUG mode is on" FORCE)
Copy link
Copy Markdown
Contributor

@aerorahul aerorahul Nov 20, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we debugging MPI implementation here?

I see we are hard-wiring flags such as:

if(DEBUG)
     if(OPENMP)
        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -link_mpi=dbg_mt")
      else()
        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -link_mpi=dbg")
      endif()
endif()

This should be handled by linking to the runtime debug MPI library by setting an environment variable such as I_MPI_LINK

By adding those flags at compile time, we are essentially debugging the MPI implementation.

42 changes: 42 additions & 0 deletions modulefiles/wcoss2/fv3
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#%Module

proc ModulesHelp {} {
puts stderr "\tcit - loads modules required for building and running UFS Model on the Dell side of WCOSS"
}

module-whatis "loads UFS Model prerequisites on Venus and Mars"

module unload cpe-cray cce
module load cpe-intel intel

module load cmake/3.17.3
setenv CMAKE_C_COMPILER cc
setenv CMAKE_CXX_COMPILER CC
setenv CMAKE_Fortran_COMPILER ftn
setenv CMAKE_Platform wcoss2

module use /lfs/h1/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack

module load hpc/1.0.0-beta1
module load hpc-intel/19.1.1.217
module load hpc-cray-mpich/8.0.15

module load jasper/2.0.22
module load zlib/1.2.11
module load png/1.6.35

module load hdf5/1.10.6
module load netcdf/4.7.4
module load pio/2.5.1
module load esmf/8_1_0_beta_snapshot_27

module load bacio/2.4.1
module load crtm/2.3.0
module load g2/3.4.1
module load g2tmpl/1.9.1
module load ip/3.3.3
module load nemsio/2.5.2
module load sp/2.3.3
module load w3emc/2.7.3
module load w3nco/2.4.1
module load upp/10.0.0
42 changes: 42 additions & 0 deletions modulefiles/wcoss2/fv3_debug
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#%Module

proc ModulesHelp {} {
puts stderr "\tcit - loads modules required for building and running UFS Model on the Dell side of WCOSS"
}

module-whatis "loads UFS Model prerequisites on Venus and Mars"

module unload cpe-cray cce
module load cpe-intel intel

module load cmake/3.17.3
setenv CMAKE_C_COMPILER cc
setenv CMAKE_CXX_COMPILER CC
setenv CMAKE_Fortran_COMPILER ftn
setenv CMAKE_Platform wcoss2

Copy link
Copy Markdown
Contributor

@aerorahul aerorahul Nov 20, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One could just add I_MPI_LINK environment variable in the debug modulefile and eliminate the hard-wired compile time flags.

module use /lfs/h1/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack

module load hpc/1.0.0-beta1
module load hpc-intel/19.1.1.217
module load hpc-cray-mpich/8.0.15

module load jasper/2.0.22
module load zlib/1.2.11
module load png/1.6.35

module load hdf5/1.10.6
module load netcdf/4.7.4
module load pio/2.5.1
module load esmf/8_1_0_beta_snapshot_27

module load bacio/2.4.1
module load crtm/2.3.0
module load g2/3.4.1
module load g2tmpl/1.9.1
module load ip/3.3.3
module load nemsio/2.5.2
module load sp/2.3.3
module load w3emc/2.7.3
module load w3nco/2.4.1
module load upp/10.0.0
3 changes: 3 additions & 0 deletions tests/compile.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ set +x
if [[ $MACHINE_ID == macosx.* ]] || [[ $MACHINE_ID == linux.* ]]; then
source $PATHTR/modulefiles/${MACHINE_ID}/fv3
else
if [[ $MACHINE_ID == wcoss2 ]]; then
source /apps/prod/lmodules/startLmod
Copy link
Copy Markdown
Contributor

@MinsukJi-NOAA MinsukJi-NOAA Nov 20, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess this will eventually be moved to ../NEMS/src/conf/module-setup.*sh.inc?

Copy link
Copy Markdown
Collaborator Author

@DusanJovic-NOAA DusanJovic-NOAA Nov 20, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. Or if they switch to Lmod version of modules, sourcing this script might not be necessary at all.

fi
# Activate lua environment for gaea
if [[ $MACHINE_ID == gaea.* ]] ; then
source /lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.sh
Expand Down
4 changes: 2 additions & 2 deletions tests/default_vars.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@
#
###############################################################################

if [ $MACHINE_ID = wcoss_cray ]; then
if [[ $MACHINE_ID = wcoss_cray ]]; then

TASKS_dflt=150 ; TPN_dflt=24 ; INPES_dflt=3 ; JNPES_dflt=8
TASKS_thrd=84 ; TPN_thrd=12 ; INPES_thrd=3 ; JNPES_thrd=4
TASKS_stretch=48 ; TPN_stretch=24 ; INPES_stretch=2 ; JNPES_stretch=4
TASKS_strnest=96 ; TPN_strnest=24 ; INPES_strnest=2 ; JNPES_strnest=4

elif [ $MACHINE_ID = wcoss_dell_p3 ]; then
elif [[ $MACHINE_ID = wcoss_dell_p3 || $MACHINE_ID = wcoss2 ]]; then

TASKS_dflt=150 ; TPN_dflt=28 ; INPES_dflt=3 ; JNPES_dflt=8
TASKS_thrd=84 ; TPN_thrd=14 ; INPES_thrd=3 ; JNPES_thrd=4
Expand Down
3 changes: 3 additions & 0 deletions tests/detect_machine.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ case $(hostname -f) in
m72a2.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### mars
m72a3.ncep.noaa.gov) MACHINE_ID=wcoss_dell_p3 ;; ### mars

alogin01) MACHINE_ID=wcoss2 ;; ### acorn
alogin02) MACHINE_ID=wcoss2 ;; ### acorn

gaea9) MACHINE_ID=gaea ;; ### gaea9
gaea10) MACHINE_ID=gaea ;; ### gaea10
gaea11) MACHINE_ID=gaea ;; ### gaea11
Expand Down
20 changes: 20 additions & 0 deletions tests/fv3_conf/compile_qsub.IN_wcoss2
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/bin/bash

#PBS -o out
#PBS -e err
#PBS -N @[JBNME]
# #PBS -A @[ACCNR]
#PBS -q @[QUEUE]
#PBS -l select=1:ncpus=8:mpiprocs=1
#PBS -l walltime=00:30:00

set -eux

cd $PBS_O_WORKDIR
export CRAY_CONFIG_DIR=$HOME

echo "Compile started: " `date`

@[PATHRT]/compile.sh @[MACHINE_ID] "@[MAKE_OPT]" @[COMPILE_NR]

echo "Compile ended: " `date`
32 changes: 32 additions & 0 deletions tests/fv3_conf/fv3_qsub.IN_wcoss2
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/bin/bash

#PBS -o out
#PBS -e err
#PBS -N @[JBNME]
# #PBS -A @[ACCNR]
#PBS -q @[QUEUE]
#PBS -l place=vscatter,select=@[NODES]:ncpus=@[TPN]:mpiprocs=@[TPN]
#PBS -l walltime=00:@[WLCLK]:00

set -eux

cd $PBS_O_WORKDIR
export CRAY_CONFIG_DIR=$HOME

source /apps/prod/lmodules/startLmod
module use $( pwd -P )
module load modules.fv3
module list

echo "Model started: " `date`

export MPI_TYPE_DEPTH=20
export OMP_STACKSIZE=512M
export OMP_NUM_THREADS=@[THRD]
export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4

cray aprun -n @[TASKS] ./fv3.exe

echo "Model ended: " `date`

exit
33 changes: 33 additions & 0 deletions tests/rt.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,33 @@ elif [[ $MACHINE_ID = wcoss_dell_p3 ]]; then
cp fv3_conf/fv3_bsub.IN_wcoss_dell_p3 fv3_conf/fv3_bsub.IN
cp fv3_conf/compile_bsub.IN_wcoss_dell_p3 fv3_conf/compile_bsub.IN

elif [[ $MACHINE_ID = wcoss2 ]]; then

source /apps/prod/lmodules/startLmod
Copy link
Copy Markdown
Contributor

@MinsukJi-NOAA MinsukJi-NOAA Nov 20, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Once NEMS/src/conf/module-setup.*sh.inc is modified to include wcoss2, this line will not be needed.


#module use /usrx/local/dev/emc_rocoto/modulefiles
#module load ruby/2.5.1 rocoto/1.3.0rc2
#ROCOTORUN=$(which rocotorun)
#ROCOTOSTAT=$(which rocotostat)
#ROCOTOCOMPLETE=$(which rocotocomplete)
#ROCOTO_SCHEDULER=lsf

#module load ips/18.0.1.163
#module load ecflow/4.7.1
#ECFLOW_START=${ECF_ROOT}/intel/bin/ecflow_start.sh
#ECF_PORT=$(grep $USER /usrx/local/sys/ecflow/assigned_ports.txt | awk '{print $2}')

DISKNM=/lfs/h1/emc/ptmp/Dusan.Jovic/RT
QUEUE=workq
COMPILE_QUEUE=workq
PARTITION=
ACCNR=GFS-DEV
STMP=/lfs/h1/emc/stmp
PTMP=/lfs/h1/emc/ptmp
SCHEDULER=pbs
cp fv3_conf/fv3_qsub.IN_wcoss2 fv3_conf/fv3_qsub.IN
cp fv3_conf/compile_qsub.IN_wcoss2 fv3_conf/compile_qsub.IN

elif [[ $MACHINE_ID = gaea.* ]]; then

module load cray-python/3.7.3.2
Expand Down Expand Up @@ -451,6 +478,10 @@ if [[ $ROCOTO == true ]]; then
QUEUE=dev
COMPILE_QUEUE=dev_transfer
ROCOTO_SCHEDULER=lsf
elif [[ $MACHINE_ID = wcoss2 ]]; then
QUEUE=workq
COMPILE_QUEUE=workq
ROCOTO_SCHEDULER=pbs
elif [[ $MACHINE_ID = hera.* ]]; then
QUEUE=batch
COMPILE_QUEUE=batch
Expand Down Expand Up @@ -512,6 +543,8 @@ EOF
QUEUE=dev
elif [[ $MACHINE_ID = wcoss_dell_p3 ]]; then
QUEUE=dev
elif [[ $MACHINE_ID = wcoss2 ]]; then
QUEUE=workq
elif [[ $MACHINE_ID = hera.* ]]; then
QUEUE=batch
elif [[ $MACHINE_ID = orion.* ]]; then
Expand Down
52 changes: 52 additions & 0 deletions tests/rt_acorn.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
#######################################################################################################################################################################################
# CCPP PROD tests #
#######################################################################################################################################################################################

COMPILE | CCPP=Y SUITES=FV3_GFS_2017 | standard | | fv3 |

RUN | fv3_ccpp_control | standard | | fv3 |
RUN | fv3_ccpp_decomp | standard | | |
RUN | fv3_ccpp_2threads | standard | | |
RUN | fv3_ccpp_restart | standard | | fv3 |
RUN | fv3_ccpp_read_inc | standard | | fv3 |
RUN | fv3_ccpp_wrtGauss_netcdf_esmf | standard | | fv3 |
RUN | fv3_ccpp_wrtGauss_netcdf | standard | | fv3 |
RUN | fv3_ccpp_wrtGlatlon_netcdf | standard | | fv3 |
RUN | fv3_ccpp_wrtGauss_nemsio | standard | | fv3 |
RUN | fv3_ccpp_wrtGauss_nemsio_c192 | standard | | fv3 |
RUN | fv3_ccpp_stochy | standard | | fv3 |
RUN | fv3_ccpp_iau | standard | | fv3 |
RUN | fv3_ccpp_lheatstrg | standard | | fv3 |

COMPILE | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp_regional,FV3_GFS_2017_gfdlmp_regional_c768 32BIT=Y | standard | | fv3 |
RUN | fv3_ccpp_regional_control | standard | | fv3 |
RUN | fv3_ccpp_regional_restart | standard | | fv3 | fv3_ccpp_regional_control
RUN | fv3_ccpp_regional_quilt | standard | | fv3 |

COMPILE | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_GFS_2017_gfdlmp_noahmp | standard | | fv3 |
RUN | fv3_ccpp_gfdlmp | standard | | fv3 |
RUN | fv3_ccpp_gfdlmprad_gwd | standard | | fv3 |
RUN | fv3_ccpp_gfdlmprad_noahmp | standard | | fv3 |

COMPILE | CCPP=Y SUITES=FV3_GFS_2017_csawmgshoc,FV3_GFS_2017_csawmg,FV3_GFS_2017_satmedmf,FV3_GFS_2017_satmedmfq | standard | | fv3 |
RUN | fv3_ccpp_csawmg | standard | | fv3 |
RUN | fv3_ccpp_satmedmf | standard | | fv3 |
RUN | fv3_ccpp_satmedmfq | standard | | fv3 |

COMPILE | CCPP=Y SUITES=FV3_GFS_2017_gfdlmp,FV3_CPT_v0,FV3_GSD_v0,FV3_GFS_v15_thompson,FV3_RAP,FV3_HRRR,FV3_RRFS_v1beta 32BIT=Y | standard | | fv3 |
RUN | fv3_ccpp_gfdlmp_32bit | standard | | fv3 |
RUN | fv3_ccpp_gfdlmprad_32bit_post | standard | | fv3 |
RUN | fv3_ccpp_cpt | standard | | fv3 |
RUN | fv3_ccpp_gsd | standard | | fv3 |
RUN | fv3_ccpp_thompson | standard | | fv3 |
RUN | fv3_ccpp_thompson_no_aero | standard | | fv3 |
RUN | fv3_ccpp_rrfs_v1beta | standard | | fv3 |

#######################################################################################################################################################################################
# CPLD tests
#######################################################################################################################################################################################

COMPILE | CCPP=Y SUITES=FV3_GFS_2017_coupled,FV3_GFS_2017_satmedmf_coupled,FV3_GFS_v15p2_coupled S2S=Y | standard | | fv3 |
RUN | cpld_control | standard | | fv3 |
RUN | cpld_2threads | standard | | |
RUN | cpld_decomp | standard | | |