From 36914649adf9b1e50610dc5758c7d87d26d784ee Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Tue, 10 Aug 2021 13:25:50 -0400 Subject: [PATCH 1/6] Port to WCOSS2/Acorn On branch feature/gefs_v12_port2acorn modified: model/bin/cmplr.env modified: model/bin/w3_setup modified: model/esmf/Makefile Refs: #445 --- model/bin/cmplr.env | 9 +++++++-- model/bin/w3_setup | 2 ++ model/esmf/Makefile | 8 ++++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/model/bin/cmplr.env b/model/bin/cmplr.env index fdaf4621cc..0deeee20b1 100644 --- a/model/bin/cmplr.env +++ b/model/bin/cmplr.env @@ -110,7 +110,7 @@ if [ "$cmplr" == "intel" ] || [ "$cmplr" == "intel_debug" ] || [ "$c [ "$cmplr" == "so_intel" ] || [ "$cmplr" == "so_intel_debug" ] || [ "$cmplr" == "so_intel_prof" ] || \ [ "$cmplr" == "datarmor_intel" ] || [ "$cmplr" == "datarmor_intel_debug" ] || [ "$cmplr" == "datarmor_intel_prof" ] || \ [ "$cmplr" == "wcoss_phase2" ] || [ "$cmplr" == "wcoss_cray" ] || [ "$cmplr" == "wcoss_dell_p3" ] || \ - [ "$cmplr" == "theia" ] || [ "$cmplr" == "hera" ] ; then + [ "$cmplr" == "theia" ] || [ "$cmplr" == "hera" ] || [ "$cmplr" == "wcoss2" ]; then # COMPILER - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -151,7 +151,7 @@ if [ "$cmplr" == "intel" ] || [ "$cmplr" == "intel_debug" ] || [ "$c if [ "$list" == 'yes' ] ; then optc="$optc -list"; fi # omp options - if [ "$cmplr" == "hera" ] || [ "$cmplr" == "wcoss_dell_p3" ]; then + if [ "$cmplr" == "hera" ] || [ "$cmplr" == "wcoss_dell_p3" ] || [ "$cmplr" == "wcoss2" ]; then optomp="-qopenmp" else optomp="-openmp" @@ -213,6 +213,11 @@ if [ "$cmplr" == "gnu" ] || [ "$cmplr" == "gnu_debug" ] || [ "$cmplr" == "gnu_pr comp_mpi='ftn' fi + # WCOSS2 compiler + if [ ! -z "$(echo $cmplr | grep wcoss2)" ] ; then + comp_seq='ftn' + comp_mpi='ftn' + fi # OPTIONS - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/model/bin/w3_setup b/model/bin/w3_setup index 1c3a2c1846..40bb9ed115 100755 --- a/model/bin/w3_setup +++ b/model/bin/w3_setup @@ -417,6 +417,7 @@ then [ "$cmplr" == "theia" ] || [ "$cmplr" == "wcoss_cray" ] || \ [ "$cmplr" == "hera" ] || \ [ "$cmplr" == "wcoss_phase2" ] || [ "$cmplr" == "wcoss_dell_p3" ] || \ + [ "$cmplr" == "wcoss2" ] || \ [ "$cmplr" == "datarmor_gnu" ] || [ "$cmplr" == "datarmor_gnu_debug" ] || \ [ "$cmplr" == "pgi" ] || [ "$cmplr" == "pgi_debug" ] || \ [ "$cmplr" == "datarmor_pgi" ] || [ "$cmplr" == "datarmor_pgi_debug" ] || \ @@ -443,6 +444,7 @@ then [ "$cmplr" == "theia" ] || [ "$cmplr" == "wcoss_cray" ] || \ [ "$cmplr" == "hera" ] || \ [ "$cmplr" == "wcoss_phase2" ] || [ "$cmplr" == "wcoss_dell_p3" ] || \ + [ "$cmplr" == "wcoss2" ] || \ [ "$cmplr" == "datarmor_gnu" ] || [ "$cmplr" == "datarmor_gnu_debug" ] || \ [ "$cmplr" == "pgi" ] || [ "$cmplr" == "pgi_debug" ] || \ [ "$cmplr" == "datarmor_pgi" ] || [ "$cmplr" == "datarmor_pgi_debug" ] || \ diff --git a/model/esmf/Makefile b/model/esmf/Makefile index 64b3aec965..7aa1ad8f09 100644 --- a/model/esmf/Makefile +++ b/model/esmf/Makefile @@ -8,8 +8,8 @@ WW3_BINDIR := $(WW3_DIR)/bin WW3_TMPDIR := $(WW3_DIR)/tmp WW3_EXEDIR := $(WW3_DIR)/exe WW3_COMP ?= gnu -WW3_CC := gcc -WW3_F90 := gfortran +WW3_CC := ftn +WW3_F90 := ftn SWITCHES := $(shell cat switch) WWATCH3_ENV := $(WW3_BINDIR)/wwatch3.env @@ -37,7 +37,7 @@ else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","pgi" "datarmor_pgi" "datarmor_p # intel else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","theia" "Intel" "hera")) ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian -else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","wcoss_phase2" "wcoss_cray" "wcoss_dell_p3")) +else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","wcoss_phase2" "wcoss_cray" "wcoss_dell_p3" "wcoss2")) ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","intel" "datarmor_intel" "datarmor_intel_debug")) ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian @@ -96,7 +96,7 @@ ww3_nemslibonly: env setup switch ww3_multi_esmf: esmApp.o $(DEP_LINK_OBJS) $(ESMF_F90LINKER) $(ESMF_F90LINKOPTS) -o $(EXE) $^ \ - $(ESMF_F90LINKPATHS) $(ESMF_F90LINKRPATHS) $(ESMF_F90ESMFLINKLIBS) + $(ESMF_F90LINKPATHS) $(ESMF_F90LINKRPATHS) -lesmf -cxxlib -lrt -ldl -lnetcdff -lnetcdf #$(ESMF_F90ESMFLINKLIBS) # module dependencies: esmApp.o: macros.h utl.o esm.o From 22f0dafcb1b472961f78585beb746e017c613eb1 Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Wed, 15 Sep 2021 13:23:16 +0000 Subject: [PATCH 2/6] update esmf makefile --- model/esmf/Makefile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/model/esmf/Makefile b/model/esmf/Makefile index 7aa1ad8f09..996860f8a3 100644 --- a/model/esmf/Makefile +++ b/model/esmf/Makefile @@ -8,8 +8,8 @@ WW3_BINDIR := $(WW3_DIR)/bin WW3_TMPDIR := $(WW3_DIR)/tmp WW3_EXEDIR := $(WW3_DIR)/exe WW3_COMP ?= gnu -WW3_CC := ftn -WW3_F90 := ftn +WW3_CC := gcc +WW3_F90 := gfortran SWITCHES := $(shell cat switch) WWATCH3_ENV := $(WW3_BINDIR)/wwatch3.env @@ -37,8 +37,12 @@ else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","pgi" "datarmor_pgi" "datarmor_p # intel else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","theia" "Intel" "hera")) ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian -else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","wcoss_phase2" "wcoss_cray" "wcoss_dell_p3" "wcoss2")) +else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","wcoss_phase2" "wcoss_cray" "wcoss_dell_p3")) ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian +else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","wcoss2")) + ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian + WW3_CC := cc + WW3_F90 := ftn else ifeq ("$(WW3_COMP)",$(filter "$(WW3_COMP)","intel" "datarmor_intel" "datarmor_intel_debug")) ESMF_F90COMPILEOPTS := $(ESMF_F90COMPILEOPTS) -convert big_endian # mpt @@ -96,7 +100,7 @@ ww3_nemslibonly: env setup switch ww3_multi_esmf: esmApp.o $(DEP_LINK_OBJS) $(ESMF_F90LINKER) $(ESMF_F90LINKOPTS) -o $(EXE) $^ \ - $(ESMF_F90LINKPATHS) $(ESMF_F90LINKRPATHS) -lesmf -cxxlib -lrt -ldl -lnetcdff -lnetcdf #$(ESMF_F90ESMFLINKLIBS) + $(ESMF_F90LINKPATHS) $(ESMF_F90LINKRPATHS) $(ESMF_F90ESMFLINKLIBS) # module dependencies: esmApp.o: macros.h utl.o esm.o From c0f0230098444bf43bff4e5fdaa1a329c38ff94f Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Wed, 15 Sep 2021 13:27:38 +0000 Subject: [PATCH 3/6] update cmplr.env to make sure we are using the right settings for wcoss2 --- model/bin/cmplr.env | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/model/bin/cmplr.env b/model/bin/cmplr.env index 0deeee20b1..4385f336d2 100644 --- a/model/bin/cmplr.env +++ b/model/bin/cmplr.env @@ -125,11 +125,16 @@ if [ "$cmplr" == "intel" ] || [ "$cmplr" == "intel_debug" ] || [ "$c comp_mpi='ftn' fi + # WCOSS2 compiler + if [ ! -z "$(echo $cmplr | grep wcoss2)" ] ; then + comp_seq='ftn' + comp_mpi='ftn' + fi # OPTIONS - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # common options - optc='-c -module $path_m -no-fma -ip -g -i4 -real-size 32 -fp-model precise -assume byterecl -convert big_endian -fno-alias -fno-fnalias -sox' + optc='-c -module $path_m -no-fma -ip -g -traceback -i4 -real-size 32 -fp-model precise -assume byterecl -convert big_endian -fno-alias -fno-fnalias -sox' optl='-o $prog -g' if [ ! -z "$(echo $cmplr | grep datarmor)" ] ; then @@ -175,15 +180,6 @@ if [ "$cmplr" == "intel" ] || [ "$cmplr" == "intel_debug" ] || [ "$c optl="$optl -O0 -traceback" fi - # system-dependant options - if [ ! -z "$(echo $cmplr | grep datarmor)" ] ; then - optc="$optc -xcore-avx2" - optl="$optl -xcore-avx2" - else - optc="$optc -xhost" - optl="$optl -xhost" - fi - if [ ! -z "$(echo $cmplr | grep so)" ] ; then optc="$optc -fPIC" optl='-o $prog -g' @@ -213,12 +209,6 @@ if [ "$cmplr" == "gnu" ] || [ "$cmplr" == "gnu_debug" ] || [ "$cmplr" == "gnu_pr comp_mpi='ftn' fi - # WCOSS2 compiler - if [ ! -z "$(echo $cmplr | grep wcoss2)" ] ; then - comp_seq='ftn' - comp_mpi='ftn' - fi - # OPTIONS - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # common options From eccd4a854fc4f73296566d78c51f5529cbcad3b4 Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Wed, 15 Sep 2021 13:45:49 +0000 Subject: [PATCH 4/6] make sure wcoss2 is not using any xhost --- model/bin/cmplr.env | 3 +++ 1 file changed, 3 insertions(+) diff --git a/model/bin/cmplr.env b/model/bin/cmplr.env index 4385f336d2..0193dc80f7 100644 --- a/model/bin/cmplr.env +++ b/model/bin/cmplr.env @@ -146,6 +146,9 @@ if [ "$cmplr" == "intel" ] || [ "$cmplr" == "intel_debug" ] || [ "$c elif [ ! -z "$(echo $cmplr | grep wcoss_dell_p3)" ] ; then optc="$optc -xHOST" optl="$optl -xHOST" + elif [ ! -z "$(echo $cmplr | grep wcoss2)" ] ; then + optc="$optc" + optl="$optl" else optc="$optc -xhost" optl="$optl -xhost" From 7b09afce5c2c2e57a03e0e5e8beed61646671ae8 Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Wed, 15 Sep 2021 13:48:45 +0000 Subject: [PATCH 5/6] try flags suggested for optimization --- model/bin/cmplr.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/bin/cmplr.env b/model/bin/cmplr.env index 0193dc80f7..f167b6da7d 100644 --- a/model/bin/cmplr.env +++ b/model/bin/cmplr.env @@ -147,7 +147,7 @@ if [ "$cmplr" == "intel" ] || [ "$cmplr" == "intel_debug" ] || [ "$c optc="$optc -xHOST" optl="$optl -xHOST" elif [ ! -z "$(echo $cmplr | grep wcoss2)" ] ; then - optc="$optc" + optc="$optc -fp-modelprecise-ftz-fast-transcendentals-no-prec-div- no-prec-sqrt -align array64byte" optl="$optl" else optc="$optc -xhost" From 80ddef062b4fc48687c5633d1903d9fe21087ed4 Mon Sep 17 00:00:00 2001 From: "Xianwu.Xue" Date: Thu, 16 Sep 2021 14:59:21 -0400 Subject: [PATCH 6/6] Fix typo in the added flags suggested for opt Changes to be committed: modified: model/bin/cmplr.env Refs: #445 --- model/bin/cmplr.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/bin/cmplr.env b/model/bin/cmplr.env index f167b6da7d..5aebe7f420 100644 --- a/model/bin/cmplr.env +++ b/model/bin/cmplr.env @@ -147,7 +147,7 @@ if [ "$cmplr" == "intel" ] || [ "$cmplr" == "intel_debug" ] || [ "$c optc="$optc -xHOST" optl="$optl -xHOST" elif [ ! -z "$(echo $cmplr | grep wcoss2)" ] ; then - optc="$optc -fp-modelprecise-ftz-fast-transcendentals-no-prec-div- no-prec-sqrt -align array64byte" + optc="$optc -fp-model precise -ftz -fast-transcendentals -no-prec-div -no-prec-sqrt -align array64byte" optl="$optl" else optc="$optc -xhost"