Skip to content

CCPP-acceptance bit-for-bit csawmg and gfdlmp-32bit #211

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
climbfuji merged 3 commits into from
Feb 26, 2019
Merged

CCPP-acceptance bit-for-bit csawmg and gfdlmp-32bit #211

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c8e22a8
CMakeLists.txt: introduce SCHEMES_SFX list that contains files with s…
climbfuji Feb 25, 2019
540b17a
Merge branch 'master' of https://github.com/NCAR/ccpp-physics into cs…
climbfuji Feb 25, 2019
a55fca0
CMakeLists.txt: bugfix, add back in -fimf-arch-consistency=true in DE…
climbfuji Feb 26, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 60 additions & 49 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,8 @@ list(APPEND LIBS "ccpp")
include(./CCPP_SCHEMES.cmake)
# Set the sources: physics scheme caps
include(./CCPP_CAPS.cmake)
# Create empty lists for schemes with special compiler flags
set(SCHEMES_SFX "")

#------------------------------------------------------------------------------
if (${CMAKE_Fortran_COMPILER_ID} STREQUAL "GNU")
Expand All @@ -111,10 +113,7 @@ if (${CMAKE_Fortran_COMPILER_ID} STREQUAL "GNU")
SET_SOURCE_FILES_PROPERTIES(./physics/aer_cloud.F ./physics/wv_saturation.F ./physics/cldwat2m_micro.F ./physics/surface_perturbation.F90 PROPERTIES COMPILE_FLAGS "-fdefault-real-8 -fdefault-double-8")
if (PROJECT STREQUAL "CCPP-FV3")
if (DYN32)
set(CMAKE_Fortran_FLAGS_OPT32BIT ${CMAKE_Fortran_FLAGS})
string(REPLACE "-fdefault-real-8" "" CMAKE_Fortran_FLAGS_OPT32BIT "${CMAKE_Fortran_FLAGS_OPT32BIT}")
SET_SOURCE_FILES_PROPERTIES(./physics/gfdl_fv_sat_adj.F90
PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS_OPT32BIT}")
message (FATAL_ERROR "The current build system does not allow building fast physics with 32-bit precision when the GNU compilers are used")
endif (DYN32)
endif (PROJECT STREQUAL "CCPP-FV3")
elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel")
Expand All @@ -133,6 +132,7 @@ elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel")
./physics/wv_saturation.F
./physics/cldwat2m_micro.F
./physics/surface_perturbation.F90
./physics/radiation_aerosols.f
./physics/cu_gf_deep.F90
./physics/cu_gf_sh.F90
./physics/module_bl_mynn.F90
Expand All @@ -141,11 +141,10 @@ elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel")
./physics/module_MYNNSFC_wrapper.F90
./physics/module_MYNNrad_pre.F90
./physics/module_MYNNrad_post.F90
PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS} -r8 -ftz")
# Replace -xHost or -xCORE-AVX2 with -xCORE-AVX-I for certain files (following FV3/gfsphysics/makefile)
# for bit-for-bit reproducibility with non-CCPP builds. These may go in the future once the CCPP solution
# is fully accepted.
set(CMAKE_Fortran_FLAGS_LOPT1 ${CMAKE_Fortran_FLAGS})
PROPERTIES COMPILE_FLAGS "-r8 -ftz")

# Replace -xHost or -xCORE-AVX2 with -xCORE-AVX-I for certain files
set(CMAKE_Fortran_FLAGS_LOPT1 ${CMAKE_Fortran_FLAGS_OPT})
string(REPLACE "-xHOST" "-xCORE-AVX-I"
CMAKE_Fortran_FLAGS_LOPT1
"${CMAKE_Fortran_FLAGS_LOPT1}")
Expand All @@ -156,35 +155,34 @@ elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel")
CMAKE_Fortran_FLAGS_LOPT1
"${CMAKE_Fortran_FLAGS_LOPT1}")
SET_SOURCE_FILES_PROPERTIES(./physics/radiation_aerosols.f
PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS_LOPT1} -r8 -ftz")
PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS_LOPT1}")
# Add all of the above files to the list of schemes with special compiler flags
list(APPEND SCHEMES_SFX ./physics/radiation_aerosols.f)

# Force consistent results of math calculations for MG microphysics;
# in Debug/Bitforbit) mode; without this flag, the results of the
# in Debug/Bitforbit mode; without this flag, the results of the
# intrinsic gamma function are different for the non-CCPP and CCPP
# version (on Theia with Intel 18). Note this is only required with
# dynamic CCPP builds (hybrid, standalone), not with static CCPP builds.
if (${CMAKE_BUILD_TYPE} MATCHES "Debug")
SET_SOURCE_FILES_PROPERTIES(./physics/micro_mg2_0.F90
./physics/micro_mg3_0.F90
PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS} -fimf-arch-consistency=true")
PROPERTIES COMPILE_FLAGS "-fimf-arch-consistency=true")
# Add all of the above files to the list of schemes with special compiler flags
list(APPEND SCHEMES_SFX ./physics/micro_mg2_0.F90
./physics/micro_mg3_0.F90)
elseif (${CMAKE_BUILD_TYPE} MATCHES "Bitforbit")
SET_SOURCE_FILES_PROPERTIES(./physics/micro_mg2_0.F90
./physics/micro_mg3_0.F90
PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS} -fimf-arch-consistency=true")
endif (${CMAKE_BUILD_TYPE} MATCHES "Debug")
if (DYN32)
set(CMAKE_Fortran_FLAGS_OPT32BIT ${CMAKE_Fortran_FLAGS})
string(REPLACE "-real-size 64" "-real-size 32" CMAKE_Fortran_FLAGS_OPT32BIT "${CMAKE_Fortran_FLAGS_OPT32BIT}")
string(REPLACE "-r8" "-r4" CMAKE_Fortran_FLAGS_OPT32BIT "${CMAKE_Fortran_FLAGS_OPT32BIT}")
SET_SOURCE_FILES_PROPERTIES(./physics/gfdl_fv_sat_adj.F90
PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS_OPT32BIT}")
endif (DYN32)

# For CCPP acceptance: selective reduction of optimization flags, hopefully
# to be removed once established that this is not a reasonable approach.
if (TRANSITION)
# Replace "-no-prec-div -no-prec-sqrt" with "-prec-div -prec-sqrt",
# replace "CORE-AVX2" with "CORE-AVX-I"
set(CMAKE_Fortran_FLAGS_LOPT2 ${CMAKE_Fortran_FLAGS})
PROPERTIES COMPILE_FLAGS "-fimf-arch-consistency=true")
# Add all of the above files to the list of schemes with special compiler flags
list(APPEND SCHEMES_SFX ./physics/micro_mg2_0.F90
./physics/micro_mg3_0.F90)
elseif (TRANSITION)
# Replace -xHost or -xCORE-AVX2 with -xCORE-AVX-I, -no-prec-div with -prec-div, and
# -no-prec-sqrt with -prec-sqrt for certain files for bit-for-bit reproducibility
# with non-CCPP builds. These may go in the future once the CCPP solution is fully accepted.
set(CMAKE_Fortran_FLAGS_LOPT2 ${CMAKE_Fortran_FLAGS_OPT})
string(REPLACE "-no-prec-div" "-prec-div"
CMAKE_Fortran_FLAGS_LOPT2
"${CMAKE_Fortran_FLAGS_LOPT2}")
Expand All @@ -197,21 +195,39 @@ elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel")
string(REPLACE "-axSSE4.2,AVX,CORE-AVX2" "-axSSE4.2,AVX,CORE-AVX-I"
CMAKE_Fortran_FLAGS_LOPT2
"${CMAKE_Fortran_FLAGS_LOPT2}")
SET_SOURCE_FILES_PROPERTIES(./physics/module_gfdl_cloud_microphys.F90
SET_SOURCE_FILES_PROPERTIES(./physics/micro_mg2_0.F90
./physics/micro_mg3_0.F90
./physics/aer_cloud.F
./physics/cldmacro.F
./physics/gfdl_fv_sat_adj.F90
./physics/module_gfdl_cloud_microphys.F90
./physics/sflx.f
./physics/satmedmfvdif.F
./physics/cs_conv.F90
PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS_LOPT2}")
# Replace "CORE-AVX2" with "CORE-AVX-I"
set(CMAKE_Fortran_FLAGS_LOPT3 ${CMAKE_Fortran_FLAGS})
string(REPLACE "-xCORE-AVX2" "-xCORE-AVX-I"
CMAKE_Fortran_FLAGS_LOPT3
"${CMAKE_Fortran_FLAGS_LOPT3}")
string(REPLACE "-axSSE4.2,AVX,CORE-AVX2" "-axSSE4.2,AVX,CORE-AVX-I"
CMAKE_Fortran_FLAGS_LOPT3
"${CMAKE_Fortran_FLAGS_LOPT3}")
SET_SOURCE_FILES_PROPERTIES(./physics/gfdl_fv_sat_adj.F90
PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS_LOPT3}")
endif (TRANSITION)
# Add all of the above files to the list of schemes with special compiler flags
list(APPEND SCHEMES_SFX ./physics/micro_mg2_0.F90
./physics/micro_mg3_0.F90
./physics/aer_cloud.F
./physics/cldmacro.F
./physics/module_gfdl_cloud_microphys.F90
./physics/sflx.f
./physics/satmedmfvdif.F
./physics/cs_conv.F90
./physics/gfdl_fv_sat_adj.F90)
endif (${CMAKE_BUILD_TYPE} MATCHES "Debug")

# Remove files with special compiler flags from list of files with standard compiler flags
list(REMOVE_ITEM SCHEMES ${SCHEMES_SFX})
# Assign standard compiler flags to all remaining schemes and caps
SET_SOURCE_FILES_PROPERTIES(${SCHEMES} ${CAPS}
PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS_OPT}")

# This has to come last: append 32-bit dynamics flags to certain files that are executed
# in the dynamics (fast physics part); this will overwrite any preceding -real-size 64
if (DYN32)
SET_PROPERTY(SOURCE ./physics/gfdl_fv_sat_adj.F90 APPEND_STRING PROPERTY COMPILE_FLAGS " -real-size 32 ")
endif (DYN32)

else (PROJECT STREQUAL "CCPP-FV3")
SET_SOURCE_FILES_PROPERTIES(./physics/module_bfmicrophysics.f ./physics/rascnvv2.f ./physics/sflx.f ./physics/sfc_diff.f ./physics/sfc_diag.f PROPERTIES COMPILE_FLAGS -r8)
Expand All @@ -228,10 +244,7 @@ elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "PGI")
SET_SOURCE_FILES_PROPERTIES(./physics/aer_cloud.F ./physics/wv_saturation.F ./physics/cldwat2m_micro.F ./physics/surface_perturbation.F90 PROPERTIES COMPILE_FLAGS "-r8")
if (PROJECT STREQUAL "CCPP-FV3")
if (DYN32)
set(CMAKE_Fortran_FLAGS_OPT32BIT ${CMAKE_Fortran_FLAGS})
string(REPLACE "-r8" "-r4" CMAKE_Fortran_FLAGS_OPT32BIT "${CMAKE_Fortran_FLAGS_OPT32BIT}")
SET_SOURCE_FILES_PROPERTIES(./physics/gfdl_fv_sat_adj.F90
PROPERTIES COMPILE_FLAGS "${CMAKE_Fortran_FLAGS_OPT32BIT}")
SET_PROPERTY(SOURCE ./physics/gfdl_fv_sat_adj.F90 APPEND_STRING PROPERTY COMPILE_FLAGS " -r4 ")
endif (DYN32)
endif (PROJECT STREQUAL "CCPP-FV3")
else (${CMAKE_Fortran_COMPILER_ID} STREQUAL "GNU")
Expand Down Expand Up @@ -280,15 +293,15 @@ endif (PROJECT STREQUAL "CCPP-FV3")
#------------------------------------------------------------------------------

if(STATIC)
add_library(ccppphys STATIC ${SCHEMES} ${CAPS})
add_library(ccppphys STATIC ${SCHEMES} ${SCHEMES_SFX} ${CAPS})
# Generate list of Fortran modules from defined sources
foreach(source_f90 ${CAPS})
string(REGEX REPLACE ".F90" ".mod" tmp_module_f90 ${source_f90})
string(TOLOWER ${tmp_module_f90} module_f90)
list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/../${module_f90})
endforeach()
else(STATIC)
add_library(ccppphys ${SCHEMES} ${CAPS})
add_library(ccppphys ${SCHEMES} ${SCHEMES_SFX} ${CAPS})
endif(STATIC)

if (PROJECT STREQUAL "CCPP-FV3")
Expand All @@ -297,9 +310,7 @@ elseif (PROJECT STREQUAL "CCPP-SCM")
target_link_libraries(ccppphys LINK_PUBLIC ${LIBS} w3 sp bacio)
endif (PROJECT STREQUAL "CCPP-FV3")
set_target_properties(ccppphys PROPERTIES VERSION ${PROJECT_VERSION}
SOVERSION ${PROJECT_VERSION_MAJOR}
COMPILE_FLAGS "${CMAKE_Fortran_FLAGS}"
LINK_FLAGS "${CMAKE_Fortran_FLAGS}")
SOVERSION ${PROJECT_VERSION_MAJOR})

# DH* Hack for PGI compiler: rename objects in scheme cap object files for ISO_C compliancy,
# this is only needed for dynamics builds - static build generates plain Fortran code.
Expand Down