From 75b9cefb7fb208b6cfb6e6b476b0b1f720a03830 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Thu, 24 Feb 2022 07:59:19 -0500 Subject: [PATCH 1/6] Adapted to linux-centos-intel General Linux-CentOS (7), Intel compilers and MPI --- build_stack.sh | 1 + libs/build_libtiff.sh | 56 +++++++++++++++++++ libs/build_proj.sh | 11 ++++ .../compilerVersion/libtiff/libtiff.lua | 32 +++++++++++ stack/stack_custom.yaml | 8 ++- stack_helpers.sh | 5 ++ 6 files changed, 111 insertions(+), 2 deletions(-) create mode 100755 libs/build_libtiff.sh create mode 100644 modulefiles/compiler/compilerName/compilerVersion/libtiff/libtiff.lua diff --git a/build_stack.sh b/build_stack.sh index 6de97e41..2aa7ff31 100755 --- a/build_stack.sh +++ b/build_stack.sh @@ -140,6 +140,7 @@ build_lib libpng build_lib szip build_lib jasper build_lib sqlite +build_lib libtiff build_lib proj build_lib geos diff --git a/libs/build_libtiff.sh b/libs/build_libtiff.sh new file mode 100755 index 00000000..4f93d554 --- /dev/null +++ b/libs/build_libtiff.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -eux + +name="libtiff" +version=${1:-${STACK_libtiff_version}} + +[[ ${STACK_libtiff_shared:-} =~ [yYtT] ]] && enable_shared=YES || enable_shared=NO + +# Hyphenated version used for install prefix +compiler=$(echo $HPC_COMPILER | sed 's/\//-/g') + +# manage package dependencies here +if $MODULES; then + set +x + source $MODULESHOME/init/bash + module load hpc-$HPC_COMPILER + module try-load cmake + module load zlib + module list + set -x + + prefix="${PREFIX:-"/opt/modules"}/$compiler/$name/$version" + if [[ -d $prefix ]]; then + [[ $OVERWRITE =~ [yYtT] ]] && ( echo "WARNING: $prefix EXISTS: OVERWRITING!";$SUDO rm -rf $prefix ) \ + || ( echo "WARNING: $prefix EXISTS, SKIPPING"; exit 1 ) + fi + +else + prefix=${LIBTIFF_ROOT:-"/usr/local"} +fi + +export CC=$SERIAL_CC +export CFLAGS="${STACK_CFLAGS:-} ${STACK_libtiff_CFLAGS:-} -fPIC" + +cd ${HPC_STACK_ROOT}/${PKGDIR:-"pkg"} + +software=$name-$version +URL="https://gitlab.com/lib${name}/lib${name}.git" +[[ -d $software ]] || ( git clone $URL $software ) +[[ ${DOWNLOAD_ONLY} =~ [yYtT] ]] && exit 0 +[[ -d $software ]] && cd $software || ( echo "$software does not exist, ABORT!"; exit 1 ) + +sourceDir=$PWD +cd build +cmake $sourceDir \ + -DCMAKE_INSTALL_PREFIX=$prefix \ + -DCMAKE_BUILD_TYPE=RELEASE \ + -DZLIB_ROOT=${ZLIB_ROOT} +# +make -j${NTHREADS:-4} +$SUDO make install + +# generate modulefile from template +$MODULES && update_modules compiler $name $version +echo $name $version $URL >> ${HPC_STACK_ROOT}/hpc-stack-contents.log diff --git a/libs/build_proj.sh b/libs/build_proj.sh index 2f4a5e99..12fd1cc2 100755 --- a/libs/build_proj.sh +++ b/libs/build_proj.sh @@ -17,6 +17,7 @@ if $MODULES; then module load hpc-$HPC_COMPILER module try-load cmake module load sqlite + module try-load tiff module list set -x @@ -57,6 +58,16 @@ CMAKE_OPTS=${STACK_proj_cmake_opts:-""} [[ $MAKE_CHECK =~ [yYtT] ]] || CMAKE_OPTS+=" -DBUILD_TESTING=OFF" +if [[ -n ${LIBTIFF_ROOT-} ]] ; then + CMAKE_OPTS+=" -DLIBTIFF_INCLUDE_DIR=${LIBTIFF_ROOT}/include " + [[ -f ${LIBTIFF_ROOT}/lib64/libtiff.so ]] && CMAKE_OPTS+=" -DLIBTIFF_LIBRARY=${LIBTIFF_ROOT}/lib64/libtiff.so " || CMAKE_OPTS+=" -DLIBTIFF_LIBRARY=${LIBTIFF_ROOT}/lib/libtiff.so " +fi + +if [[ -n ${SQLITE_ROOT-} ]] ; then + CMAKE_OPTS+=" -DSQLITE3_INCLUDE_DIR=${SQLITE_ROOT}/include " + CMAKE_OPTS+=" -DSQLITE3_LIBRARY=${SQLITE_ROOT}/lib/libsqlite3.so " +fi + LIB_DIR=${SQLITE_ROOT:-} cmake -H. -Bbuild -DCMAKE_INSTALL_PREFIX=$prefix $CMAKE_OPTS cd build VERBOSE=$MAKE_VERBOSE make -j${NTHREADS:-4} diff --git a/modulefiles/compiler/compilerName/compilerVersion/libtiff/libtiff.lua b/modulefiles/compiler/compilerName/compilerVersion/libtiff/libtiff.lua new file mode 100644 index 00000000..80843342 --- /dev/null +++ b/modulefiles/compiler/compilerName/compilerVersion/libtiff/libtiff.lua @@ -0,0 +1,32 @@ +help([[ +]]) + +local pkgName = myModuleName() +local pkgVersion = myModuleVersion() +local pkgNameVer = myModuleFullName() + +local hierA = hierarchyA(pkgNameVer,1) +local compNameVer = hierA[1] +local compNameVerD = compNameVer:gsub("/","-") + +conflict(pkgName) + +local opt = os.getenv("HPC_OPT") or os.getenv("OPT") or "/opt/modules" + +local base = pathJoin(opt,compNameVerD,pkgName,pkgVersion) + +prepend_path("PATH", pathJoin(base,"bin")) +prepend_path("LD_LIBRARY_PATH", pathJoin(base,"${CMAKE_INSTALL_LIBDIR}")) +prepend_path("DYLD_LIBRARY_PATH", pathJoin(base,"${CMAKE_INSTALL_LIBDIR}")) +prepend_path("CPATH", pathJoin(base,"include")) +prepend_path("MANPATH", pathJoin(base,"share","man")) + +setenv("LIBTIFF_ROOT", base) +setenv("LIBTIFF_VERSION", pkgVersion) + +prepend_path("PKG_CONFIG_PATH", pathJoin(base,"${CMAKE_INSTALL_LIBDIR}","pkgconfig")) + +whatis("Name: ".. pkgName) +whatis("Version: " .. pkgVersion) +whatis("Category: library") +whatis("Description: LIBTIFF (libtiff library) are Functions to read, write and display bitmap images stored in the LIBTIFF format. It can read and write both files and in-memory raw vectors, including native image representation") diff --git a/stack/stack_custom.yaml b/stack/stack_custom.yaml index c4edcafc..ca393cc8 100644 --- a/stack/stack_custom.yaml +++ b/stack/stack_custom.yaml @@ -77,7 +77,7 @@ pio: esmf: build: YES - version: 8_1_1 + version: 8_2_0 shared: YES enable_pnetcdf: NO debug: NO @@ -313,7 +313,7 @@ cmakemodules: gftl_shared: build: YES repo: Goddard-Fortran-Ecosystem - version: v1.3.0 + version: v1.3.3 yafyaml: build: YES @@ -329,6 +329,10 @@ geos: build: YES version: 3.8.1 +libtiff: + build: YES + version: 4.0.3 + sqlite: build: YES version: 3.32.3 diff --git a/stack_helpers.sh b/stack_helpers.sh index 759858b2..66b8558b 100755 --- a/stack_helpers.sh +++ b/stack_helpers.sh @@ -129,6 +129,11 @@ function no_modules { export MPI_CXX=${MPI_CXX:-"CC"} export MPI_FC=${MPI_FC:-"ftn"} ;; + mpt ) + export MPI_CC=${MPI_CC:-"mpicc"} + export MPI_CXX=${MPI_CXX:-"CC"} + export MPI_FC=${MPI_FC:-"mpif90"} + ;; * ) echo "Unknown MPI option = $mpiName, ABORT!" local abort=Y From c032c9f66221a0a4d091110a74801df7682ccb86 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Thu, 24 Feb 2022 09:42:27 -0500 Subject: [PATCH 2/6] add config_linux_centos_intel.sh file --- config/config_linux_centos_intel.sh | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 config/config_linux_centos_intel.sh diff --git a/config/config_linux_centos_intel.sh b/config/config_linux_centos_intel.sh new file mode 100644 index 00000000..eafcafd9 --- /dev/null +++ b/config/config_linux_centos_intel.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Compiler/MPI combination +export HPC_PYTHON=${HPC_PYTHON:-"python/3.6.8"} +export HPC_COMPILER=${HPC_COMPILER:-"intel/2020.0"} +export HPC_MPI=${HPC_MPI:-"impi/2020u0"} + +# Build options +export USE_SUDO=N +export PKGDIR=pkg +export LOGDIR=log +export OVERWRITE=N +export NTHREADS=4 +export MAKE_CHECK=N +export MAKE_VERBOSE=Y +export MAKE_CLEAN=N +export DOWNLOAD_ONLY=N +export STACK_EXIT_ON_FAIL=Y +export WGET="wget -nv --no-check-certificate " + +module purge +export BASH_ENV=$HOME/apps/lmod/lmod/init/bash # Point to the new definition of Lmod + +source $BASH_ENV # Redefine the module command to point + # to the new Lmod +module use $HOME/modulefiles +module load cmake +module load intel/2020.0 +module load impi/2020u0 +# + +export CC=icc +export FC=ifort +export CXX=icpc + +export SERIAL_CC=icc +export SERIAL_FC=ifort +export SERIAL_CXX=icpc + +export MPI_CC=mpiicc +export MPI_FC=mpiifort +export MPI_CXX=mpiicpc + +# From 255ef2c94eed6f4c42a4db0fcabf2a7bc4a26dbd Mon Sep 17 00:00:00 2001 From: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> Date: Thu, 24 Feb 2022 10:32:32 -0500 Subject: [PATCH 3/6] Update build_libtiff.sh corrected the address for libtiff download, removed double "lib" in the address --- libs/build_libtiff.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/build_libtiff.sh b/libs/build_libtiff.sh index 4f93d554..ec71f7db 100755 --- a/libs/build_libtiff.sh +++ b/libs/build_libtiff.sh @@ -36,7 +36,7 @@ export CFLAGS="${STACK_CFLAGS:-} ${STACK_libtiff_CFLAGS:-} -fPIC" cd ${HPC_STACK_ROOT}/${PKGDIR:-"pkg"} software=$name-$version -URL="https://gitlab.com/lib${name}/lib${name}.git" +URL="https://gitlab.com/${name}/${name}.git" [[ -d $software ]] || ( git clone $URL $software ) [[ ${DOWNLOAD_ONLY} =~ [yYtT] ]] && exit 0 [[ -d $software ]] && cd $software || ( echo "$software does not exist, ABORT!"; exit 1 ) From dee2f60924a8c3eb0df2e9251c176edfab7164aa Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Mon, 28 Feb 2022 17:17:40 -0500 Subject: [PATCH 4/6] corrected build_libtiff.sh as requested by PR reviewers --- libs/build_libtiff.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/libs/build_libtiff.sh b/libs/build_libtiff.sh index ec71f7db..ef198209 100755 --- a/libs/build_libtiff.sh +++ b/libs/build_libtiff.sh @@ -22,8 +22,13 @@ if $MODULES; then prefix="${PREFIX:-"/opt/modules"}/$compiler/$name/$version" if [[ -d $prefix ]]; then - [[ $OVERWRITE =~ [yYtT] ]] && ( echo "WARNING: $prefix EXISTS: OVERWRITING!";$SUDO rm -rf $prefix ) \ - || ( echo "WARNING: $prefix EXISTS, SKIPPING"; exit 1 ) + if [[ $OVERWRITE =~ [yYtT] ]]; then + echo "WARNING: $prefix EXISTS: OVERWRITING!" + $SUDO rm -rf $prefix + else + echo "WARNING: $prefix EXISTS, SKIPPING" + exit 0 + fi fi else From 4fb48ff61720fb786eed86610827a2b691be1c96 Mon Sep 17 00:00:00 2001 From: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> Date: Tue, 1 Mar 2022 05:15:27 -0500 Subject: [PATCH 5/6] Update build_proj.sh --- libs/build_proj.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/build_proj.sh b/libs/build_proj.sh index 12fd1cc2..a16c20a1 100755 --- a/libs/build_proj.sh +++ b/libs/build_proj.sh @@ -17,7 +17,7 @@ if $MODULES; then module load hpc-$HPC_COMPILER module try-load cmake module load sqlite - module try-load tiff + module try-load libtiff module list set -x From 44db0953668b47122a378695cfdd92030e593966 Mon Sep 17 00:00:00 2001 From: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> Date: Tue, 1 Mar 2022 09:52:02 -0500 Subject: [PATCH 6/6] Update build_proj.sh --- libs/build_proj.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/build_proj.sh b/libs/build_proj.sh index a16c20a1..57f05b75 100755 --- a/libs/build_proj.sh +++ b/libs/build_proj.sh @@ -59,8 +59,8 @@ CMAKE_OPTS=${STACK_proj_cmake_opts:-""} [[ $MAKE_CHECK =~ [yYtT] ]] || CMAKE_OPTS+=" -DBUILD_TESTING=OFF" if [[ -n ${LIBTIFF_ROOT-} ]] ; then - CMAKE_OPTS+=" -DLIBTIFF_INCLUDE_DIR=${LIBTIFF_ROOT}/include " - [[ -f ${LIBTIFF_ROOT}/lib64/libtiff.so ]] && CMAKE_OPTS+=" -DLIBTIFF_LIBRARY=${LIBTIFF_ROOT}/lib64/libtiff.so " || CMAKE_OPTS+=" -DLIBTIFF_LIBRARY=${LIBTIFF_ROOT}/lib/libtiff.so " + CMAKE_OPTS+=" -DTIFF_INCLUDE_DIR=${LIBTIFF_ROOT}/include " + [[ -f ${LIBTIFF_ROOT}/lib64/libtiff.so ]] && CMAKE_OPTS+=" -DTIFF_LIBRARY=${LIBTIFF_ROOT}/lib64/libtiff.so " || CMAKE_OPTS+=" -DTIFF_LIBRARY=${LIBTIFF_ROOT}/lib/libtiff.so " fi if [[ -n ${SQLITE_ROOT-} ]] ; then