From 21a69252120ea0cd73e7acbd610f9877524f3e4e Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sat, 9 May 2020 15:45:28 +0200 Subject: [PATCH 1/5] [cmake] all-in-one: update suitesparse and make BUILD_SHARED_LIBS explicit --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 72724251b1..0bc2903999 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,8 @@ option(AV_BUILD_ALICEVISION "Enable building of AliceVision" ON) option(AV_USE_CUDA "Enable CUDA" ON) +option(BUILD_SHARED_LIBS "Build shared libraries" ON) + if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type for AliceVision" FORCE) endif() @@ -234,7 +236,7 @@ set(SUITESPARSE_TARGET suitesparse) set(SUITESPARSE_INTERNAL_MAKE_CMD LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR} make BLAS="${BLAS_LIBRARIES}" LAPACK="${LAPACK_LIBRARIES}") ExternalProject_Add(${SUITESPARSE_TARGET} # URL https://github.com/DrTimothyAldenDavis/SuiteSparse/archive/v4.5.6.tar.gz - URL https://github.com/DrTimothyAldenDavis/SuiteSparse/archive/v5.6.0.tar.gz # requires gxx >= 4.9, centos 7 use gxx-4.8.5 by default + URL https://github.com/DrTimothyAldenDavis/SuiteSparse/archive/v5.7.2.tar.gz # requires gxx >= 4.9, centos 7 use gxx-4.8.5 by default PREFIX ${BUILD_DIR} BUILD_IN_SOURCE 0 BUILD_ALWAYS 0 From 4787326e66e692944927408810f400b7997d3441 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Mon, 11 May 2020 00:11:58 +0200 Subject: [PATCH 2/5] [cmake] all-in-one: setup eigen flags on all dependencies using eigen --- CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0bc2903999..a20c3f6318 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,6 +180,7 @@ set(TBB_CMAKE_FLAGS -DTBB_INCLUDE_DIRS:PATH=${CMAKE_INSTALL_PREFIX}/include -DTB # Add Eigen set(EIGEN_TARGET eigen) +set(EIGEN_DISABLE_ALIGN_FLAGS -DEIGEN_MAX_ALIGN_BYTES=0 -DEIGEN_MAX_STATIC_ALIGN_BYTES=0) ExternalProject_Add(${EIGEN_TARGET} URL https://github.com/eigenteam/eigen-git-mirror/archive/3.3.7.tar.gz PREFIX ${BUILD_DIR} @@ -189,9 +190,9 @@ ExternalProject_Add(${EIGEN_TARGET} SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/eigen BINARY_DIR ${BUILD_DIR}/eigen_build INSTALL_DIR ${CMAKE_INSTALL_PREFIX} - CONFIGURE_COMMAND ${CMAKE_COMMAND} ${CMAKE_CORE_BUILD_FLAGS} -DCMAKE_INSTALL_PREFIX:PATH= + CONFIGURE_COMMAND ${CMAKE_COMMAND} ${CMAKE_CORE_BUILD_FLAGS} ${EIGEN_DISABLE_ALIGN_FLAGS} -DCMAKE_INSTALL_PREFIX:PATH= ) -set(EIGEN_CMAKE_FLAGS -DEigen3_DIR:PATH=${CMAKE_INSTALL_PREFIX}/share/eigen3/cmake -DEIGEN3_INCLUDE_DIR=${CMAKE_INSTALL_PREFIX}/include/eigen3 -DEIGEN_INCLUDE_DIR=${CMAKE_INSTALL_PREFIX}/include/eigen3 -DEigen_INCLUDE_DIR=${CMAKE_INSTALL_PREFIX}/include/eigen3) +set(EIGEN_CMAKE_FLAGS ${EIGEN_DISABLE_ALIGN_FLAGS} -DEigen3_DIR:PATH=${CMAKE_INSTALL_PREFIX}/share/eigen3/cmake -DEIGEN3_INCLUDE_DIR=${CMAKE_INSTALL_PREFIX}/include/eigen3 -DEIGEN_INCLUDE_DIR=${CMAKE_INSTALL_PREFIX}/include/eigen3 -DEigen_INCLUDE_DIR=${CMAKE_INSTALL_PREFIX}/include/eigen3) if(AV_BUILD_OPENGV) set(OPENGV_TARGET opengv) From a9b3b2b0477748a6438eb84584e1e89457c637b3 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Mon, 11 May 2020 00:13:16 +0200 Subject: [PATCH 3/5] [cmake] eigen: use new eigen flags to disable alignment The previous ones are still valid for now but are deprecated. --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b071993647..e1b61ed36b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -345,7 +345,7 @@ find_package(Eigen3 3.3 REQUIRED) if(Eigen3_FOUND OR EIGEN3_FOUND) message(STATUS "EIGEN_INCLUDE_DIR: ${EIGEN_INCLUDE_DIR}") # See https://eigen.tuxfamily.org/dox/group__TopicUnalignedArrayAssert.html - set(AV_EIGEN_DEFINITIONS EIGEN_DONT_ALIGN_STATICALLY EIGEN_DONT_VECTORIZE) + set(AV_EIGEN_DEFINITIONS -DEIGEN_MAX_ALIGN_BYTES=0 -DEIGEN_MAX_STATIC_ALIGN_BYTES=0) else() message(FATAL_ERROR " EIGEN NOT FOUND. EIGEN_INCLUDE_DIR: ${EIGEN_INCLUDE_DIR}") endif() From 89eb50baeb8b9b52cd4633dea81166aba2bd1173 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Mon, 11 May 2020 00:16:47 +0200 Subject: [PATCH 4/5] [docker] devtoolset-6 has been removed from default centos-7 packages Use an external mirror site. Use one yum command per package to create an error if one package is missing. Disable doc installation. --- Dockerfile_deps | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/Dockerfile_deps b/Dockerfile_deps index 95875cee13..9b8ec8522b 100644 --- a/Dockerfile_deps +++ b/Dockerfile_deps @@ -22,25 +22,34 @@ ENV AV_DEV=/opt/AliceVisionDeps_git \ AV_INSTALL=/opt/AliceVision_install \ AV_BUNDLE=/opt/AliceVision_bundle \ PATH="${PATH}:${AV_BUNDLE}" \ - VERBOSE=1 + VERBOSE=1 \ + YUM_INSTALL='yum install -y --setopt=tsflags=nodocs' +# # Install all compilation tools # - file and openssl are needed for cmake -RUN yum -y install centos-release-scl -RUN yum -y install \ - devtoolset-6 \ - devtoolset-6-make \ - devtoolset-6-gcc-gfortran \ - file \ - git \ - wget \ - unzip \ - yasm \ - pkg-config \ - libtool \ - nasm \ - automake \ - openssl-devel +# +# Workaround to give access to deprecated devtoolset-6 is taken from ASWF container: +# https://github.com/AcademySoftwareFoundation/aswf-docker/blob/master/scripts/common/install_yumpackages.sh#L119 +# +# Install packages one by one with yum to ensure that it creates an error if a package is missing. +# +RUN $YUM_INSTALL centos-release-scl-rh && \ + $YUM_INSTALL yum-utils && \ + sed -i 's/7/7.6.1810/g; s|^#\s*\(baseurl=http://\)mirror|\1vault|g; /mirrorlist/d' /etc/yum.repos.d/CentOS-SCLo-*.repo && \ + yum clean all && \ + $YUM_INSTALL devtoolset-6-toolchain --nogpgcheck && \ + $YUM_INSTALL --enablerepo=extras epel-release && \ + $YUM_INSTALL file && \ + $YUM_INSTALL git && \ + $YUM_INSTALL wget && \ + $YUM_INSTALL unzip && \ + $YUM_INSTALL yasm && \ + $YUM_INSTALL pkgconfig && \ + $YUM_INSTALL libtool && \ + $YUM_INSTALL nasm && \ + $YUM_INSTALL automake && \ + $YUM_INSTALL openssl-devel # Okay, change our shell to specifically use our software collections. # (default was SHELL [ "/bin/sh", "-c" ]) From 80e232061c6ba642f1dd9d132e21c98d8644c3f4 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Mon, 11 May 2020 00:18:49 +0200 Subject: [PATCH 5/5] [docker] reduce docker image size clean intermediate build files build and clean in one RUN to avoid intermediate cache --- Dockerfile | 13 ++++++++++--- Dockerfile_deps | 29 +++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8e2d8893e8..c286bd5321 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,14 +24,21 @@ ENV AV_DEV=/opt/AliceVision_git \ COPY . "${AV_DEV}" WORKDIR "${AV_BUILD}" + RUN cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS:BOOL=ON -DTARGET_ARCHITECTURE=core \ -DALICEVISION_BUILD_DEPENDENCIES:BOOL=OFF \ -DCMAKE_PREFIX_PATH:PATH="${AV_INSTALL}" \ -DCMAKE_INSTALL_PREFIX:PATH="${AV_INSTALL}" -DALICEVISION_BUNDLE_PREFIX="${AV_BUNDLE}" \ -DALICEVISION_USE_ALEMBIC=ON -DMINIGLOG=ON -DALICEVISION_USE_CCTAG=ON -DALICEVISION_USE_OPENCV=ON -DALICEVISION_USE_OPENGV=ON \ -DALICEVISION_USE_POPSIFT=ON -DALICEVISION_USE_CUDA=ON -DALICEVISION_BUILD_DOC=OFF -DALICEVISION_BUILD_EXAMPLES=OFF \ - "${AV_DEV}" + "${AV_DEV}" && \ +\ +make install && \ +\ +make bundle && \ +\ +cd /opt && \ +\ +rm -rf "${AV_BUILD}" -RUN make install && make bundle -# && cd /opt && rm -rf "${AV_BUILD}" diff --git a/Dockerfile_deps b/Dockerfile_deps index 9b8ec8522b..b8a90da5e7 100644 --- a/Dockerfile_deps +++ b/Dockerfile_deps @@ -17,8 +17,8 @@ LABEL maintainer="AliceVision Team alicevision-team@googlegroups.com" # OS/Version (FILE): cat /etc/issue.net # Cuda version (ENV): $CUDA_VERSION -ENV AV_DEV=/opt/AliceVisionDeps_git \ - AV_BUILD=/tmp/AliceVisionDeps_build \ +ENV AV_DEPS_DEV=/opt/AliceVisionDeps_git \ + AV_DEPS_BUILD=/tmp/AliceVisionDeps_build \ AV_INSTALL=/opt/AliceVision_install \ AV_BUNDLE=/opt/AliceVision_bundle \ PATH="${PATH}:${AV_BUNDLE}" \ @@ -69,12 +69,27 @@ RUN wget https://cmake.org/files/v3.16/cmake-3.16.3.tar.gz && tar zxvf cmake-3.1 WORKDIR "${AV_BUNDLE}/share/aliceVision" RUN wget https://gitlab.com/alicevision/trainedVocabularyTreeData/raw/master/vlfeat_K80L3.SIFT.tree -COPY . "${AV_DEV}" +COPY . "${AV_DEPS_DEV}" -WORKDIR "${AV_BUILD}" -RUN cmake "${AV_DEV}" -DCMAKE_BUILD_TYPE=Release -DALICEVISION_BUILD_DEPENDENCIES:BOOL=ON -DAV_BUILD_ALICEVISION:BOOL=OFF -DCMAKE_INSTALL_PREFIX="${AV_INSTALL}" -DALICEVISION_BUNDLE_PREFIX="${AV_BUNDLE}" +WORKDIR "${AV_DEPS_BUILD}" +# Build and clear build files in one line to reduce image size (avoid intermediate cache) +RUN \ +\ +cmake "${AV_DEPS_DEV}" -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS:BOOL=ON -DALICEVISION_BUILD_DEPENDENCIES:BOOL=ON -DAV_DEPS_BUILD_ALICEVISION:BOOL=OFF -DCMAKE_INSTALL_PREFIX="${AV_INSTALL}" -DALICEVISION_BUNDLE_PREFIX="${AV_BUNDLE}" && \ +\ +make -j8 && \ +\ +make install && \ +\ +mv ${AV_INSTALL}/bin ${AV_INSTALL}/bin_deps && \ +\ +cd /opt && \ +\ +rm -rf "${AV_DEPS_BUILD}" -WORKDIR "${AV_BUILD}" +## To debug a specific library + +# WORKDIR "${AV_DEPS_BUILD}" # RUN make zlib # RUN make geogram # RUN make tbb @@ -93,5 +108,3 @@ WORKDIR "${AV_BUILD}" # RUN make alembic # RUN make popsift -RUN make install && mv ${AV_INSTALL}/bin ${AV_INSTALL}/bin_deps -# && cd /opt && rm -rf "${AV_BUILD}"