From e29559273c32a910c45ce7dcfe42569bf404e82d Mon Sep 17 00:00:00 2001 From: Akila Premachandra Date: Fri, 9 Aug 2019 17:35:53 -0600 Subject: [PATCH 1/2] Adding OpenMP and pthreads to rocBLAS cmake files --- CMakeLists.txt | 3 ++ Jenkinsfile | 2 +- clients/CMakeLists.txt | 10 ++++++ clients/benchmarks/CMakeLists.txt | 48 +++++++++++++++++++++++------ clients/gtest/CMakeLists.txt | 41 ++++++++++++++++++------ clients/samples/CMakeLists.txt | 3 ++ docker/dockerfile-build-centos | 1 + docker/dockerfile-build-ubuntu-rock | 2 ++ install.sh | 16 +++++----- library/CMakeLists.txt | 3 ++ library/src/CMakeLists.txt | 3 ++ 11 files changed, 103 insertions(+), 29 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 508276df3..f6b125a81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,9 @@ endif() project( rocblas LANGUAGES CXX ) +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + # ######################################################################## # NOTE: CUDA compiling path # ######################################################################## diff --git a/Jenkinsfile b/Jenkinsfile index 5bef76620..feff1d3ce 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,7 +32,7 @@ rocBLASCI: rocblas.paths.build_command = './install.sh -lasm_ci -c' // Define test architectures, optional rocm version argument is available - def nodes = new dockerNodes(['gfx900 && ubuntu', 'gfx906 && centos7'], rocblas) + def nodes = new dockerNodes(['gfx900 && ubuntu', 'gfx906 && ubuntu'], rocblas) boolean formatCheck = true diff --git a/clients/CMakeLists.txt b/clients/CMakeLists.txt index 060453744..dafff9bff 100755 --- a/clients/CMakeLists.txt +++ b/clients/CMakeLists.txt @@ -24,6 +24,16 @@ endif() # This project may compile dependencies for clients project( rocblas-clients LANGUAGES CXX ) +if(EXISTS /etc/redhat-release) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp=libgomp -pthread") +else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -pthread") +endif() + +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + + list( APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ) include( build-options ) diff --git a/clients/benchmarks/CMakeLists.txt b/clients/benchmarks/CMakeLists.txt index 1b175009e..14acd794b 100644 --- a/clients/benchmarks/CMakeLists.txt +++ b/clients/benchmarks/CMakeLists.txt @@ -8,6 +8,12 @@ set( Boost_DETAILED_FAILURE_MSG ON ) set( Boost_ADDITIONAL_VERSIONS 1.65.1 1.65 ) set( Boost_USE_STATIC_LIBS OFF ) +if(EXISTS /etc/redhat-release) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp=libgomp -pthread") +else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -pthread") +endif() + find_package( Boost COMPONENTS program_options ) if( NOT Boost_FOUND ) @@ -20,6 +26,9 @@ if( NOT Boost_FOUND ) endif( ) endif( ) +set( THREADS_PREFER_PTHREAD_FLAG ON ) +find_package( Threads REQUIRED ) + # Linking lapack library requires fortran flags enable_language( Fortran ) find_package( cblas CONFIG REQUIRED ) @@ -49,16 +58,35 @@ target_include_directories( rocblas-bench $ ) -# External header includes included as system files -target_include_directories( rocblas-bench - SYSTEM PRIVATE - $ - $ - $ - $ - ) - -target_link_libraries( rocblas-bench PRIVATE ${Boost_LIBRARIES} cblas lapack roc::rocblas ) +if( EXISTS /etc/redhat-release) + set( OPENMP_INCLUDE_DIR /opt/rh/devtoolset-7/root/usr/lib/gcc/x86_64-redhat-linux/7/include ) + set( OPENMP_LIBRARY /opt/rh/devtoolset-7/root/usr/lib/gcc/x86_64-redhat-linux/7/libgomp.so ) + set( CLANG_INCLUDE_DIR /opt/rocm/hcc/lib/clang/9.0.0/include ) + + # External header includes included as system files + target_include_directories( rocblas-bench + SYSTEM PRIVATE + $ + $ + $ + $ + $ + $ + ) + + target_link_libraries( rocblas-bench PRIVATE ${Boost_LIBRARIES} ${OPENMP_LIBRARY} cblas lapack roc::rocblas ) +else() + # External header includes included as system files + target_include_directories( rocblas-bench + SYSTEM PRIVATE + $ + $ + $ + $ + ) + + target_link_libraries( rocblas-bench PRIVATE ${Boost_LIBRARIES} cblas lapack roc::rocblas ) +endif() get_target_property( HIPHCC_LOCATION hip::hip_hcc IMPORTED_LOCATION_RELEASE ) diff --git a/clients/gtest/CMakeLists.txt b/clients/gtest/CMakeLists.txt index a6c602bda..3523c25af 100644 --- a/clients/gtest/CMakeLists.txt +++ b/clients/gtest/CMakeLists.txt @@ -4,6 +4,7 @@ # For debugging, uncomment this # set( CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -g -O0" ) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp") # set( Boost_DEBUG ON ) set( Boost_USE_MULTITHREADED ON ) @@ -83,17 +84,37 @@ target_include_directories( rocblas-test $ ) -# External header includes included as system files -target_include_directories( rocblas-test - SYSTEM PRIVATE - $ - $ - $ - $ - $ - ) +if( EXISTS /etc/redhat-release) + set( OPENMP_INCLUDE_DIR /opt/rh/devtoolset-7/root/usr/lib/gcc/x86_64-redhat-linux/7/include ) + set( OPENMP_LIBRARY /opt/rh/devtoolset-7/root/usr/lib/gcc/x86_64-redhat-linux/7/libgomp.so ) + set( CLANG_INCLUDE_DIR /opt/rocm/hcc/lib/clang/9.0.0/include ) + + # External header includes included as system files + target_include_directories( rocblas-test + SYSTEM PRIVATE + $ + $ + $ + $ + $ + $ + $ + ) + + target_link_libraries( rocblas-test PRIVATE ${GTEST_LIBRARIES} ${Boost_LIBRARIES} ${OPENMP_LIBRARY} cblas lapack roc::rocblas ) +else() + # External header includes included as system files + target_include_directories( rocblas-test + SYSTEM PRIVATE + $ + $ + $ + $ + $ + ) -target_link_libraries( rocblas-test PRIVATE ${GTEST_LIBRARIES} ${Boost_LIBRARIES} cblas lapack roc::rocblas Threads::Threads ) + target_link_libraries( rocblas-test PRIVATE ${GTEST_LIBRARIES} ${Boost_LIBRARIES} cblas lapack roc::rocblas Threads::Threads ) +endif() get_target_property( HIPHCC_LOCATION hip::hip_hcc IMPORTED_LOCATION_RELEASE ) diff --git a/clients/samples/CMakeLists.txt b/clients/samples/CMakeLists.txt index 377e3a8dd..2ab2d75cf 100644 --- a/clients/samples/CMakeLists.txt +++ b/clients/samples/CMakeLists.txt @@ -6,6 +6,9 @@ get_target_property( HIPHCC_LOCATION hip::hip_hcc IMPORTED_LOCATION_RELEASE ) set( rocblas_samples_common ../common/utility.cpp ) +set( THREADS_PREFER_PTHREAD_FLAG ON ) +find_package( Threads REQUIRED ) + add_executable( example-sscal example_sscal.cpp ${rocblas_samples_common} ) add_executable( example-sgemm example_sgemm.cpp ${rocblas_samples_common} ) add_executable( example-sgemm-strided-batched example_sgemm_strided_batched.cpp ${rocblas_samples_common} ) diff --git a/docker/dockerfile-build-centos b/docker/dockerfile-build-centos index a924c6038..32bf06457 100644 --- a/docker/dockerfile-build-centos +++ b/docker/dockerfile-build-centos @@ -35,6 +35,7 @@ RUN yum install -y \ boost-devel \ numactl-libs \ rpm-build \ + libgomp \ zlib-devel \ deltarpm diff --git a/docker/dockerfile-build-ubuntu-rock b/docker/dockerfile-build-ubuntu-rock index 49cee3bb7..d310ebf26 100644 --- a/docker/dockerfile-build-ubuntu-rock +++ b/docker/dockerfile-build-ubuntu-rock @@ -30,6 +30,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins gfortran \ libboost-program-options-dev \ libnuma1 \ + libpthread-stubs0-dev \ + libomp-dev \ zlib1g-dev \ && \ apt-get clean && \ diff --git a/install.sh b/install.sh index 2cac31b60..1ab50df4b 100755 --- a/install.sh +++ b/install.sh @@ -121,16 +121,16 @@ install_packages( ) # dependencies needed for rocblas and clients to build local library_dependencies_ubuntu=( "make" "cmake-curses-gui" "pkg-config" "python2.7" "python3" "python-yaml" "python3-yaml" - "llvm-6.0-dev" + "llvm-6.0-dev" "libomp-dev" "hip_hcc" "rocm_smi64" "zlib1g-dev") local library_dependencies_centos=( "epel-release" "make" "cmake3" "rpm-build" "python34" "PyYAML" "python3*-PyYAML" "gcc-c++" "llvm7.0-devel" "llvm7.0-static" - "hip_hcc" "rocm_smi64" "zlib-devel" ) + "hip_hcc" "rocm_smi64" "libgomp" "zlib-devel" ) local library_dependencies_fedora=( "make" "cmake" "rpm-build" "python34" "PyYAML" "python3*-PyYAML" - "gcc-c++" "libcxx-devel" + "gcc-c++" "libcxx-devel" "libgomp" "hip_hcc" "rocm_smi64" "zlib-devel" ) if [[ "${build_cuda}" == true ]]; then @@ -140,9 +140,9 @@ install_packages( ) library_dependencies_fedora+=( "" ) # how to install cuda on fedora? fi - local client_dependencies_ubuntu=( "gfortran" "libboost-program-options-dev" ) - local client_dependencies_centos=( "gcc-gfortran" "boost-devel" ) - local client_dependencies_fedora=( "gcc-gfortran" "boost-devel" ) + local client_dependencies_ubuntu=( "gfortran" "libboost-program-options-dev" "libomp-dev") + local client_dependencies_centos=( "gcc-gfortran" "boost-devel" "libgomp") + local client_dependencies_fedora=( "gcc-gfortran" "boost-devel" "libgomp") case "${ID}" in ubuntu) @@ -323,7 +323,7 @@ if [[ "${install_dependencies}" == true ]]; then pushd . printf "\033[32mBuilding \033[33mgoogletest & lapack\033[32m from source; installing into \033[33m/usr/local\033[0m\n" mkdir -p ${build_dir}/deps && cd ${build_dir}/deps - ${cmake_executable} -DBUILD_BOOST=OFF ../../deps + ${cmake_executable} -lpthread -DBUILD_BOOST=OFF ../../deps make -j$(nproc) elevate_if_not_root make install popd @@ -339,7 +339,7 @@ pushd . # ################################################# cmake_common_options="" cmake_client_options="" - cmake_common_options="${cmake_common_options} -DTensile_LOGIC=${tensile_logic} -DTensile_CODE_OBJECT_VERSION=${tensile_cov}" + cmake_common_options="${cmake_common_options} -lpthread -DTensile_LOGIC=${tensile_logic} -DTensile_CODE_OBJECT_VERSION=${tensile_cov}" # build type diff --git a/library/CMakeLists.txt b/library/CMakeLists.txt index 5d7822659..62586c224 100755 --- a/library/CMakeLists.txt +++ b/library/CMakeLists.txt @@ -21,6 +21,9 @@ function( target_compile_features target_name ) endif( ) endfunction( ) +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + # ######################################################################## # target_link_libraries() override # ######################################################################## diff --git a/library/src/CMakeLists.txt b/library/src/CMakeLists.txt index f692bd56f..92caa694c 100755 --- a/library/src/CMakeLists.txt +++ b/library/src/CMakeLists.txt @@ -23,6 +23,9 @@ endfunction( ) # package_targets is used as a list of install target set( package_targets rocblas ) +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + # Set up Tensile Dependency if( BUILD_WITH_TENSILE ) # If we want to build a shared rocblas lib, force Tensile to build as a static lib to absorb into rocblas From 213e8921ae0dd567ede5f01d815d9ad90b2bcf3f Mon Sep 17 00:00:00 2001 From: Akila Premachandra Date: Sun, 11 Aug 2019 16:52:32 -0600 Subject: [PATCH 2/2] added llvm to dockerfile --- docker/dockerfile-build-centos | 2 ++ docker/dockerfile-build-ubuntu-rock | 1 + 2 files changed, 3 insertions(+) diff --git a/docker/dockerfile-build-centos b/docker/dockerfile-build-centos index 32bf06457..d3dceaba7 100644 --- a/docker/dockerfile-build-centos +++ b/docker/dockerfile-build-centos @@ -31,6 +31,8 @@ RUN yum install -y \ python36-pytest \ python36-setuptools \ PyYAML \ + llvm7.0-devel \ + llvm7.0-static \ libcxx-devel \ boost-devel \ numactl-libs \ diff --git a/docker/dockerfile-build-ubuntu-rock b/docker/dockerfile-build-ubuntu-rock index d310ebf26..496122e48 100644 --- a/docker/dockerfile-build-ubuntu-rock +++ b/docker/dockerfile-build-ubuntu-rock @@ -28,6 +28,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins python-yaml \ python3-yaml \ gfortran \ + llvm-6.0-dev \ libboost-program-options-dev \ libnuma1 \ libpthread-stubs0-dev \