Using Blis for test validation

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "extern/blis"]
+	path = extern/blis
+	url = https://github.com/amd/blis.git
+	branch = 1.3

Jenkinsfile

@@ -1,6 +1,6 @@
 #!/usr/bin/env groovy
 // This shared library is available at https://github.com/ROCmSoftwarePlatform/rocJENKINS/
-@Library('rocJenkins') _
+@Library('rocJenkins@hyperthread2') _
 
 // This is file for internal AMD use.
 // If you are interested in running your own Jenkins, please raise a github issue for assistance.
@@ -90,7 +90,7 @@ rocBLASCI:
                         LD_LIBRARY_PATH=/opt/rocm/hcc/lib ./example-sscal
                         LD_LIBRARY_PATH=/opt/rocm/hcc/lib GTEST_LISTENER=NO_PASS_LINE_IN_LOG sudo ./rocblas-test --gtest_output=xml --gtest_color=yes  --gtest_filter=*quick*:*pre_checkin*-*known_bug* #--gtest_filter=*checkin*
                     """
-
+                
                 platform.runCommand(this, command)
                 junit "${project.paths.project_build_prefix}/build/release/clients/staging/*.xml"
             }

clients/CMakeLists.txt

@@ -33,7 +33,6 @@ endif()
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-
 list( APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake )
 
 include( build-options )

clients/benchmarks/CMakeLists.txt

@@ -39,11 +39,12 @@ endif( )
 set( rocblas_benchmark_common
       ../common/utility.cpp
       ../common/cblas_interface.cpp
+      ../common/blis_interface.cpp
       ../common/rocblas_parse_data.cpp
     )
 
 add_executable( rocblas-bench client.cpp ${rocblas_benchmark_common} )
-target_compile_features( rocblas-bench PRIVATE cxx_static_assert cxx_nullptr cxx_auto_type )
+target_compile_features( rocblas-bench PRIVATE cxx_static_assert cxx_nullptr cxx_auto_type)
 
 if( BUILD_WITH_TENSILE )
     target_compile_definitions( rocblas-bench PRIVATE BUILD_WITH_TENSILE=1 )
@@ -57,23 +58,27 @@ target_include_directories( rocblas-bench
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include>
 )
 
+set( BLIS_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/build/deps/blis/include/blis )
+set( BLIS_LIBRARY ${CMAKE_SOURCE_DIR}/build/deps/blis/lib/libblis.so )
+
 if( EXISTS /etc/redhat-release)
     set( OPENMP_INCLUDE_DIR /opt/rh/devtoolset-7/root/usr/lib/gcc/x86_64-redhat-linux/7/include )
     set( OPENMP_LIBRARY /opt/rh/devtoolset-7/root/usr/lib/gcc/x86_64-redhat-linux/7/libgomp.so ) 
-    set( CLANG_INCLUDE_DIR /opt/rocm/hcc/lib/clang/9.0.0/include )
- 
+    set( CLANG_INCLUDE_DIR /opt/rocm/hcc/lib/clang/9.0.0/include ) 
+
     # External header includes included as system files
     target_include_directories( rocblas-bench
       SYSTEM PRIVATE
-        $<BUILD_INTERFACE:${CLANG_INCLUDE_DIR}>
+        $<BUILD_INTERFACE:${CLANG_INCLUDE_DIR}> 
+        $<BUILD_INTERFACE:${BLIS_INCLUDE_DIR}>
         $<BUILD_INTERFACE:${HIP_INCLUDE_DIRS}>
         $<BUILD_INTERFACE:${HCC_INCLUDE_DIRS}>
         $<BUILD_INTERFACE:${Boost_INCLUDE_DIRS}>
         $<BUILD_INTERFACE:${CBLAS_INCLUDE_DIRS}>
-        $<BUILD_INTERFACE:${OPENMP_INCLUDE_DIR}>
-      )
+        $<BUILD_INTERFACE:${OPENMP_INCLUDE_DIR}> 
+	)
 
-    target_link_libraries( rocblas-bench PRIVATE ${Boost_LIBRARIES} ${OPENMP_LIBRARY} cblas lapack roc::rocblas )
+    target_link_libraries( rocblas-bench PRIVATE ${Boost_LIBRARIES} ${BLIS_LIBRARY} ${OPENMP_LIBRARY} cblas lapack roc::rocblas )
 else()
     # External header includes included as system files
     target_include_directories( rocblas-bench
@@ -82,9 +87,10 @@ else()
         $<BUILD_INTERFACE:${HCC_INCLUDE_DIRS}>
         $<BUILD_INTERFACE:${Boost_INCLUDE_DIRS}>
         $<BUILD_INTERFACE:${CBLAS_INCLUDE_DIRS}>
-      )
-
-    target_link_libraries( rocblas-bench PRIVATE ${Boost_LIBRARIES} cblas lapack roc::rocblas )
+        $<BUILD_INTERFACE:${BLIS_INCLUDE_DIR}>
+    )
+
+    target_link_libraries( rocblas-bench PRIVATE ${Boost_LIBRARIES} ${BLIS_LIBRARY} cblas lapack roc::rocblas )
 endif()
 
 get_target_property( HIPHCC_LOCATION hip::hip_hcc IMPORTED_LOCATION_RELEASE )

clients/common/blis_interface.cpp

@@ -0,0 +1,9 @@
+#include "blis.h"
+#include "omp.h"
+
+void setup_blis()
+{
+    bli_init();
+}
+
+static int initialize_blis = (setup_blis(), 0);

clients/gtest/CMakeLists.txt

@@ -66,6 +66,7 @@ set(rocblas_test_source
 set( rocblas_benchmark_common
       ../common/utility.cpp
       ../common/cblas_interface.cpp
+      ../common/blis_interface.cpp
       ../common/rocblas_parse_data.cpp
     )
 
@@ -83,6 +84,20 @@ target_include_directories( rocblas-test
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include>
 )
 
+# External header includes included as system files
+target_include_directories( rocblas-test
+  SYSTEM PRIVATE
+    $<BUILD_INTERFACE:${HIP_INCLUDE_DIRS}>
+    $<BUILD_INTERFACE:${HCC_INCLUDE_DIRS}>
+    $<BUILD_INTERFACE:${Boost_INCLUDE_DIRS}>
+    $<BUILD_INTERFACE:${GTEST_INCLUDE_DIRS}>
+    $<BUILD_INTERFACE:${CBLAS_INCLUDE_DIRS}>
+    $<BUILD_INTERFACE:${BLIS_INCLUDE_DIR}>
+    )
+
+set( BLIS_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/build/deps/blis/include/blis )
+set( BLIS_LIBRARY ${CMAKE_SOURCE_DIR}/build/deps/blis/lib/libblis.so )
+
 if( EXISTS /etc/redhat-release)
     set( OPENMP_INCLUDE_DIR /opt/rh/devtoolset-7/root/usr/lib/gcc/x86_64-redhat-linux/7/include )
     set( OPENMP_LIBRARY /opt/rh/devtoolset-7/root/usr/lib/gcc/x86_64-redhat-linux/7/libgomp.so )
@@ -92,6 +107,7 @@ if( EXISTS /etc/redhat-release)
     target_include_directories( rocblas-test
       SYSTEM PRIVATE
         $<BUILD_INTERFACE:${CLANG_INCLUDE_DIR}> 
+        $<BUILD_INTERFACE:${BLIS_INCLUDE_DIR}>
         $<BUILD_INTERFACE:${HIP_INCLUDE_DIRS}>
         $<BUILD_INTERFACE:${HCC_INCLUDE_DIRS}>
         $<BUILD_INTERFACE:${Boost_INCLUDE_DIRS}>
@@ -100,7 +116,7 @@ if( EXISTS /etc/redhat-release)
         $<BUILD_INTERFACE:${OPENMP_INCLUDE_DIR}> 
         )
 
-    target_link_libraries( rocblas-test PRIVATE ${GTEST_LIBRARIES} ${Boost_LIBRARIES} ${OPENMP_LIBRARY} cblas lapack roc::rocblas )
+    target_link_libraries( rocblas-test PRIVATE ${GTEST_LIBRARIES} ${Boost_LIBRARIES} ${BLIS_LIBRARY} ${OPENMP_LIBRARY} cblas lapack roc::rocblas )
 else()
     # External header includes included as system files
     target_include_directories( rocblas-test
@@ -110,9 +126,10 @@ else()
         $<BUILD_INTERFACE:${Boost_INCLUDE_DIRS}>
         $<BUILD_INTERFACE:${GTEST_INCLUDE_DIRS}>
         $<BUILD_INTERFACE:${CBLAS_INCLUDE_DIRS}>
-      )
-
-    target_link_libraries( rocblas-test PRIVATE ${GTEST_LIBRARIES} ${Boost_LIBRARIES} cblas lapack roc::rocblas Threads::Threads )
+        $<BUILD_INTERFACE:${BLIS_INCLUDE_DIR}>
+    )
+
+    target_link_libraries( rocblas-test PRIVATE ${GTEST_LIBRARIES} ${Boost_LIBRARIES} ${BLIS_LIBRARY} cblas lapack roc::rocblas )
 endif()
 
 get_target_property( HIPHCC_LOCATION hip::hip_hcc IMPORTED_LOCATION_RELEASE )

clients/include/testing_iamax_iamin.hpp

@@ -201,6 +201,17 @@ namespace rocblas_cblas
         return half_to_float(x) < half_to_float(y);
     }
 
+    template <typename T>
+    bool greatherthan(T x, T y)
+    {
+        return x > y;
+    }
+
+    bool greatherthan(rocblas_half x, rocblas_half y)
+    {
+        return half_to_float(x) > half_to_float(y);
+    }
+
     template <typename T>
     void cblas_iamin(rocblas_int N, const T* X, rocblas_int incx, rocblas_int* result)
     {
@@ -222,12 +233,33 @@ namespace rocblas_cblas
         *result = minpos;
     }
 
+    template <typename T>
+    void cblas_iamax(rocblas_int N, const T* X, rocblas_int incx, rocblas_int* result)
+    {
+        rocblas_int maxpos = -1;
+        if(N > 0 && incx > 0)
+        {
+            auto max = asum(X[0]);
+            maxpos   = 0;
+            for(size_t i = 1; i < N; ++i)
+            {
+                auto a = asum(X[i * incx]);
+                if(greatherthan(a, max))
+                {
+                    max    = a;
+                    maxpos = i;
+                }
+            }
+        }
+        *result = maxpos;
+    }
+
 } // namespace rocblas_cblas
 
 template <typename T>
 void testing_iamax(const Arguments& arg)
 {
-    testing_iamax_iamin<T, rocblas_iamax<T>, cblas_iamax<T>>(arg);
+    testing_iamax_iamin<T, rocblas_iamax<T>, rocblas_cblas::cblas_iamax<T>>(arg);
 }
 
 template <typename T>

docker/dockerfile-build-centos

@@ -35,19 +35,18 @@ RUN yum install -y \
     llvm7.0-static \
     libcxx-devel \
     boost-devel \
+    zlib-devel \ 
+    libgomp \
     numactl-libs \
     rpm-build \
-    libgomp \
-    zlib-devel \ 
     deltarpm
 
 RUN echo '#!/bin/bash' | tee /etc/profile.d/devtoolset7.sh && echo \
     'source scl_source enable devtoolset-7' >>/etc/profile.d/devtoolset7.sh
 
 RUN pip3 install wheel && pip3 install tox pyyaml
-
-# docker pipeline runs containers with particular uid
-# create a jenkins user with this specific uid so it can use sudo priviledges
+
+# Create a jenkins user with this specific uid so it can use sudo priviledges
 # Grant any member of sudo group password-less sudo privileges
 RUN useradd --create-home -u ${user_uid} -o -G video --shell /bin/bash jenkins && \
     echo '%video ALL=(ALL) NOPASSWD:ALL' | tee /etc/sudoers.d/sudo-nopasswd && \

docker/dockerfile-build-ubuntu-rock

@@ -27,6 +27,10 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
     python3 \
     python-yaml \
     python3-yaml \
+    python3-dev \
+    python3-pip \
+    python3-pytest \
+    python3-setuptools \
     gfortran \
     llvm-6.0-dev \
     libboost-program-options-dev \
@@ -38,12 +42,14 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+RUN pip3 install wheel && pip3 install tox pyyaml
+
 # docker pipeline runs containers with particular uid
 # create a jenkins user with this specific uid so it can use sudo priviledges
 # Grant any member of sudo group password-less sudo privileges
-RUN useradd --create-home -u ${user_uid} -o -G sudo --shell /bin/bash jenkins && \
-    mkdir -p /etc/sudoers.d/ && \
-    echo '%sudo   ALL=(ALL) NOPASSWD:ALL' | tee /etc/sudoers.d/sudo-nopasswd
+RUN useradd --create-home -u ${user_uid} -o -G video --shell /bin/bash jenkins && \
+    echo '%video   ALL=(ALL) NOPASSWD:ALL' | tee /etc/sudoers.d/sudo-nopasswd && \
+    chmod 400 /etc/sudoers.d/sudo-nopasswd
 
 ARG ROCBLAS_SRC_ROOT=/usr/local/src/rocBLAS
 

install.sh

@@ -327,6 +327,21 @@ if [[ "${install_dependencies}" == true ]]; then
     make -j$(nproc)
     elevate_if_not_root make install
   popd
+
+fi
+
+if [[ ! -f "${build_dir}/deps/blis/lib/libblis.a" ]]; then
+  git submodule update --init
+  cd extern/blis
+  if [[ -e "/etc/redhat-release" ]]; then  
+    echo 'CentOS detected'
+    ./configure --prefix=../../${build_dir}/deps/blis --enable-threading=openmp auto
+  else
+    echo 'Ubuntu detected'
+     ./configure --prefix=../../${build_dir}/deps/blis --enable-threading=openmp CC=/opt/rocm/hcc/bin/clang auto
+  fi
+  make install
+  cd ../..
 fi
 
 # We append customary rocm path; if user provides custom rocm path in ${path}, our
@@ -339,8 +354,8 @@ pushd .
   # #################################################
   cmake_common_options=""
   cmake_client_options=""
-  cmake_common_options="${cmake_common_options} -lpthread -DTensile_LOGIC=${tensile_logic} -DTensile_CODE_OBJECT_VERSION=${tensile_cov}"
 
+  cmake_common_options="${cmake_common_options} -lpthread -DTensile_LOGIC=${tensile_logic} -DTensile_CODE_OBJECT_VERSION=${tensile_cov}"
 
   # build type
   if [[ "${build_release}" == true ]]; then