julia: migrate build process to cmake

cmake/Modules/FindOpenBLAS.cmake

@@ -46,14 +46,15 @@ SET(Open_BLAS_LIB_SEARCH_PATHS
         /usr/local/opt/openblas/lib
         ${PROJECT_SOURCE_DIR}/3rdparty/OpenBLAS/lib
         ${PROJECT_SOURCE_DIR}/thirdparty/OpenBLAS/lib
-	${OpenBLAS_DIR}
-	${OpenBLAS_DIR}/lib
+        ${OpenBLAS_DIR}
+        ${OpenBLAS_DIR}/lib
         ${OpenBLAS_HOME}
         ${OpenBLAS_HOME}/lib
  )
 
 FIND_PATH(OpenBLAS_INCLUDE_DIR NAMES cblas.h PATHS ${Open_BLAS_INCLUDE_SEARCH_PATHS})
-FIND_LIBRARY(OpenBLAS_LIB NAMES openblas PATHS ${Open_BLAS_LIB_SEARCH_PATHS})
+# the Julia's private OpenBLAS is named as `libopenblas64_.so` on x86-64 Linux
+FIND_LIBRARY(OpenBLAS_LIB NAMES openblas64_ openblas PATHS ${Open_BLAS_LIB_SEARCH_PATHS})
 IF(NOT OpenBLAS_LIB)
 	FIND_FILE(OpenBLAS_LIB NAMES libopenblas.dll.a PATHS ${Open_BLAS_LIB_SEARCH_PATHS})
 ENDIF()
@@ -89,3 +90,21 @@ MARK_AS_ADVANCED(
     OpenBLAS
 )
 
+#   Check ILP64 data model for the case of Julia self-shipped `libopenblas64_.so`
+SET(detect_interface64_src "
+    #include <string.h>
+    char* openblas_get_config64_(void)\;
+    int main() {
+        return strstr(openblas_get_config64_(), \"USE64BITINT\") == NULL\;
+    }
+")
+FILE(WRITE "${CMAKE_CURRENT_BINARY_DIR}/detect_interface64.c" ${detect_interface64_src})
+TRY_RUN(
+    OpenBLAS_INTERFACE64 compile_detect_interface64
+    "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/detect_interface64.c"
+    LINK_LIBRARIES ${OpenBLAS_LIB}
+)
+IF(OpenBLAS_INTERFACE64 EQUAL 0)
+    add_definitions(-DOPENBLAS_INTERFACE64=1)  # see julia/deps/cblas.h
+ENDIF(OpenBLAS_INTERFACE64 EQUAL 0)
+FILE(REMOVE "${CMAKE_CURRENT_BINARY_DIR}/detect_interface64.c")

julia/Project.toml

@@ -5,6 +5,7 @@ version = "1.6.0"
 
 [deps]
 BinDeps = "9e28174c-4ba2-5203-b857-d8d62c4213ee"
+CMake = "631607c0-34d2-5d66-819e-eb0f9aa2061a"
 Formatting = "59287772-0a20-5a39-b81b-1366585eb4c0"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

julia/deps/build.jl

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+using CMake
 using JSON
 using Libdl
 using LinearAlgebra
@@ -93,19 +94,19 @@ else
 end
 
 # propagate more build flags from ENV
-const CC  = get(ENV, "CC", nothing)
-const CXX = get(ENV, "CXX", nothing)
-const ADD_CFLAGS  = get(ENV, "ADD_CFLAGS", nothing)
-const ADD_LDFLAGS = get(ENV, "ADD_LDFLAGS", nothing)
-const USE_JEMALLOC = get(ENV, "USE_JEMALLOC", nothing)  # "0" or "1"
+const USE_JEMALLOC = get(ENV, "USE_JEMALLOC", nothing)  # "ON" or "OFF"
 
-function get_cpucore()
-    if haskey(ENV, "TRAVIS")  # on travis-ci
-        2
-    else
-        min(Sys.CPU_THREADS, 32)
-    end
-end
+get_cpucore() = min(Sys.CPU_THREADS, 32)
+
+cmake_bool(x::Bool) = ifelse(x ≡ true, "ON", "OFF")
+
+cmake_jemalloc(::Nothing) = ""
+cmake_jemalloc(x::Bool)   =  "-DUSE_JEMALLOC=" * cmake_bool(x)
+
+cmake_cuda_path(::Nothing) = ""
+cmake_cuda_path(x::String) = "-DUSE_CUDA_PATH=" * x
+
+cmake_jl_blas(x::Bool, blas_path) = ifelse(x, "-DOpenBLAS_LIB=$blas_path", "")
 
 using BinDeps
 @BinDeps.setup
@@ -137,7 +138,7 @@ if !libmxnet_detected
 
     run(download_cmd(package_url, "mxnet.7z"))
     # this command will create the dir "usr\\lib"
-    run(`$exe7z e mxnet.7z *\\build\\* *\\lib\\* -y -ousr\\lib`)
+    run(`$exe7z e mxnet.7z "*\\build\\*" "*\\lib\\*" -y -ousr\\lib`)
 
     run(download_cmd(base_url, "mxnet_base.7z"))
     run(`$exe7z x mxnet_base.7z -y -ousr`)
@@ -146,7 +147,7 @@ if !libmxnet_detected
     # testing
     run(`cmd /c dir "usr\\lib"`)
     return
-  end
+  end  # if Sys.iswindows()
 
   ################################################################################
   # If not found, try to build automatically using BinDeps
@@ -155,131 +156,66 @@ if !libmxnet_detected
   blas_path = Libdl.dlpath(Libdl.dlopen(Base.libblas_name))
   blas_vendor = LinearAlgebra.BLAS.vendor()
 
-  ilp64 = ""
-  if blas_vendor == :openblas64
-    ilp64 = "-DINTERFACE64"
-  end
+  USE_JULIA_BLAS = (blas_vendor in (:openblas, :openblas64))
+  @info "USE_JULIA_BLAS -> $USE_JULIA_BLAS"
 
-  FORCE_LAPACK = false
-  if blas_vendor == :unknown
-    @info("Julia is built with an unkown blas library ($blas_path).")
-    @info("Attempting build without reusing the blas library")
-    USE_JULIA_BLAS = false
-  elseif !(blas_vendor in (:openblas, :openblas64))
-    @info("Unsure if we can build against $blas_vendor.")
-    @info("Attempting build anyway.")
-    USE_JULIA_BLAS = true
-  else
-    USE_JULIA_BLAS = true
-    FORCE_LAPACK = true
-  end
-  @info("USE_JULIA_BLAS -> $USE_JULIA_BLAS")
-
-  blas_name = blas_vendor == :openblas64 ? "openblas" : string(blas_vendor)
-  MSHADOW_LDFLAGS = "MSHADOW_LDFLAGS=-lm $blas_path"
+  blas_name = occursin("openblas", string(blas_vendor)) ?  "open" : string(blas_vendor)
 
   #--------------------------------------------------------------------------------
   # Build libmxnet
   mxnet = library_dependency("mxnet", aliases=["mxnet", "libmxnet", "libmxnet.so"])
 
-  _prefix = joinpath(BinDeps.depsdir(mxnet), "usr")
+  _blddir = joinpath(BinDeps.depsdir(mxnet), "build")
   _srcdir = joinpath(BinDeps.depsdir(mxnet), "src")
   _mxdir  = joinpath(_srcdir, "mxnet")
-  _libdir = joinpath(_prefix, "lib")
-  # We have do eagerly delete the installed libmxnet.so
+
+  # We have to eagerly delete the build directory.
   # Otherwise we won't rebuild on an update.
-  run(`rm -f $_libdir/libmxnet.$(Libdl.dlext)`)
+  rm(_blddir, recursive=true, force=true)
+
+  @debug "build dir -> $_blddir"
+
   provides(BuildProcess,
     (@build_steps begin
+      CreateDirectory(_blddir)
       CreateDirectory(_srcdir)
-      CreateDirectory(_libdir)
       @build_steps begin
         BinDeps.DirectoryRule(_mxdir, @build_steps begin
           ChangeDirectory(_srcdir)
-          `git clone https://github.com/apache/incubator-mxnet mxnet`
+          `git clone --recursive https://github.com/apache/incubator-mxnet mxnet`
         end)
         @build_steps begin
           ChangeDirectory(_mxdir)
           `git fetch`
+          `git submodule update --recursive --force`
           if libmxnet_curr_ver != "master"
             `git checkout $libmxnet_curr_ver`
           else
             `git checkout origin/$libmxnet_curr_ver`
           end
-          `git submodule update --init --recursive`
-          `git -C 3rdparty/mshadow checkout -- make/mshadow.mk`
-          `cp -v ../../cblas.h include/cblas.h`
-          `sed -i -s "s/MSHADOW_CFLAGS = \(.*\)/MSHADOW_CFLAGS = \1 $ilp64/" 3rdparty/mshadow/make/mshadow.mk`
-
-          # Copy config.mk, always override the file
-          if Sys.isapple()
-            `cp make/osx.mk config.mk`
-          else
-            `cp make/config.mk config.mk`
-          end
-
-          # Configure OpenCV
-          `sed -i -s 's/USE_OPENCV = 1/USE_OPENCV = 0/' config.mk`
-
-          # Configure CUDA
-          if HAS_CUDA
-            @build_steps begin
-              `sed -i -s 's/USE_CUDA = 0/USE_CUDA = 1/' config.mk`
-              # address https://github.com/apache/incubator-mxnet/pull/7856
-              `sed -i -s "s/ADD_LDFLAGS =\(.*\)/ADD_LDFLAGS =\1 -lcublas -lcusolver -lcurand -lcudart/" config.mk`
-              if haskey(ENV, "CUDA_HOME")
-                `sed -i -s "s@USE_CUDA_PATH = NONE@USE_CUDA_PATH = $(ENV["CUDA_HOME"])@" config.mk`
-              end
-              if haskey(ENV, "CUDA_HOME")
-                # address https://github.com/apache/incubator-mxnet/pull/7838
-                flag = "-L$(ENV["CUDA_HOME"])/lib64 -L$(ENV["CUDA_HOME"])/lib"
-                `sed -i -s "s@ADD_LDFLAGS =\(.*\)@ADD_LDFLAGS =\1 $flag@" config.mk`
-              end
-              if HAS_CUDNN
-                `sed -i -s 's/USE_CUDNN = 0/USE_CUDNN = 1/' config.mk`
-              end
-            end
-          end
-
-          # Force enable LAPACK build
-          # Julia's OpenBLAS has LAPACK functionality already
-          if FORCE_LAPACK
-            if Sys.isapple()
-              MSHADOW_LDFLAGS *= " -framework Accelerate"
-            end
-            `sed -i -s 's/ADD_CFLAGS =\(.*\)/ADD_CFLAGS =\1 -DMXNET_USE_LAPACK/' config.mk`
-          end
-
-          # propagate more build flags from ENV
-          if CC != nothing
-            `sed -i -s "s@^export CC =\(.*\)@export CC = $CC@" config.mk`
-          end
-          if CXX != nothing
-            `sed -i -s "s@^export CXX =\(.*\)@export CXX = $CXX@" config.mk`
-          end
-          if ADD_CFLAGS != nothing
-            `sed -i -s "s@ADD_CFLAGS =\(.*\)@ADD_CFLAGS =\1 $ADD_CFLAGS@" config.mk`
-          end
-          if ADD_LDFLAGS != nothing
-            `sed -i -s "s@ADD_LDFLAGS =\(.*\)@ADD_LDFLAGS =\1 $ADD_LDFLAGS@" config.mk`
-          end
-          if USE_JEMALLOC != nothing
-            `sed -i -s "s@USE_JEMALLOC =\(.*\)@USE_JEMALLOC = $USE_JEMALLOC@" config.mk`
-          end
-
-          if USE_JULIA_BLAS
-            `make -j$(get_cpucore()) USE_BLAS=$blas_name $MSHADOW_LDFLAGS`
-          else
-            `make -j$(get_cpucore())`
-          end
+          `cp -f -v julia/deps/include/cblas.h include/cblas.h`
+        end
+        @build_steps begin
+          ChangeDirectory(_blddir)
+          `$cmake
+            -DCMAKE_BUILD_TYPE=$(libmxnet_curr_ver == "master" ? "Debug" : "Release")
+            -DUSE_BLAS=$blas_name
+            -DUSE_OPENCV=$(cmake_bool(false))
+            -DUSE_CUDA=$(cmake_bool(HAS_CUDA))
+            -DUSE_CUDNN=$(cmake_bool(HAS_CUDNN))
+            $(cmake_jemalloc(USE_JEMALLOC))
+            $(cmake_cuda_path(get(ENV, "CUDA_HOME", nothing)))
+            $(cmake_jl_blas(USE_JULIA_BLAS, blas_path))
+            $_mxdir`
+          `make -j$(get_cpucore()) VERBOSE=$(Int(libmxnet_curr_ver == "master"))`
         end
-        FileRule(joinpath(_libdir, "libmxnet.$(Libdl.dlext)"), @build_steps begin
-          # the output file on macos is still in `.so` suffix
-          # so we rename it
-          `cp $_mxdir/lib/libmxnet.so $_libdir/libmxnet.$(Libdl.dlext)`
+        FileRule(joinpath(_blddir, "libmxnet.$(Libdl.dlext)"), @build_steps begin
+          # the output file on macos is still in `.so` suffix,
+          # so we create a soft link for it.
+          `ln -s libmxnet.so $_blddir/libmxnet.$(Libdl.dlext)`
         end)
       end
-    end), mxnet, installed_libpath=_libdir)
+    end), mxnet, installed_libpath=_blddir)
 
   @BinDeps.install Dict(:mxnet => :mxnet)
 end

julia/deps/cblas.h → julia/deps/include/cblas.h

@@ -52,7 +52,7 @@ typedef long BLASLONG;
 typedef unsigned long BLASULONG;
 #endif
 
-#ifdef INTERFACE64
+#ifdef OPENBLAS_INTERFACE64
 typedef BLASLONG blasint;
 #else
 typedef int blasint;
@@ -74,7 +74,7 @@ typedef int blasint;
   typedef struct { double real, imag; } openblas_complex_double;
 #endif
 
-#ifdef INTERFACE64
+#ifdef OPENBLAS_INTERFACE64
 # define cblas_sdsdot cblas_sdsdot64_
 # define cblas_dsdot cblas_dsdot64_
 # define cblas_sdot cblas_sdot64_
@@ -240,7 +240,7 @@ typedef int blasint;
 # define cblas_dgeadd cblas_dgeadd64_
 # define cblas_cgeadd cblas_cgeadd64_
 # define cblas_zgeadd cblas_zgeadd64_
-#endif
+#endif  // OPENBLAS_INTERFACE64
 
 #define CBLAS_INDEX size_t
 

julia/docs/src/user-guide/install.md

@@ -50,11 +50,12 @@ There are several environment variables that change this behaviour.
   (e.g. `a0b1c2d3`).
 - `CC`: The path of C compiler.
 - `CXX`: The path of C++ compiler.
-- `ADD_CFLAGS`: Additional C flags. For instance,
+- `CFLAGS`: Additional C flags. For instance,
   if you need to point non-standard include directory, please set it as
-  `ENV["ADD_CFLAGS"] = "-I'/path/to/include/dir'"`.
-- `ADD_LDFLAGS`: Additional linker flags.
-- `USE_JEMALLOC`: Default is enabled if jemalloc available.
+  `ENV["CFLAGS"] = "-I'/path/to/include/dir'"`.
+- `LDFLAGS`: Additional linker flags.
+- `USE_JEMALLOC`: Set it to `ON` or `OFF`.
+  Default is enabled if jemalloc available.
   If you ran into segfault cause by jemalloc,
   Please try to disable it.
 

julia/src/base.jl

@@ -40,23 +40,40 @@ const grad_req_map = Dict{Symbol,GRAD_REQ}(
 ################################################################################
 # Initialization and library API entrance
 ################################################################################
-const MXNET_LIB = Libdl.find_library(["libmxnet.$(Libdl.dlext)", "libmxnet.so"],  # see build.jl
-                                     [joinpath(get(ENV, "MXNET_HOME", ""), "lib"),
-                                      get(ENV, "MXNET_HOME", ""),
-                                      joinpath(@__DIR__, "..",
-                                               "deps", "usr", "lib")])
-const LIB_VERSION = Ref{Cint}(0)
+function _get_search_names()
+  MXNET_LIBRARY_PATH = get(ENV, "MXNET_LIBRARY_PATH", "")
+  A = ["libmxnet.$(Libdl.dlext)", "libmxnet.so"]  # see build.jl
+  if !isempty(MXNET_LIBRARY_PATH)
+    !isabspath(MXNET_LIBRARY_PATH) && error("MXNET_LIBRARY_PATH should be a absolute path")
+    pushfirst!(A, MXNET_LIBRARY_PATH)
+  end
+  A
+end
+
+function _get_search_dirs()
+  # TODO: remove MXNET_HOME backward compatibility in v2.0
+  if haskey(ENV, "MXNET_HOME")
+    @warn "The environment variable `MXNET_HOME` has been renamed, please use `MXNET_ROOT` instead."
+  end
+  A = [joinpath(@__DIR__, "..", "deps", "build")]
+  MXNET_ROOT = get(ENV, "MXNET_ROOT", get(ENV, "MXNET_HOME", ""))
+  if !isempty(MXNET_ROOT)
+    !isabspath(MXNET_ROOT) && error("MXNET_ROOT should be a absolute path")
+    prepend!(A, [joinpath(MXNET_ROOT, "lib"), MXNET_ROOT])
+  end
+  A
+end
+
+const MXNET_LIB   = Libdl.find_library(_get_search_names(), _get_search_dirs())
+const LIB_VERSION = Ref{Cint}(C_NULL)
 
 if isempty(MXNET_LIB)
-  # touch this file, so that after the user properly build libmxnet, the precompiled
-  # MXNet.ji will be re-compiled to get MXNET_LIB properly.
-  touch(@__FILE__)
   error("Cannot find or load libmxnet.$(Libdl.dlext). " *
         "Please see the document on how to build it.")
-else
-  include_dependency(MXNET_LIB)
 end
 
+include_dependency(MXNET_LIB)
+
 function __init__()
   # TODO: bug in nnvm, if do not call this, call get handle "_copyto" will fail
   _get_libmx_op_names()
@@ -65,12 +82,12 @@ function __init__()
 
   atexit() do
     # notify libmxnet we are shutting down
-    ccall( ("MXNotifyShutdown", MXNET_LIB), Cint, () )
+    ccall(("MXNotifyShutdown", MXNET_LIB), Cint, ())
   end
 end
 
 function mx_get_last_error()
-  msg = ccall( ("MXGetLastError", MXNET_LIB), char_p, () )
+  msg = ccall(("MXGetLastError", MXNET_LIB), char_p, ())
   if msg == C_NULL
     throw(MXError("Failed to get last error message"))
   end