Julia v0.7/1.0 support and drop v0.6 support (#12845)

ci/docker/install/ubuntu_julia.sh

@@ -22,16 +22,22 @@
 
 set -ex
 
-export JLBINARY='julia.tar.gz'
-export JULIADIR='/work/julia'
-export JULIA="${JULIADIR}/bin/julia"
+function install_julia() {
+    local suffix=`echo $1 | sed 's/\.//'`  # 0.7 -> 07; 1.0 -> 10
+    local JLBINARY="julia-$1.tar.gz"
+    local JULIADIR="/work/julia$suffix"
+    local JULIA="${JULIADIR}/bin/julia"
 
-mkdir -p $JULIADIR
-# The julia version in Ubuntu repo is too old
-# We download the tarball from the official link:
-#   https://julialang.org/downloads/
-wget -O $JLBINARY https://julialang-s3.julialang.org/bin/linux/x64/0.6/julia-0.6.2-linux-x86_64.tar.gz
-tar xzvf $JLBINARY -C $JULIADIR --strip 1
-rm $JLBINARY
+    mkdir -p $JULIADIR
+    # The julia version in Ubuntu repo is too old
+    # We download the tarball from the official link:
+    #   https://julialang.org/downloads/
+    wget -O $JLBINARY https://julialang-s3.julialang.org/bin/linux/x64/$1/julia-$2-linux-x86_64.tar.gz
+    tar xzvf $JLBINARY -C $JULIADIR --strip 1
+    rm $JLBINARY
 
-$JULIA -e 'versioninfo()'
+    $JULIA -e 'using InteractiveUtils; versioninfo()'
+}
+
+install_julia 0.7 0.7.0
+install_julia 1.0 1.0.3

ci/docker/runtime_functions.sh

@@ -907,36 +907,46 @@ unittest_ubuntu_gpu_R() {
     make rpkgtest R_LIBS=/tmp/r-site-library R_GPU_ENABLE=1
 }
 
-unittest_ubuntu_cpu_julia06() {
+unittest_ubuntu_cpu_julia() {
     set -ex
-    export PATH="/work/julia/bin:$PATH"
+    export PATH="$1/bin:$PATH"
     export MXNET_HOME='/work/mxnet'
-    export JULIA_PKGDIR='/work/julia-pkg'
-    export DEPDIR=`julia -e 'print(Pkg.dir())'`
+    export JULIA_DEPOT_PATH='/work/julia-depot'
+    export DEVDIR="$JULIA_DEPOT_PATH/dev"
 
-    julia -e 'versioninfo()'
-    julia -e 'Pkg.init()'
+    julia -e 'using InteractiveUtils; versioninfo()'
 
     # install package
-    ln -sf ${MXNET_HOME}/julia ${DEPDIR}/MXNet
+    mkdir -p $DEVDIR
+    ln -sf ${MXNET_HOME}/julia ${DEVDIR}/MXNet
 
-    # install dependencies
-    julia -e 'Pkg.resolve()'
+    # register MXNet.jl and dependencies
+    julia -e 'using Pkg; Pkg.develop("MXNet")'
 
     # FIXME
     export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so'
     export LD_LIBRARY_PATH=/work/mxnet/lib:$LD_LIBRARY_PATH
 
     # use the prebuilt binary from $MXNET_HOME/lib
-    julia -e 'Pkg.build("MXNet")'
+    julia -e 'using Pkg; Pkg.build("MXNet")'
 
     # run the script `julia/test/runtests.jl`
-    julia -e 'Pkg.test("MXNet")'
+    julia -e 'using Pkg; Pkg.test("MXNet")'
 
     # See https://github.com/dmlc/MXNet.jl/pull/303#issuecomment-341171774
     julia -e 'using MXNet; mx._sig_checker()'
 }
 
+unittest_ubuntu_cpu_julia07() {
+    set -ex
+    unittest_ubuntu_cpu_julia /work/julia07
+}
+
+unittest_ubuntu_cpu_julia10() {
+    set -ex
+    unittest_ubuntu_cpu_julia /work/julia10
+}
+
 unittest_centos7_cpu() {
     set -ex
     cd /work/mxnet
@@ -1248,25 +1258,19 @@ deploy_docs() {
 
 deploy_jl_docs() {
     set -ex
-    export PATH="/work/julia/bin:$PATH"
+    export PATH="/work/julia10/bin:$PATH"
     export MXNET_HOME='/work/mxnet'
-    export JULIA_PKGDIR='/work/julia-pkg'
-    export DEPDIR=`julia -e 'print(Pkg.dir())'`
+    export JULIA_DEPOT_PATH='/work/julia-depot'
+    export DEVDIR="$JULIA_DEPOT_PATH/dev"
 
-    julia -e 'versioninfo()'
-    julia -e 'Pkg.init()'
-    ln -sf ${MXNET_HOME}/julia ${DEPDIR}/MXNet
-    julia -e 'Pkg.resolve()'
+    julia -e 'using InteractiveUtils; versioninfo()'
+    mkdir -p $DEVDIR
 
     # FIXME
     export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so'
     export LD_LIBRARY_PATH=/work/mxnet/lib:$LD_LIBRARY_PATH
 
-    # use the prebuilt binary from $MXNET_HOME/lib
-    julia -e 'Pkg.build("MXNet")'
-    # build docs
-    julia -e 'Pkg.add("Documenter")'
-    julia -e 'cd(Pkg.dir("MXNet")); include(joinpath("docs", "make.jl"))'
+    make -C julia/docs
 
     # TODO: make Jenkins worker push to MXNet.jl ph-pages branch if master build
     # ...

ci/jenkins/Jenkins_steps.groovy

@@ -944,13 +944,26 @@ def test_unix_r_gpu() {
     }]
 }
 
-def test_unix_julia_cpu() {
-    return ['Julia 0.6: CPU': {
+def test_unix_julia07_cpu() {
+    return ['Julia 0.7: CPU': {
       node(NODE_LINUX_CPU) {
-        ws('workspace/ut-julia06-cpu') {
+        ws('workspace/ut-julia07-cpu') {
           timeout(time: max_time, unit: 'MINUTES') {
             utils.unpack_and_init('cpu', mx_lib)
-            utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_julia06', false)
+            utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_julia07', false)
+          }
+        }
+      }
+    }]
+}
+
+def test_unix_julia10_cpu() {
+    return ['Julia 1.0: CPU': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/ut-julia10-cpu') {
+          timeout(time: max_time, unit: 'MINUTES') {
+            utils.unpack_and_init('cpu', mx_lib)
+            utils.docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_julia10', false)
           }
         }
       }

ci/jenkins/Jenkinsfile_unix_cpu

@@ -39,7 +39,7 @@ core_logic: {
     custom_steps.compile_unix_mkl_cpu(),
     custom_steps.compile_unix_mkldnn_cpu(),
     custom_steps.compile_unix_mkldnn_mkl_cpu()
-  ]) 
+  ])
 
   utils.parallel_stage('Tests', [
     custom_steps.test_unix_python2_cpu(),
@@ -53,7 +53,8 @@ core_logic: {
     custom_steps.test_unix_scala_mkldnn_cpu(),
     custom_steps.test_unix_clojure_cpu(),
     custom_steps.test_unix_r_cpu(),
-    custom_steps.test_unix_julia_cpu(),
+    custom_steps.test_unix_julia07_cpu(),
+    custom_steps.test_unix_julia10_cpu(),
     custom_steps.test_unix_onnx_cpu(),
     custom_steps.test_unix_cpp_cpu(),
     /*  Disabled due to master build failure:

julia/.gitignore

@@ -7,7 +7,4 @@ data
 deps/src
 deps/usr
 deps/deps.jl
-docs/_build
-docs/build/
-docs/site/
 .vscode

julia/README.md

@@ -50,7 +50,7 @@ labels = reduce(
 labels .= labels .+ 1
 
 # Now we use compute the accuracy
-pred = map(i -> indmax(probs[1:10, i]), 1:size(probs, 2))
+pred = map(i -> argmax(probs[1:10, i]), 1:size(probs, 2))
 correct = sum(pred .== labels)
 accuracy = 100correct/length(labels)
 @printf "Accuracy on eval set: %.2f%%\n" accuracy

julia/REQUIRE

@@ -3,5 +3,4 @@ Formatting
 BinDeps
 JSON
 MacroTools
-TakingBroadcastSeriously
 Reexport

julia/deps/build.jl

@@ -15,7 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import JSON
+using JSON
+using Libdl
+using LinearAlgebra
 
 ################################################################################
 # First try to detect and load existing libmxnet
@@ -26,65 +28,67 @@ curr_win = "20180211"  # v1.1.0
 
 if haskey(ENV, "MXNET_HOME")
   MXNET_HOME = ENV["MXNET_HOME"]
-  info("MXNET_HOME environment detected: $MXNET_HOME")
-  info("Trying to load existing libmxnet...")
+  @info("MXNET_HOME environment detected: $MXNET_HOME")
+  @info("Trying to load existing libmxnet...")
   # In case of macOS, if user build libmxnet from source and set the MXNET_HOME,
   # the output is still named as `libmxnet.so`.
   lib = Libdl.find_library(["libmxnet.$(Libdl.dlext)", "libmxnet.so"],
                            [joinpath(MXNET_HOME, "lib"), MXNET_HOME])
   if !isempty(lib)
-    info("Existing libmxnet detected at $lib, skip building...")
+    @info("Existing libmxnet detected at $lib, skip building...")
     libmxnet_detected = true
   else
-    info("Failed to load existing libmxnet, trying to build from source...")
+    @info("Failed to load existing libmxnet, trying to build from source...")
   end
 end
 
 # Try to find cuda
 CUDAPATHS = String[]
 if haskey(ENV, "CUDA_HOME")
   push!(CUDAPATHS, joinpath(ENV["CUDA_HOME"], "lib64"))
-elseif is_linux()
+elseif Sys.islinux()
   append!(CUDAPATHS, ["/opt/cuda/lib64", "/usr/local/cuda/lib64"])
 end
 
-if is_unix()
+if Sys.isunix()
   try
-    push!(CUDAPATHS, replace(strip(readstring(`which nvcc`)), "bin/nvcc", "lib64"))
+    push!(CUDAPATHS, replace(strip(read(`which nvcc`, String)), "bin/nvcc", "lib64"))
+  catch
   end
 end
 
 HAS_CUDA = false
 HAS_CUDNN = false
 let cudalib = Libdl.find_library(["libcuda", "nvcuda.dll"], CUDAPATHS)
-  HAS_CUDA = !isempty(cudalib) && Libdl.dlopen_e(cudalib) != C_NULL
+  global HAS_CUDA = !isempty(cudalib) && Libdl.dlopen_e(cudalib) != C_NULL
 end
 
-if !HAS_CUDA && is_windows()
+if !HAS_CUDA && Sys.iswindows()
   # TODO: this needs to be improved.
   try
     run(`nvcc --version`)
-    HAS_CUDA = true
+    global HAS_CUDA = true
+  catch
   end
 end
 
 if HAS_CUDA  # then check cudnn
   let cudnnlib = Libdl.find_library("libcudnn", CUDAPATHS)
-    HAS_CUDNN = !isempty(cudnnlib) && Libdl.dlopen_e(cudnnlib) != C_NULL
+    global HAS_CUDNN = !isempty(cudnnlib) && Libdl.dlopen_e(cudnnlib) != C_NULL
     if HAS_CUDNN && !haskey(ENV, "CUDA_HOME")  # inference `CUDA_HOME`
       ENV["CUDA_HOME"] = dirname(dirname(cudnnlib))
     end
   end
 end
 
 if HAS_CUDA
-  info("Found a CUDA installation.")
+  @info("Found a CUDA installation.")
   if HAS_CUDNN
-    info("Found a CuDNN installation.")
+    @info("Found a CuDNN installation.")
   end
-  info("CUDA_HOME -> $(get(ENV, "CUDA_HOME", nothing))")
+  @info("CUDA_HOME -> $(get(ENV, "CUDA_HOME", nothing))")
 else
-  info("Did not find a CUDA installation, using CPU-only version of MXNet.")
+  @info("Did not find a CUDA installation, using CPU-only version of MXNet.")
 end
 
 # propagate more build flags from ENV
@@ -98,26 +102,26 @@ function get_cpucore()
     if haskey(ENV, "TRAVIS")  # on travis-ci
         2
     else
-        min(Sys.CPU_CORES, 32)
+        min(Sys.CPU_THREADS, 32)
     end
 end
 
 using BinDeps
 @BinDeps.setup
 if !libmxnet_detected
-  if is_windows()
+  if Sys.iswindows()
     if Sys.ARCH != :x86_64
-      info("Prebuilt windows binaries are only available on 64bit. You will have to built MXNet yourself.")
+      @info("Prebuilt windows binaries are only available on 64bit. You will have to built MXNet yourself.")
       return
     end
-    info("Downloading pre-built packages for Windows.")
+    @info("Downloading pre-built packages for Windows.")
     base_url = "https://github.com/yajiedesign/mxnet/releases/download/weekly_binary_build_v2/prebuildbase_win10_x64_vc14_v2.7z"
 
     if libmxnet_curr_ver == "master"
       # download_cmd uses powershell 2, but we need powershell 3 to do this
       run(`powershell -NoProfile -Command Invoke-WebRequest -Uri "https://api.github.com/repos/yajiedesign/mxnet/releases/latest" -OutFile "mxnet.json"`)
       curr_win = JSON.parsefile("mxnet.json")["tag_name"]
-      info("Can't use MXNet master on Windows, using latest binaries from $curr_win.")
+      @info("Can't use MXNet master on Windows, using latest binaries from $curr_win.")
     end
     # TODO: Get url from JSON.
     name = "mxnet_x64_vc14_$(HAS_CUDA ? "gpu" : "cpu").7z"
@@ -144,8 +148,7 @@ if !libmxnet_detected
   ################################################################################
 
   blas_path = Libdl.dlpath(Libdl.dlopen(Base.libblas_name))
-
-  blas_vendor = Base.BLAS.vendor()
+  blas_vendor = LinearAlgebra.BLAS.vendor()
 
   ilp64 = ""
   if blas_vendor == :openblas64
@@ -154,18 +157,18 @@ if !libmxnet_detected
 
   FORCE_LAPACK = false
   if blas_vendor == :unknown
-    info("Julia is built with an unkown blas library ($blas_path).")
-    info("Attempting build without reusing the blas library")
+    @info("Julia is built with an unkown blas library ($blas_path).")
+    @info("Attempting build without reusing the blas library")
     USE_JULIA_BLAS = false
   elseif !(blas_vendor in (:openblas, :openblas64))
-    info("Unsure if we can build against $blas_vendor.")
-    info("Attempting build anyway.")
+    @info("Unsure if we can build against $blas_vendor.")
+    @info("Attempting build anyway.")
     USE_JULIA_BLAS = true
   else
     USE_JULIA_BLAS = true
     FORCE_LAPACK = true
   end
-  info("USE_JULIA_BLAS -> $USE_JULIA_BLAS")
+  @info("USE_JULIA_BLAS -> $USE_JULIA_BLAS")
 
   blas_name = blas_vendor == :openblas64 ? "openblas" : string(blas_vendor)
   MSHADOW_LDFLAGS = "MSHADOW_LDFLAGS=-lm $blas_path"
@@ -188,7 +191,7 @@ if !libmxnet_detected
       @build_steps begin
         BinDeps.DirectoryRule(_mxdir, @build_steps begin
           ChangeDirectory(_srcdir)
-          `git clone https://github.com/apache/incubator-mxnet`
+          `git clone https://github.com/apache/incubator-mxnet mxnet`
         end)
         @build_steps begin
           ChangeDirectory(_mxdir)
@@ -199,15 +202,12 @@ if !libmxnet_detected
             `git checkout origin/$libmxnet_curr_ver`
           end
           `git submodule update --init --recursive`
-          `git -C mshadow checkout -- make/mshadow.mk`
-
-          # copying on changed, make travis caching happy
-          `../../cpcblas.sh`
-
-          `sed -i -s "s/MSHADOW_CFLAGS = \(.*\)/MSHADOW_CFLAGS = \1 $ilp64/" mshadow/make/mshadow.mk`
+          `git -C 3rdparty/mshadow checkout -- make/mshadow.mk`
+          `cp -v ../../cblas.h include/cblas.h`
+          `sed -i -s "s/MSHADOW_CFLAGS = \(.*\)/MSHADOW_CFLAGS = \1 $ilp64/" 3rdparty/mshadow/make/mshadow.mk`
 
           # Copy config.mk, always override the file
-          if is_apple()
+          if Sys.isapple()
             `cp make/osx.mk config.mk`
           else
             `cp make/config.mk config.mk`
@@ -239,7 +239,7 @@ if !libmxnet_detected
           # Force enable LAPACK build
           # Julia's OpenBLAS has LAPACK functionality already
           if FORCE_LAPACK
-            if is_apple()
+            if Sys.isapple()
               MSHADOW_LDFLAGS *= " -framework Accelerate"
             end
             `sed -i -s 's/ADD_CFLAGS =\(.*\)/ADD_CFLAGS =\1 -DMXNET_USE_LAPACK/' config.mk`