Julia: add windows-cpu build (apache#13937)

- Julia v0.7
- Julia v1.0

ci/docker/runtime_functions.sh

@@ -922,29 +922,21 @@ unittest_ubuntu_cpu_julia() {
     export PATH="$1/bin:$PATH"
     export MXNET_HOME='/work/mxnet'
     export JULIA_DEPOT_PATH='/work/julia-depot'
-    export DEVDIR="$JULIA_DEPOT_PATH/dev"
 
     julia -e 'using InteractiveUtils; versioninfo()'
 
-    # install package
-    mkdir -p $DEVDIR
-    ln -sf ${MXNET_HOME}/julia ${DEVDIR}/MXNet
-
-    # register MXNet.jl and dependencies
-    julia -e 'using Pkg; Pkg.develop("MXNet")'
-
     # FIXME
     export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so'
     export LD_LIBRARY_PATH=/work/mxnet/lib:$LD_LIBRARY_PATH
 
     # use the prebuilt binary from $MXNET_HOME/lib
-    julia -e 'using Pkg; Pkg.build("MXNet")'
+    julia --project=./julia -e 'using Pkg; Pkg.build("MXNet")'
 
     # run the script `julia/test/runtests.jl`
-    julia -e 'using Pkg; Pkg.test("MXNet")'
+    julia --project=./julia -e 'using Pkg; Pkg.test("MXNet")'
 
     # See https://github.com/dmlc/MXNet.jl/pull/303#issuecomment-341171774
-    julia -e 'using MXNet; mx._sig_checker()'
+    julia --project=./julia -e 'using MXNet; mx._sig_checker()'
 }
 
 unittest_ubuntu_cpu_julia07() {
@@ -1280,10 +1272,8 @@ deploy_jl_docs() {
     export PATH="/work/julia10/bin:$PATH"
     export MXNET_HOME='/work/mxnet'
     export JULIA_DEPOT_PATH='/work/julia-depot'
-    export DEVDIR="$JULIA_DEPOT_PATH/dev"
 
     julia -e 'using InteractiveUtils; versioninfo()'
-    mkdir -p $DEVDIR
 
     # FIXME
     export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so'

ci/jenkins/Jenkins_steps.groovy

@@ -1210,6 +1210,34 @@ def test_windows_python3_cpu() {
     }]
 }
 
+def test_windows_julia07_cpu() {
+    return ['Julia 0.7: CPU Win': {
+      node(NODE_WINDOWS_CPU) {
+        ws('workspace/ut-julia07-cpu') {
+          timeout(time: max_time, unit: 'MINUTES') {
+            utils.init_git_win()
+            unstash 'windows_package_cpu'
+            powershell 'ci/windows/test_jl07_cpu.ps1'
+          }
+        }
+      }
+    }]
+}
+
+def test_windows_julia10_cpu() {
+    return ['Julia 1.0: CPU Win': {
+      node(NODE_WINDOWS_CPU) {
+        ws('workspace/ut-julia10-cpu') {
+          timeout(time: max_time, unit: 'MINUTES') {
+            utils.init_git_win()
+            unstash 'windows_package_cpu'
+            powershell 'ci/windows/test_jl10_cpu.ps1'
+          }
+        }
+      }
+    }]
+}
+
 def test_qemu_armv7_cpu() {
     return ['ARMv7 QEMU': {
       node(NODE_LINUX_CPU) {

ci/jenkins/Jenkinsfile_windows_cpu

@@ -35,12 +35,14 @@ utils.main_wrapper(
 core_logic: {
   utils.parallel_stage('Build', [
     custom_steps.compile_windows_cpu()
-  ]) 
+  ])
 
   utils.parallel_stage('Tests', [
     custom_steps.test_windows_python2_cpu(),
-    custom_steps.test_windows_python3_cpu()
-  ]) 
+    custom_steps.test_windows_python3_cpu(),
+    custom_steps.test_windows_julia07_cpu(),
+    custom_steps.test_windows_julia10_cpu()
+  ])
 }
 ,
 failure_handler: {

ci/windows/test_jl07_cpu.ps1

@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+7z x -y windows_package.7z
+
+# set default output encoding to utf8
+$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8'
+
+$env:MXNET_HOME = [System.IO.Path]::GetFullPath('.\windows_package')
+$env:JULIA_URL = "https://julialang-s3.julialang.org/bin/winnt/x64/0.7/julia-0.7.0-win64.exe"
+$env:JULIA_DEPOT_PATH = [System.IO.Path]::GetFullPath('.\julia-depot')
+
+$JULIA_DIR = [System.IO.Path]::GetFullPath('.\julia07')
+$JULIA = "$JULIA_DIR\bin\julia"
+
+# Download most recent Julia Windows binary
+[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12
+(New-Object System.Net.WebClient).DownloadFile($env:JULIA_URL, "julia-binary.exe")
+if (! $?) { Throw ("Error on downloading Julia Windows binary") }
+
+# Run installer silently, output to C:\julia07\julia
+Start-Process -Wait "julia-binary.exe" -ArgumentList "/S /D=$JULIA_DIR"
+if (! $?) { Throw ("Error on installing Julia") }
+
+& $JULIA -e "using InteractiveUtils; versioninfo()"
+
+dir
+
+$src='
+    using Pkg
+    Pkg.activate(".\\julia")
+    Pkg.build()
+    Pkg.test()
+'
+
+$src > .\ci-build.jl
+
+# Redirect all stderr output to stdout,
+# since Julia loggers output stuffs to stderr.
+# Then, stderr triggers powershell NativeCommandError.
+& $JULIA .\ci-build.jl 2>&1 | %{ "$_" }
+if ($LastExitCode -eq 1) { Throw ("Error") }

ci/windows/test_jl10_cpu.ps1

@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+7z x -y windows_package.7z
+
+# set default output encoding to utf8
+$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8'
+
+$env:MXNET_HOME = [System.IO.Path]::GetFullPath('.\windows_package')
+$env:JULIA_URL = "https://julialang-s3.julialang.org/bin/winnt/x64/1.0/julia-1.0.3-win64.exe"
+$env:JULIA_DEPOT_PATH = [System.IO.Path]::GetFullPath('.\julia-depot')
+
+$JULIA_DIR = [System.IO.Path]::GetFullPath('.\julia10')
+$JULIA = "$JULIA_DIR\bin\julia"
+
+# Download most recent Julia Windows binary
+[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12
+(New-Object System.Net.WebClient).DownloadFile($env:JULIA_URL, "julia-binary.exe")
+if (! $?) { Throw ("Error on downloading Julia Windows binary") }
+
+# Run installer silently, output to C:\julia10\julia
+Start-Process -Wait "julia-binary.exe" -ArgumentList "/S /D=$JULIA_DIR"
+if (! $?) { Throw ("Error on installing Julia") }
+
+& $JULIA -e "using InteractiveUtils; versioninfo()"
+
+dir
+
+$src='
+    using Pkg
+    Pkg.activate(".\\julia")
+    Pkg.build()
+    Pkg.test()
+'
+
+$src > .\ci-build.jl
+
+# Redirect all stderr output to stdout,
+# since Julia loggers output stuffs to stderr.
+# Then, stderr triggers powershell NativeCommandError.
+& $JULIA .\ci-build.jl 2>&1 | %{ "$_" }
+if ($LastExitCode -eq 1) { Throw ("Error") }

julia/.gitignore

@@ -8,3 +8,4 @@ deps/src
 deps/usr
 deps/deps.jl
 .vscode
+/Manifest.toml

julia/Project.toml

@@ -0,0 +1,26 @@
+name = "MXNet"
+uuid = "a7949054-b901-59c6-b8e3-7238c29bf7f0"
+authors = ["Chiyuan Zhang <[email protected]>", "Valentin Churavy <[email protected]>", "Iblis Lin <[email protected]>"]
+version = "1.5.0"
+
+[deps]
+BinDeps = "9e28174c-4ba2-5203-b857-d8d62c4213ee"
+Formatting = "59287772-0a20-5a39-b81b-1366585eb4c0"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
+Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
+Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+
+[compat]
+julia = "≥0.7"
+
+[extras]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Test"]

julia/REQUIRE

@@ -1,4 +1,4 @@
-julia 0.6
+julia 0.7
 Formatting
 BinDeps
 JSON

julia/deps/build.jl

@@ -86,7 +86,7 @@ if HAS_CUDA
   if HAS_CUDNN
     @info("Found a CuDNN installation.")
   end
-  @info("CUDA_HOME -> $(get(ENV, "CUDA_HOME", nothing))")
+  @info("CUDA_HOME -> $(get(ENV, "CUDA_HOME", "nothing"))")
 else
   @info("Did not find a CUDA installation, using CPU-only version of MXNet.")
 end

julia/src/util.jl

@@ -35,12 +35,12 @@ function get_mnist_ubyte()
   filenames = Dict((x[1] => joinpath(mnist_dir, x[2]) for x ∈ pairs(filenames)))
   if !all(isfile, values(filenames))
     cd(mnist_dir) do
-      mnist_dir = download("http://data.mxnet.io/mxnet/data/mnist.zip", "mnist.zip")
+      data = download("http://data.mxnet.io/mxnet/data/mnist.zip", "mnist.zip")
         try
-          run(`unzip -u $mnist_dir`)
+          run(`unzip -u $data`)
         catch
           try
-            run(pipe(`7z x $mnist_dir`,stdout = devnull))
+            run(pipeline(`7z x $data`,stdout = devnull))
           catch
             error("Extraction Failed:No extraction program found in path")
           end

tests/nightly/apache_rat_license_check/rat-excludes

@@ -48,6 +48,7 @@ moderngpu/*
 deformable_im2col.cuh
 deformable_im2col.h
 REQUIRE
+Project.toml
 include/*
 .*.iml
 .*.json.ref

tools/license_header.py

@@ -91,6 +91,9 @@
 
                # Licensed under 2-Clause BSD in header
                'example/ssd/dataset/pycocotools/coco.py',
+
+               # Julia package metadata, generated by Pkg3.jl
+               'julia/Project.toml',
                ]
 
 # language extensions and the according commment mark