From 298b6eca9d31253d65d38419bb98cec6f6b9f9a7 Mon Sep 17 00:00:00 2001 From: Iblis Lin Date: Sun, 20 Jan 2019 10:01:21 +0800 Subject: [PATCH] Julia: add windows-cpu build - Julia v0.7 - Julia v1.0 --- ci/jenkins/Jenkins_steps.groovy | 28 +++++++++++++++ ci/jenkins/Jenkinsfile_windows_cpu | 8 +++-- ci/windows/test_jl07_cpu.ps1 | 54 ++++++++++++++++++++++++++++ ci/windows/test_jl10_cpu.ps1 | 54 ++++++++++++++++++++++++++++ julia/appveyor.yml | 56 ------------------------------ julia/deps/build.jl | 2 +- julia/src/util.jl | 6 ++-- 7 files changed, 145 insertions(+), 63 deletions(-) create mode 100644 ci/windows/test_jl07_cpu.ps1 create mode 100644 ci/windows/test_jl10_cpu.ps1 delete mode 100644 julia/appveyor.yml diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 8b0e83e1ab19..a6cc2b2d32dc 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -1169,6 +1169,34 @@ def test_windows_python3_cpu() { }] } +def test_windows_julia07_cpu() { + return ['Julia 0.7: CPU Win': { + node(NODE_WINDOWS_CPU) { + ws('workspace/ut-julia07-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git_win() + unstash 'windows_package_cpu' + powershell 'ci/windows/test_jl07_cpu.ps1' + } + } + } + }] +} + +def test_windows_julia10_cpu() { + return ['Julia 1.0: CPU Win': { + node(NODE_WINDOWS_CPU) { + ws('workspace/ut-julia10-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git_win() + unstash 'windows_package_cpu' + powershell 'ci/windows/test_jl10_cpu.ps1' + } + } + } + }] +} + def test_qemu_armv7_cpu() { return ['ARMv7 QEMU': { node(NODE_LINUX_CPU) { diff --git a/ci/jenkins/Jenkinsfile_windows_cpu b/ci/jenkins/Jenkinsfile_windows_cpu index a8746db73d34..5bc40d625930 100644 --- a/ci/jenkins/Jenkinsfile_windows_cpu +++ b/ci/jenkins/Jenkinsfile_windows_cpu @@ -35,12 +35,14 @@ utils.main_wrapper( core_logic: { utils.parallel_stage('Build', [ custom_steps.compile_windows_cpu() - ]) + ]) utils.parallel_stage('Tests', [ custom_steps.test_windows_python2_cpu(), - custom_steps.test_windows_python3_cpu() - ]) + custom_steps.test_windows_python3_cpu(), + custom_steps.test_windows_julia07_cpu(), + custom_steps.test_windows_julia10_cpu() + ]) } , failure_handler: { diff --git a/ci/windows/test_jl07_cpu.ps1 b/ci/windows/test_jl07_cpu.ps1 new file mode 100644 index 000000000000..85ff223ba4ac --- /dev/null +++ b/ci/windows/test_jl07_cpu.ps1 @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +7z x -y windows_package.7z + +# set default output encoding to utf8 +$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8' + +$env:MXNET_HOME = [System.IO.Path]::GetFullPath('.\windows_package') +$env:JULIA_URL = "https://julialang-s3.julialang.org/bin/winnt/x64/0.7/julia-0.7.0-win64.exe" +$env:JULIA_DEPOT_PATH = [System.IO.Path]::GetFullPath('.\julia-depot') + +$JULIA_DIR = [System.IO.Path]::GetFullPath('.\julia07') +$JULIA = "$JULIA_DIR\bin\julia" + +# Download most recent Julia Windows binary +[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12 +(New-Object System.Net.WebClient).DownloadFile($env:JULIA_URL, "julia-binary.exe") +if (! $?) { Throw ("Error on downloading Julia Windows binary") } + +# Run installer silently, output to C:\julia07\julia +Start-Process -Wait "julia-binary.exe" -ArgumentList "/S /D=$JULIA_DIR" +if (! $?) { Throw ("Error on installing Julia") } + +& $JULIA -e "using InteractiveUtils; versioninfo()" + +# workaround a bug of Julia 0.7 +# use PackageSpec(name = "MXNet", path = "...") if we drop the 0.7 support +Copy-Item -Path .\julia -Recurse -Destination "$env:JULIA_DEPOT_PATH\dev\MXNet" + +$src=' + using Pkg + Pkg.develop("MXNet") + Pkg.build("MXNet") + Pkg.test("MXNet") +' + +$src > .\ci-build.jl +& $JULIA .\ci-build.jl +if (! $?) { Throw ("Error") } diff --git a/ci/windows/test_jl10_cpu.ps1 b/ci/windows/test_jl10_cpu.ps1 new file mode 100644 index 000000000000..4dce8276cdf1 --- /dev/null +++ b/ci/windows/test_jl10_cpu.ps1 @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +7z x -y windows_package.7z + +# set default output encoding to utf8 +$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8' + +$env:MXNET_HOME = [System.IO.Path]::GetFullPath('.\windows_package') +$env:JULIA_URL = "https://julialang-s3.julialang.org/bin/winnt/x64/1.0/julia-1.0.3-win64.exe" +$env:JULIA_DEPOT_PATH = [System.IO.Path]::GetFullPath('.\julia-depot') + +$JULIA_DIR = [System.IO.Path]::GetFullPath('.\julia10') +$JULIA = "$JULIA_DIR\bin\julia" + +# Download most recent Julia Windows binary +[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12 +(New-Object System.Net.WebClient).DownloadFile($env:JULIA_URL, "julia-binary.exe") +if (! $?) { Throw ("Error on downloading Julia Windows binary") } + +# Run installer silently, output to C:\julia10\julia +Start-Process -Wait "julia-binary.exe" -ArgumentList "/S /D=$JULIA_DIR" +if (! $?) { Throw ("Error on installing Julia") } + +& $JULIA -e "using InteractiveUtils; versioninfo()" + +# workaround a bug of Julia 0.7 +# use PackageSpec(name = "MXNet", path = "...") if we drop the 0.7 support +Copy-Item -Path .\julia -Recurse -Destination "$env:JULIA_DEPOT_PATH\dev\MXNet" + +$src=' + using Pkg + Pkg.develop("MXNet") + Pkg.build("MXNet") + Pkg.test("MXNet") +' + +$src > .\ci-build.jl +& $JULIA .\ci-build.jl +if (! $?) { Throw ("Error") } diff --git a/julia/appveyor.yml b/julia/appveyor.yml deleted file mode 100644 index 50e275cfa8a8..000000000000 --- a/julia/appveyor.yml +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -environment: - matrix: - - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe" - -branches: - only: - - master - - stable - -notifications: - - provider: Email - on_build_success: false - on_build_failure: false - on_build_status_changed: false - -install: - - ps: "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12" -# If there's a newer build queued for the same PR, cancel this one - - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` - https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` - Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` - throw "There are newer queued builds for this pull request, failing early." } - -# Download most recent Julia Windows binary - - ps: (new-object net.webclient).DownloadFile( - $env:JULIA_URL, - "C:\projects\julia-binary.exe") -# Run installer silently, output to C:\projects\julia - - C:\projects\julia-binary.exe /S /D=C:\projects\julia - -build_script: -# Need to convert from shallow to complete for Pkg.clone to work - - IF EXIST .git\shallow (git fetch --unshallow) - - C:\projects\julia\bin\julia -e "versioninfo(); - Pkg.clone(pwd(), \"MXNet\"); Pkg.build(\"MXNet\")" - -test_script: - - C:\projects\julia\bin\julia --check-bounds=yes -e "Pkg.test(\"MXNet\")" - diff --git a/julia/deps/build.jl b/julia/deps/build.jl index 7a37803f306a..9a719be850f5 100644 --- a/julia/deps/build.jl +++ b/julia/deps/build.jl @@ -86,7 +86,7 @@ if HAS_CUDA if HAS_CUDNN @info("Found a CuDNN installation.") end - @info("CUDA_HOME -> $(get(ENV, "CUDA_HOME", nothing))") + @info("CUDA_HOME -> $(get(ENV, "CUDA_HOME", "nothing"))") else @info("Did not find a CUDA installation, using CPU-only version of MXNet.") end diff --git a/julia/src/util.jl b/julia/src/util.jl index ae2e50b92c55..ac7f4fc71653 100644 --- a/julia/src/util.jl +++ b/julia/src/util.jl @@ -35,12 +35,12 @@ function get_mnist_ubyte() filenames = Dict((x[1] => joinpath(mnist_dir, x[2]) for x ∈ pairs(filenames))) if !all(isfile, values(filenames)) cd(mnist_dir) do - mnist_dir = download("http://data.mxnet.io/mxnet/data/mnist.zip", "mnist.zip") + data = download("http://data.mxnet.io/mxnet/data/mnist.zip", "mnist.zip") try - run(`unzip -u $mnist_dir`) + run(`unzip -u $data`) catch try - run(pipe(`7z x $mnist_dir`,stdout = devnull)) + run(pipeline(`7z x $data`,stdout = devnull)) catch error("Extraction Failed:No extraction program found in path") end