From 3d7bcfd9ec49187ee01a022343081b09b91dc6c9 Mon Sep 17 00:00:00 2001 From: Qing Date: Wed, 16 Jan 2019 21:16:50 -0800 Subject: [PATCH 1/7] add python unit test --- ci/docker/runtime_functions.sh | 8 ++++++++ ci/jenkins/Jenkins_steps.groovy | 13 +++++++++++++ ci/jenkins/Jenkinsfile_unix_cpu | 1 + ci/publish/python/build.sh | 26 ++++++++++++++++++++++++++ tools/staticbuild/build_wheel.sh | 5 +---- 5 files changed, 49 insertions(+), 4 deletions(-) create mode 100755 ci/publish/python/build.sh diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 4c70c1accd02..7d2720bf37ad 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1305,6 +1305,14 @@ build_scala_static_mkl() { popd } +publish_python_build() { + set -ex + pushd . + export mxnet_variant=mkl + ./ci/publish/python/build.sh + popd +} + publish_scala_build() { set -ex pushd . diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 9db3588a346d..953043e02c92 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -534,6 +534,19 @@ def test_static_scala_cpu() { }] } +def test_static_python_cpu() { + return ['Static build CPU 14.04 Python' : { + node(NODE_LINUX_CPU) { + ws('workspace/ut-publish-python-cpu') { + timeout(time: max_time, unit: 'MINUTES') { + utils.init_git() + utils.docker_run("publish.ubuntu1404_cpu", 'publish_python_build', false) + } + } + } + }] +} + def test_unix_python2_cpu() { return ['Python2: CPU': { node(NODE_LINUX_CPU) { diff --git a/ci/jenkins/Jenkinsfile_unix_cpu b/ci/jenkins/Jenkinsfile_unix_cpu index 9446348cd370..234a65b3fac5 100644 --- a/ci/jenkins/Jenkinsfile_unix_cpu +++ b/ci/jenkins/Jenkinsfile_unix_cpu @@ -60,6 +60,7 @@ core_logic: { custom_steps.test_unix_onnx_cpu(), custom_steps.test_unix_cpp_cpu(), custom_steps.test_static_scala_cpu(), + custom_steps.test_static_python_cpu(), /* Disabled due to master build failure: * http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/ * https://github.com/apache/incubator-mxnet/issues/11801 diff --git a/ci/publish/python/build.sh b/ci/publish/python/build.sh new file mode 100755 index 000000000000..61549896c0b7 --- /dev/null +++ b/ci/publish/python/build.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +source tools/staticbuild/build.sh $mxnet_variant pip + +set -ex + +# Compile tests for discovery later +source tools/staticbuild/build_wheel.sh diff --git a/tools/staticbuild/build_wheel.sh b/tools/staticbuild/build_wheel.sh index a79634117c21..6b7d59330e2d 100755 --- a/tools/staticbuild/build_wheel.sh +++ b/tools/staticbuild/build_wheel.sh @@ -18,10 +18,7 @@ # under the License. # This script builds the wheel for binary distribution and performs sanity check. - -cd mxnet-build -echo $(git rev-parse HEAD) >> python/mxnet/COMMIT_HASH -cd - +cd python/ # Make wheel for testing python setup.py bdist_wheel From 7f5b3c154cc452f690b78c127b2e424566b17c1a Mon Sep 17 00:00:00 2001 From: Qing Date: Wed, 16 Jan 2019 21:40:42 -0800 Subject: [PATCH 2/7] address comments --- ci/docker/runtime_functions.sh | 2 +- ci/jenkins/Jenkins_steps.groovy | 2 +- tools/staticbuild/build_wheel.sh | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 7d2720bf37ad..7a5192d8e5d3 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1305,7 +1305,7 @@ build_scala_static_mkl() { popd } -publish_python_build() { +build_static_python() { set -ex pushd . export mxnet_variant=mkl diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index 953043e02c92..ab035fdf95ca 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -540,7 +540,7 @@ def test_static_python_cpu() { ws('workspace/ut-publish-python-cpu') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.docker_run("publish.ubuntu1404_cpu", 'publish_python_build', false) + utils.docker_run("publish.ubuntu1404_cpu", 'build_static_python', false) } } } diff --git a/tools/staticbuild/build_wheel.sh b/tools/staticbuild/build_wheel.sh index 6b7d59330e2d..433a0a862603 100755 --- a/tools/staticbuild/build_wheel.sh +++ b/tools/staticbuild/build_wheel.sh @@ -18,6 +18,7 @@ # under the License. # This script builds the wheel for binary distribution and performs sanity check. +echo $(git rev-parse HEAD) >> python/mxnet/COMMIT_HASH cd python/ # Make wheel for testing From be87814d509afb2703495de8f39e33cf1056f6da Mon Sep 17 00:00:00 2001 From: Qing Date: Wed, 16 Jan 2019 22:30:15 -0800 Subject: [PATCH 3/7] switch sanity test to Gluon module test --- tools/staticbuild/build_wheel.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/staticbuild/build_wheel.sh b/tools/staticbuild/build_wheel.sh index 433a0a862603..d2cb927cf729 100755 --- a/tools/staticbuild/build_wheel.sh +++ b/tools/staticbuild/build_wheel.sh @@ -26,4 +26,5 @@ python setup.py bdist_wheel wheel_name=$(ls -t dist | head -n 1) pip install -U --user --force-reinstall dist/$wheel_name -python sanity_test.py +cd .. +python nosetests -v $(ls tests/python/unittest/test_gluon*.py | grep -v data | grep -v model_zoo | grep -v utils | grep -v rnn) From fc3c901e1cab05eced73c4f63a19f5d6aa08b01e Mon Sep 17 00:00:00 2001 From: Qing Date: Wed, 16 Jan 2019 23:42:36 -0800 Subject: [PATCH 4/7] =?UTF-8?q?We=20don't=20run=20tests=20(=E2=95=AF?= =?UTF-8?q?=E2=80=B5=E2=96=A1=E2=80=B2)=E2=95=AF=EF=B8=B5=E2=94=BB?= =?UTF-8?q?=E2=94=81=E2=94=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/staticbuild/build_wheel.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/staticbuild/build_wheel.sh b/tools/staticbuild/build_wheel.sh index d2cb927cf729..9c1803b3dad7 100755 --- a/tools/staticbuild/build_wheel.sh +++ b/tools/staticbuild/build_wheel.sh @@ -26,5 +26,3 @@ python setup.py bdist_wheel wheel_name=$(ls -t dist | head -n 1) pip install -U --user --force-reinstall dist/$wheel_name -cd .. -python nosetests -v $(ls tests/python/unittest/test_gluon*.py | grep -v data | grep -v model_zoo | grep -v utils | grep -v rnn) From a245e075cb843a0e1f654c61b1edaf13f5c72641 Mon Sep 17 00:00:00 2001 From: Qing Date: Thu, 17 Jan 2019 11:25:44 -0800 Subject: [PATCH 5/7] add variant in the environment variable --- ci/docker/runtime_functions.sh | 3 ++- ci/jenkins/Jenkins_steps.groovy | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 7a5192d8e5d3..909147621f44 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1295,6 +1295,7 @@ deploy_jl_docs() { # ... } +<<<<<<< HEAD build_scala_static_mkl() { set -ex pushd . @@ -1305,7 +1306,7 @@ build_scala_static_mkl() { popd } -build_static_python() { +build_static_python_mkl() { set -ex pushd . export mxnet_variant=mkl diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy index ab035fdf95ca..0343ce7f09ff 100644 --- a/ci/jenkins/Jenkins_steps.groovy +++ b/ci/jenkins/Jenkins_steps.groovy @@ -540,7 +540,7 @@ def test_static_python_cpu() { ws('workspace/ut-publish-python-cpu') { timeout(time: max_time, unit: 'MINUTES') { utils.init_git() - utils.docker_run("publish.ubuntu1404_cpu", 'build_static_python', false) + utils.docker_run("publish.ubuntu1404_cpu", 'build_static_python_mkl', false) } } } From 22f7ecebc4bfa81afe05f9e4cffd4e6dcbd0bf68 Mon Sep 17 00:00:00 2001 From: Qing Date: Tue, 22 Jan 2019 15:00:38 -0800 Subject: [PATCH 6/7] add document improvement --- ci/publish/README.md | 16 ++++++++-------- tools/staticbuild/README.md | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ci/publish/README.md b/ci/publish/README.md index 2344cdff4cd2..f1ece6f84097 100644 --- a/ci/publish/README.md +++ b/ci/publish/README.md @@ -1,9 +1,9 @@ # MXNet Publish Settings -This folder contains the configuration of restricted node on Jenkins for the publish. It also contains a folder called `scala` that contains everything required for scala publish. In this `README`, we would bring a brief walkthrough of the Jenkins configuration as well as the usages of the scala deployment files. +This folder contains the configuration for restricted nodes on Jenkins for the publishing MXNet artifacts. It also contains a folder called `scala` that contains everything required for publishing to Maven. In this `README`, we provide a brief walkthrough of the Jenkins configuration as well as the usage of the Scala deployment files. Python publishing is TBD. ## Jenkins -Currently, Jenkins contains three build stages, namely `Build Packages`, `Test Packages` and `Deploy Packages`. During the `build package` stages, all dependencies will be built and a Scala package would be created. In the second stage, the package created from the previous stage would move to this stage to specifically run the tests. In the final stage, the packages passed the test would be deployed by the instances. +Currently, Jenkins contains three build stages, namely `Build Packages`, `Test Packages` and `Deploy Packages`. During the `build package` stages, all dependencies are built and a Scala package are created. In the second stage, the package created from the previous stage moves to this stage to specifically run the tests. In the final stage, the packages that pass the tests are deployed by the instances. The job is scheduled to be triggered every 24 hours on a [restricted instance](http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/restricted-publish-artifacts). @@ -17,11 +17,11 @@ All packages are currently built in `Ubuntu 14.04`. All Dockerfile used for publ Apart from that, the script used to create the environment and publish are available under `ci/docker/install`: -- `ubuntu_publish.sh` install all required dependencies for Ubuntu 14.04 for publishing -- `ubuntu_base.sh` install minimum dependencies required to run the published packages +- `ubuntu_publish.sh` installs all required dependencies for Ubuntu 14.04 for publishing +- `ubuntu_base.sh` installs minimum dependencies required to run the published packages -## Scala publish -Currently Scala publish on Linux is fully supported on jenkins. The `scala/` folder contains all files needed to do the publish. Here is a breif instroduction of the files: +## Scala publishing +Currently Scala publish on Linux is fully supported on Jenkins. The `scala/` folder contains all files needed for publishing. Here is a brief introduction of the files: - `build.sh` Main executable files to build the backend as well as scala package - `buildkey.py` Main file used to extract password from the system and configure the maven @@ -29,5 +29,5 @@ Currently Scala publish on Linux is fully supported on jenkins. The `scala/` fol - `fullDeploy.sh` Used by CI to make full publish - `test.sh` Make Scala test on CI -## Python -We plans to support Python build on Jenkins soon \ No newline at end of file +## Python publishing +Python build support is TBD. diff --git a/tools/staticbuild/README.md b/tools/staticbuild/README.md index 7cfeb7855a55..2def768a1f1e 100644 --- a/tools/staticbuild/README.md +++ b/tools/staticbuild/README.md @@ -1,9 +1,9 @@ -# MXNet Static build +# MXNet Static Build -This folder contains the core script used to build the static library. This README would bring you the information and usages of the script in here. Please be aware, all of the scripts are designed to be run under the root folder. +This folder contains the core script used to build the static library. This README provides information on how to use the scripts in this folder. Please be aware, all of the scripts are designed to be run under the root folder. ## `build.sh` -This script is a wrapper around `build_lib.sh` aimed to simplify the usage of it. It would automatically identify the system version, number of cores and all environment variable settings. Here are the examples you can run this script: +This script is a wrapper around `build_lib.sh. It simplifies the things by automatically identifing the system version, number of cores, and all environment variable settings. Here are examples you can run with this script: ``` tools/staticbuild/build.sh cu92 maven @@ -17,7 +17,7 @@ This would build the mxnet package based on MKLDNN and and pypi configuration se As the result, users would have a complete static dependencies in `/staticdeps` in the root folder as well as a static-linked `libmxnet.so` file lives in `lib`. You can build your language binding by using the `libmxnet.so`. ## `build_lib.sh` -This script would clone the most up-to-date master and build the MXNet backend with static library. In order to run that, you should have prepare the the following environment variable: +This script clones the most up-to-date master and builds the MXNet backend with a static library. In order to run the static library, you must set the the following environment variables: - `DEPS_PATH` Path to your static dependencies - `STATIC_BUILD_TARGET` Either `pip` or `maven` as your publish platform @@ -29,4 +29,4 @@ It is not recommended to run this file alone since there are a bunch of variable After running this script, you would have everything you need ready in the `/lib` folder. ## `build_wheel.sh` -This script is used to build the python package as well as running a sanity test \ No newline at end of file +This script builds the python package. It also runs a sanity test. \ No newline at end of file From 6ff0bf3e819be3022d52cf4320bc459dd4d86a47 Mon Sep 17 00:00:00 2001 From: Qing Date: Tue, 22 Jan 2019 15:02:34 -0800 Subject: [PATCH 7/7] kill the conflict --- ci/docker/runtime_functions.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 909147621f44..a4d9956d8d87 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1295,7 +1295,6 @@ deploy_jl_docs() { # ... } -<<<<<<< HEAD build_scala_static_mkl() { set -ex pushd .