From 9fe4efa8e418a857b787834f08a2f85dce200344 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Wed, 12 Sep 2018 16:10:51 -0700 Subject: [PATCH 01/20] Initial Commit for docker automation python --- .../docker-python/Dockerfile.mxnet.python.cpu | 8 ++ .../Dockerfile.mxnet.python.cpu.mkl | 8 ++ .../docker-python/Dockerfile.mxnet.python.gpu | 8 ++ .../Dockerfile.mxnet.python.gpu.mkl | 8 ++ docker/docker-python/README.md | 11 ++ .../docker-python/build_python_dockerfile.sh | 108 ++++++++++++++++++ 6 files changed, 151 insertions(+) create mode 100644 docker/docker-python/Dockerfile.mxnet.python.cpu create mode 100644 docker/docker-python/Dockerfile.mxnet.python.cpu.mkl create mode 100644 docker/docker-python/Dockerfile.mxnet.python.gpu create mode 100644 docker/docker-python/Dockerfile.mxnet.python.gpu.mkl create mode 100644 docker/docker-python/README.md create mode 100755 docker/docker-python/build_python_dockerfile.sh diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu b/docker/docker-python/Dockerfile.mxnet.python.cpu new file mode 100644 index 000000000000..52b269548a59 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python.cpu @@ -0,0 +1,8 @@ +FROM ubuntu:16.04 + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet \ No newline at end of file diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl new file mode 100644 index 000000000000..cec1daec53d0 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl @@ -0,0 +1,8 @@ +FROM ubuntu:16.04 + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-mkl \ No newline at end of file diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu b/docker/docker-python/Dockerfile.mxnet.python.gpu new file mode 100644 index 000000000000..9248ee843db1 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu @@ -0,0 +1,8 @@ +FROM nvidia/cuda:9.0-cudnn7-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu90 \ No newline at end of file diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.mkl new file mode 100644 index 000000000000..72400c701a0c --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.mkl @@ -0,0 +1,8 @@ +FROM nvidia/cuda:9.0-cudnn7-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu90mkl \ No newline at end of file diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md new file mode 100644 index 000000000000..6353fd3b4163 --- /dev/null +++ b/docker/docker-python/README.md @@ -0,0 +1,11 @@ +It uses the latest pip binaries to build docker images. +The latest released mxnet version will be used by the dockerfiles. +If you want to use another pip binary, feel free to modify the dockerfile before running the build script. + + +use as : ./build_python_dockerfile.sh [tag] + +The parameter tag inputs the mxnet version which is used to tag the generated docker image. This does not select a different pip binary itself. + + +Tests run: diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh new file mode 100755 index 000000000000..6d5f60d9860e --- /dev/null +++ b/docker/docker-python/build_python_dockerfile.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# function to exit script if command fails +runme() { + cmd=$* + echo "$cmd" + $cmd + ret=$? + if [[ ${ret} != 0 ]]; then + echo " " + echo "ERROR: Return value non-zero for: $cmd" + echo " " + exit 1 + fi +} + +docker_build_image(){ + runme docker build -t mxnet/python:${1} -f ${2} . +} + +docker_tag_image(){ + docker tag mxnet/python:${1} ${2} + +} + +docker_test_image_cpu(){ + test_dir="/Users/mbaijal/Documents/mxnet/mxnet_repo2/incubator-mxnet" + runme docker run -it -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py" + runme docker run -it -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python -c 'import mxnet'" + +} + +docker_test_image_gpu(){ + test_dir="/Users/mbaijal/Documents/mxnet/mxnet_repo2/incubator-mxnet" + runme nvidia-docker run -it -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py --gpu" + runme nvidia-docker run -it -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python -c 'import mxnet'" + + +} + +docker_account_login(){ + docker login +} + +docker_account_logout(){ + docker logout +} + +docker_push_image(){ + runme docker push mxnet/python:${1} +} + +# Script to take one parameter +echo "Building Docker Images for Apache MXNet (Incubating) v$1" + + +# Build and Test dockerfiles - CPU +docker_build_image "${1}_cpu" "Dockerfile.mxnet.python.cpu" +docker_test_image_cpu "${1}_cpu" + +docker_build_image "${1}_cpu_mkl" "Dockerfile.mxnet.python.cpu.mkl" +docker_test_image_cpu "${1}_cpu_mkl" + +docker_tag_image "${1}_cpu" "latest" +docker_test_image_cpu "latest" + + +#Build and Test dockerfiles - GPU +docker_build_image "${1}_gpu_cu90" "Dockerfile.mxnet.python.gpu" +docker_test_image_gpu "${1}_gpu_cu90" + +docker_build_image "${1}_gpu_mkl_cu90" "Dockerfile.mxnet.python.gpu.mkl" +docker_test_image_gpu "${1}_gpu_cu90" + +docker_tag_image "${1}_gpu" "gpu" +docker_test_image_gpu "${1}_gpu_cu90" + + +# Push dockerfiles +docker_account_login + +docker_push_image "${1}_cpu" +docker_push_image "${1}_cpu_mkl" +docker_push_image "latest" +docker_push_image "${1}_gpu_cu90" +docker_push_image "${1}_gpu_cu90" +docker_push_image "${1}_gpu_cu90" + +docker_account_logout + + From cc4bc35139547483f6aa5c7dfb2cdb38b39521b5 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Wed, 12 Sep 2018 16:55:21 -0700 Subject: [PATCH 02/20] Fixes --- .../docker-python/build_python_dockerfile.sh | 30 +++++++++---------- docker/docker-python/test_mxnet.py | 21 +++++++++++++ 2 files changed, 35 insertions(+), 16 deletions(-) create mode 100644 docker/docker-python/test_mxnet.py diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index 6d5f60d9860e..7a0eee52250f 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -17,6 +17,13 @@ # specific language governing permissions and limitations # under the License. +set -ex + +# Script to take two parameters +echo "Building Docker Images for Apache MXNet (Incubating) v$1" +test_dir="/Users/mbaijal/Documents/mxnet/mxnet_repo2/incubator-mxnet" +#test_dir="${2}" + # function to exit script if command fails runme() { cmd=$* @@ -37,22 +44,18 @@ docker_build_image(){ docker_tag_image(){ docker tag mxnet/python:${1} ${2} - } docker_test_image_cpu(){ - test_dir="/Users/mbaijal/Documents/mxnet/mxnet_repo2/incubator-mxnet" - runme docker run -it -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py" - runme docker run -it -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python -c 'import mxnet'" - + docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py" + docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py" + docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py" } docker_test_image_gpu(){ - test_dir="/Users/mbaijal/Documents/mxnet/mxnet_repo2/incubator-mxnet" - runme nvidia-docker run -it -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py --gpu" - runme nvidia-docker run -it -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python -c 'import mxnet'" - - + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py'" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py --gpu" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py" } docker_account_login(){ @@ -67,9 +70,6 @@ docker_push_image(){ runme docker push mxnet/python:${1} } -# Script to take one parameter -echo "Building Docker Images for Apache MXNet (Incubating) v$1" - # Build and Test dockerfiles - CPU docker_build_image "${1}_cpu" "Dockerfile.mxnet.python.cpu" @@ -103,6 +103,4 @@ docker_push_image "${1}_gpu_cu90" docker_push_image "${1}_gpu_cu90" docker_push_image "${1}_gpu_cu90" -docker_account_logout - - +docker_account_logout \ No newline at end of file diff --git a/docker/docker-python/test_mxnet.py b/docker/docker-python/test_mxnet.py new file mode 100644 index 000000000000..88c7b81dab3d --- /dev/null +++ b/docker/docker-python/test_mxnet.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx + +print mx.__version__ + From c1fe93e62b66488ab07af30cf41e220b25ecbe79 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Wed, 12 Sep 2018 16:59:48 -0700 Subject: [PATCH 03/20] change dir for tests --- docker/docker-python/build_python_dockerfile.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index 7a0eee52250f..c1a99e70be56 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -21,8 +21,7 @@ set -ex # Script to take two parameters echo "Building Docker Images for Apache MXNet (Incubating) v$1" -test_dir="/Users/mbaijal/Documents/mxnet/mxnet_repo2/incubator-mxnet" -#test_dir="${2}" +test_dir="${2}" # function to exit script if command fails runme() { From 58c21b54ca2f0a28c145a65f79f1b7f0eb0df03c Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Wed, 12 Sep 2018 17:51:17 -0700 Subject: [PATCH 04/20] Fix more issues --- docker/docker-python/build_python_dockerfile.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index c1a99e70be56..3a0701ed4bbf 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -52,9 +52,9 @@ docker_test_image_cpu(){ } docker_test_image_gpu(){ - nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py'" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py" nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py --gpu" - nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py --gpus 2" } docker_account_login(){ @@ -85,11 +85,11 @@ docker_test_image_cpu "latest" docker_build_image "${1}_gpu_cu90" "Dockerfile.mxnet.python.gpu" docker_test_image_gpu "${1}_gpu_cu90" -docker_build_image "${1}_gpu_mkl_cu90" "Dockerfile.mxnet.python.gpu.mkl" -docker_test_image_gpu "${1}_gpu_cu90" +docker_build_image "${1}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.mkl" +docker_test_image_gpu "${1}_gpu_cu90_mkl" -docker_tag_image "${1}_gpu" "gpu" -docker_test_image_gpu "${1}_gpu_cu90" +docker_tag_image "${1}_gpu_cu90" "gpu" +docker_test_image_gpu "gpu" # Push dockerfiles @@ -99,7 +99,7 @@ docker_push_image "${1}_cpu" docker_push_image "${1}_cpu_mkl" docker_push_image "latest" docker_push_image "${1}_gpu_cu90" -docker_push_image "${1}_gpu_cu90" -docker_push_image "${1}_gpu_cu90" +docker_push_image "${1}_gpu_cu90_mkl" +docker_push_image "gpu" docker_account_logout \ No newline at end of file From 76f0ce11908a4f598276843b762b74c790713915 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Wed, 12 Sep 2018 18:13:31 -0700 Subject: [PATCH 05/20] fix docker tag command --- docker/docker-python/build_python_dockerfile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index 3a0701ed4bbf..2b91cc3ca2ad 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -42,7 +42,7 @@ docker_build_image(){ } docker_tag_image(){ - docker tag mxnet/python:${1} ${2} + docker tag mxnet/python:${1} mxnet/python:${2} } docker_test_image_cpu(){ From b256f6fa0bc3494a6e35cc80c5f8d21c206bb857 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Wed, 12 Sep 2018 19:19:25 -0700 Subject: [PATCH 06/20] cosmetic changes --- .../docker-python/build_python_dockerfile.sh | 56 ++++++++----------- docker/docker-python/test_mxnet.py | 4 ++ 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index 2b91cc3ca2ad..fde4ab48eee0 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -17,28 +17,17 @@ # specific language governing permissions and limitations # under the License. -set -ex +set -e # Script to take two parameters echo "Building Docker Images for Apache MXNet (Incubating) v$1" +mxnet_version="${1}" test_dir="${2}" -# function to exit script if command fails -runme() { - cmd=$* - echo "$cmd" - $cmd - ret=$? - if [[ ${ret} != 0 ]]; then - echo " " - echo "ERROR: Return value non-zero for: $cmd" - echo " " - exit 1 - fi -} docker_build_image(){ - runme docker build -t mxnet/python:${1} -f ${2} . + echo "Building docker image mxnet/python:${1}" + docker build -t mxnet/python:${1} -f ${2} . } docker_tag_image(){ @@ -46,13 +35,15 @@ docker_tag_image(){ } docker_test_image_cpu(){ - docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py" + echo "Running tests on mxnet/python:${1}" + docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py" docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py" } docker_test_image_gpu(){ - nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py" + echo "Running tests on mxnet/python:${1}" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py --gpu" nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py --gpus 2" } @@ -66,40 +57,41 @@ docker_account_logout(){ } docker_push_image(){ - runme docker push mxnet/python:${1} + docker push mxnet/python:${1} } # Build and Test dockerfiles - CPU -docker_build_image "${1}_cpu" "Dockerfile.mxnet.python.cpu" -docker_test_image_cpu "${1}_cpu" +docker_build_image "${mxnet_version}_cpu" "Dockerfile.mxnet.python.cpu" +docker_test_image_cpu "${mxnet_version}_cpu" -docker_build_image "${1}_cpu_mkl" "Dockerfile.mxnet.python.cpu.mkl" -docker_test_image_cpu "${1}_cpu_mkl" +docker_build_image "${mxnet_version}_cpu_mkl" "Dockerfile.mxnet.python.cpu.mkl" +docker_test_image_cpu "${mxnet_version}_cpu_mkl" -docker_tag_image "${1}_cpu" "latest" +docker_tag_image "${mxnet_version}_cpu" "latest" docker_test_image_cpu "latest" #Build and Test dockerfiles - GPU -docker_build_image "${1}_gpu_cu90" "Dockerfile.mxnet.python.gpu" -docker_test_image_gpu "${1}_gpu_cu90" +docker_build_image "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu" +docker_test_image_gpu "${mxnet_version}_gpu_cu90" -docker_build_image "${1}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.mkl" -docker_test_image_gpu "${1}_gpu_cu90_mkl" +docker_build_image "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.mkl" +docker_test_image_gpu "${mxnet_version}_gpu_cu90_mkl" -docker_tag_image "${1}_gpu_cu90" "gpu" +docker_tag_image "${mxnet_version}_gpu_cu90" "gpu" docker_test_image_gpu "gpu" # Push dockerfiles +echo "All images were successfully built. Now login to dockerhub and push images" docker_account_login -docker_push_image "${1}_cpu" -docker_push_image "${1}_cpu_mkl" +docker_push_image "${mxnet_version}_cpu" +docker_push_image "${mxnet_version}_cpu_mkl" docker_push_image "latest" -docker_push_image "${1}_gpu_cu90" -docker_push_image "${1}_gpu_cu90_mkl" +docker_push_image "${mxnet_version}_gpu_cu90" +docker_push_image "${mxnet_version}_gpu_cu90_mkl" docker_push_image "gpu" docker_account_logout \ No newline at end of file diff --git a/docker/docker-python/test_mxnet.py b/docker/docker-python/test_mxnet.py index 88c7b81dab3d..15fc9742dfad 100644 --- a/docker/docker-python/test_mxnet.py +++ b/docker/docker-python/test_mxnet.py @@ -15,7 +15,11 @@ # specific language governing permissions and limitations # under the License. +# WIP import mxnet as mx +import sys print mx.__version__ +print sys.argv[1] + From f75535eb136e32750118d6774e547d2d56203ada Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Wed, 12 Sep 2018 20:01:43 -0700 Subject: [PATCH 07/20] update README --- docker/docker-python/README.md | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md index 6353fd3b4163..94b0e9eed1fa 100644 --- a/docker/docker-python/README.md +++ b/docker/docker-python/README.md @@ -1,11 +1,30 @@ -It uses the latest pip binaries to build docker images. -The latest released mxnet version will be used by the dockerfiles. -If you want to use another pip binary, feel free to modify the dockerfile before running the build script. +# Release Python Docker Images for MXNet +The `docker-python` directory can be used to release mxnet python docker images to dockerhub after any mxnet release. +It uses the appropriate pip binaries to build different docker images as - +* cpu +* gpu_cu90 +* cpu_mkl +* gpu_cu90_mkl +* latest (same as cpu) +* gpu (same as gpu_cu90) -use as : ./build_python_dockerfile.sh [tag] -The parameter tag inputs the mxnet version which is used to tag the generated docker image. This does not select a different pip binary itself. +** Note: If you want to use a different pip binary (specific mxnet or cuda version, etc), you can edit the last line of the cpu or gpu dockerfile as required. +Refer: https://pypi.org/project/mxnet/ -Tests run: +###Usage +`./build_python_dockerfile.sh ` + +For example: +`./build_python_dockerfile.sh 1.3.0 ~/build-docker/incubator-mxnet` + +** Note: The build script picks up the latest pip binaries. This means, it uses the latest released mxnet version. The version specified as a parameter to the script is only used to tag the built image correctly. + +###Tests run +* `/mxnet/tests/python/train/test_conv.py` +* `/mxnet/example/image-classification/train_mnist.py` +* `/mxnet/docker/docker-python/test_mxnet.py`: This script is used to make sure that the docker image builds the expected mxnet version. That is, the version picked by pip is the same as as the version passed as a parameter. + +Dockerhub credentials will be required to upload images at the end of this script. From 61bed69337858fe1c9ffdc2d6e7dd3f9a284c13c Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Wed, 12 Sep 2018 23:54:42 -0700 Subject: [PATCH 08/20] update test to fail on version mismatch --- .../docker-python/build_python_dockerfile.sh | 26 ++++++++++++++++++- docker/docker-python/test_mxnet.py | 9 ++++--- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index fde4ab48eee0..97553cb3ffa4 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -19,12 +19,36 @@ set -e -# Script to take two parameters +# Check Params +programname=$0 + +function usage { + echo "usage: $programname [version] [path]" + echo " [version] Mxnet Version to build" + echo " [path] Path to MXNet repository (to run tests)" + echo " " + exit 1 +} + +if [ $# -le 1 ] +then + usage + exit 1 +fi + +# Two params provided echo "Building Docker Images for Apache MXNet (Incubating) v$1" mxnet_version="${1}" test_dir="${2}" +# Run in debug mode, if set +if [ "${debug}" == "1" ]; then + echo "debug mode on" + set -x +fi + + docker_build_image(){ echo "Building docker image mxnet/python:${1}" docker build -t mxnet/python:${1} -f ${2} . diff --git a/docker/docker-python/test_mxnet.py b/docker/docker-python/test_mxnet.py index 15fc9742dfad..d5fec992f605 100644 --- a/docker/docker-python/test_mxnet.py +++ b/docker/docker-python/test_mxnet.py @@ -15,11 +15,14 @@ # specific language governing permissions and limitations # under the License. -# WIP +# This checks that the version of mxnet imported matches the parameter passed to the build script. import mxnet as mx import sys -print mx.__version__ -print sys.argv[1] +pip_version = mx.__version__ +expected_version = sys.argv[1] + +if pip_version != expected_version: + raise ValueError("ERROR: Incorrect pip version. Please check the parameter passed or pip binary used.") From 60df42cfec31e11a633bba0001dff600682d44c3 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Wed, 12 Sep 2018 23:55:22 -0700 Subject: [PATCH 09/20] remove debug mode --- docker/docker-python/build_python_dockerfile.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index 97553cb3ffa4..38d0aac2edd6 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -41,14 +41,6 @@ echo "Building Docker Images for Apache MXNet (Incubating) v$1" mxnet_version="${1}" test_dir="${2}" - -# Run in debug mode, if set -if [ "${debug}" == "1" ]; then - echo "debug mode on" - set -x -fi - - docker_build_image(){ echo "Building docker image mxnet/python:${1}" docker build -t mxnet/python:${1} -f ${2} . From 284d8452e81584040ae9ebf2ab93ffddf5e9de03 Mon Sep 17 00:00:00 2001 From: Meghna Baijal <30911248+mbaijal@users.noreply.github.com> Date: Thu, 13 Sep 2018 12:10:29 -0700 Subject: [PATCH 10/20] Update README.md --- docker/docker-python/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md index 94b0e9eed1fa..68f689b21775 100644 --- a/docker/docker-python/README.md +++ b/docker/docker-python/README.md @@ -14,7 +14,7 @@ It uses the appropriate pip binaries to build different docker images as - Refer: https://pypi.org/project/mxnet/ -###Usage +##Usage `./build_python_dockerfile.sh ` For example: @@ -22,7 +22,7 @@ For example: ** Note: The build script picks up the latest pip binaries. This means, it uses the latest released mxnet version. The version specified as a parameter to the script is only used to tag the built image correctly. -###Tests run +##Tests run * `/mxnet/tests/python/train/test_conv.py` * `/mxnet/example/image-classification/train_mnist.py` * `/mxnet/docker/docker-python/test_mxnet.py`: This script is used to make sure that the docker image builds the expected mxnet version. That is, the version picked by pip is the same as as the version passed as a parameter. From 6d396a022e72dacff8d33439f2925087c5f4c9b1 Mon Sep 17 00:00:00 2001 From: Meghna Baijal <30911248+mbaijal@users.noreply.github.com> Date: Thu, 13 Sep 2018 12:10:51 -0700 Subject: [PATCH 11/20] Update README.md --- docker/docker-python/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md index 68f689b21775..221ea8fe7415 100644 --- a/docker/docker-python/README.md +++ b/docker/docker-python/README.md @@ -14,7 +14,7 @@ It uses the appropriate pip binaries to build different docker images as - Refer: https://pypi.org/project/mxnet/ -##Usage +### Usage `./build_python_dockerfile.sh ` For example: @@ -22,7 +22,7 @@ For example: ** Note: The build script picks up the latest pip binaries. This means, it uses the latest released mxnet version. The version specified as a parameter to the script is only used to tag the built image correctly. -##Tests run +### Tests run * `/mxnet/tests/python/train/test_conv.py` * `/mxnet/example/image-classification/train_mnist.py` * `/mxnet/docker/docker-python/test_mxnet.py`: This script is used to make sure that the docker image builds the expected mxnet version. That is, the version picked by pip is the same as as the version passed as a parameter. From 742a3636980a5be2dd1da69e9c145f0ce4a6ce25 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Thu, 13 Sep 2018 14:15:16 -0700 Subject: [PATCH 12/20] update README --- docker/docker-python/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md index 221ea8fe7415..e3817ed0e903 100644 --- a/docker/docker-python/README.md +++ b/docker/docker-python/README.md @@ -3,10 +3,10 @@ The `docker-python` directory can be used to release mxnet python docker images to dockerhub after any mxnet release. It uses the appropriate pip binaries to build different docker images as - * cpu -* gpu_cu90 * cpu_mkl -* gpu_cu90_mkl * latest (same as cpu) +* gpu_cu90 +* gpu_cu90_mkl * gpu (same as gpu_cu90) @@ -20,7 +20,7 @@ Refer: https://pypi.org/project/mxnet/ For example: `./build_python_dockerfile.sh 1.3.0 ~/build-docker/incubator-mxnet` -** Note: The build script picks up the latest pip binaries. This means, it uses the latest released mxnet version. The version specified as a parameter to the script is only used to tag the built image correctly. +** Note: The build script picks up the latest pip binaries. This means it uses the latest released mxnet version. The version specified as a parameter to the script is only used to tag the built image correctly. ### Tests run * `/mxnet/tests/python/train/test_conv.py` From b3e1b96e481a2d73628b023d3b60da172f48ac5c Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Thu, 13 Sep 2018 16:19:04 -0700 Subject: [PATCH 13/20] Add Licenses --- .../docker-python/Dockerfile.mxnet.python.cpu | 20 +++++++++++++++++++ .../Dockerfile.mxnet.python.cpu.mkl | 20 +++++++++++++++++++ .../docker-python/Dockerfile.mxnet.python.gpu | 20 +++++++++++++++++++ .../Dockerfile.mxnet.python.gpu.mkl | 20 +++++++++++++++++++ 4 files changed, 80 insertions(+) diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu b/docker/docker-python/Dockerfile.mxnet.python.cpu index 52b269548a59..cc148611650d 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.cpu +++ b/docker/docker-python/Dockerfile.mxnet.python.cpu @@ -1,3 +1,23 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for CPU + FROM ubuntu:16.04 RUN apt-get update diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl index cec1daec53d0..25b2d4547717 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl @@ -1,3 +1,23 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet CPU with MKL + FROM ubuntu:16.04 RUN apt-get update diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu b/docker/docker-python/Dockerfile.mxnet.python.gpu index 9248ee843db1..7cda6187dab0 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu @@ -1,3 +1,23 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU + FROM nvidia/cuda:9.0-cudnn7-devel RUN apt-get update diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.mkl index 72400c701a0c..2f4bda80018a 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.mkl @@ -1,3 +1,23 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU with MKL + FROM nvidia/cuda:9.0-cudnn7-devel RUN apt-get update From cf6bdcf9bf38d072a57a6ce0432493dd1369e036 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Fri, 14 Sep 2018 17:01:06 -0700 Subject: [PATCH 14/20] Some review comments --- docker/docker-python/Dockerfile.mxnet.python.cpu | 2 +- docker/docker-python/Dockerfile.mxnet.python.cpu.mkl | 2 +- docker/docker-python/Dockerfile.mxnet.python.gpu | 2 +- docker/docker-python/Dockerfile.mxnet.python.gpu.mkl | 2 +- docker/docker-python/build_python_dockerfile.sh | 4 ++-- docker/docker-python/test_mxnet.py | 2 -- 6 files changed, 6 insertions(+), 8 deletions(-) diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu b/docker/docker-python/Dockerfile.mxnet.python.cpu index cc148611650d..eb437f3c2334 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.cpu +++ b/docker/docker-python/Dockerfile.mxnet.python.cpu @@ -25,4 +25,4 @@ RUN apt-get install -y wget python gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python get-pip.py -RUN pip install mxnet \ No newline at end of file +RUN pip install mxnet diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl index 25b2d4547717..043932ff7c8e 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl @@ -25,4 +25,4 @@ RUN apt-get install -y wget python gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python get-pip.py -RUN pip install mxnet-mkl \ No newline at end of file +RUN pip install mxnet-mkl diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu b/docker/docker-python/Dockerfile.mxnet.python.gpu index 7cda6187dab0..1e3d9869ac63 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu @@ -25,4 +25,4 @@ RUN apt-get install -y wget python gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python get-pip.py -RUN pip install mxnet-cu90 \ No newline at end of file +RUN pip install mxnet-cu90 diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.mkl index 2f4bda80018a..d82abd7cf523 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.mkl @@ -25,4 +25,4 @@ RUN apt-get install -y wget python gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python get-pip.py -RUN pip install mxnet-cu90mkl \ No newline at end of file +RUN pip install mxnet-cu90mkl diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index 38d0aac2edd6..405265d94071 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -30,7 +30,7 @@ function usage { exit 1 } -if [ $# -le 1 ] +if [ $# -le 1 ] || [ $# -ge 3 ] then usage exit 1 @@ -110,4 +110,4 @@ docker_push_image "${mxnet_version}_gpu_cu90" docker_push_image "${mxnet_version}_gpu_cu90_mkl" docker_push_image "gpu" -docker_account_logout \ No newline at end of file +docker_account_logout diff --git a/docker/docker-python/test_mxnet.py b/docker/docker-python/test_mxnet.py index d5fec992f605..657205832728 100644 --- a/docker/docker-python/test_mxnet.py +++ b/docker/docker-python/test_mxnet.py @@ -24,5 +24,3 @@ if pip_version != expected_version: raise ValueError("ERROR: Incorrect pip version. Please check the parameter passed or pip binary used.") - - From 21671e51225b58ed17d820f8d648267bde19de84 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Tue, 18 Sep 2018 18:08:54 -0700 Subject: [PATCH 15/20] Add Cuda80 and cuda92 dockerfiles and build steps --- ...n.gpu => Dockerfile.mxnet.python.gpu.cu80} | 4 +-- ...l => Dockerfile.mxnet.python.gpu.cu80.mkl} | 4 +-- .../Dockerfile.mxnet.python.gpu.cu92 | 28 +++++++++++++++++++ .../Dockerfile.mxnet.python.gpu.cu92.mkl | 28 +++++++++++++++++++ .../docker-python/build_python_dockerfile.sh | 22 +++++++++++++-- 5 files changed, 80 insertions(+), 6 deletions(-) rename docker/docker-python/{Dockerfile.mxnet.python.gpu => Dockerfile.mxnet.python.gpu.cu80} (94%) rename docker/docker-python/{Dockerfile.mxnet.python.gpu.mkl => Dockerfile.mxnet.python.gpu.cu80.mkl} (93%) create mode 100644 docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 create mode 100644 docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 similarity index 94% rename from docker/docker-python/Dockerfile.mxnet.python.gpu rename to docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 index 1e3d9869ac63..8c83ece434a3 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 @@ -18,11 +18,11 @@ # # Dockerfile to build MXNet for GPU -FROM nvidia/cuda:9.0-cudnn7-devel +FROM nvidia/cuda:8.0-cudnn5-devel RUN apt-get update RUN apt-get install -y wget python gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python get-pip.py -RUN pip install mxnet-cu90 +RUN pip install mxnet-cu80 diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl similarity index 93% rename from docker/docker-python/Dockerfile.mxnet.python.gpu.mkl rename to docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl index d82abd7cf523..a057c1d20cb1 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl @@ -18,11 +18,11 @@ # # Dockerfile to build MXNet for GPU with MKL -FROM nvidia/cuda:9.0-cudnn7-devel +FROM nvidia/cuda:8.0-cudnn5-devel RUN apt-get update RUN apt-get install -y wget python gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python get-pip.py -RUN pip install mxnet-cu90mkl +RUN pip install mxnet-cu80mkl diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 new file mode 100644 index 000000000000..ba5c54a2a2aa --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU + +FROM nvidia/cuda:9.2-cudnn7-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu92 diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl new file mode 100644 index 000000000000..96a943980b59 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU with MKL + +FROM nvidia/cuda:9.2-cudnn7-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu92mkl diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index 405265d94071..f77cd0ea5e09 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -89,15 +89,27 @@ docker_test_image_cpu "latest" #Build and Test dockerfiles - GPU -docker_build_image "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu" +docker_build_image "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu.cu90" docker_test_image_gpu "${mxnet_version}_gpu_cu90" -docker_build_image "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.mkl" +docker_build_image "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.cu90.mkl" docker_test_image_gpu "${mxnet_version}_gpu_cu90_mkl" docker_tag_image "${mxnet_version}_gpu_cu90" "gpu" docker_test_image_gpu "gpu" +docker_build_image "${mxnet_version}_gpu_cu80" "Dockerfile.mxnet.python.gpu.cu80" +docker_test_image_gpu "${mxnet_version}_gpu_cu80" + +docker_build_image "${mxnet_version}_gpu_cu80_mkl" "Dockerfile.mxnet.python.gpu.cu80.mkl" +docker_test_image_gpu "${mxnet_version}_gpu_cu80_mkl" + +docker_build_image "${mxnet_version}_gpu_cu92" "Dockerfile.mxnet.python.gpu.cu92" +docker_test_image_gpu "${mxnet_version}_gpu_cu92" + +docker_build_image "${mxnet_version}_gpu_cu92_mkl" "Dockerfile.mxnet.python.gpu.cu92.mkl" +docker_test_image_gpu "${mxnet_version}_gpu_cu92_mkl" + # Push dockerfiles echo "All images were successfully built. Now login to dockerhub and push images" @@ -109,5 +121,11 @@ docker_push_image "latest" docker_push_image "${mxnet_version}_gpu_cu90" docker_push_image "${mxnet_version}_gpu_cu90_mkl" docker_push_image "gpu" +docker_push_image "${mxnet_version}_gpu_cu80" +docker_push_image "${mxnet_version}_gpu_cu80_mkl" +docker_push_image "${mxnet_version}_gpu_cu92" +docker_push_image "${mxnet_version}_gpu_cu92_mkl" docker_account_logout + +echo "Successfully Built, Tested and Pushed all Images to Dockerhub. Link: https://hub.docker.com/r/mxnet/python/tags/" From 08c21d171106fdf700e7bd702797b4d34da159da Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Tue, 18 Sep 2018 18:22:36 -0700 Subject: [PATCH 16/20] Add renamed and hence untracked files for cu90 --- .../Dockerfile.mxnet.python.gpu.cu90 | 28 +++++++++++++++++++ .../Dockerfile.mxnet.python.gpu.cu90.mkl | 28 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 create mode 100644 docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 new file mode 100644 index 000000000000..1e3d9869ac63 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU + +FROM nvidia/cuda:9.0-cudnn7-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu90 diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl new file mode 100644 index 000000000000..d82abd7cf523 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU with MKL + +FROM nvidia/cuda:9.0-cudnn7-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu90mkl From b7425677956202d29860df933cc8ba27a998df16 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Wed, 19 Sep 2018 22:26:28 -0700 Subject: [PATCH 17/20] Update README --- docker/docker-python/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md index e3817ed0e903..e1d41e97f8b9 100644 --- a/docker/docker-python/README.md +++ b/docker/docker-python/README.md @@ -8,6 +8,10 @@ It uses the appropriate pip binaries to build different docker images as - * gpu_cu90 * gpu_cu90_mkl * gpu (same as gpu_cu90) +* gpu_cu80 +* gpu_cu80_mkl +* gpu_cu92 +* gpu_cu92_mkl ** Note: If you want to use a different pip binary (specific mxnet or cuda version, etc), you can edit the last line of the cpu or gpu dockerfile as required. From 3d0f5cae17c6b254096dbd9cbe8bc439a888ef3b Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Mon, 24 Sep 2018 15:06:18 -0700 Subject: [PATCH 18/20] More ways to login --- docker/docker-python/build_python_dockerfile.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index f77cd0ea5e09..24a44c28970c 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -64,8 +64,14 @@ docker_test_image_gpu(){ nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py --gpus 2" } +# if both $MXNET_DOCKERHUB_PASSWORD and $MXNET_DOCKERHUB_USERNAME environment variables are set, docker will automatically login +# if env variables are not set, login will be interactive. docker_account_login(){ - docker login + if [[ -z $MXNET_DOCKERHUB_PASSWORD ]] || [[ -z $MXNET_DOCKERHUB_USERNAME ]]; then + docker login + else + echo $MXNET_DOCKERHUB_PASSWORD | docker login -u $MXNET_DOCKERHUB_USERNAME --password-stdin + fi } docker_account_logout(){ From 6d938eaf1e745e960c752af3ebaa4b761f5dccdd Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Tue, 25 Sep 2018 11:54:31 -0700 Subject: [PATCH 19/20] Update README with login options --- docker/docker-python/README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md index e1d41e97f8b9..da7f1ddb36f3 100644 --- a/docker/docker-python/README.md +++ b/docker/docker-python/README.md @@ -31,4 +31,11 @@ For example: * `/mxnet/example/image-classification/train_mnist.py` * `/mxnet/docker/docker-python/test_mxnet.py`: This script is used to make sure that the docker image builds the expected mxnet version. That is, the version picked by pip is the same as as the version passed as a parameter. -Dockerhub credentials will be required to upload images at the end of this script. +### Dockerhub Credentials +Dockerhub credentials will be required to push images at the end of this script. +Credentials can be provided in the following ways: +* **Interactive Login:** Run the script as is and it will ask you for credentials interactively. +* **Be Already Logged in:** Login to the mxnet dockerhub account before you run the build script and the script will complete build, test and push. +* **Set Environment Variables:** Set the following environment variables which the script will pick up to login to dockerhub at runtime - + * $MXNET_DOCKERHUB_PASSWORD + * $MXNET_DOCKERHUB_USERNAME From a05316153bc3c6cf233f5377e29dc964fefa68a2 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Fri, 28 Sep 2018 18:42:27 -0700 Subject: [PATCH 20/20] Update README with links to test. test_mxnet link will work only after merge --- docker/docker-python/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md index da7f1ddb36f3..f806d5d6f459 100644 --- a/docker/docker-python/README.md +++ b/docker/docker-python/README.md @@ -27,9 +27,9 @@ For example: ** Note: The build script picks up the latest pip binaries. This means it uses the latest released mxnet version. The version specified as a parameter to the script is only used to tag the built image correctly. ### Tests run -* `/mxnet/tests/python/train/test_conv.py` -* `/mxnet/example/image-classification/train_mnist.py` -* `/mxnet/docker/docker-python/test_mxnet.py`: This script is used to make sure that the docker image builds the expected mxnet version. That is, the version picked by pip is the same as as the version passed as a parameter. +* [test_conv.py](https://github.com/apache/incubator-mxnet/blob/master/tests/python/train/test_conv.py) +* [train_mnist.py](https://github.com/apache/incubator-mxnet/blob/master/example/image-classification/train_mnist.py) +* [test_mxnet.py](https://github.com/apache/incubator-mxnet/blob/master/docker/docker-python/test_mxnet.py): This script is used to make sure that the docker image builds the expected mxnet version. That is, the version picked by pip is the same as as the version passed as a parameter. ### Dockerhub Credentials Dockerhub credentials will be required to push images at the end of this script.