From d452d88c5aba47536b58fb255476edc2789ea19a Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Tue, 9 Oct 2018 16:29:08 -0700 Subject: [PATCH 01/19] Add dockerfiles for python3 --- .../Dockerfile.mxnet.python3.cpu | 28 ++++++++++ .../Dockerfile.mxnet.python3.cpu.mkl | 28 ++++++++++ .../Dockerfile.mxnet.python3.gpu.cu80 | 28 ++++++++++ .../Dockerfile.mxnet.python3.gpu.cu80.mkl | 28 ++++++++++ .../Dockerfile.mxnet.python3.gpu.cu90 | 28 ++++++++++ .../Dockerfile.mxnet.python3.gpu.cu90.mkl | 28 ++++++++++ .../Dockerfile.mxnet.python3.gpu.cu92 | 28 ++++++++++ .../Dockerfile.mxnet.python3.gpu.cu92.mkl | 28 ++++++++++ .../docker-python/build_python_dockerfile.sh | 54 +++++++++++++++++++ 9 files changed, 278 insertions(+) create mode 100644 docker/docker-python/Dockerfile.mxnet.python3.cpu create mode 100644 docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl create mode 100644 docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 create mode 100644 docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl create mode 100644 docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 create mode 100644 docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl create mode 100644 docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 create mode 100644 docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl diff --git a/docker/docker-python/Dockerfile.mxnet.python3.cpu b/docker/docker-python/Dockerfile.mxnet.python3.cpu new file mode 100644 index 000000000000..39344aef53a8 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python3.cpu @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for CPU + +FROM ubuntu:16.04 + +RUN apt-get update +RUN apt-get install -y wget python3 gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python3 get-pip.py + +RUN pip3 install mxnet diff --git a/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl new file mode 100644 index 000000000000..043932ff7c8e --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet CPU with MKL + +FROM ubuntu:16.04 + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-mkl diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 new file mode 100644 index 000000000000..8c83ece434a3 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU + +FROM nvidia/cuda:8.0-cudnn5-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu80 diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl new file mode 100644 index 000000000000..a057c1d20cb1 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU with MKL + +FROM nvidia/cuda:8.0-cudnn5-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu80mkl diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 new file mode 100644 index 000000000000..1e3d9869ac63 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU + +FROM nvidia/cuda:9.0-cudnn7-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu90 diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl new file mode 100644 index 000000000000..d82abd7cf523 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU with MKL + +FROM nvidia/cuda:9.0-cudnn7-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu90mkl diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 new file mode 100644 index 000000000000..ba5c54a2a2aa --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU + +FROM nvidia/cuda:9.2-cudnn7-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu92 diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl new file mode 100644 index 000000000000..96a943980b59 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU with MKL + +FROM nvidia/cuda:9.2-cudnn7-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu92mkl diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index 24a44c28970c..ede2dc40e71f 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -64,6 +64,20 @@ docker_test_image_gpu(){ nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py --gpus 2" } +docker_test_image_cpu_py3(){ + echo "Running tests on mxnet/python:${1}" + docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python3 /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" + docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python3 /mxnet/tests/python/train/test_conv.py" + docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python3 /mxnet/example/image-classification/train_mnist.py" +} + +docker_test_image_gpu_py3(){ + echo "Running tests on mxnet/python:${1}" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python3 /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python3 /mxnet/tests/python/train/test_conv.py --gpu" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python3 /mxnet/example/image-classification/train_mnist.py --gpus 2" +} + # if both $MXNET_DOCKERHUB_PASSWORD and $MXNET_DOCKERHUB_USERNAME environment variables are set, docker will automatically login # if env variables are not set, login will be interactive. docker_account_login(){ @@ -117,10 +131,40 @@ docker_build_image "${mxnet_version}_gpu_cu92_mkl" "Dockerfile.mxnet.python.gpu. docker_test_image_gpu "${mxnet_version}_gpu_cu92_mkl" + +# Build and Test Python3 dockerfiles - CPU +docker_build_image "${mxnet_version}_cpu_python3" "Dockerfile.mxnet.python3.cpu" +docker_test_image_cpu_py3 "${mxnet_version}_cpu_python3" + +docker_build_image "${mxnet_version}_cpu_mkl_python3" "Dockerfile.mxnet.python3.cpu.mkl" +docker_test_image_cpu_py3 "${mxnet_version}_cpu_mkl_python3" + + +#Build and Test Python3 dockerfiles - GPU +docker_build_image "${mxnet_version}_gpu_cu90_py3" "Dockerfile.mxnet.python3.gpu.cu90" +docker_test_image_gpu_py3 "${mxnet_version}_gpu_cu90_py3" + +docker_build_image "${mxnet_version}_gpu_cu90_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu90.mkl" +docker_test_image_gpu_py3 "${mxnet_version}_gpu_cu90_mkl_py3" + +docker_build_image "${mxnet_version}_gpu_cu80_py3" "Dockerfile.mxnet.python3.gpu.cu80" +docker_test_image_gpu_py3 "${mxnet_version}_gpu_cu80_py3" + +docker_build_image "${mxnet_version}_gpu_cu80_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu80.mkl" +docker_test_image_gpu_py3 "${mxnet_version}_gpu_cu80_mkl_py3" + +docker_build_image "${mxnet_version}_gpu_cu92_py3" "Dockerfile.mxnet.python3.gpu.cu92" +docker_test_image_gpu_py3 "${mxnet_version}_gpu_cu92_py3" + +docker_build_image "${mxnet_version}_gpu_cu92_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu92.mkl" +docker_test_image_gpu_py3 "${mxnet_version}_gpu_cu92_mkl_py3" + + # Push dockerfiles echo "All images were successfully built. Now login to dockerhub and push images" docker_account_login +# Python2 docker_push_image "${mxnet_version}_cpu" docker_push_image "${mxnet_version}_cpu_mkl" docker_push_image "latest" @@ -132,6 +176,16 @@ docker_push_image "${mxnet_version}_gpu_cu80_mkl" docker_push_image "${mxnet_version}_gpu_cu92" docker_push_image "${mxnet_version}_gpu_cu92_mkl" +# Python3 +docker_push_image "${mxnet_version}_cpu_py3" +docker_push_image "${mxnet_version}_cpu_mkl_py3" +docker_push_image "${mxnet_version}_gpu_cu90_py3" +docker_push_image "${mxnet_version}_gpu_cu90_mkl_py3" +docker_push_image "${mxnet_version}_gpu_cu80_py3" +docker_push_image "${mxnet_version}_gpu_cu80_mkl_py3" +docker_push_image "${mxnet_version}_gpu_cu92_py3" +docker_push_image "${mxnet_version}_gpu_cu92_mkl_py3" + docker_account_logout echo "Successfully Built, Tested and Pushed all Images to Dockerhub. Link: https://hub.docker.com/r/mxnet/python/tags/" From b716b7c08ee34a1d8fb85c905ffd8bed8015ce37 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Wed, 10 Oct 2018 15:18:34 -0700 Subject: [PATCH 02/19] Fix --- docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl | 6 +++--- docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 | 6 +++--- docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl | 6 +++--- docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 | 6 +++--- docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl | 6 +++--- docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 | 6 +++--- docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl | 6 +++--- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl index 043932ff7c8e..c2f117907328 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl @@ -21,8 +21,8 @@ FROM ubuntu:16.04 RUN apt-get update -RUN apt-get install -y wget python gcc +RUN apt-get install -y wget python3 gcc RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN python3 get-pip.py -RUN pip install mxnet-mkl +RUN pip3 install mxnet-mkl diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 index 8c83ece434a3..58047283238d 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 @@ -21,8 +21,8 @@ FROM nvidia/cuda:8.0-cudnn5-devel RUN apt-get update -RUN apt-get install -y wget python gcc +RUN apt-get install -y wget python3 gcc RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN python3 get-pip.py -RUN pip install mxnet-cu80 +RUN pip3 install mxnet-cu80 diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl index a057c1d20cb1..85e146c22377 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl @@ -21,8 +21,8 @@ FROM nvidia/cuda:8.0-cudnn5-devel RUN apt-get update -RUN apt-get install -y wget python gcc +RUN apt-get install -y wget python3 gcc RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN python3 get-pip.py -RUN pip install mxnet-cu80mkl +RUN pip3 install mxnet-cu80mkl diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 index 1e3d9869ac63..9b3c85955993 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 @@ -21,8 +21,8 @@ FROM nvidia/cuda:9.0-cudnn7-devel RUN apt-get update -RUN apt-get install -y wget python gcc +RUN apt-get install -y wget python3 gcc RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN python3 get-pip.py -RUN pip install mxnet-cu90 +RUN pip3 install mxnet-cu90 diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl index d82abd7cf523..7d845c27de98 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl @@ -21,8 +21,8 @@ FROM nvidia/cuda:9.0-cudnn7-devel RUN apt-get update -RUN apt-get install -y wget python gcc +RUN apt-get install -y wget python3 gcc RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN python3 get-pip.py -RUN pip install mxnet-cu90mkl +RUN pip3 install mxnet-cu90mkl diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 index ba5c54a2a2aa..b3f117fc60fc 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 @@ -21,8 +21,8 @@ FROM nvidia/cuda:9.2-cudnn7-devel RUN apt-get update -RUN apt-get install -y wget python gcc +RUN apt-get install -y wget python3 gcc RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN python3 get-pip.py -RUN pip install mxnet-cu92 +RUN pip3 install mxnet-cu92 diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl index 96a943980b59..84b789f253b7 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl @@ -21,8 +21,8 @@ FROM nvidia/cuda:9.2-cudnn7-devel RUN apt-get update -RUN apt-get install -y wget python gcc +RUN apt-get install -y wget python3 gcc RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN python3 get-pip.py -RUN pip install mxnet-cu92mkl +RUN pip3 install mxnet-cu92mkl From 35a459599beaa943c426f9ce6815aa3773f3379e Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Wed, 10 Oct 2018 20:26:20 -0700 Subject: [PATCH 03/19] Fix the dockerfile names --- docker/docker-python/build_python_dockerfile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index ede2dc40e71f..d64a3d48d53d 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -137,7 +137,7 @@ docker_build_image "${mxnet_version}_cpu_python3" "Dockerfile.mxnet.python3.cpu" docker_test_image_cpu_py3 "${mxnet_version}_cpu_python3" docker_build_image "${mxnet_version}_cpu_mkl_python3" "Dockerfile.mxnet.python3.cpu.mkl" -docker_test_image_cpu_py3 "${mxnet_version}_cpu_mkl_python3" +docker_test_image_cpu_py3 "${mxnet_version}_cpu_mkl_py3" #Build and Test Python3 dockerfiles - GPU From 3e7262f00ccc1fa4e03d9e56dc1deaee7b0c658c Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Wed, 10 Oct 2018 20:40:16 -0700 Subject: [PATCH 04/19] Update the README with python3 images --- docker/docker-python/README.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md index f806d5d6f459..a5c7812daa36 100644 --- a/docker/docker-python/README.md +++ b/docker/docker-python/README.md @@ -1,7 +1,7 @@ # Release Python Docker Images for MXNet The `docker-python` directory can be used to release mxnet python docker images to dockerhub after any mxnet release. -It uses the appropriate pip binaries to build different docker images as - +It uses the appropriate pip binaries to build different docker images. Both python2 (default) and python3 images are available as - * cpu * cpu_mkl * latest (same as cpu) @@ -12,13 +12,21 @@ It uses the appropriate pip binaries to build different docker images as - * gpu_cu80_mkl * gpu_cu92 * gpu_cu92_mkl +* cpu_py3 +* cpu_mkl_py3 +* gpu_cu90_py3 +* gpu_cu90_mkl_py3 +* gpu_cu80_py3 +* gpu_cu80_mkl_py3 +* gpu_cu92_py3 +* gpu_cu92_mkl_py3 ** Note: If you want to use a different pip binary (specific mxnet or cuda version, etc), you can edit the last line of the cpu or gpu dockerfile as required. Refer: https://pypi.org/project/mxnet/ -### Usage +### Using the Build Script `./build_python_dockerfile.sh ` For example: @@ -39,3 +47,10 @@ Credentials can be provided in the following ways: * **Set Environment Variables:** Set the following environment variables which the script will pick up to login to dockerhub at runtime - * $MXNET_DOCKERHUB_PASSWORD * $MXNET_DOCKERHUB_USERNAME + + +### Using the Docker Images +* The MXNet Python Docker images can be found here: https://hub.docker.com/r/mxnet/python/ + +* Docker Pull Command: `docker pull mxnet/python:` +* Get started: `docker run -it mxnet/python: bash` From 1bbd0228b86d67b3aaf80ee8fd9b3a431f594600 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Fri, 12 Oct 2018 12:43:42 -0700 Subject: [PATCH 05/19] Parameterize the mxnet version --- .../docker-python/Dockerfile.mxnet.python.cpu | 3 +- .../Dockerfile.mxnet.python.cpu.mkl | 3 +- .../Dockerfile.mxnet.python.gpu.cu80 | 3 +- .../Dockerfile.mxnet.python.gpu.cu80.mkl | 3 +- .../Dockerfile.mxnet.python.gpu.cu90 | 3 +- .../Dockerfile.mxnet.python.gpu.cu90.mkl | 3 +- .../Dockerfile.mxnet.python.gpu.cu92 | 3 +- .../Dockerfile.mxnet.python.gpu.cu92.mkl | 3 +- .../Dockerfile.mxnet.python3.cpu | 3 +- .../Dockerfile.mxnet.python3.cpu.mkl | 3 +- .../Dockerfile.mxnet.python3.gpu.cu80 | 3 +- .../Dockerfile.mxnet.python3.gpu.cu80.mkl | 3 +- .../Dockerfile.mxnet.python3.gpu.cu90 | 3 +- .../Dockerfile.mxnet.python3.gpu.cu90.mkl | 3 +- .../Dockerfile.mxnet.python3.gpu.cu92 | 3 +- .../Dockerfile.mxnet.python3.gpu.cu92.mkl | 3 +- .../docker-python/build_python_dockerfile.sh | 28 +++++++++++-------- 17 files changed, 48 insertions(+), 28 deletions(-) diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu b/docker/docker-python/Dockerfile.mxnet.python.cpu index eb437f3c2334..856a3d665bbd 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.cpu +++ b/docker/docker-python/Dockerfile.mxnet.python.cpu @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for CPU FROM ubuntu:16.04 +ARG version RUN apt-get update RUN apt-get install -y wget python gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python get-pip.py -RUN pip install mxnet +RUN pip install mxnet==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl index 043932ff7c8e..3b1cc06cd066 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl @@ -19,10 +19,11 @@ # Dockerfile to build MXNet CPU with MKL FROM ubuntu:16.04 +ARG version RUN apt-get update RUN apt-get install -y wget python gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python get-pip.py -RUN pip install mxnet-mkl +RUN pip install mxnet-mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 index 8c83ece434a3..b0a201f200ca 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU FROM nvidia/cuda:8.0-cudnn5-devel +ARG version RUN apt-get update RUN apt-get install -y wget python gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python get-pip.py -RUN pip install mxnet-cu80 +RUN pip install mxnet-cu80==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl index a057c1d20cb1..d77f6ade9793 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU with MKL FROM nvidia/cuda:8.0-cudnn5-devel +ARG version RUN apt-get update RUN apt-get install -y wget python gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python get-pip.py -RUN pip install mxnet-cu80mkl +RUN pip install mxnet-cu80mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 index 1e3d9869ac63..a2a23fb94b8e 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU FROM nvidia/cuda:9.0-cudnn7-devel +ARG version RUN apt-get update RUN apt-get install -y wget python gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python get-pip.py -RUN pip install mxnet-cu90 +RUN pip install mxnet-cu90==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl index d82abd7cf523..5c1e2dc7d49b 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU with MKL FROM nvidia/cuda:9.0-cudnn7-devel +ARG version RUN apt-get update RUN apt-get install -y wget python gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python get-pip.py -RUN pip install mxnet-cu90mkl +RUN pip install mxnet-cu90mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 index ba5c54a2a2aa..be876eb3d72c 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU FROM nvidia/cuda:9.2-cudnn7-devel +ARG version RUN apt-get update RUN apt-get install -y wget python gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python get-pip.py -RUN pip install mxnet-cu92 +RUN pip install mxnet-cu92==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl index 96a943980b59..25f69ff52790 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU with MKL FROM nvidia/cuda:9.2-cudnn7-devel +ARG version RUN apt-get update RUN apt-get install -y wget python gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python get-pip.py -RUN pip install mxnet-cu92mkl +RUN pip install mxnet-cu92mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.cpu b/docker/docker-python/Dockerfile.mxnet.python3.cpu index 39344aef53a8..66c630e2c14c 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.cpu +++ b/docker/docker-python/Dockerfile.mxnet.python3.cpu @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for CPU FROM ubuntu:16.04 +ARG version RUN apt-get update RUN apt-get install -y wget python3 gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python3 get-pip.py -RUN pip3 install mxnet +RUN pip3 install mxnet==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl index c2f117907328..19e3e8d8acc4 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl @@ -19,10 +19,11 @@ # Dockerfile to build MXNet CPU with MKL FROM ubuntu:16.04 +ARG version RUN apt-get update RUN apt-get install -y wget python3 gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python3 get-pip.py -RUN pip3 install mxnet-mkl +RUN pip3 install mxnet-mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 index 58047283238d..3b24555cfa5f 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU FROM nvidia/cuda:8.0-cudnn5-devel +ARG version RUN apt-get update RUN apt-get install -y wget python3 gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python3 get-pip.py -RUN pip3 install mxnet-cu80 +RUN pip3 install mxnet-cu80==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl index 85e146c22377..75aaf994006f 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU with MKL FROM nvidia/cuda:8.0-cudnn5-devel +ARG version RUN apt-get update RUN apt-get install -y wget python3 gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python3 get-pip.py -RUN pip3 install mxnet-cu80mkl +RUN pip3 install mxnet-cu80mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 index 9b3c85955993..3b67eda90620 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU FROM nvidia/cuda:9.0-cudnn7-devel +ARG version RUN apt-get update RUN apt-get install -y wget python3 gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python3 get-pip.py -RUN pip3 install mxnet-cu90 +RUN pip3 install mxnet-cu90==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl index 7d845c27de98..ce152dda4357 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU with MKL FROM nvidia/cuda:9.0-cudnn7-devel +ARG version RUN apt-get update RUN apt-get install -y wget python3 gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python3 get-pip.py -RUN pip3 install mxnet-cu90mkl +RUN pip3 install mxnet-cu90mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 index b3f117fc60fc..cd5e73883fa1 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU FROM nvidia/cuda:9.2-cudnn7-devel +ARG version RUN apt-get update RUN apt-get install -y wget python3 gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python3 get-pip.py -RUN pip3 install mxnet-cu92 +RUN pip3 install mxnet-cu92==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl index 84b789f253b7..3afb3f8b34bf 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl @@ -19,10 +19,11 @@ # Dockerfile to build MXNet for GPU with MKL FROM nvidia/cuda:9.2-cudnn7-devel +ARG version RUN apt-get update RUN apt-get install -y wget python3 gcc RUN wget https://bootstrap.pypa.io/get-pip.py RUN python3 get-pip.py -RUN pip3 install mxnet-cu92mkl +RUN pip3 install mxnet-cu92mkl==$version diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index d64a3d48d53d..945d42dd9a9c 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -23,14 +23,15 @@ set -e programname=$0 function usage { - echo "usage: $programname [version] [path]" + echo "usage: $programname [version] [pip_tag] [path]" echo " [version] Mxnet Version to build" + echo " [pip_tag] Pip Tag to use" echo " [path] Path to MXNet repository (to run tests)" echo " " exit 1 } -if [ $# -le 1 ] || [ $# -ge 3 ] +if [ $# -le 2 ] || [ $# -ge 4 ] then usage exit 1 @@ -39,16 +40,8 @@ fi # Two params provided echo "Building Docker Images for Apache MXNet (Incubating) v$1" mxnet_version="${1}" -test_dir="${2}" - -docker_build_image(){ - echo "Building docker image mxnet/python:${1}" - docker build -t mxnet/python:${1} -f ${2} . -} - -docker_tag_image(){ - docker tag mxnet/python:${1} mxnet/python:${2} -} +pip_tag="${2}" +test_dir="${3}" docker_test_image_cpu(){ echo "Running tests on mxnet/python:${1}" @@ -96,6 +89,17 @@ docker_push_image(){ docker push mxnet/python:${1} } +docker_build_image(){ + echo "Building docker image mxnet/python:${1}" + docker build --build-arg mversion=${pip_tag} -t mxnet/python:${1} -f ${2} . + +} + +docker_tag_image(){ + docker tag mxnet/python:${1} mxnet/python:${2} +} + + # Build and Test dockerfiles - CPU docker_build_image "${mxnet_version}_cpu" "Dockerfile.mxnet.python.cpu" From af3bcec8bc0b15d6f9dd06430c3794c01fea2412 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Fri, 12 Oct 2018 15:28:28 -0700 Subject: [PATCH 06/19] Fix typo --- docker/docker-python/build_python_dockerfile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index 945d42dd9a9c..66898159f08e 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -91,7 +91,7 @@ docker_push_image(){ docker_build_image(){ echo "Building docker image mxnet/python:${1}" - docker build --build-arg mversion=${pip_tag} -t mxnet/python:${1} -f ${2} . + docker build --build-arg version=${pip_tag} -t mxnet/python:${1} -f ${2} . } From 1f12903a5066106828c11e8582e3e8b0b5e28534 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Mon, 15 Oct 2018 14:52:47 -0700 Subject: [PATCH 07/19] Reduce the number of docker layers --- docker/docker-python/Dockerfile.mxnet.python.cpu | 8 ++++---- docker/docker-python/Dockerfile.mxnet.python.cpu.mkl | 8 ++++---- docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 | 8 ++++---- docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl | 8 ++++---- docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 | 8 ++++---- docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl | 8 ++++---- docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 | 8 ++++---- docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl | 8 ++++---- docker/docker-python/Dockerfile.mxnet.python3.cpu | 8 ++++---- docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl | 8 ++++---- docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 | 8 ++++---- .../docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl | 8 ++++---- docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 | 8 ++++---- .../docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl | 8 ++++---- docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 | 8 ++++---- .../docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl | 8 ++++---- 16 files changed, 64 insertions(+), 64 deletions(-) diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu b/docker/docker-python/Dockerfile.mxnet.python.cpu index 856a3d665bbd..aec338655e72 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.cpu +++ b/docker/docker-python/Dockerfile.mxnet.python.cpu @@ -21,9 +21,9 @@ FROM ubuntu:16.04 ARG version -RUN apt-get update -RUN apt-get install -y wget python gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN apt-get update && \ + apt-get install -y wget python gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py RUN pip install mxnet==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl index 3b1cc06cd066..5e87aacd4936 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl @@ -21,9 +21,9 @@ FROM ubuntu:16.04 ARG version -RUN apt-get update -RUN apt-get install -y wget python gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN apt-get update && \ + apt-get install -y wget python gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py RUN pip install mxnet-mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 index b0a201f200ca..69f8ccb2acff 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 @@ -21,9 +21,9 @@ FROM nvidia/cuda:8.0-cudnn5-devel ARG version -RUN apt-get update -RUN apt-get install -y wget python gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN apt-get update && \ + apt-get install -y wget python gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py RUN pip install mxnet-cu80==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl index d77f6ade9793..84bbdfecbc49 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl @@ -21,9 +21,9 @@ FROM nvidia/cuda:8.0-cudnn5-devel ARG version -RUN apt-get update -RUN apt-get install -y wget python gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN apt-get update && \ + apt-get install -y wget python gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py RUN pip install mxnet-cu80mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 index a2a23fb94b8e..409742a8ac73 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 @@ -21,9 +21,9 @@ FROM nvidia/cuda:9.0-cudnn7-devel ARG version -RUN apt-get update -RUN apt-get install -y wget python gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN apt-get update && \ + apt-get install -y wget python gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py RUN pip install mxnet-cu90==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl index 5c1e2dc7d49b..64a9f80ef659 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl @@ -21,9 +21,9 @@ FROM nvidia/cuda:9.0-cudnn7-devel ARG version -RUN apt-get update -RUN apt-get install -y wget python gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN apt-get update && \ + apt-get install -y wget python gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py RUN pip install mxnet-cu90mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 index be876eb3d72c..828308bddc7e 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 @@ -21,9 +21,9 @@ FROM nvidia/cuda:9.2-cudnn7-devel ARG version -RUN apt-get update -RUN apt-get install -y wget python gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN apt-get update && \ + apt-get install -y wget python gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py RUN pip install mxnet-cu92==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl index 25f69ff52790..09b2a2164cef 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl @@ -21,9 +21,9 @@ FROM nvidia/cuda:9.2-cudnn7-devel ARG version -RUN apt-get update -RUN apt-get install -y wget python gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python get-pip.py +RUN apt-get update && \ + apt-get install -y wget python gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py RUN pip install mxnet-cu92mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.cpu b/docker/docker-python/Dockerfile.mxnet.python3.cpu index 66c630e2c14c..c416355d35b6 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.cpu +++ b/docker/docker-python/Dockerfile.mxnet.python3.cpu @@ -21,9 +21,9 @@ FROM ubuntu:16.04 ARG version -RUN apt-get update -RUN apt-get install -y wget python3 gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python3 get-pip.py +RUN apt-get update && \ + apt-get install -y wget python3 gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py RUN pip3 install mxnet==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl index 19e3e8d8acc4..4f837d23e097 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl @@ -21,9 +21,9 @@ FROM ubuntu:16.04 ARG version -RUN apt-get update -RUN apt-get install -y wget python3 gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python3 get-pip.py +RUN apt-get update && \ + apt-get install -y wget python3 gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py RUN pip3 install mxnet-mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 index 3b24555cfa5f..432277a83216 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 @@ -21,9 +21,9 @@ FROM nvidia/cuda:8.0-cudnn5-devel ARG version -RUN apt-get update -RUN apt-get install -y wget python3 gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python3 get-pip.py +RUN apt-get update && \ + apt-get install -y wget python3 gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py RUN pip3 install mxnet-cu80==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl index 75aaf994006f..9f0bd7b3c3c8 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl @@ -21,9 +21,9 @@ FROM nvidia/cuda:8.0-cudnn5-devel ARG version -RUN apt-get update -RUN apt-get install -y wget python3 gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python3 get-pip.py +RUN apt-get update && \ + apt-get install -y wget python3 gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py RUN pip3 install mxnet-cu80mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 index 3b67eda90620..3b5f633a3c14 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 @@ -21,9 +21,9 @@ FROM nvidia/cuda:9.0-cudnn7-devel ARG version -RUN apt-get update -RUN apt-get install -y wget python3 gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python3 get-pip.py +RUN apt-get update && \ + apt-get install -y wget python3 gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py RUN pip3 install mxnet-cu90==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl index ce152dda4357..6e193784dabd 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl @@ -21,9 +21,9 @@ FROM nvidia/cuda:9.0-cudnn7-devel ARG version -RUN apt-get update -RUN apt-get install -y wget python3 gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python3 get-pip.py +RUN apt-get update && \ + apt-get install -y wget python3 gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py RUN pip3 install mxnet-cu90mkl==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 index cd5e73883fa1..c80b76d7244b 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 @@ -21,9 +21,9 @@ FROM nvidia/cuda:9.2-cudnn7-devel ARG version -RUN apt-get update -RUN apt-get install -y wget python3 gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python3 get-pip.py +RUN apt-get update && \ + apt-get install -y wget python3 gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py RUN pip3 install mxnet-cu92==$version diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl index 3afb3f8b34bf..ced92576ba7a 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl @@ -21,9 +21,9 @@ FROM nvidia/cuda:9.2-cudnn7-devel ARG version -RUN apt-get update -RUN apt-get install -y wget python3 gcc -RUN wget https://bootstrap.pypa.io/get-pip.py -RUN python3 get-pip.py +RUN apt-get update && \ + apt-get install -y wget python3 gcc && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python3 get-pip.py RUN pip3 install mxnet-cu92mkl==$version From 74f622b6198b18fe2b0b97ba13741a73c85f704a Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Mon, 15 Oct 2018 18:08:12 -0700 Subject: [PATCH 08/19] Combine build and Test, run builds in parallel. Use variable names instead of numbers --- .../docker-python/build_python_dockerfile.sh | 147 ++++++++---------- 1 file changed, 65 insertions(+), 82 deletions(-) diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index 66898159f08e..e0e05fb0f0e0 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -31,7 +31,7 @@ function usage { exit 1 } -if [ $# -le 2 ] || [ $# -ge 4 ] +if [ $# -le 2 ] || [ $# -ge 5 ] then usage exit 1 @@ -44,31 +44,21 @@ pip_tag="${2}" test_dir="${3}" docker_test_image_cpu(){ - echo "Running tests on mxnet/python:${1}" - docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" - docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py" - docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py" + image_tag="${1}" + python_version="${2}" + echo "Running tests on mxnet/python:${image_tag}" + docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" + docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/tests/python/train/test_conv.py" + docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/example/image-classification/train_mnist.py" } docker_test_image_gpu(){ + image_tag="${1}" + python_version="${2}" echo "Running tests on mxnet/python:${1}" - nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" - nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py --gpu" - nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py --gpus 2" -} - -docker_test_image_cpu_py3(){ - echo "Running tests on mxnet/python:${1}" - docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python3 /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" - docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python3 /mxnet/tests/python/train/test_conv.py" - docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python3 /mxnet/example/image-classification/train_mnist.py" -} - -docker_test_image_gpu_py3(){ - echo "Running tests on mxnet/python:${1}" - nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python3 /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" - nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python3 /mxnet/tests/python/train/test_conv.py --gpu" - nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python3 /mxnet/example/image-classification/train_mnist.py --gpus 2" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/tests/python/train/test_conv.py --gpu" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/example/image-classification/train_mnist.py --gpus 2" } # if both $MXNET_DOCKERHUB_PASSWORD and $MXNET_DOCKERHUB_USERNAME environment variables are set, docker will automatically login @@ -86,83 +76,76 @@ docker_account_logout(){ } docker_push_image(){ - docker push mxnet/python:${1} + image_tag="${1}" + docker push mxnet/python:${image_tag} } -docker_build_image(){ - echo "Building docker image mxnet/python:${1}" - docker build --build-arg version=${pip_tag} -t mxnet/python:${1} -f ${2} . +docker_generate_image_cpu(){ + image_tag="${1}" + dockerfile="${2}" + python_version="${3}" + echo "Building docker image mxnet/python:${image_tag}" + docker build --build-arg version=${pip_tag} -t mxnet/python:${image_tag} -f ${dockerfile} . + docker_test_image_cpu ${image_tag} ${python_version} +} +docker_tag_image_cpu(){ + original_tag="${1}" + image_tag="${2}" + python_version="${3}" + docker tag mxnet/python:${original_tag} mxnet/python:${image_tag} + docker_test_image_cpu ${image_tag} ${python_version} } -docker_tag_image(){ - docker tag mxnet/python:${1} mxnet/python:${2} +docker_generate_image_gpu(){ + image_tag="${1}" + dockerfile="${2}" + python_version="${3}" + echo "Building docker image mxnet/python:${1}" + docker build --build-arg version=${pip_tag} -t mxnet/python:${image_tag} -f ${dockerfile} . + docker_test_image_gpu ${image_tag} ${python_version} } +docker_tag_image_gpu(){ + original_tag="${1}" + image_tag="${2}" + python_version="${3}" + docker tag mxnet/python:${original_tag} mxnet/python:${image_tag} + docker_test_image_gpu ${image_tag} ${python_version} +} # Build and Test dockerfiles - CPU -docker_build_image "${mxnet_version}_cpu" "Dockerfile.mxnet.python.cpu" -docker_test_image_cpu "${mxnet_version}_cpu" - -docker_build_image "${mxnet_version}_cpu_mkl" "Dockerfile.mxnet.python.cpu.mkl" -docker_test_image_cpu "${mxnet_version}_cpu_mkl" - -docker_tag_image "${mxnet_version}_cpu" "latest" -docker_test_image_cpu "latest" - +docker_generate_image_cpu "${mxnet_version}_cpu" "Dockerfile.mxnet.python.cpu" "python" & +docker_generate_image_cpu "${mxnet_version}_cpu_mkl" "Dockerfile.mxnet.python.cpu.mkl" "python" & +docker_tag_image_cpu "${mxnet_version}_cpu" "latest" "python" & #Build and Test dockerfiles - GPU -docker_build_image "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu.cu90" -docker_test_image_gpu "${mxnet_version}_gpu_cu90" - -docker_build_image "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.cu90.mkl" -docker_test_image_gpu "${mxnet_version}_gpu_cu90_mkl" - -docker_tag_image "${mxnet_version}_gpu_cu90" "gpu" -docker_test_image_gpu "gpu" - -docker_build_image "${mxnet_version}_gpu_cu80" "Dockerfile.mxnet.python.gpu.cu80" -docker_test_image_gpu "${mxnet_version}_gpu_cu80" - -docker_build_image "${mxnet_version}_gpu_cu80_mkl" "Dockerfile.mxnet.python.gpu.cu80.mkl" -docker_test_image_gpu "${mxnet_version}_gpu_cu80_mkl" - -docker_build_image "${mxnet_version}_gpu_cu92" "Dockerfile.mxnet.python.gpu.cu92" -docker_test_image_gpu "${mxnet_version}_gpu_cu92" - -docker_build_image "${mxnet_version}_gpu_cu92_mkl" "Dockerfile.mxnet.python.gpu.cu92.mkl" -docker_test_image_gpu "${mxnet_version}_gpu_cu92_mkl" - +docker_generate_image_gpu "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu.cu90" "python" & +docker_generate_image_gpu "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.cu90.mkl" "python" & +docker_tag_image_gpu "${mxnet_version}_gpu_cu90" "gpu" "python" & +docker_generate_image_gpu "${mxnet_version}_gpu_cu80" "Dockerfile.mxnet.python.gpu.cu80" "python" & +docker_generate_image_gpu "${mxnet_version}_gpu_cu80_mkl" "Dockerfile.mxnet.python.gpu.cu80.mkl" "python" & +docker_generate_image_gpu "${mxnet_version}_gpu_cu92" "Dockerfile.mxnet.python.gpu.cu92" "python" & +docker_generate_image_gpu "${mxnet_version}_gpu_cu92_mkl" "Dockerfile.mxnet.python.gpu.cu92.mkl" "python" +echo "Waiting for MXNet Python2 Docker Images to Build" +wait # Build and Test Python3 dockerfiles - CPU -docker_build_image "${mxnet_version}_cpu_python3" "Dockerfile.mxnet.python3.cpu" -docker_test_image_cpu_py3 "${mxnet_version}_cpu_python3" - -docker_build_image "${mxnet_version}_cpu_mkl_python3" "Dockerfile.mxnet.python3.cpu.mkl" -docker_test_image_cpu_py3 "${mxnet_version}_cpu_mkl_py3" - +docker_generate_image_cpu "${mxnet_version}_cpu_py3" "Dockerfile.mxnet.python3.cpu" "python3" & +docker_generate_image_cpu "${mxnet_version}_cpu_mkl_py3" "Dockerfile.mxnet.python3.cpu.mkl" "python3" & #Build and Test Python3 dockerfiles - GPU -docker_build_image "${mxnet_version}_gpu_cu90_py3" "Dockerfile.mxnet.python3.gpu.cu90" -docker_test_image_gpu_py3 "${mxnet_version}_gpu_cu90_py3" - -docker_build_image "${mxnet_version}_gpu_cu90_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu90.mkl" -docker_test_image_gpu_py3 "${mxnet_version}_gpu_cu90_mkl_py3" - -docker_build_image "${mxnet_version}_gpu_cu80_py3" "Dockerfile.mxnet.python3.gpu.cu80" -docker_test_image_gpu_py3 "${mxnet_version}_gpu_cu80_py3" - -docker_build_image "${mxnet_version}_gpu_cu80_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu80.mkl" -docker_test_image_gpu_py3 "${mxnet_version}_gpu_cu80_mkl_py3" - -docker_build_image "${mxnet_version}_gpu_cu92_py3" "Dockerfile.mxnet.python3.gpu.cu92" -docker_test_image_gpu_py3 "${mxnet_version}_gpu_cu92_py3" - -docker_build_image "${mxnet_version}_gpu_cu92_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu92.mkl" -docker_test_image_gpu_py3 "${mxnet_version}_gpu_cu92_mkl_py3" - +docker_generate_image_gpu "${mxnet_version}_gpu_cu90_py3" "Dockerfile.mxnet.python3.gpu.cu90" "python3" & +docker_generate_image_gpu "${mxnet_version}_gpu_cu90_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu90.mkl" "python3" & +docker_generate_image_gpu "${mxnet_version}_gpu_cu80_py3" "Dockerfile.mxnet.python3.gpu.cu80" "python3" & +docker_generate_image_gpu "${mxnet_version}_gpu_cu80_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu80.mkl" "python3" & +docker_generate_image_gpu "${mxnet_version}_gpu_cu92_py3" "Dockerfile.mxnet.python3.gpu.cu92" "python3" & +docker_generate_image_gpu "${mxnet_version}_gpu_cu92_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu92.mkl" "python3" + +echo "Waiting for MXNet Python3 Docker Images to Build" +wait # Push dockerfiles echo "All images were successfully built. Now login to dockerhub and push images" From 7aa944436f23163d0921181aedd5b798f913cfb9 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Tue, 16 Oct 2018 11:47:30 -0700 Subject: [PATCH 09/19] minor fix --- docker/docker-python/build_python_dockerfile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index e0e05fb0f0e0..c653bc28b503 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -31,7 +31,7 @@ function usage { exit 1 } -if [ $# -le 2 ] || [ $# -ge 5 ] +if [ $# -le 2 ] || [ $# -ge 4 ] then usage exit 1 From dcf2836ee918780093fc03be43ae30a7d687c8bc Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Thu, 18 Oct 2018 16:09:15 -0700 Subject: [PATCH 10/19] Update README for build command --- docker/docker-python/README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md index a5c7812daa36..fc19507c1606 100644 --- a/docker/docker-python/README.md +++ b/docker/docker-python/README.md @@ -21,18 +21,14 @@ It uses the appropriate pip binaries to build different docker images. Both pyth * gpu_cu92_py3 * gpu_cu92_mkl_py3 - -** Note: If you want to use a different pip binary (specific mxnet or cuda version, etc), you can edit the last line of the cpu or gpu dockerfile as required. - Refer: https://pypi.org/project/mxnet/ ### Using the Build Script -`./build_python_dockerfile.sh ` +`./build_python_dockerfile.sh ` For example: -`./build_python_dockerfile.sh 1.3.0 ~/build-docker/incubator-mxnet` +`./build_python_dockerfile.sh 1.3.0 1.3.0.post0 ~/build-docker/incubator-mxnet` -** Note: The build script picks up the latest pip binaries. This means it uses the latest released mxnet version. The version specified as a parameter to the script is only used to tag the built image correctly. ### Tests run * [test_conv.py](https://github.com/apache/incubator-mxnet/blob/master/tests/python/train/test_conv.py) From 1e72e4f63fa5fdab84248a7232cac53f61dbdbd3 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Thu, 25 Oct 2018 17:11:07 -0700 Subject: [PATCH 11/19] Bug Fix: Script should fail if any background process returns non-zero code --- .../docker-python/build_python_dockerfile.sh | 56 +++++++++++++------ 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index c653bc28b503..478d3f55e16f 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -43,6 +43,15 @@ mxnet_version="${1}" pip_tag="${2}" test_dir="${3}" +# Remove the logs directory if it already exists else it may fail due to old logs. +LOGDIR=~/temp/docker_logs +if [ -d "${LOGDIR}" ]; then + rm -rf ${LOGDIR} +fi +mkdir ${LOGDIR} + + +# Functions docker_test_image_cpu(){ image_tag="${1}" python_version="${2}" @@ -116,37 +125,45 @@ docker_tag_image_gpu(){ # Build and Test dockerfiles - CPU -docker_generate_image_cpu "${mxnet_version}_cpu" "Dockerfile.mxnet.python.cpu" "python" & -docker_generate_image_cpu "${mxnet_version}_cpu_mkl" "Dockerfile.mxnet.python.cpu.mkl" "python" & -docker_tag_image_cpu "${mxnet_version}_cpu" "latest" "python" & +docker_generate_image_cpu "${mxnet_version}_cpu" "Dockerfile.mxnet.python.cpu" "python" > ${LOGDIR}/docker_cpu.out 2>&1 & +docker_generate_image_cpu "${mxnet_version}_cpu_mkl" "Dockerfile.mxnet.python.cpu.mkl" "python" > ${LOGDIR}/docker_cpu_mkl.out 2>&1 & +docker_tag_image_cpu "${mxnet_version}_cpu" "latest" "python" > ${LOGDIR}/docker_latest.out 2>&1 & #Build and Test dockerfiles - GPU -docker_generate_image_gpu "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu.cu90" "python" & -docker_generate_image_gpu "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.cu90.mkl" "python" & -docker_tag_image_gpu "${mxnet_version}_gpu_cu90" "gpu" "python" & -docker_generate_image_gpu "${mxnet_version}_gpu_cu80" "Dockerfile.mxnet.python.gpu.cu80" "python" & -docker_generate_image_gpu "${mxnet_version}_gpu_cu80_mkl" "Dockerfile.mxnet.python.gpu.cu80.mkl" "python" & -docker_generate_image_gpu "${mxnet_version}_gpu_cu92" "Dockerfile.mxnet.python.gpu.cu92" "python" & -docker_generate_image_gpu "${mxnet_version}_gpu_cu92_mkl" "Dockerfile.mxnet.python.gpu.cu92.mkl" "python" +docker_generate_image_gpu "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu.cu90" "python" > ${LOGDIR}/docker_gpu_cu90.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.cu90.mkl" "python" > ${LOGDIR}/docker_gpu_cu90_mkl.out 2>&1 & +docker_tag_image_gpu "${mxnet_version}_gpu_cu90" "gpu" "python" > ${LOGDIR}/docker_gpu_cu90.out & +docker_generate_image_gpu "${mxnet_version}_gpu_cu80" "Dockerfile.mxnet.python.gpu.cu80" "python" > ${LOGDIR}/docker_gpu_cu80.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu80_mkl" "Dockerfile.mxnet.python.gpu.cu80.mkl" "python" > ${LOGDIR}/docker_gpu_cu80_mkl.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu92" "Dockerfile.mxnet.python.gpu.cu92" "python" > ${LOGDIR}/docker_gpu_cu92.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu92_mkl" "Dockerfile.mxnet.python.gpu.cu92.mkl" "python" > ${LOGDIR}/docker_gpu_cu92_mkl.out 2>&1 echo "Waiting for MXNet Python2 Docker Images to Build" wait # Build and Test Python3 dockerfiles - CPU -docker_generate_image_cpu "${mxnet_version}_cpu_py3" "Dockerfile.mxnet.python3.cpu" "python3" & -docker_generate_image_cpu "${mxnet_version}_cpu_mkl_py3" "Dockerfile.mxnet.python3.cpu.mkl" "python3" & +docker_generate_image_cpu "${mxnet_version}_cpu_py3" "Dockerfile.mxnet.python3.cpu" "python3" > ${LOGDIR}/docker_cpu_py3.out 2>&1 & +docker_generate_image_cpu "${mxnet_version}_cpu_mkl_py3" "Dockerfile.mxnet.python3.cpu.mkl" "python3" > ${LOGDIR}/docker_cpu_mkl_py3.out 2>&1 & #Build and Test Python3 dockerfiles - GPU -docker_generate_image_gpu "${mxnet_version}_gpu_cu90_py3" "Dockerfile.mxnet.python3.gpu.cu90" "python3" & -docker_generate_image_gpu "${mxnet_version}_gpu_cu90_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu90.mkl" "python3" & -docker_generate_image_gpu "${mxnet_version}_gpu_cu80_py3" "Dockerfile.mxnet.python3.gpu.cu80" "python3" & -docker_generate_image_gpu "${mxnet_version}_gpu_cu80_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu80.mkl" "python3" & -docker_generate_image_gpu "${mxnet_version}_gpu_cu92_py3" "Dockerfile.mxnet.python3.gpu.cu92" "python3" & -docker_generate_image_gpu "${mxnet_version}_gpu_cu92_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu92.mkl" "python3" +docker_generate_image_gpu "${mxnet_version}_gpu_cu90_py3" "Dockerfile.mxnet.python3.gpu.cu90" "python3" > ${LOGDIR}/docker_gpu_cu90_py3.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu90_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu90.mkl" "python3" > ${LOGDIR}/docker_gpu_cu90_mkl_py3.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu80_py3" "Dockerfile.mxnet.python3.gpu.cu80" "python3" > ${LOGDIR}/docker_gpu_cu80_py3.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu80_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu80.mkl" "python3" > ${LOGDIR}/docker_gpu_cu80_mkl_py3.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu92_py3" "Dockerfile.mxnet.python3.gpu.cu92" "python3" > ${LOGDIR}/docker_gpu_cu92_py3.out 2>&1 & +docker_generate_image_gpu "${mxnet_version}_gpu_cu92_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu92.mkl" "python3" > ${LOGDIR}/docker_gpu_cu92_mkl_py3.out 2>&1 echo "Waiting for MXNet Python3 Docker Images to Build" wait +# Parse all the docker logfiles to make sure there is no error. Fail script if error is found. +egrep -i "not found|error|returned a non-zero code|fail" ${LOGDIR}/docker* +if [ $? -eq 0 ] +then + echo "ERROR: One of the build/test commands failed. Refer to the filename above to see which image tag caused it." + exit 1 +fi + # Push dockerfiles echo "All images were successfully built. Now login to dockerhub and push images" docker_account_login @@ -176,3 +193,6 @@ docker_push_image "${mxnet_version}_gpu_cu92_mkl_py3" docker_account_logout echo "Successfully Built, Tested and Pushed all Images to Dockerhub. Link: https://hub.docker.com/r/mxnet/python/tags/" + +#Delete the log directory since everything succeeded: +rm -rf ${LOGDIR} \ No newline at end of file From f487314f1927dd1673763fe142f2675978c584d6 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Tue, 6 Nov 2018 15:16:32 -0800 Subject: [PATCH 12/19] Changes needed for Benchmarking, 4 new tags, change python to python-dev --- .../docker-python/Dockerfile.mxnet.python.cpu | 2 +- .../Dockerfile.mxnet.python.cpu.mkl | 2 +- .../Dockerfile.mxnet.python.gpu.cu80 | 2 +- .../Dockerfile.mxnet.python.gpu.cu80.mkl | 2 +- .../Dockerfile.mxnet.python.gpu.cu90 | 2 +- .../Dockerfile.mxnet.python.gpu.cu90.mkl | 2 +- .../Dockerfile.mxnet.python.gpu.cu92 | 2 +- .../Dockerfile.mxnet.python.gpu.cu92.mkl | 2 +- .../Dockerfile.mxnet.python3.cpu | 2 +- .../Dockerfile.mxnet.python3.cpu.mkl | 2 +- .../Dockerfile.mxnet.python3.gpu.cu80 | 2 +- .../Dockerfile.mxnet.python3.gpu.cu80.mkl | 2 +- .../Dockerfile.mxnet.python3.gpu.cu90 | 2 +- .../Dockerfile.mxnet.python3.gpu.cu90.mkl | 2 +- .../Dockerfile.mxnet.python3.gpu.cu92 | 2 +- .../Dockerfile.mxnet.python3.gpu.cu92.mkl | 2 +- docker/docker-python/README.md | 43 +++++++++++-------- .../docker-python/build_python_dockerfile.sh | 9 +++- 18 files changed, 49 insertions(+), 35 deletions(-) diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu b/docker/docker-python/Dockerfile.mxnet.python.cpu index aec338655e72..0858e99e2d7d 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.cpu +++ b/docker/docker-python/Dockerfile.mxnet.python.cpu @@ -22,7 +22,7 @@ FROM ubuntu:16.04 ARG version RUN apt-get update && \ - apt-get install -y wget python gcc && \ + apt-get install -y wget python-dev gcc && \ wget https://bootstrap.pypa.io/get-pip.py && \ python get-pip.py diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl index 5e87aacd4936..dbb7d29f5037 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl @@ -22,7 +22,7 @@ FROM ubuntu:16.04 ARG version RUN apt-get update && \ - apt-get install -y wget python gcc && \ + apt-get install -y wget python-dev gcc && \ wget https://bootstrap.pypa.io/get-pip.py && \ python get-pip.py diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 index 69f8ccb2acff..cb22721f48b9 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 @@ -22,7 +22,7 @@ FROM nvidia/cuda:8.0-cudnn5-devel ARG version RUN apt-get update && \ - apt-get install -y wget python gcc && \ + apt-get install -y wget python-dev gcc && \ wget https://bootstrap.pypa.io/get-pip.py && \ python get-pip.py diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl index 84bbdfecbc49..eda96c90d181 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl @@ -22,7 +22,7 @@ FROM nvidia/cuda:8.0-cudnn5-devel ARG version RUN apt-get update && \ - apt-get install -y wget python gcc && \ + apt-get install -y wget python-dev gcc && \ wget https://bootstrap.pypa.io/get-pip.py && \ python get-pip.py diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 index 409742a8ac73..cd36b8afdbd8 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 @@ -22,7 +22,7 @@ FROM nvidia/cuda:9.0-cudnn7-devel ARG version RUN apt-get update && \ - apt-get install -y wget python gcc && \ + apt-get install -y wget python-dev gcc && \ wget https://bootstrap.pypa.io/get-pip.py && \ python get-pip.py diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl index 64a9f80ef659..0b274c4e0d3f 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl @@ -22,7 +22,7 @@ FROM nvidia/cuda:9.0-cudnn7-devel ARG version RUN apt-get update && \ - apt-get install -y wget python gcc && \ + apt-get install -y wget python-dev gcc && \ wget https://bootstrap.pypa.io/get-pip.py && \ python get-pip.py diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 index 828308bddc7e..2c43187faf79 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 @@ -22,7 +22,7 @@ FROM nvidia/cuda:9.2-cudnn7-devel ARG version RUN apt-get update && \ - apt-get install -y wget python gcc && \ + apt-get install -y wget python-dev gcc && \ wget https://bootstrap.pypa.io/get-pip.py && \ python get-pip.py diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl index 09b2a2164cef..db204897ef60 100644 --- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl @@ -22,7 +22,7 @@ FROM nvidia/cuda:9.2-cudnn7-devel ARG version RUN apt-get update && \ - apt-get install -y wget python gcc && \ + apt-get install -y wget python-dev gcc && \ wget https://bootstrap.pypa.io/get-pip.py && \ python get-pip.py diff --git a/docker/docker-python/Dockerfile.mxnet.python3.cpu b/docker/docker-python/Dockerfile.mxnet.python3.cpu index c416355d35b6..8ad9950a2f21 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.cpu +++ b/docker/docker-python/Dockerfile.mxnet.python3.cpu @@ -22,7 +22,7 @@ FROM ubuntu:16.04 ARG version RUN apt-get update && \ - apt-get install -y wget python3 gcc && \ + apt-get install -y wget python3-dev gcc && \ wget https://bootstrap.pypa.io/get-pip.py && \ python3 get-pip.py diff --git a/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl index 4f837d23e097..c6312891c6ea 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl @@ -22,7 +22,7 @@ FROM ubuntu:16.04 ARG version RUN apt-get update && \ - apt-get install -y wget python3 gcc && \ + apt-get install -y wget python3-dev gcc && \ wget https://bootstrap.pypa.io/get-pip.py && \ python3 get-pip.py diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 index 432277a83216..58af6bfeb273 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80 @@ -22,7 +22,7 @@ FROM nvidia/cuda:8.0-cudnn5-devel ARG version RUN apt-get update && \ - apt-get install -y wget python3 gcc && \ + apt-get install -y wget python3-dev gcc && \ wget https://bootstrap.pypa.io/get-pip.py && \ python3 get-pip.py diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl index 9f0bd7b3c3c8..059f002c8560 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl @@ -22,7 +22,7 @@ FROM nvidia/cuda:8.0-cudnn5-devel ARG version RUN apt-get update && \ - apt-get install -y wget python3 gcc && \ + apt-get install -y wget python3-dev gcc && \ wget https://bootstrap.pypa.io/get-pip.py && \ python3 get-pip.py diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 index 3b5f633a3c14..a860de918054 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90 @@ -22,7 +22,7 @@ FROM nvidia/cuda:9.0-cudnn7-devel ARG version RUN apt-get update && \ - apt-get install -y wget python3 gcc && \ + apt-get install -y wget python3-dev gcc && \ wget https://bootstrap.pypa.io/get-pip.py && \ python3 get-pip.py diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl index 6e193784dabd..c0b6145b28e7 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl @@ -22,7 +22,7 @@ FROM nvidia/cuda:9.0-cudnn7-devel ARG version RUN apt-get update && \ - apt-get install -y wget python3 gcc && \ + apt-get install -y wget python3-dev gcc && \ wget https://bootstrap.pypa.io/get-pip.py && \ python3 get-pip.py diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 index c80b76d7244b..6d877961db9f 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92 @@ -22,7 +22,7 @@ FROM nvidia/cuda:9.2-cudnn7-devel ARG version RUN apt-get update && \ - apt-get install -y wget python3 gcc && \ + apt-get install -y wget python3-dev gcc && \ wget https://bootstrap.pypa.io/get-pip.py && \ python3 get-pip.py diff --git a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl index ced92576ba7a..b73df97e5950 100644 --- a/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl +++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl @@ -22,7 +22,7 @@ FROM nvidia/cuda:9.2-cudnn7-devel ARG version RUN apt-get update && \ - apt-get install -y wget python3 gcc && \ + apt-get install -y wget python3-dev gcc && \ wget https://bootstrap.pypa.io/get-pip.py && \ python3 get-pip.py diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md index fc19507c1606..4c85eafdb665 100644 --- a/docker/docker-python/README.md +++ b/docker/docker-python/README.md @@ -2,24 +2,31 @@ The `docker-python` directory can be used to release mxnet python docker images to dockerhub after any mxnet release. It uses the appropriate pip binaries to build different docker images. Both python2 (default) and python3 images are available as - -* cpu -* cpu_mkl -* latest (same as cpu) -* gpu_cu90 -* gpu_cu90_mkl -* gpu (same as gpu_cu90) -* gpu_cu80 -* gpu_cu80_mkl -* gpu_cu92 -* gpu_cu92_mkl -* cpu_py3 -* cpu_mkl_py3 -* gpu_cu90_py3 -* gpu_cu90_mkl_py3 -* gpu_cu80_py3 -* gpu_cu80_mkl_py3 -* gpu_cu92_py3 -* gpu_cu92_mkl_py3 +* {version}_cpu +* {version}_cpu_mkl +* {version}_gpu_cu90 +* {version}_gpu_cu90_mkl +* {version}_gpu (same as gpu_cu90) +* {version}_gpu_cu80 +* {version}_gpu_cu80_mkl +* {version}_gpu_cu92 +* {version}_gpu_cu92_mkl +* {version}_cpu_py3 +* {version}_cpu_mkl_py3 +* {version}_gpu_cu90_py3 +* {version}_gpu_cu90_mkl_py3 +* {version}_gpu_cu80_py3 +* {version}_gpu_cu80_mkl_py3 +* {version}_gpu_cu92_py3 +* {version}_gpu_cu92_mkl_py3 + +And the following tags will be available without the version string in the image name (for Benchmarking and other use cases): +* latest (same as {version}_cpu) +* gpu (same as {version}_gpu_cu90) +* latest_cpu_mkl_py2 (same as {version}_cpu_mkl) +* latest_cpu_mkl_py3 (same as {version}_cpu_mkl_py3) +* latest_gpu_mkl_py2 (same as {version}_gpu_cu90_mkl) +* latest_gpu_mkl_py3 (same as {version}_gpu_cu90_mkl_py3) Refer: https://pypi.org/project/mxnet/ diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index 478d3f55e16f..dacc9fc4dd3c 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -132,7 +132,7 @@ docker_tag_image_cpu "${mxnet_version}_cpu" "latest" "python" > ${LOGDIR}/docker #Build and Test dockerfiles - GPU docker_generate_image_gpu "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu.cu90" "python" > ${LOGDIR}/docker_gpu_cu90.out 2>&1 & docker_generate_image_gpu "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.cu90.mkl" "python" > ${LOGDIR}/docker_gpu_cu90_mkl.out 2>&1 & -docker_tag_image_gpu "${mxnet_version}_gpu_cu90" "gpu" "python" > ${LOGDIR}/docker_gpu_cu90.out & +docker_tag_image_gpu "${mxnet_version}_gpu_cu90" "gpu" "python" > ${LOGDIR}/docker_gpu_cu90.out 2>&1 & docker_generate_image_gpu "${mxnet_version}_gpu_cu80" "Dockerfile.mxnet.python.gpu.cu80" "python" > ${LOGDIR}/docker_gpu_cu80.out 2>&1 & docker_generate_image_gpu "${mxnet_version}_gpu_cu80_mkl" "Dockerfile.mxnet.python.gpu.cu80.mkl" "python" > ${LOGDIR}/docker_gpu_cu80_mkl.out 2>&1 & docker_generate_image_gpu "${mxnet_version}_gpu_cu92" "Dockerfile.mxnet.python.gpu.cu92" "python" > ${LOGDIR}/docker_gpu_cu92.out 2>&1 & @@ -156,6 +156,13 @@ docker_generate_image_gpu "${mxnet_version}_gpu_cu92_mkl_py3" "Dockerfile.mxnet. echo "Waiting for MXNet Python3 Docker Images to Build" wait +echo "Re-Tag 4 images with version-free names for Benchmarking" +docker_tag_image_cpu "${mxnet_version}_cpu_mkl" "latest_cpu_mkl_py2" "python" > ${LOGDIR}/docker_latest_cpu_mkl_py2.out 2>&1 & +docker_tag_image_cpu "${mxnet_version}_cpu_mkl_py3" "latest_cpu_mkl_py3" "python3" > ${LOGDIR}/docker_latest_cpu_mkl_py3.out 2>&1 & +docker_tag_image_gpu "${mxnet_version}_gpu_cu90_mkl" "latest_gpu_mkl_py2" "python" > ${LOGDIR}/docker_latest_gpu_mkl_py2.out 2>&1 & +docker_tag_image_gpu "${mxnet_version}_gpu_cu90_mkl_py3" "latest_gpu_mkl_py3" "python3" > ${LOGDIR}/docker_latest_gpu_mkl_py3.out 2>&1 +wait + # Parse all the docker logfiles to make sure there is no error. Fail script if error is found. egrep -i "not found|error|returned a non-zero code|fail" ${LOGDIR}/docker* if [ $? -eq 0 ] From 7a92fe6ab85c89b9cd9ce57a2b6cea9fb7ca1736 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Fri, 16 Nov 2018 15:45:12 -0800 Subject: [PATCH 13/19] Minor typos --- docker/docker-python/README.md | 1 - docker/docker-python/build_python_dockerfile.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md index 4c85eafdb665..d84d23955b1a 100644 --- a/docker/docker-python/README.md +++ b/docker/docker-python/README.md @@ -6,7 +6,6 @@ It uses the appropriate pip binaries to build different docker images. Both pyth * {version}_cpu_mkl * {version}_gpu_cu90 * {version}_gpu_cu90_mkl -* {version}_gpu (same as gpu_cu90) * {version}_gpu_cu80 * {version}_gpu_cu80_mkl * {version}_gpu_cu92 diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index dacc9fc4dd3c..c9f66af40c89 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -132,7 +132,7 @@ docker_tag_image_cpu "${mxnet_version}_cpu" "latest" "python" > ${LOGDIR}/docker #Build and Test dockerfiles - GPU docker_generate_image_gpu "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu.cu90" "python" > ${LOGDIR}/docker_gpu_cu90.out 2>&1 & docker_generate_image_gpu "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.cu90.mkl" "python" > ${LOGDIR}/docker_gpu_cu90_mkl.out 2>&1 & -docker_tag_image_gpu "${mxnet_version}_gpu_cu90" "gpu" "python" > ${LOGDIR}/docker_gpu_cu90.out 2>&1 & +docker_tag_image_gpu "${mxnet_version}_gpu_cu90" "gpu" "python" > ${LOGDIR}/docker_gpu 2>&1 & docker_generate_image_gpu "${mxnet_version}_gpu_cu80" "Dockerfile.mxnet.python.gpu.cu80" "python" > ${LOGDIR}/docker_gpu_cu80.out 2>&1 & docker_generate_image_gpu "${mxnet_version}_gpu_cu80_mkl" "Dockerfile.mxnet.python.gpu.cu80.mkl" "python" > ${LOGDIR}/docker_gpu_cu80_mkl.out 2>&1 & docker_generate_image_gpu "${mxnet_version}_gpu_cu92" "Dockerfile.mxnet.python.gpu.cu92" "python" > ${LOGDIR}/docker_gpu_cu92.out 2>&1 & From 98c3d455806ac5a8d300a4c951a71281a6963809 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Sun, 18 Nov 2018 19:21:56 -0800 Subject: [PATCH 14/19] Minor Rearrangement - sometimes tagging fails --- docker/docker-python/build_python_dockerfile.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index c9f66af40c89..8e0f34536319 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -67,7 +67,7 @@ docker_test_image_gpu(){ echo "Running tests on mxnet/python:${1}" nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/tests/python/train/test_conv.py --gpu" - nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/example/image-classification/train_mnist.py --gpus 2" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/example/image-classification/train_mnist.py --gpus 0,1,2,3" } # if both $MXNET_DOCKERHUB_PASSWORD and $MXNET_DOCKERHUB_USERNAME environment variables are set, docker will automatically login @@ -127,12 +127,11 @@ docker_tag_image_gpu(){ # Build and Test dockerfiles - CPU docker_generate_image_cpu "${mxnet_version}_cpu" "Dockerfile.mxnet.python.cpu" "python" > ${LOGDIR}/docker_cpu.out 2>&1 & docker_generate_image_cpu "${mxnet_version}_cpu_mkl" "Dockerfile.mxnet.python.cpu.mkl" "python" > ${LOGDIR}/docker_cpu_mkl.out 2>&1 & -docker_tag_image_cpu "${mxnet_version}_cpu" "latest" "python" > ${LOGDIR}/docker_latest.out 2>&1 & + #Build and Test dockerfiles - GPU docker_generate_image_gpu "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu.cu90" "python" > ${LOGDIR}/docker_gpu_cu90.out 2>&1 & docker_generate_image_gpu "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.cu90.mkl" "python" > ${LOGDIR}/docker_gpu_cu90_mkl.out 2>&1 & -docker_tag_image_gpu "${mxnet_version}_gpu_cu90" "gpu" "python" > ${LOGDIR}/docker_gpu 2>&1 & docker_generate_image_gpu "${mxnet_version}_gpu_cu80" "Dockerfile.mxnet.python.gpu.cu80" "python" > ${LOGDIR}/docker_gpu_cu80.out 2>&1 & docker_generate_image_gpu "${mxnet_version}_gpu_cu80_mkl" "Dockerfile.mxnet.python.gpu.cu80.mkl" "python" > ${LOGDIR}/docker_gpu_cu80_mkl.out 2>&1 & docker_generate_image_gpu "${mxnet_version}_gpu_cu92" "Dockerfile.mxnet.python.gpu.cu92" "python" > ${LOGDIR}/docker_gpu_cu92.out 2>&1 & @@ -156,7 +155,9 @@ docker_generate_image_gpu "${mxnet_version}_gpu_cu92_mkl_py3" "Dockerfile.mxnet. echo "Waiting for MXNet Python3 Docker Images to Build" wait -echo "Re-Tag 4 images with version-free names for Benchmarking" +echo "Re-Tag 6 images with version-free names (for Benchmarking) - only after previous builds complete. " +docker_tag_image_cpu "${mxnet_version}_cpu" "latest" "python" > ${LOGDIR}/docker_latest.out 2>&1 & +docker_tag_image_gpu "${mxnet_version}_gpu_cu90" "gpu" "python" > ${LOGDIR}/docker_gpu.out 2>&1 & docker_tag_image_cpu "${mxnet_version}_cpu_mkl" "latest_cpu_mkl_py2" "python" > ${LOGDIR}/docker_latest_cpu_mkl_py2.out 2>&1 & docker_tag_image_cpu "${mxnet_version}_cpu_mkl_py3" "latest_cpu_mkl_py3" "python3" > ${LOGDIR}/docker_latest_cpu_mkl_py3.out 2>&1 & docker_tag_image_gpu "${mxnet_version}_gpu_cu90_mkl" "latest_gpu_mkl_py2" "python" > ${LOGDIR}/docker_latest_gpu_mkl_py2.out 2>&1 & From 6be49682e9876cb3df9562c0464a42a8aa756f7f Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Mon, 19 Nov 2018 05:06:11 -0800 Subject: [PATCH 15/19] Move error checking to a method --- .../docker-python/build_python_dockerfile.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index 8e0f34536319..d23e0f016e3d 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -123,6 +123,17 @@ docker_tag_image_gpu(){ docker_test_image_gpu ${image_tag} ${python_version} } +check_errors(){ + set +e + egrep -i "not found|error|returned a non-zero code|fail" ${LOGDIR}/docker* + if [ $? -eq 0 ]; then + echo "ERROR: One of the build/test commands failed. Refer to the filename above to see which image tag caused it." + exit 1 + else + echo "Success: No errors found" + fi + set -e +} # Build and Test dockerfiles - CPU docker_generate_image_cpu "${mxnet_version}_cpu" "Dockerfile.mxnet.python.cpu" "python" > ${LOGDIR}/docker_cpu.out 2>&1 & @@ -165,12 +176,7 @@ docker_tag_image_gpu "${mxnet_version}_gpu_cu90_mkl_py3" "latest_gpu_mkl_py3" "p wait # Parse all the docker logfiles to make sure there is no error. Fail script if error is found. -egrep -i "not found|error|returned a non-zero code|fail" ${LOGDIR}/docker* -if [ $? -eq 0 ] -then - echo "ERROR: One of the build/test commands failed. Refer to the filename above to see which image tag caused it." - exit 1 -fi +check_errors # Push dockerfiles echo "All images were successfully built. Now login to dockerhub and push images" From c80763b1a4b74bef6e0dadcbf8dc71e6f3a43b11 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Mon, 19 Nov 2018 05:35:47 -0800 Subject: [PATCH 16/19] Push BAI images to dockerhub --- docker/docker-python/build_python_dockerfile.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index d23e0f016e3d..2c6c9f47352b 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -185,10 +185,8 @@ docker_account_login # Python2 docker_push_image "${mxnet_version}_cpu" docker_push_image "${mxnet_version}_cpu_mkl" -docker_push_image "latest" docker_push_image "${mxnet_version}_gpu_cu90" docker_push_image "${mxnet_version}_gpu_cu90_mkl" -docker_push_image "gpu" docker_push_image "${mxnet_version}_gpu_cu80" docker_push_image "${mxnet_version}_gpu_cu80_mkl" docker_push_image "${mxnet_version}_gpu_cu92" @@ -204,6 +202,14 @@ docker_push_image "${mxnet_version}_gpu_cu80_mkl_py3" docker_push_image "${mxnet_version}_gpu_cu92_py3" docker_push_image "${mxnet_version}_gpu_cu92_mkl_py3" +docker_push_image "latest" +docker_push_image "gpu" +docker_push_image "latest_cpu_mkl_py2" +docker_push_image "latest_cpu_mkl_py3" +docker_push_image "latest_gpu_mkl_py2" +docker_push_image "latest_gpu_mkl_py3" + + docker_account_logout echo "Successfully Built, Tested and Pushed all Images to Dockerhub. Link: https://hub.docker.com/r/mxnet/python/tags/" From 8bd23dc865441631ba9941e0ff96ba3e96013dde Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Mon, 19 Nov 2018 11:10:49 -0800 Subject: [PATCH 17/19] Create ~/temp dir --- docker/docker-python/build_python_dockerfile.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index 2c6c9f47352b..251933717ad0 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -48,6 +48,9 @@ LOGDIR=~/temp/docker_logs if [ -d "${LOGDIR}" ]; then rm -rf ${LOGDIR} fi + +# Create ~/temp if it does not exist +mkdir -p ~/temp mkdir ${LOGDIR} From cc956b63ccd036dcb2d516cb537910dd19250c68 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Mon, 19 Nov 2018 12:14:59 -0800 Subject: [PATCH 18/19] Bug Fix --- docker/docker-python/build_python_dockerfile.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh index 251933717ad0..d0f24c8d6a5e 100755 --- a/docker/docker-python/build_python_dockerfile.sh +++ b/docker/docker-python/build_python_dockerfile.sh @@ -17,8 +17,6 @@ # specific language governing permissions and limitations # under the License. -set -e - # Check Params programname=$0 @@ -127,7 +125,6 @@ docker_tag_image_gpu(){ } check_errors(){ - set +e egrep -i "not found|error|returned a non-zero code|fail" ${LOGDIR}/docker* if [ $? -eq 0 ]; then echo "ERROR: One of the build/test commands failed. Refer to the filename above to see which image tag caused it." @@ -135,7 +132,6 @@ check_errors(){ else echo "Success: No errors found" fi - set -e } # Build and Test dockerfiles - CPU From 0f135cc75332aa5d0fb70bf2fae3f5bb69efaade Mon Sep 17 00:00:00 2001 From: perdasilva Date: Tue, 19 Feb 2019 14:50:52 +0100 Subject: [PATCH 19/19] Fix merge issues I don't know if this is tha main merge issue the PR is complaining about on mxnet, but maybe it will help --- docker/docker-python/README.md | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md index d84d23955b1a..4467b3a8f255 100644 --- a/docker/docker-python/README.md +++ b/docker/docker-python/README.md @@ -1,14 +1,33 @@ + + # Release Python Docker Images for MXNet The `docker-python` directory can be used to release mxnet python docker images to dockerhub after any mxnet release. It uses the appropriate pip binaries to build different docker images. Both python2 (default) and python3 images are available as - * {version}_cpu * {version}_cpu_mkl -* {version}_gpu_cu90 +* {version}_gpu_cu90 * {version}_gpu_cu90_mkl -* {version}_gpu_cu80 +* {version}_gpu_cu80 * {version}_gpu_cu80_mkl -* {version}_gpu_cu92 +* {version}_gpu_cu92 * {version}_gpu_cu92_mkl * {version}_cpu_py3 * {version}_cpu_mkl_py3