[MXNET-1093] Add python3 Docker images for each MXNet release (apache…

…#12791)

* Add dockerfiles for python3

* Fix

* Fix the dockerfile names

* Update the README with python3 images

* Parameterize the mxnet version

* Fix typo

* Reduce the number of docker layers

* Combine build and Test, run builds in parallel. Use variable names instead of numbers

* minor fix

* Update README for build command

* Bug Fix: Script should fail if any background process returns non-zero code

* Changes needed for Benchmarking, 4 new tags, change python to python-dev

* Minor typos

* Minor Rearrangement - sometimes tagging fails

* Move error checking to a method

* Push BAI images to dockerhub

* Create ~/temp dir

* Bug Fix

* Fix merge issues

I don't know if this is tha main merge issue the PR is complaining about on mxnet, but maybe it will help

docker/docker-python/Dockerfile.mxnet.python.cpu

@@ -19,10 +19,11 @@
 # Dockerfile to build MXNet for CPU
 
 FROM ubuntu:16.04
+ARG version
 
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+    apt-get install -y wget python-dev gcc && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python get-pip.py
 
-RUN pip install mxnet
+RUN pip install mxnet==$version

docker/docker-python/Dockerfile.mxnet.python.cpu.mkl

@@ -19,10 +19,11 @@
 # Dockerfile to build MXNet CPU with MKL
 
 FROM ubuntu:16.04
+ARG version
 
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+    apt-get install -y wget python-dev gcc && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python get-pip.py
 
-RUN pip install mxnet-mkl
+RUN pip install mxnet-mkl==$version

docker/docker-python/Dockerfile.mxnet.python.gpu.cu80

@@ -19,10 +19,11 @@
 # Dockerfile to build MXNet for GPU
 
 FROM nvidia/cuda:8.0-cudnn5-devel
+ARG version
 
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+    apt-get install -y wget python-dev gcc && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python get-pip.py
 
-RUN pip install mxnet-cu80
+RUN pip install mxnet-cu80==$version

docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl

@@ -19,10 +19,11 @@
 # Dockerfile to build MXNet for GPU with MKL
 
 FROM nvidia/cuda:8.0-cudnn5-devel
+ARG version
 
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+    apt-get install -y wget python-dev gcc && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python get-pip.py
 
-RUN pip install mxnet-cu80mkl
+RUN pip install mxnet-cu80mkl==$version

docker/docker-python/Dockerfile.mxnet.python.gpu.cu90

@@ -19,10 +19,11 @@
 # Dockerfile to build MXNet for GPU
 
 FROM nvidia/cuda:9.0-cudnn7-devel
+ARG version
 
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+    apt-get install -y wget python-dev gcc && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python get-pip.py
 
-RUN pip install mxnet-cu90
+RUN pip install mxnet-cu90==$version

docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl

@@ -19,10 +19,11 @@
 # Dockerfile to build MXNet for GPU with MKL
 
 FROM nvidia/cuda:9.0-cudnn7-devel
+ARG version
 
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+    apt-get install -y wget python-dev gcc && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python get-pip.py
 
-RUN pip install mxnet-cu90mkl
+RUN pip install mxnet-cu90mkl==$version

docker/docker-python/Dockerfile.mxnet.python.gpu.cu92

@@ -19,10 +19,11 @@
 # Dockerfile to build MXNet for GPU
 
 FROM nvidia/cuda:9.2-cudnn7-devel
+ARG version
 
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+    apt-get install -y wget python-dev gcc && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python get-pip.py
 
-RUN pip install mxnet-cu92
+RUN pip install mxnet-cu92==$version

docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl

@@ -19,10 +19,11 @@
 # Dockerfile to build MXNet for GPU with MKL
 
 FROM nvidia/cuda:9.2-cudnn7-devel
+ARG version
 
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+    apt-get install -y wget python-dev gcc && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python get-pip.py
 
-RUN pip install mxnet-cu92mkl
+RUN pip install mxnet-cu92mkl==$version

docker/docker-python/Dockerfile.mxnet.python3.cpu

@@ -0,0 +1,29 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for CPU
+
+FROM ubuntu:16.04
+ARG version
+
+RUN apt-get update && \
+    apt-get install -y wget python3-dev gcc && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python3 get-pip.py
+
+RUN pip3 install mxnet==$version

docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl

@@ -0,0 +1,29 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet CPU with MKL
+
+FROM ubuntu:16.04
+ARG version
+
+RUN apt-get update && \
+    apt-get install -y wget python3-dev gcc && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python3 get-pip.py
+
+RUN pip3 install mxnet-mkl==$version

docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80

@@ -0,0 +1,29 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU
+
+FROM nvidia/cuda:8.0-cudnn5-devel
+ARG version
+
+RUN apt-get update && \
+    apt-get install -y wget python3-dev gcc && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python3 get-pip.py
+
+RUN pip3 install mxnet-cu80==$version

docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl

@@ -0,0 +1,29 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU with MKL
+
+FROM nvidia/cuda:8.0-cudnn5-devel
+ARG version
+
+RUN apt-get update && \
+    apt-get install -y wget python3-dev gcc && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python3 get-pip.py
+
+RUN pip3 install mxnet-cu80mkl==$version

docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90

@@ -0,0 +1,29 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU
+
+FROM nvidia/cuda:9.0-cudnn7-devel
+ARG version
+
+RUN apt-get update && \
+    apt-get install -y wget python3-dev gcc && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python3 get-pip.py
+
+RUN pip3 install mxnet-cu90==$version

docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl

@@ -0,0 +1,29 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU with MKL
+
+FROM nvidia/cuda:9.0-cudnn7-devel
+ARG version
+
+RUN apt-get update && \
+    apt-get install -y wget python3-dev gcc && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python3 get-pip.py
+
+RUN pip3 install mxnet-cu90mkl==$version

docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92

@@ -0,0 +1,29 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU
+
+FROM nvidia/cuda:9.2-cudnn7-devel
+ARG version
+
+RUN apt-get update && \
+    apt-get install -y wget python3-dev gcc && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python3 get-pip.py
+
+RUN pip3 install mxnet-cu92==$version

docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl

@@ -0,0 +1,29 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU with MKL
+
+FROM nvidia/cuda:9.2-cudnn7-devel
+ARG version
+
+RUN apt-get update && \
+    apt-get install -y wget python3-dev gcc && \
+    wget https://bootstrap.pypa.io/get-pip.py && \
+    python3 get-pip.py
+
+RUN pip3 install mxnet-cu92mkl==$version