diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu b/docker/docker-python/Dockerfile.mxnet.python.cpu new file mode 100644 index 000000000000..eb437f3c2334 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python.cpu @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for CPU + +FROM ubuntu:16.04 + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl new file mode 100644 index 000000000000..043932ff7c8e --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet CPU with MKL + +FROM ubuntu:16.04 + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-mkl diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 new file mode 100644 index 000000000000..8c83ece434a3 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU + +FROM nvidia/cuda:8.0-cudnn5-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu80 diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl new file mode 100644 index 000000000000..a057c1d20cb1 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU with MKL + +FROM nvidia/cuda:8.0-cudnn5-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu80mkl diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 new file mode 100644 index 000000000000..1e3d9869ac63 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU + +FROM nvidia/cuda:9.0-cudnn7-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu90 diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl new file mode 100644 index 000000000000..d82abd7cf523 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU with MKL + +FROM nvidia/cuda:9.0-cudnn7-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu90mkl diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 new file mode 100644 index 000000000000..ba5c54a2a2aa --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU + +FROM nvidia/cuda:9.2-cudnn7-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu92 diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl new file mode 100644 index 000000000000..96a943980b59 --- /dev/null +++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl @@ -0,0 +1,28 @@ +# -*- mode: dockerfile -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Dockerfile to build MXNet for GPU with MKL + +FROM nvidia/cuda:9.2-cudnn7-devel + +RUN apt-get update +RUN apt-get install -y wget python gcc +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python get-pip.py + +RUN pip install mxnet-cu92mkl diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md new file mode 100644 index 000000000000..f806d5d6f459 --- /dev/null +++ b/docker/docker-python/README.md @@ -0,0 +1,41 @@ +# Release Python Docker Images for MXNet + +The `docker-python` directory can be used to release mxnet python docker images to dockerhub after any mxnet release. +It uses the appropriate pip binaries to build different docker images as - +* cpu +* cpu_mkl +* latest (same as cpu) +* gpu_cu90 +* gpu_cu90_mkl +* gpu (same as gpu_cu90) +* gpu_cu80 +* gpu_cu80_mkl +* gpu_cu92 +* gpu_cu92_mkl + + +** Note: If you want to use a different pip binary (specific mxnet or cuda version, etc), you can edit the last line of the cpu or gpu dockerfile as required. + +Refer: https://pypi.org/project/mxnet/ + +### Usage +`./build_python_dockerfile.sh ` + +For example: +`./build_python_dockerfile.sh 1.3.0 ~/build-docker/incubator-mxnet` + +** Note: The build script picks up the latest pip binaries. This means it uses the latest released mxnet version. The version specified as a parameter to the script is only used to tag the built image correctly. + +### Tests run +* [test_conv.py](https://github.com/apache/incubator-mxnet/blob/master/tests/python/train/test_conv.py) +* [train_mnist.py](https://github.com/apache/incubator-mxnet/blob/master/example/image-classification/train_mnist.py) +* [test_mxnet.py](https://github.com/apache/incubator-mxnet/blob/master/docker/docker-python/test_mxnet.py): This script is used to make sure that the docker image builds the expected mxnet version. That is, the version picked by pip is the same as as the version passed as a parameter. + +### Dockerhub Credentials +Dockerhub credentials will be required to push images at the end of this script. +Credentials can be provided in the following ways: +* **Interactive Login:** Run the script as is and it will ask you for credentials interactively. +* **Be Already Logged in:** Login to the mxnet dockerhub account before you run the build script and the script will complete build, test and push. +* **Set Environment Variables:** Set the following environment variables which the script will pick up to login to dockerhub at runtime - + * $MXNET_DOCKERHUB_PASSWORD + * $MXNET_DOCKERHUB_USERNAME diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh new file mode 100755 index 000000000000..24a44c28970c --- /dev/null +++ b/docker/docker-python/build_python_dockerfile.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +# Check Params +programname=$0 + +function usage { + echo "usage: $programname [version] [path]" + echo " [version] Mxnet Version to build" + echo " [path] Path to MXNet repository (to run tests)" + echo " " + exit 1 +} + +if [ $# -le 1 ] || [ $# -ge 3 ] +then + usage + exit 1 +fi + +# Two params provided +echo "Building Docker Images for Apache MXNet (Incubating) v$1" +mxnet_version="${1}" +test_dir="${2}" + +docker_build_image(){ + echo "Building docker image mxnet/python:${1}" + docker build -t mxnet/python:${1} -f ${2} . +} + +docker_tag_image(){ + docker tag mxnet/python:${1} mxnet/python:${2} +} + +docker_test_image_cpu(){ + echo "Running tests on mxnet/python:${1}" + docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" + docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py" + docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py" +} + +docker_test_image_gpu(){ + echo "Running tests on mxnet/python:${1}" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py --gpu" + nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py --gpus 2" +} + +# if both $MXNET_DOCKERHUB_PASSWORD and $MXNET_DOCKERHUB_USERNAME environment variables are set, docker will automatically login +# if env variables are not set, login will be interactive. +docker_account_login(){ + if [[ -z $MXNET_DOCKERHUB_PASSWORD ]] || [[ -z $MXNET_DOCKERHUB_USERNAME ]]; then + docker login + else + echo $MXNET_DOCKERHUB_PASSWORD | docker login -u $MXNET_DOCKERHUB_USERNAME --password-stdin + fi +} + +docker_account_logout(){ + docker logout +} + +docker_push_image(){ + docker push mxnet/python:${1} +} + + +# Build and Test dockerfiles - CPU +docker_build_image "${mxnet_version}_cpu" "Dockerfile.mxnet.python.cpu" +docker_test_image_cpu "${mxnet_version}_cpu" + +docker_build_image "${mxnet_version}_cpu_mkl" "Dockerfile.mxnet.python.cpu.mkl" +docker_test_image_cpu "${mxnet_version}_cpu_mkl" + +docker_tag_image "${mxnet_version}_cpu" "latest" +docker_test_image_cpu "latest" + + +#Build and Test dockerfiles - GPU +docker_build_image "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu.cu90" +docker_test_image_gpu "${mxnet_version}_gpu_cu90" + +docker_build_image "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.cu90.mkl" +docker_test_image_gpu "${mxnet_version}_gpu_cu90_mkl" + +docker_tag_image "${mxnet_version}_gpu_cu90" "gpu" +docker_test_image_gpu "gpu" + +docker_build_image "${mxnet_version}_gpu_cu80" "Dockerfile.mxnet.python.gpu.cu80" +docker_test_image_gpu "${mxnet_version}_gpu_cu80" + +docker_build_image "${mxnet_version}_gpu_cu80_mkl" "Dockerfile.mxnet.python.gpu.cu80.mkl" +docker_test_image_gpu "${mxnet_version}_gpu_cu80_mkl" + +docker_build_image "${mxnet_version}_gpu_cu92" "Dockerfile.mxnet.python.gpu.cu92" +docker_test_image_gpu "${mxnet_version}_gpu_cu92" + +docker_build_image "${mxnet_version}_gpu_cu92_mkl" "Dockerfile.mxnet.python.gpu.cu92.mkl" +docker_test_image_gpu "${mxnet_version}_gpu_cu92_mkl" + + +# Push dockerfiles +echo "All images were successfully built. Now login to dockerhub and push images" +docker_account_login + +docker_push_image "${mxnet_version}_cpu" +docker_push_image "${mxnet_version}_cpu_mkl" +docker_push_image "latest" +docker_push_image "${mxnet_version}_gpu_cu90" +docker_push_image "${mxnet_version}_gpu_cu90_mkl" +docker_push_image "gpu" +docker_push_image "${mxnet_version}_gpu_cu80" +docker_push_image "${mxnet_version}_gpu_cu80_mkl" +docker_push_image "${mxnet_version}_gpu_cu92" +docker_push_image "${mxnet_version}_gpu_cu92_mkl" + +docker_account_logout + +echo "Successfully Built, Tested and Pushed all Images to Dockerhub. Link: https://hub.docker.com/r/mxnet/python/tags/" diff --git a/docker/docker-python/test_mxnet.py b/docker/docker-python/test_mxnet.py new file mode 100644 index 000000000000..657205832728 --- /dev/null +++ b/docker/docker-python/test_mxnet.py @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This checks that the version of mxnet imported matches the parameter passed to the build script. +import mxnet as mx +import sys + +pip_version = mx.__version__ +expected_version = sys.argv[1] + +if pip_version != expected_version: + raise ValueError("ERROR: Incorrect pip version. Please check the parameter passed or pip binary used.")