From 3b6d9462c59da3319993b5d8aed5ec85d85f4321 Mon Sep 17 00:00:00 2001 From: zha0q1 Date: Thu, 18 Feb 2021 00:32:08 +0000 Subject: [PATCH 1/6] attemp to fix docker --- cd/python/docker/Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cd/python/docker/Dockerfile b/cd/python/docker/Dockerfile index ed97bdc8316a..51727ce98cbe 100644 --- a/cd/python/docker/Dockerfile +++ b/cd/python/docker/Dockerfile @@ -23,11 +23,8 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -ARG PYTHON_CMD=python3 -RUN apt-get update && \ - apt-get install -y wget ${PYTHON_CMD}-dev gcc && \ - wget https://bootstrap.pypa.io/get-pip.py && \ - ${PYTHON_CMD} get-pip.py +COPY ../../../ci/docker/install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh ARG MXNET_COMMIT_ID ENV MXNET_COMMIT_ID=${MXNET_COMMIT_ID} From 0d17e224939ca246be358c3e7b888460a493180a Mon Sep 17 00:00:00 2001 From: zha0q1 Date: Thu, 18 Feb 2021 04:54:22 +0000 Subject: [PATCH 2/6] move file to cd --- cd/python/docker/Dockerfile | 2 +- cd/python/docker/ubuntu_python.sh | 36 +++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100755 cd/python/docker/ubuntu_python.sh diff --git a/cd/python/docker/Dockerfile b/cd/python/docker/Dockerfile index 51727ce98cbe..a45df9eeb605 100644 --- a/cd/python/docker/Dockerfile +++ b/cd/python/docker/Dockerfile @@ -23,7 +23,7 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -COPY ../../../ci/docker/install/ubuntu_python.sh /work/ +COPY ./ubuntu_python.sh /work/ RUN /work/ubuntu_python.sh ARG MXNET_COMMIT_ID diff --git a/cd/python/docker/ubuntu_python.sh b/cd/python/docker/ubuntu_python.sh new file mode 100755 index 000000000000..ea930678523d --- /dev/null +++ b/cd/python/docker/ubuntu_python.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build and install are separated so changes to build don't invalidate +# the whole docker cache for the image + +set -ex +# install libraries for mxnet's python package on ubuntu +apt-get update || true +apt-get install -y software-properties-common +add-apt-repository -y ppa:deadsnakes/ppa +apt-get update || true +apt-get install -y python3.7-dev python3.7-distutils virtualenv wget +# setup symlink in /usr/local/bin to override python3 version +ln -sf /usr/bin/python3.7 /usr/local/bin/python3 + +# the version of the pip shipped with ubuntu may be too lower, install a recent version here +wget -nv https://bootstrap.pypa.io/get-pip.py +python3 get-pip.py +pip3 install -r /work/requirements From 1180993e34e8d521dc4af08d10b0b9d88e1d6c06 Mon Sep 17 00:00:00 2001 From: zha0q1 Date: Thu, 18 Feb 2021 22:16:35 +0000 Subject: [PATCH 3/6] fix docker image --- cd/python/docker/Dockerfile | 11 ++++++++-- cd/python/docker/ubuntu_python.sh | 36 ------------------------------- 2 files changed, 9 insertions(+), 38 deletions(-) delete mode 100755 cd/python/docker/ubuntu_python.sh diff --git a/cd/python/docker/Dockerfile b/cd/python/docker/Dockerfile index a45df9eeb605..c7a13a490991 100644 --- a/cd/python/docker/Dockerfile +++ b/cd/python/docker/Dockerfile @@ -23,8 +23,15 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -COPY ./ubuntu_python.sh /work/ -RUN /work/ubuntu_python.sh +RUN apt-get update || true +RUN apt-get install -y software-properties-common +RUN add-apt-repository -y ppa:deadsnakes/ppa +RUN apt-get update || true +RUN apt-get install -y python3.7-dev python3.7-distutils virtualenv wget +RUN ln -sf /usr/bin/python3.7 /usr/local/bin/python3 + +RUN wget -nv https://bootstrap.pypa.io/get-pip.py +RUN python3 get-pip.py ARG MXNET_COMMIT_ID ENV MXNET_COMMIT_ID=${MXNET_COMMIT_ID} diff --git a/cd/python/docker/ubuntu_python.sh b/cd/python/docker/ubuntu_python.sh deleted file mode 100755 index ea930678523d..000000000000 --- a/cd/python/docker/ubuntu_python.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image - -set -ex -# install libraries for mxnet's python package on ubuntu -apt-get update || true -apt-get install -y software-properties-common -add-apt-repository -y ppa:deadsnakes/ppa -apt-get update || true -apt-get install -y python3.7-dev python3.7-distutils virtualenv wget -# setup symlink in /usr/local/bin to override python3 version -ln -sf /usr/bin/python3.7 /usr/local/bin/python3 - -# the version of the pip shipped with ubuntu may be too lower, install a recent version here -wget -nv https://bootstrap.pypa.io/get-pip.py -python3 get-pip.py -pip3 install -r /work/requirements From 866bc7a0243668107e9fcae4523407ed7d736724 Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Thu, 18 Feb 2021 20:12:03 -0800 Subject: [PATCH 4/6] Update Dockerfile.build.ubuntu_gpu_cu102 --- ci/docker/Dockerfile.build.ubuntu_gpu_cu102 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu102 b/ci/docker/Dockerfile.build.ubuntu_gpu_cu102 index f3b3f49fbd0b..190659590f33 100644 --- a/ci/docker/Dockerfile.build.ubuntu_gpu_cu102 +++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu102 @@ -64,10 +64,10 @@ RUN /work/ubuntu_docs.sh COPY install/ubuntu_tutorials.sh /work/ RUN /work/ubuntu_tutorials.sh -ENV CUDA_VERSION=10.2.89 -ENV CUDNN_VERSION=8.0.4.30 -COPY install/ubuntu_cudnn.sh /work/ -RUN /work/ubuntu_cudnn.sh +#ENV CUDA_VERSION=10.2.89 +#ENV CUDNN_VERSION=8.0.4.30 +#COPY install/ubuntu_cudnn.sh /work/ +#RUN /work/ubuntu_cudnn.sh # Always last ARG USER_ID=0 From 6d9f2668b0ac045f6a1f42c30316c0ba52c8dc68 Mon Sep 17 00:00:00 2001 From: Zhaoqi Zhu Date: Fri, 19 Feb 2021 15:27:36 -0800 Subject: [PATCH 5/6] Update ubuntu_cudnn.sh --- ci/docker/install/ubuntu_cudnn.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/install/ubuntu_cudnn.sh b/ci/docker/install/ubuntu_cudnn.sh index 0699d80eb86e..1cc0b722bfc7 100755 --- a/ci/docker/install/ubuntu_cudnn.sh +++ b/ci/docker/install/ubuntu_cudnn.sh @@ -38,7 +38,7 @@ case ${CUDA_VERSION} in export libcudnn_dev_version="${CUDNN_VERSION}-1+cuda11.0" ;; 10\.2*) - export libcudnn_package="libcudnn8" + export libcudnn_package="libcudnn7" export libcudnn_version="${CUDNN_VERSION}-1+cuda10.2" export libcudnn_dev_version="${CUDNN_VERSION}-1+cuda10.2" ;; From 0429b2b5ef5a361354ba1b5a914c2697495281e4 Mon Sep 17 00:00:00 2001 From: zha0q1 Date: Sat, 20 Feb 2021 01:23:10 +0000 Subject: [PATCH 6/6] update cudnn to 8 in build stage as well --- ci/docker/Dockerfile.build.ubuntu_gpu_cu102 | 8 ++++---- ci/docker/install/ubuntu_cudnn.sh | 2 +- tools/setup_gpu_build_tools.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu102 b/ci/docker/Dockerfile.build.ubuntu_gpu_cu102 index 190659590f33..f3b3f49fbd0b 100644 --- a/ci/docker/Dockerfile.build.ubuntu_gpu_cu102 +++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu102 @@ -64,10 +64,10 @@ RUN /work/ubuntu_docs.sh COPY install/ubuntu_tutorials.sh /work/ RUN /work/ubuntu_tutorials.sh -#ENV CUDA_VERSION=10.2.89 -#ENV CUDNN_VERSION=8.0.4.30 -#COPY install/ubuntu_cudnn.sh /work/ -#RUN /work/ubuntu_cudnn.sh +ENV CUDA_VERSION=10.2.89 +ENV CUDNN_VERSION=8.0.4.30 +COPY install/ubuntu_cudnn.sh /work/ +RUN /work/ubuntu_cudnn.sh # Always last ARG USER_ID=0 diff --git a/ci/docker/install/ubuntu_cudnn.sh b/ci/docker/install/ubuntu_cudnn.sh index 1cc0b722bfc7..0699d80eb86e 100755 --- a/ci/docker/install/ubuntu_cudnn.sh +++ b/ci/docker/install/ubuntu_cudnn.sh @@ -38,7 +38,7 @@ case ${CUDA_VERSION} in export libcudnn_dev_version="${CUDNN_VERSION}-1+cuda11.0" ;; 10\.2*) - export libcudnn_package="libcudnn7" + export libcudnn_package="libcudnn8" export libcudnn_version="${CUDNN_VERSION}-1+cuda10.2" export libcudnn_dev_version="${CUDNN_VERSION}-1+cuda10.2" ;; diff --git a/tools/setup_gpu_build_tools.sh b/tools/setup_gpu_build_tools.sh index 6a50dd505a54..b2f80412bfa7 100755 --- a/tools/setup_gpu_build_tools.sh +++ b/tools/setup_gpu_build_tools.sh @@ -42,7 +42,7 @@ elif [[ $VARIANT == cu102* ]]; then CUDA_VERSION='10.2.89-1' CUDA_PATCH_VERSION='10.2.2.89-1' LIBCUDA_VERSION='440.33.01-0ubuntu1' - LIBCUDNN_VERSION='7.6.5.32-1+cuda10.2' + LIBCUDNN_VERSION='8.0.4.30-1+cuda10.2' LIBNCCL_VERSION='2.5.6-1+cuda10.2' elif [[ $VARIANT == cu101* ]]; then CUDA_VERSION='10.1.105-1'