diff --git a/ci/docker/Dockerfile.build.ubuntu_base_gpu b/ci/docker/Dockerfile.build.ubuntu_base_gpu
index adf9b2809f21..98e3669c8273 100644
--- a/ci/docker/Dockerfile.build.ubuntu_base_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_base_gpu
@@ -21,13 +21,11 @@
 
 FROM nvidia/cuda:10.0-devel-ubuntu16.04
 
-ENV CUDNN_VERSION=7.5.1.10
-
 WORKDIR /work/deps
 
 RUN apt-get update && apt-get -y install sudo
 
-ENV CUDNN_VERSION=7.3.1.20
+ENV CUDNN_VERSION=7.5.1.10
 COPY install/ubuntu_cudnn.sh /work/
 RUN /work/ubuntu_cudnn.sh
 
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda
index 0c1a3c33c121..ad1a1c4558b5 100644
--- a/ci/docker/Dockerfile.build.ubuntu_build_cuda
+++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda
@@ -23,8 +23,6 @@
 
 FROM nvidia/cuda:10.0-devel-ubuntu16.04
 
-ENV CUDNN_VERSION=7.5.1.10
-
 WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
@@ -45,6 +43,8 @@ COPY install/ubuntu_clang.sh /work/
 RUN /work/ubuntu_clang.sh
 COPY install/ubuntu_mklml.sh /work/
 RUN /work/ubuntu_mklml.sh
+
+ENV CUDNN_VERSION=7.5.1.10
 COPY install/ubuntu_cudnn.sh /work/
 RUN /work/ubuntu_cudnn.sh
 
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu100 b/ci/docker/Dockerfile.build.ubuntu_gpu_cu100
index c9908a5eda4e..7c93ebeffed3 100644
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_cu100
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu100
@@ -67,7 +67,7 @@ RUN /work/ubuntu_docs.sh
 COPY install/ubuntu_tutorials.sh /work/
 RUN /work/ubuntu_tutorials.sh
 
-ENV CUDNN_VERSION=7.3.1.20
+ENV CUDNN_VERSION=7.5.1.10
 COPY install/ubuntu_cudnn.sh /work/
 RUN /work/ubuntu_cudnn.sh
 
@@ -77,10 +77,6 @@ ARG GROUP_ID=0
 COPY install/ubuntu_adduser.sh /work/
 RUN /work/ubuntu_adduser.sh
 
-ENV CUDNN_VERSION=7.5.1.10
-COPY install/ubuntu_cudnn.sh /work/
-RUN /work/ubuntu_cudnn.sh
-
 COPY runtime_functions.sh /work/
 
 WORKDIR /work/mxnet
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu90 b/ci/docker/Dockerfile.build.ubuntu_gpu_cu90
index 16e8d998393a..ad3c54df1bbd 100644
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_cu90
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu90
@@ -67,7 +67,7 @@ RUN /work/ubuntu_docs.sh
 COPY install/ubuntu_tutorials.sh /work/
 RUN /work/ubuntu_tutorials.sh
 
-ENV CUDNN_VERSION=7.3.1.20
+ENV CUDNN_VERSION=7.5.1.20
 COPY install/ubuntu_cudnn.sh /work/
 RUN /work/ubuntu_cudnn.sh
 
@@ -77,10 +77,6 @@ ARG GROUP_ID=0
 COPY install/ubuntu_adduser.sh /work/
 RUN /work/ubuntu_adduser.sh
 
-ENV CUDNN_VERSION=7.5.1.10
-COPY install/ubuntu_cudnn.sh /work/
-RUN /work/ubuntu_cudnn.sh
-
 COPY runtime_functions.sh /work/
 
 WORKDIR /work/mxnet
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu92 b/ci/docker/Dockerfile.build.ubuntu_gpu_cu92
index 1618e68693e7..cd01c5996273 100644
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_cu92
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_cu92
@@ -67,7 +67,7 @@ RUN /work/ubuntu_docs.sh
 COPY install/ubuntu_tutorials.sh /work/
 RUN /work/ubuntu_tutorials.sh
 
-ENV CUDNN_VERSION=7.3.1.20
+ENV CUDNN_VERSION=7.5.1.10
 COPY install/ubuntu_cudnn.sh /work/
 RUN /work/ubuntu_cudnn.sh
 
@@ -76,10 +76,6 @@ ARG GROUP_ID=0
 COPY install/ubuntu_adduser.sh /work/
 RUN /work/ubuntu_adduser.sh
 
-ENV CUDNN_VERSION=7.5.1.10
-COPY install/ubuntu_cudnn.sh /work/
-RUN /work/ubuntu_cudnn.sh
-
 COPY runtime_functions.sh /work/
 
 WORKDIR /work/mxnet
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
index 82301bdbff6d..48b562584914 100644
--- a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
@@ -20,8 +20,6 @@
 
 FROM nvidia/cuda:10.0-devel-ubuntu16.04
 
-ENV CUDNN_VERSION=7.5.1.10
-
 WORKDIR /work/deps
 
 COPY install/ubuntu_core.sh /work/
@@ -72,6 +70,7 @@ RUN /work/ubuntu_tutorials.sh
 COPY install/ubuntu_nightly_tests.sh /work/
 RUN /work/ubuntu_nightly_tests.sh
 
+ENV CUDNN_VERSION=7.5.1.10
 COPY install/ubuntu_cudnn.sh /work/
 RUN /work/ubuntu_cudnn.sh