diff --git a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml
index 4856e88819e17..3f49575f8effd 100644
--- a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml
+++ b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml
@@ -33,7 +33,7 @@ jobs:
       condition: always()
 
 - job: Ubuntu1604_py_GPU_Wheels
-  pool: Linux-GPU
+  pool: Linux-GPU-CUDA10
   strategy:
     matrix:
       Python35:
@@ -47,7 +47,7 @@ jobs:
       displayName: 'Run build script'
       inputs:
         scriptPath: 'tools/ci_build/github/linux/run_dockerbuild.sh'
-        args: '-c Release -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) -p $(python.version) -x "--build_wheel --use_mkldnn"'
+        args: '-c Release -o ubuntu16.04 -d gpu -c cuda9.1-cudnn7.1 -r $(Build.BinariesDirectory) -p $(python.version) -x "--build_wheel --use_mkldnn"'
 
     - task: CopyFiles@2
       displayName: 'Copy Python Wheel to:  $(Build.ArtifactStagingDirectory)'
@@ -118,7 +118,7 @@ jobs:
       condition: always()
 
 - job:  Windows_py_GPU_Wheels
-  pool: Win-GPU
+  pool: Win-GPU-CUDA10
   variables:
     buildDirectory: '$(Build.SourcesDirectory)\build'
     CUDA_VERSION: '9.1'
diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml
new file mode 100644
index 0000000000000..c28d4c8d7a94b
--- /dev/null
+++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml
@@ -0,0 +1,12 @@
+jobs:
+- job: Linux_CI_GPU_Dev
+  pool: Linux-GPU-CUDA10
+  steps:
+    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -c cuda9.1-cudnn7.1 -r $(Build.BinariesDirectory)'
+      displayName: 'Command Line Script'
+      env:
+        AZURE_BLOB_KEY: $(onnxruntime-storage-key)
+
+    - script: 'sudo rm -rf $(Agent.BuildDirectory)'
+      displayName: 'Clean build folders/files'
+      condition: always()
\ No newline at end of file
diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline-cuda9.yml b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline-cuda9.yml
new file mode 100644
index 0000000000000..50aed9daa31d5
--- /dev/null
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline-cuda9.yml
@@ -0,0 +1,41 @@
+jobs:
+- job: Windows_CI_GPU_Dev
+  pool: Win-GPU-CUDA10
+  variables:
+    CUDA_VERSION: '9.1'
+  steps:
+    - task: PowerShell@1
+      displayName: 'Set CUDA path'
+      inputs:
+        scriptName: 'tools/ci_build/github/windows/set_cuda_path.ps1'
+        arguments: '-CudaMsbuildPath C:\local\cudaMsbuildIntegration-9.1.85-windows10-x64-0 -CudaVersion $(CUDA_VERSION)'
+    - task: BatchScript@1
+      displayName: 'Setup VS2017 env vars'
+      inputs:
+        filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat'
+        arguments: 'amd64 -vcvars_ver=14.11'
+        modifyEnvironment: true
+    - task: CmdLine@1
+      displayName: 'Get ONNX testdata'
+      inputs:
+        filename: azcopy
+        arguments: ' /S /Source:https://onnxruntimetestdata.blob.core.windows.net/onnx-model-zoo-20181018 /Dest:$(Build.SourcesDirectory)\build\Windows\Debug\models /SourceKey:%AZURE_BLOB_KEY%'
+      env:
+        AZURE_BLOB_KEY: $(onnxruntime-storage-key)
+    - task: BatchScript@1
+      inputs:
+        filename: build.bat
+        arguments: ' --use_cuda --cuda_home="C:\local\cuda-9.1.85-windows10-x64-0" --cudnn_home="C:\local\cudnn-9.1-windows10-x64-v7.1\cuda"'
+        workingFolder: "$(Build.SourcesDirectory)"
+    - task: PowerShell@1
+      displayName: 'Clean up CUDA props files'
+      inputs:
+        scriptName: 'tools/ci_build/github/windows/clean_up_cuda_prop_files.ps1'
+        arguments: '-CudaVersion $(CUDA_VERSION)'
+    - task: CmdLine@1
+      displayName: 'Clean build folders/files'
+      inputs:
+        filename: rd
+        arguments: '/s /q $(Agent.BuildDirectory)'
+      continueOnError: true
+      condition: always()
\ No newline at end of file
diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml
index 50aed9daa31d5..e27ede971cccc 100644
--- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml
@@ -2,13 +2,13 @@ jobs:
 - job: Windows_CI_GPU_Dev
   pool: Win-GPU-CUDA10
   variables:
-    CUDA_VERSION: '9.1'
+    CUDA_VERSION: '10.0'
   steps:
     - task: PowerShell@1
       displayName: 'Set CUDA path'
       inputs:
         scriptName: 'tools/ci_build/github/windows/set_cuda_path.ps1'
-        arguments: '-CudaMsbuildPath C:\local\cudaMsbuildIntegration-9.1.85-windows10-x64-0 -CudaVersion $(CUDA_VERSION)'
+        arguments: '-CudaMsbuildPath C:\local\cudaMsbuildIntegration-10.0.130-win10 -CudaVersion $(CUDA_VERSION)'
     - task: BatchScript@1
       displayName: 'Setup VS2017 env vars'
       inputs:
@@ -25,7 +25,7 @@ jobs:
     - task: BatchScript@1
       inputs:
         filename: build.bat
-        arguments: ' --use_cuda --cuda_home="C:\local\cuda-9.1.85-windows10-x64-0" --cudnn_home="C:\local\cudnn-9.1-windows10-x64-v7.1\cuda"'
+        arguments: ' --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"'
         workingFolder: "$(Build.SourcesDirectory)"
     - task: PowerShell@1
       displayName: 'Clean up CUDA props files'
diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu
index 8ecbf627df259..08806b148f7bb 100644
--- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu
+++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu
@@ -1,9 +1,8 @@
-# Tag: nvidia/cuda:9.1-cudnn7-devel-ubuntu16.04
-# Label: com.nvidia.cuda.version: 9.1.85
-# Label: com.nvidia.cudnn.version: 7.1.2.21
-# Label: com.nvidia.nccl.version: 2.2.12
+# Tag: nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04
+# Label: com.nvidia.cuda.version: 10.0.130
+# Label: com.nvidia.cudnn.version: 7.3.1.20
 # Ubuntu 16.04.5
-FROM nvidia/cuda:9.1-cudnn7-devel-ubuntu16.04
+FROM nvidia/cuda@sha256:362e4e25aa46a18dfa834360140e91b61cdb0a3a2796c8e09dadb268b9de3f6b
 
 ARG PYTHON_VERSION=3.5
 
diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu_cuda9 b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu_cuda9
new file mode 100644
index 0000000000000..4ba127c47f253
--- /dev/null
+++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu_cuda9
@@ -0,0 +1,43 @@
+# Tag: nvidia/cuda:9.1-cudnn7-devel-ubuntu16.04
+# Label: com.nvidia.cuda.version: 9.1.85
+# Label: com.nvidia.cudnn.version: 7.1.2.21
+# Ubuntu 16.04.5
+FROM nvidia/cuda@sha256:e48777124a0217001be8533123fcb8cc12ace38a4add2774b34295e611c99f10
+
+ARG PYTHON_VERSION=3.5
+
+ADD scripts /tmp/scripts
+ENV PATH="/opt/cmake/bin:${PATH}"
+RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts
+
+WORKDIR /root
+
+# Allow configure to pick up GDK and CuDNN where it expects it.
+# (Note: $CUDNN_VERSION is defined by NVidia's base image)
+RUN _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2) && \
+    mkdir -p /usr/local/cudnn-$_CUDNN_VERSION/cuda/include && \
+    ln -s /usr/include/cudnn.h /usr/local/cudnn-$_CUDNN_VERSION/cuda/include/cudnn.h && \
+    mkdir -p /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64 && \
+    ln -s /etc/alternatives/libcudnn_so /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64/libcudnn.so && \
+    ln -s /usr/local/cudnn{-$_CUDNN_VERSION,}
+
+# Build and Install LLVM
+ARG LLVM_VERSION=6.0.1
+RUN cd /tmp && \
+    wget --no-verbose http://releases.llvm.org/$LLVM_VERSION/llvm-$LLVM_VERSION.src.tar.xz && \
+    xz -d llvm-$LLVM_VERSION.src.tar.xz && \
+    tar xvf llvm-$LLVM_VERSION.src.tar && \
+    cd llvm-$LLVM_VERSION.src && \
+    mkdir -p build && \
+    cd build && \
+    cmake .. -DCMAKE_BUILD_TYPE=Release && \
+    cmake --build . -- -j$(nproc) && \
+    cmake -DCMAKE_INSTALL_PREFIX=/usr/local/llvm-$LLVM_VERSION -DBUILD_TYPE=Release -P cmake_install.cmake && \
+    cd /tmp && \
+    rm -rf llvm*
+
+ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH
+
+ARG BUILD_USER=onnxruntimedev
+WORKDIR /home/$BUILD_USER
+
diff --git a/tools/ci_build/github/linux/run_build.sh b/tools/ci_build/github/linux/run_build.sh
index 83110363ce732..d35190f1d6d34 100755
--- a/tools/ci_build/github/linux/run_build.sh
+++ b/tools/ci_build/github/linux/run_build.sh
@@ -25,13 +25,14 @@ else
 fi
 
 if [ $BUILD_DEVICE = "gpu" ]; then
+    _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2)
     python3 $SCRIPT_DIR/../../build.py --build_dir /home/onnxruntimedev \
         --config Debug Release \
         --skip_submodule_sync \
         --parallel --build_shared_lib \
         --use_cuda \
         --cuda_home /usr/local/cuda \
-        --cudnn_home /usr/local/cudnn-7.1/cuda --build_shared_lib $BUILD_EXTR_PAR
+        --cudnn_home /usr/local/cudnn-$_CUDNN_VERSION/cuda --build_shared_lib $BUILD_EXTR_PAR
     /home/onnxruntimedev/Release/onnx_test_runner -e cuda /data/onnx
 else
     python3 $SCRIPT_DIR/../../build.py --build_dir /home/onnxruntimedev \
diff --git a/tools/ci_build/github/linux/run_dockerbuild.sh b/tools/ci_build/github/linux/run_dockerbuild.sh
index 5be453555fda6..d9efaed87d42f 100755
--- a/tools/ci_build/github/linux/run_dockerbuild.sh
+++ b/tools/ci_build/github/linux/run_dockerbuild.sh
@@ -3,6 +3,7 @@ set -e -o -x
 
 SCRIPT_DIR="$( dirname "${BASH_SOURCE[0]}" )"
 SOURCE_ROOT=$(realpath $SCRIPT_DIR/../../../../)
+CUDA_VER=cuda10.0-cudnn7.3
 
 while getopts c:o:d:r:p:x: parameter_Option
 do case "${parameter_Option}"
@@ -16,6 +17,8 @@ r) BUILD_DIR=${OPTARG};;
 p) PYTHON_VER=${OPTARG};;
 # "--build_wheel --use_openblas"
 x) BUILD_EXTR_PAR=${OPTARG};;
+# "cuda10.0-cudnn7.3, cuda9.1-cudnn7.1"
+c) CUDA_VER=${OPTARG};;
 esac
 done
 
@@ -25,8 +28,12 @@ echo "bo=$BUILD_OS bd=$BUILD_DEVICE bdir=$BUILD_DIR pv=$PYTHON_VER bex=$BUILD_EX
 
 cd $SCRIPT_DIR/docker
 if [ $BUILD_DEVICE = "gpu" ]; then
-    IMAGE="ubuntu16.04-cuda9.1-cudnn7.1"
-    docker build -t "onnxruntime-$IMAGE" --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu_gpu .
+    IMAGE="ubuntu16.04-$CUDA_VER"
+    DOCKER_FILE=Dockerfile.ubuntu_gpu
+    if [ $CUDA_VER = "cuda9.1-cudnn7.1" ]; then
+    DOCKER_FILE=Dockerfile.ubuntu_gpu_cuda9
+    fi
+    docker build -t "onnxruntime-$IMAGE" --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
 else
     IMAGE="ubuntu16.04"
     docker build -t "onnxruntime-$IMAGE" --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu .