From 48f3a0bb277b241b921e4d3369ac9a15b424a24d Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 20 Oct 2023 12:38:18 -0700 Subject: [PATCH 1/5] Try downloading cuda 12 on windows --- .../templates/jobs/set-winenv.yml | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml index ca5a52fa61ed3..d06cafc744175 100644 --- a/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml @@ -6,14 +6,25 @@ parameters: type: boolean default: false -steps: -- ${{ if eq(parameters.DownloadCUDA, 'true') }}: - - powershell: | - azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" $(Agent.TempDirectory) +- jobs: + - ${{ if eq(parameters.DownloadCUDA, 'true') }}: + - job: Download CUDA + strategy: + matrix: + CUDA12: + CUDA_VERSION: '12.2' + CUDA11: + CUDA_VERSION: '11.8' + steps: -- task: BatchScript@1 - displayName: 'setup env' - inputs: - filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\${{ parameters.EnvSetupScript }}' - modifyEnvironment: true - workingFolder: '$(Build.BinariesDirectory)' + - powershell: | + azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(CUDA_VERSION)" $(Agent.TempDirectory) + - job: Setup Env + dependsOn: Download CUDA + tasks: + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\${{ parameters.EnvSetupScript }}' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' From 21c3e6fa4a646d290de3486650f25602e57f065b Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 20 Oct 2023 12:41:29 -0700 Subject: [PATCH 2/5] Try downloading cuda 12 on windows --- .../github/azure-pipelines/templates/jobs/set-winenv.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml index d06cafc744175..4c69732b9bd09 100644 --- a/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml @@ -6,20 +6,19 @@ parameters: type: boolean default: false -- jobs: +- steps: - ${{ if eq(parameters.DownloadCUDA, 'true') }}: - - job: Download CUDA + - step: Download CUDA strategy: matrix: CUDA12: CUDA_VERSION: '12.2' CUDA11: CUDA_VERSION: '11.8' - steps: - + task: - powershell: | azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(CUDA_VERSION)" $(Agent.TempDirectory) - - job: Setup Env + - step: Setup Env dependsOn: Download CUDA tasks: - task: BatchScript@1 From 04aaabaf67ea72087129668e912c0f3d9a62cf34 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 20 Oct 2023 12:51:28 -0700 Subject: [PATCH 3/5] Try downloading cuda 12 on windows --- .../templates/jobs/set-winenv.yml | 50 +++++++++---------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml index 4c69732b9bd09..71ef3b687740a 100644 --- a/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml @@ -1,29 +1,27 @@ parameters: -- name: EnvSetupScript - type: string + - name: EnvSetupScript + type: string + - name: DownloadCUDA + type: boolean + default: false + - name: PrimaryCUDAVersion + type: string + default: '11.8' + - name: SecondaryCUDAVersion + type: string + default: '12.2' -- name: DownloadCUDA - type: boolean - default: false - -- steps: +steps: - ${{ if eq(parameters.DownloadCUDA, 'true') }}: - - step: Download CUDA - strategy: - matrix: - CUDA12: - CUDA_VERSION: '12.2' - CUDA11: - CUDA_VERSION: '11.8' - task: - - powershell: | - azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(CUDA_VERSION)" $(Agent.TempDirectory) - - step: Setup Env - dependsOn: Download CUDA - tasks: - - task: BatchScript@1 - displayName: 'setup env' - inputs: - filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\${{ parameters.EnvSetupScript }}' - modifyEnvironment: true - workingFolder: '$(Build.BinariesDirectory)' + - powershell: | + azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(PrimaryCUDAVersion)" $(Agent.TempDirectory) + displayName: 'Download Primary CUDA SDK v$(PrimaryCUDAVersion)' + - powershell: | + azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(SecondaryCUDAVersion)" $(Agent.TempDirectory) + displayName: 'Download Secondary CUDA SDK v$(SecondaryCUDAVersion)' + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\${{ parameters.EnvSetupScript }}' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' From b46418bfdf67f3bbc580920481001e523b810681 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 20 Oct 2023 13:05:08 -0700 Subject: [PATCH 4/5] Try downloading cuda 12 on windows --- .../github/azure-pipelines/templates/jobs/set-winenv.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml index 71ef3b687740a..0c8fb91a24a31 100644 --- a/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml @@ -14,11 +14,11 @@ parameters: steps: - ${{ if eq(parameters.DownloadCUDA, 'true') }}: - powershell: | - azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(PrimaryCUDAVersion)" $(Agent.TempDirectory) - displayName: 'Download Primary CUDA SDK v$(PrimaryCUDAVersion)' + azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v${{ parameters.PrimaryCUDAVersion }}" $(Agent.TempDirectory) + displayName: 'Download Primary CUDA SDK v${{ parameters.PrimaryCUDAVersion }}' - powershell: | - azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(SecondaryCUDAVersion)" $(Agent.TempDirectory) - displayName: 'Download Secondary CUDA SDK v$(SecondaryCUDAVersion)' + azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v${{ parameters.SecondaryCUDAVersion }}" $(Agent.TempDirectory) + displayName: 'Download Secondary CUDA SDK v${{ parameters.SecondaryCUDAVersion }}' - task: BatchScript@1 displayName: 'setup env' inputs: From b8567877c0838123ff05a753a15fd3ea6850d58e Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 20 Oct 2023 22:13:09 -0700 Subject: [PATCH 5/5] Setting up env Path variable for cuda 12.2 --- .../azure-pipelines/c-api-noopenmp-packaging-pipelines.yml | 2 +- tools/ci_build/github/azure-pipelines/post-merge-jobs.yml | 2 +- .../ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml | 6 +++--- .../azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml | 2 +- .../windows/{setup_env_cuda_11.bat => setup_env_cuda.bat} | 6 ++++++ 5 files changed, 12 insertions(+), 6 deletions(-) rename tools/ci_build/github/windows/{setup_env_cuda_11.bat => setup_env_cuda.bat} (53%) diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index fdd8c09333737..b4edf088f31be 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -193,7 +193,7 @@ stages: DoCompliance: ${{ parameters.DoCompliance }} DoEsrp: ${{ parameters.DoEsrp }} stage_name_suffix: gpu - EnvSetupScript: setup_env_cuda_11.bat + EnvSetupScript: setup_env_cuda.bat buildArch: x64 msbuildPlatform: x64 packageName: x64-cuda diff --git a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml index d24b0e0539631..2a94499c7a268 100644 --- a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml +++ b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml @@ -67,7 +67,7 @@ stages: - template: templates/jobs/win-ci-vs-2022-job.yml parameters: BuildConfig: 'RelWithDebInfo' - EnvSetupScript: setup_env_cuda_11.bat + EnvSetupScript: setup_env_cuda.bat buildArch: x64 additionalBuildFlags: --enable_pybind --build_java --build_nodejs --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --enable_cuda_profiling --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86 msbuildPlatform: x64 diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml index 07b5388ea5cd2..ae2a4b4cead3d 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml @@ -40,7 +40,7 @@ stages: - template: templates/jobs/win-ci-vs-2022-job.yml parameters: BuildConfig: 'RelWithDebInfo' - EnvSetupScript: setup_env_cuda_11.bat + EnvSetupScript: setup_env_cuda.bat buildArch: x64 additionalBuildFlags: --enable_pybind --build_java --build_nodejs --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --enable_cuda_profiling --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86 msbuildPlatform: x64 @@ -57,7 +57,7 @@ stages: - template: templates/jobs/win-ci-vs-2022-job.yml parameters: BuildConfig: 'RelWithDebInfo' - EnvSetupScript: setup_env_cuda_11.bat + EnvSetupScript: setup_env_cuda.bat buildArch: x64 additionalBuildFlags: --enable_pybind --enable_training --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --skip_onnx_tests --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75 msbuildPlatform: x64 @@ -76,7 +76,7 @@ stages: - template: templates/jobs/win-ci-vs-2022-job.yml parameters: BuildConfig: 'RelWithDebInfo' - EnvSetupScript: setup_env_cuda_11.bat + EnvSetupScript: setup_env_cuda.bat buildArch: x64 # note: need to specify `--gen_doc` when creating the build config so it has to be in additionalBuildFlags additionalBuildFlags: --gen_doc validate --skip_tests --enable_pybind --use_dml --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86 --cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=OFF diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml index b5db8a5201405..d0f9772da7adc 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml @@ -10,7 +10,7 @@ jobs: BuildConfig: 'MinSizeRel' variables: MsbuildArguments: '-detailedsummary -maxcpucount -consoleloggerparameters:PerformanceSummary' - EnvSetupScript: setup_env_cuda_11.bat + EnvSetupScript: setup_env_cuda.bat buildArch: x64 TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] timeoutInMinutes: 120 diff --git a/tools/ci_build/github/windows/setup_env_cuda_11.bat b/tools/ci_build/github/windows/setup_env_cuda.bat similarity index 53% rename from tools/ci_build/github/windows/setup_env_cuda_11.bat rename to tools/ci_build/github/windows/setup_env_cuda.bat index 1308e43a4f6db..96569cbe0f648 100644 --- a/tools/ci_build/github/windows/setup_env_cuda_11.bat +++ b/tools/ci_build/github/windows/setup_env_cuda.bat @@ -6,4 +6,10 @@ if exist PATH=%AGENT_TEMPDIRECTORY%\v11.8\ { } else { set PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\extras\CUPTI\lib64;%PATH% } +@REM The default version is still cuda v11.8, because set cuda v12.2 after it +if exist PATH=%AGENT_TEMPDIRECTORY%\v12.2\ { + set PATH=%PATH%;%AGENT_TEMPDIRECTORY%\v12.2\bin;%AGENT_TEMPDIRECTORY%\v12.2\extras\CUPTI\lib64 +} else { + set PATH=%PATH%;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.2\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.2\extras\CUPTI\lib64 +} set GRADLE_OPTS=-Dorg.gradle.daemon=false