diff --git a/.github/workflows/windows_cuda.yml b/.github/workflows/windows_cuda.yml index c6346d9c4e932..701df5e5642e6 100644 --- a/.github/workflows/windows_cuda.yml +++ b/.github/workflows/windows_cuda.yml @@ -148,7 +148,6 @@ jobs: DocUpdateNeeded: false ONNXRUNTIME_TEST_GPU_DEVICE_ID: '0' AZCOPY_AUTO_LOGIN_TYPE: MSI - AZCOPY_MSI_CLIENT_ID: 63b63039-6328-442f-954b-5a64d124e5b4 test: name: Windows GPU CUDA CI Pipeline Test Job @@ -260,4 +259,3 @@ jobs: DocUpdateNeeded: false ONNXRUNTIME_TEST_GPU_DEVICE_ID: '0' AZCOPY_AUTO_LOGIN_TYPE: MSI - AZCOPY_MSI_CLIENT_ID: 63b63039-6328-442f-954b-5a64d124e5b4 diff --git a/.github/workflows/windows_cuda_plugin.yml b/.github/workflows/windows_cuda_plugin.yml index 538c1d783cd68..ac54c79bb8b57 100644 --- a/.github/workflows/windows_cuda_plugin.yml +++ b/.github/workflows/windows_cuda_plugin.yml @@ -47,8 +47,26 @@ jobs: - name: Download CUDA SDK v12.8 working-directory: ${{ runner.temp }} run: | - azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v12.8" . - dir + $cudaSdkDir = Join-Path $env:RUNNER_TEMP "v12.8" + $localCudaSdkDir = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8" + + if (Test-Path (Join-Path $cudaSdkDir "bin\nvcc.exe")) { + Write-Host "CUDA SDK already present at $cudaSdkDir" + } else { + azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v12.8" . + $azcopyExitCode = $lastExitCode + if (($azcopyExitCode -ne 0) -and (Test-Path (Join-Path $localCudaSdkDir "bin\nvcc.exe"))) { + Write-Host "AzCopy download failed; falling back to local CUDA SDK at $localCudaSdkDir" + New-Item -ItemType Directory -Path $cudaSdkDir -Force | Out-Null + Copy-Item -Path (Join-Path $localCudaSdkDir "*") -Destination $cudaSdkDir -Recurse -Force -ErrorAction Stop + $azcopyExitCode = 0 + } + if ($azcopyExitCode -ne 0) { + exit $azcopyExitCode + } + } + + dir $cudaSdkDir shell: pwsh - name: Add CUDA to PATH @@ -107,18 +125,35 @@ jobs: Remove-Item -Path $outputDir -Include "*.obj" -Recurse shell: pwsh + - name: Stage CUDA runtime test dependencies + shell: pwsh + run: | + $cudaStageDir = Join-Path $env:RUNNER_TEMP "cuda-test-deps\v12.8" + $cudaBinDir = Join-Path $cudaStageDir "bin" + $cuptiDir = Join-Path $cudaStageDir "extras\CUPTI\lib64" + New-Item -ItemType Directory -Path $cudaBinDir -Force | Out-Null + New-Item -ItemType Directory -Path $cuptiDir -Force | Out-Null + Copy-Item -Path "$env:RUNNER_TEMP\v12.8\bin\*" -Destination $cudaBinDir -Recurse -Force -ErrorAction Stop + Copy-Item -Path "$env:RUNNER_TEMP\v12.8\extras\CUPTI\lib64\*" -Destination $cuptiDir -Recurse -Force -ErrorAction Stop + - name: Upload build artifacts uses: actions/upload-artifact@v6 with: name: cuda-plugin-build-artifacts path: ${{ runner.temp }}\build + + - name: Upload CUDA runtime test dependencies + uses: actions/upload-artifact@v6 + with: + name: cuda-plugin-cuda-runtime + path: ${{ runner.temp }}\cuda-test-deps + if-no-files-found: error env: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true setVcvars: true ALLOW_RELEASED_ONNX_OPSET_ONLY: '0' ONNXRUNTIME_TEST_GPU_DEVICE_ID: '0' AZCOPY_AUTO_LOGIN_TYPE: MSI - AZCOPY_MSI_CLIENT_ID: 63b63039-6328-442f-954b-5a64d124e5b4 test: name: Windows CUDA Plugin EP Test @@ -141,6 +176,12 @@ jobs: name: cuda-plugin-build-artifacts path: ${{ runner.temp }}\build + - name: Download CUDA runtime test dependencies + uses: actions/download-artifact@v7 + with: + name: cuda-plugin-cuda-runtime + path: ${{ runner.temp }}\cuda-test-deps + - uses: actions/setup-python@v6 with: python-version: '3.14' @@ -161,20 +202,13 @@ jobs: working-directory: ${{ github.workspace }} shell: cmd - - name: Download CUDA SDK v12.8 - working-directory: ${{ runner.temp }} - run: | - azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v12.8" . - dir - shell: pwsh - - name: Add CUDA to PATH shell: powershell run: | Write-Host "Adding CUDA to PATH" - Write-Host "CUDA Path: $env:RUNNER_TEMP\v12.8\bin" - Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.8\bin" - Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.8\extras\CUPTI\lib64" + Write-Host "CUDA Path: $env:RUNNER_TEMP\cuda-test-deps\v12.8\bin" + Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\cuda-test-deps\v12.8\bin" + Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\cuda-test-deps\v12.8\extras\CUPTI\lib64" - name: Set OnnxRuntimeBuildDirectory shell: pwsh @@ -213,5 +247,3 @@ jobs: setVcvars: true ALLOW_RELEASED_ONNX_OPSET_ONLY: '0' ONNXRUNTIME_TEST_GPU_DEVICE_ID: '0' - AZCOPY_AUTO_LOGIN_TYPE: MSI - AZCOPY_MSI_CLIENT_ID: 63b63039-6328-442f-954b-5a64d124e5b4 diff --git a/.github/workflows/windows_gpu_doc_gen.yml b/.github/workflows/windows_gpu_doc_gen.yml index 5e50a970875fc..e75d3a61dae19 100644 --- a/.github/workflows/windows_gpu_doc_gen.yml +++ b/.github/workflows/windows_gpu_doc_gen.yml @@ -44,7 +44,6 @@ jobs: setVcvars: true ALLOW_RELEASED_ONNX_OPSET_ONLY: '0' AZCOPY_AUTO_LOGIN_TYPE: MSI - AZCOPY_MSI_CLIENT_ID: 63b63039-6328-442f-954b-5a64d124e5b4 runs-on: [ "self-hosted", "1ES.Pool=onnxruntime-github-Win2022-GPU-A10", diff --git a/.github/workflows/windows_openvino.yml b/.github/workflows/windows_openvino.yml index 52581c7d0a5f5..ce535e2061ba3 100644 --- a/.github/workflows/windows_openvino.yml +++ b/.github/workflows/windows_openvino.yml @@ -26,7 +26,6 @@ jobs: timeout-minutes: 240 env: AZCOPY_AUTO_LOGIN_TYPE: MSI - AZCOPY_MSI_CLIENT_ID: 63b63039-6328-442f-954b-5a64d124e5b4 OnnxRuntimeBuildDirectory: ${{ github.workspace }} DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true ALLOW_RELEASED_ONNX_OPSET_ONLY: '1' diff --git a/.github/workflows/windows_qnn_x64.yml b/.github/workflows/windows_qnn_x64.yml index 35c620ca6f650..fc190e1b7d2be 100644 --- a/.github/workflows/windows_qnn_x64.yml +++ b/.github/workflows/windows_qnn_x64.yml @@ -29,7 +29,6 @@ jobs: QnnLibKind: [shared_lib, static_lib] env: AZCOPY_AUTO_LOGIN_TYPE: MSI - AZCOPY_MSI_CLIENT_ID: 63b63039-6328-442f-954b-5a64d124e5b4 DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true ALLOW_RELEASED_ONNX_OPSET_ONLY: '1' diff --git a/.github/workflows/windows_tensorrt.yml b/.github/workflows/windows_tensorrt.yml index d5710795942d1..1087254f0258f 100644 --- a/.github/workflows/windows_tensorrt.yml +++ b/.github/workflows/windows_tensorrt.yml @@ -154,7 +154,6 @@ jobs: DocUpdateNeeded: false ONNXRUNTIME_TEST_GPU_DEVICE_ID: '0' AZCOPY_AUTO_LOGIN_TYPE: MSI - AZCOPY_MSI_CLIENT_ID: 63b63039-6328-442f-954b-5a64d124e5b4 test: name: Windows GPU TensorRT CI Pipeline Test Job @@ -265,4 +264,3 @@ jobs: DocUpdateNeeded: false ONNXRUNTIME_TEST_GPU_DEVICE_ID: '0' AZCOPY_AUTO_LOGIN_TYPE: MSI - AZCOPY_MSI_CLIENT_ID: 63b63039-6328-442f-954b-5a64d124e5b4 diff --git a/cmake/external/onnxruntime_external_deps.cmake b/cmake/external/onnxruntime_external_deps.cmake index ed3b0aa8192a7..3514f0555327d 100644 --- a/cmake/external/onnxruntime_external_deps.cmake +++ b/cmake/external/onnxruntime_external_deps.cmake @@ -822,6 +822,22 @@ if (onnxruntime_USE_WEBGPU) endif() onnxruntime_fetchcontent_makeavailable(dawn) + + # Pre-create the emdawnwebgpu include directory at configure time to prevent a parallel + # build race condition. Dawn's src/emdawnwebgpu/CMakeLists.txt registers add_custom_command + # rules that copy webgpu_glfw.h and webgpu_enum_class_bitmasks.h into + # gen/src/emdawnwebgpu/include/webgpu/ using cmake -E copy. With -j32 parallel builds + # those copy commands can be triggered from multiple Makefile targets simultaneously + # before DawnJSONGenerator has created the destination directory, causing: + # Error copying file "…/webgpu_enum_class_bitmasks.h" to + # "…/gen/src/emdawnwebgpu/include/webgpu/webgpu_enum_class_bitmasks.h" + # Creating the directory here (at configure time) ensures it already exists when the + # build starts, regardless of whether the dawn_parallel_build_fix.patch was applied to + # the cached Dawn source on this runner. + FetchContent_GetProperties(dawn) + if(dawn_POPULATED) + file(MAKE_DIRECTORY "${dawn_BINARY_DIR}/gen/src/emdawnwebgpu/include/webgpu") + endif() endif() if (NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten")