diff --git a/.github/actions/linux-web-init-and-check/action.yml b/.github/actions/linux-web-init-and-check/action.yml index 694f026d07d0d..41216e8c2860b 100644 --- a/.github/actions/linux-web-init-and-check/action.yml +++ b/.github/actions/linux-web-init-and-check/action.yml @@ -50,7 +50,7 @@ runs: - name: Check unformatted files run: | - node -e "a=require('child_process').execSync('git diff --name-only').toString();if(a)throw new Error('Following source files are not formatted: (did you run \"npm run format\"?)\n'+a)" + node -e "a=require('child_process').execSync('git diff --name-only -- .').toString();if(a)throw new Error('Following source files are not formatted: (did you run \"npm run format\"?)\n'+a)" shell: bash working-directory: ${{ github.workspace }}/js @@ -66,6 +66,6 @@ runs: - name: Check out of dated documents run: | - node -e "a=require('child_process').execSync('git diff --name-only').toString();if(a)throw new Error('Following documents are not up-to-date: (did you run \"npm run build:doc\"?)\n'+a)" + node -e "a=require('child_process').execSync('git diff --name-only -- .').toString();if(a)throw new Error('Following documents are not up-to-date: (did you run \"npm run build:doc\"?)\n'+a)" shell: bash working-directory: ${{ github.workspace }}/js/web diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-cuda-minimal-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-cuda-minimal-ci-pipeline.yml index a062a1fce5c1b..6cee227b70b2b 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-cuda-minimal-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-cuda-minimal-ci-pipeline.yml @@ -93,6 +93,10 @@ jobs: --volume $(Build.BinariesDirectory):/build \ --volume /data/models:/build/models:ro \ --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ + -e NPM_CONFIG_USERCONFIG=/tmp/.npmrc \ + --volume "${NPM_CONFIG_USERCONFIG}:/tmp/.npmrc:ro" \ + --volume $HOME/.m2:/home/onnxruntimedev/.m2:ro \ + --volume $HOME/.gradle:/home/onnxruntimedev/.gradle \ -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ diff --git a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml index 2fc3f30bd04c7..3211707ccf53a 100644 --- a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml @@ -19,26 +19,6 @@ parameters: type: boolean default: true -- name: enable_windows_arm64_qnn - displayName: 'Whether Windows ARM64 package with QNN EP is built.' - type: boolean - default: true - -- name: enable_windows_arm64ec_qnn - displayName: 'Whether Windows ARM64EC package with QNN EP is built.' - type: boolean - default: true - -- name: enable_windows_x64_qnn - displayName: 'Whether Windows x86_64 package with QNN EP is built.' - type: boolean - default: true - -- name: enable_linux_x64_qnn - displayName: 'Whether Linux x86_64 package with QNN EP is built.' - type: boolean - default: true - - name: build_py_parameters displayName: 'Specify extra build parameters' type: string @@ -55,11 +35,6 @@ parameters: - RelWithDebInfo - MinSizeRel -# Only applies to QNN packages. -- name: qnn_sdk_version - type: string - displayName: 'QNN SDK version. Only for QNN packages.' - default: 2.42.0.251225 trigger: none @@ -103,10 +78,5 @@ extends: enable_windows_cpu: ${{ parameters.enable_windows_cpu }} enable_mac_cpu: ${{ parameters.enable_mac_cpu }} enable_linux_arm: ${{ parameters.enable_linux_arm }} - enable_windows_arm64_qnn: ${{ parameters.enable_windows_arm64_qnn }} - enable_windows_arm64ec_qnn: ${{ parameters.enable_windows_arm64ec_qnn }} - enable_windows_x64_qnn: ${{ parameters.enable_windows_x64_qnn }} - enable_linux_x64_qnn: ${{ parameters.enable_linux_x64_qnn }} build_py_parameters: ${{ parameters.build_py_parameters }} cmake_build_type: ${{ parameters.cmake_build_type }} - qnn_sdk_version: ${{ parameters.qnn_sdk_version }} diff --git a/tools/ci_build/github/azure-pipelines/stages/nodejs-linux-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nodejs-linux-packaging-stage.yml index ff35d3e35ef6c..1a326e9dbd584 100644 --- a/tools/ci_build/github/azure-pipelines/stages/nodejs-linux-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/nodejs-linux-packaging-stage.yml @@ -33,6 +33,9 @@ stages: - checkout: self clean: true submodules: recursive + + - template: ../templates/setup-feeds-and-python-steps.yml + - template: ../templates/get-docker-image-steps.yml parameters: Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cuda${{ variables.CUDA_VERSION_MAJOR }}/Dockerfile diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml index 88d2981e2ccaa..25bded72891d6 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml @@ -25,26 +25,6 @@ parameters: type: boolean default: true -- name: enable_windows_arm64_qnn - displayName: 'Whether Windows ARM64 package with QNN EP is built.' - type: boolean - default: true - -- name: enable_windows_arm64ec_qnn - displayName: 'Whether Windows ARM64EC package with QNN EP is built.' - type: boolean - default: true - -- name: enable_windows_x64_qnn - displayName: 'Whether Windows x86_64 package with QNN EP is built.' - type: boolean - default: true - -- name: enable_linux_x64_qnn - displayName: 'Whether Linux x86_64 package with QNN EP is built.' - type: boolean - default: true - - name: cmake_build_type type: string displayName: 'Linux packages cmake build type. Linux Only.' @@ -55,12 +35,6 @@ parameters: - RelWithDebInfo - MinSizeRel -# Only applies to QNN packages. -- name: qnn_sdk_version - type: string - displayName: 'QNN SDK version. Only for QNN packages.' - default: 2.42.0.251225 - stages: - ${{ if eq(parameters.enable_windows_cpu, true) }}: - stage: Python_Packaging_Windows_CPU @@ -133,68 +107,3 @@ stages: extra_build_arg: ${{ parameters.build_py_parameters }} cmake_build_type: ${{ parameters.cmake_build_type }} is1ES: true - -- ${{ if eq(parameters.enable_windows_arm64_qnn, true) }}: - - stage: Python_Packaging_Windows_ARM64_QNN - dependsOn: [] - jobs: - - template: ../templates/py-win-arm64-qnn.yml - parameters: - MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64' - QNN_SDK: ${{ parameters.qnn_sdk_version }} - BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} - PYTHON_VERSION: '3.11' - - - template: ../templates/py-win-arm64-qnn.yml - parameters: - MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64' - QNN_SDK: ${{ parameters.qnn_sdk_version }} - BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} - PYTHON_VERSION: '3.12' - - - template: ../templates/py-win-arm64-qnn.yml - parameters: - MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64' - QNN_SDK: ${{ parameters.qnn_sdk_version }} - BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} - PYTHON_VERSION: '3.13' - - - template: ../templates/py-win-arm64-qnn.yml - parameters: - MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64' - QNN_SDK: ${{ parameters.qnn_sdk_version }} - BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} - PYTHON_VERSION: '3.14' - -- ${{ if eq(parameters.enable_windows_arm64ec_qnn, true) }}: - - stage: Python_Packaging_Windows_arm64ec_QNN - dependsOn: [] - jobs: - - template: ../templates/py-win-arm64ec-qnn.yml - parameters: - MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' - QNN_SDK: ${{ parameters.qnn_sdk_version }} - BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} - -- ${{ if eq(parameters.enable_windows_x64_qnn, true) }}: - - stage: Python_Packaging_Windows_x64_QNN - dependsOn: [] - jobs: - - template: ../templates/py-win-x64-qnn.yml - parameters: - MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' - QNN_SDK: ${{ parameters.qnn_sdk_version }} - BUILD_PY_PARAMETERS: ${{ parameters.build_py_parameters }} - is1ES: true - -- ${{ if eq(parameters.enable_linux_x64_qnn, true) }}: - - stage: Python_Packaging_Linux_x64_QNN - dependsOn: [] - jobs: - - template: ../templates/py-linux-qnn.yml - parameters: - machine_pool: 'onnxruntime-Ubuntu2404-AMD-CPU' - QnnSdk: ${{ parameters.qnn_sdk_version }} - extra_build_arg: ${{ parameters.build_py_parameters }} - cmake_build_type: ${{ parameters.cmake_build_type }} - is1ES: true diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml index f22527bb2bd51..c28ad09a7c02d 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml @@ -74,6 +74,10 @@ jobs: set -e -x mkdir -p $HOME/.onnx docker run -e SYSTEM_COLLECTIONURI --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ + -e NPM_CONFIG_USERCONFIG=/tmp/.npmrc \ + --volume "${NPM_CONFIG_USERCONFIG}:/tmp/.npmrc:ro" \ + --volume $HOME/.m2:/home/onnxruntimedev/.m2:ro \ + --volume $HOME/.gradle:/home/onnxruntimedev/.gradle \ --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging /bin/bash -c "python3 \ /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release \ --skip_submodule_sync --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --use_binskim_compliant_compile_flags --use_vcpkg --use_vcpkg_ms_internal_asset_cache --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/installed" diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-web-init-and-check.yml b/tools/ci_build/github/azure-pipelines/templates/linux-web-init-and-check.yml index 2b73f82615bba..125dc2a885433 100644 --- a/tools/ci_build/github/azure-pipelines/templates/linux-web-init-and-check.yml +++ b/tools/ci_build/github/azure-pipelines/templates/linux-web-init-and-check.yml @@ -34,9 +34,10 @@ steps: - script: | npm run format workingDirectory: '$(Build.SourcesDirectory)/js' - displayName: 'Clang-format' + displayName: 'npm run format' +# stash any changes to `NuGet.config` (expected from setup-feed auth) to prevent false positives. restore after diff check. - script: | - node -e "a=require('child_process').execSync('git diff --name-only').toString();if(a)throw new Error('Following source files are not formatted: (did you run \"npm run format\"?)\n'+a)" + node -e "a=require('child_process').execSync('git diff --name-only -- .').toString();if(a)throw new Error('Following source files are not formatted: (did you run \"npm run format\"?)\n'+a)" workingDirectory: '$(Build.SourcesDirectory)/js' displayName: 'Check unformatted files' - script: | @@ -48,6 +49,6 @@ steps: workingDirectory: '$(Build.SourcesDirectory)/js/web' displayName: 'Generating documents' - script: | - node -e "a=require('child_process').execSync('git diff --name-only').toString();if(a)throw new Error('Following documents are not up-to-date: (did you run \"npm run build:doc\"?)\n'+a)" + node -e "a=require('child_process').execSync('git diff --name-only -- .').toString();if(a)throw new Error('Following documents are not up-to-date: (did you run \"npm run build:doc\"?)\n'+a)" workingDirectory: '$(Build.SourcesDirectory)/js/web' displayName: 'Check out of dated documents' diff --git a/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml b/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml index 50628ad617d56..b72787241a0a3 100644 --- a/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml @@ -1,10 +1,11 @@ # Template for setting up the package feeds and installing python. # Feeds configured: -# - pip # - maven & gradle +# - npm # - nuget +# - pip # -# FIXME: Not idempotent. +# n.b. This template writes to user profile files. parameters: - name: artifactFeeds @@ -20,8 +21,47 @@ parameters: default: '' steps: +# HACK: Ensure there are no leftover m2 settings from previous runs. Agent pools are currently stateful. +# Ideally we'd just push/pop/layer specific splices to the settings (auth & mirror stuff), but there's no easy +# mechanism for that AFAIK. +- task: PowerShell@2 + displayName: Maven - remove existing ~/.m2/settings.xml + inputs: + targetType: 'inline' + script: | + if (Test-Path ~/.m2/settings.xml) { + Remove-Item ~/.m2/settings.xml + } + +# use `PowerShell@2` instead of `pwsh` as it is available on arm64 agents, whereas `pwsh` is not. (2026-04-15) +- task: PowerShell@2 + displayName: 'NPM - set NPM_CONFIG_USERCONFIG to agent temp config' + env: + artifact_feed: ${{ parameters.artifactFeeds }} + inputs: + targetType: 'inline' + script: | + # ensure we have a clean/empty npm config. config file must exist prior to `npmAuthenticate`. + $npmrcPath = "${env:AGENT_TEMPDIRECTORY}/.npmrc" + + Set-Content -Force -Path $npmrcPath -Value @" + registry=https://aiinfra.pkgs.visualstudio.com/_packaging/${env:artifact_feed}/npm/registry/ + always-auth=true + "@ + + Write-Host "Generated npmrc contents:" + Get-Content $npmrcPath | Write-Host + + echo "##vso[task.setvariable variable=NPM_CONFIG_USERCONFIG]${npmrcPath}" + +# FUTURE WORK: use `Npm@1` task in pipelines and specify the feed there? +- task: npmAuthenticate@0 + inputs: + workingFile: $(NPM_CONFIG_USERCONFIG) + - task: NuGetAuthenticate@1 # just auth w/ internal feeds +# Creates/touches ~/.m2/settings.xml - task: MavenAuthenticate@0 inputs: artifactsFeeds: ${{ parameters.artifactFeeds }} @@ -41,59 +81,93 @@ steps: ${{ if ne(parameters.architecture, '') }}: architecture: ${{ parameters.architecture }} -- pwsh: | - # Check if python is available in PATH - $pythonCmd = Get-Command "python${env:python_version_spec}" -ErrorAction SilentlyContinue - if ($pythonCmd) { - Write-Host "Found `python${env:python_version_spec}` at $($pythonCmd.Source)" - exit 0 - } - - $pythonCmd = Get-Command "python" -ErrorAction SilentlyContinue - if ($pythonCmd) { - Write-Host "Found `python` at $($pythonCmd.Source)." - # FUTURE WORK: validate it matches the requested version spec - exit 0 - } - - Write-Host "##vso[task.logissue type=error]Neither `python` nor `python${env:python_version_spec}` are on PATH." - exit 1 +- task: PowerShell@2 displayName: 'Verify Python ${{ parameters.versionSpec }} is on path' + inputs: + targetType: 'inline' + script: | + # Check if python is available in PATH + $pythonCmd = Get-Command "python${env:python_version_spec}" -ErrorAction SilentlyContinue + if ($pythonCmd) { + Write-Host "Found `python${env:python_version_spec}` at $($pythonCmd.Source)" + exit 0 + } + + $pythonCmd = Get-Command "python" -ErrorAction SilentlyContinue + if ($pythonCmd) { + Write-Host "Found `python` at $($pythonCmd.Source)." + # FUTURE WORK: validate it matches the requested version spec + exit 0 + } + + Write-Host "##vso[task.logissue type=error]Neither `python` nor `python${env:python_version_spec}` are on PATH." + exit 1 env: python_version_spec: ${{ parameters.versionSpec }} -- pwsh: | - # Force everything to go through & authenticate using the main feed. - # ASSUME: there is no `mirrors` section already present in the settings.xml. - # HACK: Does not escape `parameters.artifactFeeds`. Assumes pattern is [A-Za-z0-9]+. - # HACK: Assumes feed is a org-scoped feed in `aiinfra`. - (Get-Content ~/.m2/settings.xml) | ` - %{ $_ -replace '', "${env:central_maven_feed_name}1ES Feed Overridehttps://aiinfra.pkgs.visualstudio.com/_packaging/${env:central_maven_feed_name}/maven/v1*" } | ` - Set-Content ~/.m2/settings.xml - - # gradle - inject 1ES feed repo into all projects/buildscripts. Auth using creds from maven settings.xml. - $gradleInitDir = if ($env:GRADLE_HOME_DIR) { $env:GRADLE_HOME_DIR } else { "~/.gradle" } - if (!(Test-Path -Path "${gradleInitDir}/init.d")) { - New-Item -ItemType Directory -Path "${gradleInitDir}/init.d" - } - Set-Content -Path "${gradleInitDir}/init.d/1es-feed-inject.init.gradle" -Value @" - // Use official 1ES feed. Network isolation prevents us from reaching out directly to other feeds. - - def mavenSettingsServers = { - File mavenSettings = new File(System.getProperty("user.home"), ".m2/settings.xml") - return new XmlSlurper().parse(mavenSettings)."servers"."server" - } - - def oneEsFeedCredentials = { - for (x in mavenSettingsServers()) { - if (x."id".text() == "${env:central_maven_feed_name}") { - return [username: x.username.text(), password: x.password.text()] +# HACK: If agent pools are stateful then we have to ensure that `1es-feed-inject.init.gradle` file path is constant +# across all runs. +# Proper fix would be to have a post-job task to clean up our junk, but we can't do that without a full task def. +# (Or intrusively set GRADLE_USER_HOME env var to an agent temp dir. Maven doesn't have equiv facilities.) +- task: PowerShell@2 + displayName: 'Maven/Gradle - mirror / feed via `${{ parameters.artifactFeeds }}`' + env: + central_maven_feed_name: ${{ parameters.artifactFeeds }} + inputs: + targetType: 'inline' + script: | + # Force everything in Maven to go through & authenticate using the main feed. + # HACK: Does not escape `parameters.artifactFeeds`. Assumes pattern is [A-Za-z0-9]+. + # HACK: Assumes feed is a org-scoped feed in `aiinfra`. + $m2SettingsXmlPath = "~/.m2/settings.xml" + $m2SettingsContent = Get-Content $m2SettingsXmlPath -Raw + + if ($m2SettingsContent -match '') { + echo "##vso[task.logissue type=error]Configuration already contains ``, cannot inject feed mirror w/o corrupting existing config." + exit 1 + } + + echo "Injecting Maven mirrors section for 1ES feed" + $( $m2SettingsContent -replace '', "${env:central_maven_feed_name}1ES Feed Overridehttps://aiinfra.pkgs.visualstudio.com/_packaging/${env:central_maven_feed_name}/maven/v1*" ) | ` + Set-Content $m2SettingsXmlPath + + + # gradle - inject 1ES feed repo into all projects/buildscripts. Auth using creds from maven settings.xml. + # nb: see HACK note above. + $gradleInitDir = if ($env:GRADLE_HOME_DIR) { $env:GRADLE_HOME_DIR } else { "~/.gradle" } + if (!(Test-Path -Path "${gradleInitDir}/init.d")) { + New-Item -ItemType Directory -Path "${gradleInitDir}/init.d" + } + Set-Content -Path "${gradleInitDir}/init.d/1es-feed-inject.init.gradle" -Value @" + // Use official 1ES feed. Network isolation prevents us from reaching out directly to other feeds. + + def mavenSettingsServers = { + File mavenSettings = new File(System.getProperty("user.home"), ".m2/settings.xml") + return new XmlSlurper().parse(mavenSettings)."servers"."server" + } + + def oneEsFeedCredentials = { + for (x in mavenSettingsServers()) { + if (x."id".text() == "${env:central_maven_feed_name}") { + return [username: x.username.text(), password: x.password.text()] + } } } - } - allprojects { - buildscript { + allprojects { + buildscript { + repositories { + maven { + name "1ES Feed" + url uri("https://aiinfra.pkgs.visualstudio.com/_packaging/${env:central_maven_feed_name}/maven/v1") + credentials(PasswordCredentials) { + username oneEsFeedCredentials().username + password oneEsFeedCredentials().password + } + } + } + } + repositories { maven { name "1ES Feed" @@ -105,42 +179,30 @@ steps: } } } + "@ - repositories { - maven { - name "1ES Feed" - url uri("https://aiinfra.pkgs.visualstudio.com/_packaging/${env:central_maven_feed_name}/maven/v1") - credentials(PasswordCredentials) { - username oneEsFeedCredentials().username - password oneEsFeedCredentials().password - } - } - } - } - "@ - displayName: 'Maven/Gradle - mirror / feed via `${{ parameters.artifactFeeds }}`' - env: - central_maven_feed_name: ${{ parameters.artifactFeeds }} - -- pwsh: | - # NuGet appears to fail if any of the pkg sources are unreachable. - # Replace the NuGet.config w/ one that only has the internal feed. - # HACK: We'd rather splice the structure, not override the file. - Set-Content -Path "NuGet.config" -Value @" - - - - - - - - - - - - - - "@ +- task: PowerShell@2 displayName: NuGet - override `NuGet.config` to lock feed to `${{ parameters.artifactFeeds }}` env: central_maven_feed_name: ${{ parameters.artifactFeeds }} + inputs: + targetType: 'inline' + script: | + # NuGet appears to fail if any of the pkg sources are unreachable. + # Replace the NuGet.config w/ one that only has the internal feed. + # HACK: We'd rather splice the structure, not override the file. + Set-Content -Path "NuGet.config" -Value @" + + + + + + + + + + + + + + "@ diff --git a/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml index 273664aecf8c8..895e87a73ded0 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-web-ci.yml @@ -76,7 +76,7 @@ jobs: git checkout -- .gitattributes workingDirectory: '$(Build.SourcesDirectory)' displayName: 'Testing: force EOL to lf on windows for /js/**' - + - template: setup-build-tools.yml parameters: host_cpu_arch: 'x64' @@ -129,9 +129,10 @@ jobs: - script: | npm run format workingDirectory: '$(Build.SourcesDirectory)\js' - displayName: 'Clang-format' + displayName: 'npm run format' + # stash any changes to `NuGet.config` (expected from setup-feed auth) to prevent false positives. restore after diff check. - script: | - node -e "a=require('child_process').execSync('git diff --name-only').toString();if(a)throw new Error('Following source files are not formatted: (did you run \"npm run format\"?)\n'+a)" + node -e "a=require('child_process').execSync('git diff --name-only -- .').toString();if(a)throw new Error('Following source files are not formatted: (did you run \"npm run format\"?)\n'+a)" workingDirectory: '$(Build.SourcesDirectory)\js' displayName: 'Check unformatted files' - script: | @@ -139,7 +140,7 @@ jobs: workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Generating documents' - script: | - node -e "a=require('child_process').execSync('git diff --name-only').toString();if(a)throw new Error('Following documents are not up-to-date: (did you run \"npm run build:doc\"?)\n'+a)" + node -e "a=require('child_process').execSync('git diff --name-only -- .').toString();if(a)throw new Error('Following documents are not up-to-date: (did you run \"npm run build:doc\"?)\n'+a)" workingDirectory: '$(Build.SourcesDirectory)\js\web' displayName: 'Check out of dated documents' - task: Cache@2 diff --git a/tools/ci_build/github/linux/build_cuda_c_api_package.sh b/tools/ci_build/github/linux/build_cuda_c_api_package.sh index a1a2d347414c7..12a03e686d5b1 100755 --- a/tools/ci_build/github/linux/build_cuda_c_api_package.sh +++ b/tools/ci_build/github/linux/build_cuda_c_api_package.sh @@ -12,6 +12,10 @@ fi docker run -e SYSTEM_COLLECTIONURI --rm --volume \ $BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build \ +-e NPM_CONFIG_USERCONFIG=/tmp/.npmrc \ +--volume "${NPM_CONFIG_USERCONFIG}:/tmp/.npmrc:ro" \ +--volume $HOME/.m2:/home/onnxruntimedev/.m2:ro \ +--volume $HOME/.gradle:/home/onnxruntimedev/.gradle \ -e NIGHTLY_BUILD onnxruntimecuda${CUDA_VERSION_MAJOR}build \ /bin/bash -c "/usr/bin/python3 /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs \ --build_dir /build --config Release --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib \ diff --git a/tools/ci_build/github/linux/build_nodejs_package.sh b/tools/ci_build/github/linux/build_nodejs_package.sh index fac12a4342f60..3d7a7d855ef10 100755 --- a/tools/ci_build/github/linux/build_nodejs_package.sh +++ b/tools/ci_build/github/linux/build_nodejs_package.sh @@ -13,6 +13,10 @@ fi mkdir -p $HOME/.onnx docker run -e SYSTEM_COLLECTIONURI --rm --volume /data/onnx:/data/onnx:ro --volume $BUILD_SOURCESDIRECTORY:/onnxruntime_src \ --volume $BUILD_BINARIESDIRECTORY:/build --volume /data/models:/build/models:ro \ +-e NPM_CONFIG_USERCONFIG=/tmp/.npmrc \ +--volume "${NPM_CONFIG_USERCONFIG}:/tmp/.npmrc:ro" \ +--volume $HOME/.m2:/home/onnxruntimedev/.m2:ro \ +--volume $HOME/.gradle:/home/onnxruntimedev/.gradle \ --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda${CUDA_VERSION_MAJOR}xtrt86build \ /bin/bash -c "/usr/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \ --skip_tests --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --build_nodejs \ diff --git a/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh b/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh index 8c2866bd37833..b6358aa775a24 100755 --- a/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh +++ b/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh @@ -13,6 +13,10 @@ fi mkdir -p $HOME/.onnx docker run -e SYSTEM_COLLECTIONURI --rm --volume /data/onnx:/data/onnx:ro --volume $BUILD_SOURCESDIRECTORY:/onnxruntime_src \ --volume $BUILD_BINARIESDIRECTORY:/build --volume /data/models:/build/models:ro \ + -e NPM_CONFIG_USERCONFIG=/tmp/.npmrc \ + --volume "${NPM_CONFIG_USERCONFIG}:/tmp/.npmrc:ro" \ + --volume $HOME/.m2:/home/onnxruntimedev/.m2:ro \ + --volume $HOME/.gradle:/home/onnxruntimedev/.gradle \ --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda${CUDA_VERSION_MAJOR}xtrt86build \ /bin/bash -c "/usr/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_tests \ --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --build_java --build_nodejs \ diff --git a/tools/ci_build/github/linux/run_python_dockerbuild.sh b/tools/ci_build/github/linux/run_python_dockerbuild.sh index 915f8727f12cb..cc79c30969102 100755 --- a/tools/ci_build/github/linux/run_python_dockerbuild.sh +++ b/tools/ci_build/github/linux/run_python_dockerbuild.sh @@ -27,12 +27,17 @@ if [ "${BUILD_EXTR_PAR}" != "" ] ; then DOCKER_SCRIPT_OPTIONS+=("-x" "${BUILD_EXTR_PAR}") fi +# HACK: `ADDITIONAL_DOCKER_PARAMETER` is passed in via env in some pipelines docker run -e SYSTEM_COLLECTIONURI --rm \ --volume /data/onnx:/data/onnx:ro \ --volume "${BUILD_SOURCESDIRECTORY}:/onnxruntime_src" \ --volume "${BUILD_BINARIESDIRECTORY}:/build" \ --volume /data/models:/build/models:ro \ --volume "${HOME}/.onnx:/home/onnxruntimedev/.onnx" \ + -e NPM_CONFIG_USERCONFIG=/tmp/.npmrc \ + --volume "${NPM_CONFIG_USERCONFIG}:/tmp/.npmrc:ro" \ + --volume "$HOME/.m2:/home/onnxruntimedev/.m2:ro" \ + --volume "$HOME/.gradle:/home/onnxruntimedev/.gradle" \ -w /onnxruntime_src \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ diff --git a/tools/ci_build/github/linux/run_python_dockertest.sh b/tools/ci_build/github/linux/run_python_dockertest.sh index 1523824f8d05e..993ea3b20d18f 100755 --- a/tools/ci_build/github/linux/run_python_dockertest.sh +++ b/tools/ci_build/github/linux/run_python_dockertest.sh @@ -1,29 +1,37 @@ #!/bin/bash -set -e -x +set -xeuo pipefail BUILD_CONFIG="Release" -while getopts "i:d:x:c:" parameter_Option -do case "${parameter_Option}" -in -i) DOCKER_IMAGE=${OPTARG};; -d) DEVICE=${OPTARG};; -c) BUILD_CONFIG=${OPTARG};; -esac +while getopts "i:d:c:" parameter_Option; do + case "${parameter_Option}" in + i) DOCKER_IMAGE=${OPTARG} ;; + d) DEVICE=${OPTARG} ;; + c) BUILD_CONFIG=${OPTARG} ;; + *) + echo "Usage: $0 -i -d [-c ]" + exit 1 + ;; + esac done -if [ $DEVICE = "GPU" ]; then - ADDITIONAL_DOCKER_PARAMETER="--gpus all" +ADDITIONAL_DOCKER_PARAMETERS=() +if [ "$DEVICE" = "GPU" ]; then + ADDITIONAL_DOCKER_PARAMETERS+=("--gpus" "all") fi -mkdir -p $HOME/.onnx +mkdir -p "$HOME/.onnx" docker run -e SYSTEM_COLLECTIONURI --rm \ - --volume /data/onnx:/data/onnx:ro \ - --volume $BUILD_SOURCESDIRECTORY:/onnxruntime_src \ - --volume $BUILD_BINARIESDIRECTORY:/build \ - --volume /data/models:/build/models:ro \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ - -w /onnxruntime_src \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - $ADDITIONAL_DOCKER_PARAMETER \ - $DOCKER_IMAGE tools/ci_build/github/linux/run_python_tests.sh -d $DEVICE -c $BUILD_CONFIG + --volume /data/onnx:/data/onnx:ro \ + --volume "$BUILD_SOURCESDIRECTORY:/onnxruntime_src" \ + --volume "$BUILD_BINARIESDIRECTORY:/build" \ + --volume /data/models:/build/models:ro \ + --volume "$HOME/.onnx:/home/onnxruntimedev/.onnx" \ + -e NPM_CONFIG_USERCONFIG=/tmp/.npmrc \ + --volume "${NPM_CONFIG_USERCONFIG}:/tmp/.npmrc:ro" \ + --volume "$HOME/.m2:/home/onnxruntimedev/.m2:ro" \ + --volume "$HOME/.gradle:/home/onnxruntimedev/.gradle" \ + -w /onnxruntime_src \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + "${ADDITIONAL_DOCKER_PARAMETERS[@]}" \ + "$DOCKER_IMAGE" tools/ci_build/github/linux/run_python_tests.sh -d "$DEVICE" -c "$BUILD_CONFIG"