diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-cuda-minimal-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-cuda-minimal-ci-pipeline.yml index a062a1fce5c1b..6cee227b70b2b 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-cuda-minimal-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-cuda-minimal-ci-pipeline.yml @@ -93,6 +93,10 @@ jobs: --volume $(Build.BinariesDirectory):/build \ --volume /data/models:/build/models:ro \ --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ + -e NPM_CONFIG_USERCONFIG=/tmp/.npmrc \ + --volume "${NPM_CONFIG_USERCONFIG}:/tmp/.npmrc:ro" \ + --volume $HOME/.m2:/home/onnxruntimedev/.m2:ro \ + --volume $HOME/.gradle:/home/onnxruntimedev/.gradle \ -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ diff --git a/tools/ci_build/github/azure-pipelines/stages/nodejs-linux-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nodejs-linux-packaging-stage.yml index ff35d3e35ef6c..1a326e9dbd584 100644 --- a/tools/ci_build/github/azure-pipelines/stages/nodejs-linux-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/nodejs-linux-packaging-stage.yml @@ -33,6 +33,9 @@ stages: - checkout: self clean: true submodules: recursive + + - template: ../templates/setup-feeds-and-python-steps.yml + - template: ../templates/get-docker-image-steps.yml parameters: Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cuda${{ variables.CUDA_VERSION_MAJOR }}/Dockerfile diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml index f22527bb2bd51..c28ad09a7c02d 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml @@ -74,6 +74,10 @@ jobs: set -e -x mkdir -p $HOME/.onnx docker run -e SYSTEM_COLLECTIONURI --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ + -e NPM_CONFIG_USERCONFIG=/tmp/.npmrc \ + --volume "${NPM_CONFIG_USERCONFIG}:/tmp/.npmrc:ro" \ + --volume $HOME/.m2:/home/onnxruntimedev/.m2:ro \ + --volume $HOME/.gradle:/home/onnxruntimedev/.gradle \ --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging /bin/bash -c "python3 \ /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release \ --skip_submodule_sync --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --use_binskim_compliant_compile_flags --use_vcpkg --use_vcpkg_ms_internal_asset_cache --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/installed" diff --git a/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml b/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml index c074270ce5949..e02f0a3a854df 100644 --- a/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/setup-feeds-and-python-steps.yml @@ -1,8 +1,9 @@ # Template for setting up the package feeds and installing python. # Feeds configured: -# - pip # - maven & gradle +# - npm # - nuget +# - pip # # n.b. This template writes to user profile files. @@ -26,6 +27,32 @@ steps: - pwsh: if (Test-Path ~/.m2/settings.xml) { Remove-Item ~/.m2/settings.xml } displayName: Maven - remove existing ~/.m2/settings.xml +# use `PowerShell@2` instead of `pwsh` as it is available on arm64 agents, whereas `pwsh` is not. (2026-04-15) +- task: PowerShell@2 + displayName: 'NPM - set NPM_CONFIG_USERCONFIG to agent temp config' + env: + artifact_feed: ${{ parameters.artifactFeeds }} + inputs: + targetType: 'inline' + script: | + # ensure we have a clean/empty npm config. config file must exist prior to `npmAuthenticate`. + $npmrcPath = "${env:AGENT_TEMPDIRECTORY}/.npmrc" + + Set-Content -Force -Path $npmrcPath -Value @" + registry=https://aiinfra.pkgs.visualstudio.com/_packaging/${env:artifact_feed}/npm/registry/ + always-auth=true + "@ + + Write-Host "Generated npmrc contents:" + Get-Content $npmrcPath | Write-Host + + echo "##vso[task.setvariable variable=NPM_CONFIG_USERCONFIG]${npmrcPath}" + +# FUTURE WORK: use `Npm@1` task in pipelines and specify the feed there? +- task: npmAuthenticate@0 + inputs: + workingFile: $(NPM_CONFIG_USERCONFIG) + - task: NuGetAuthenticate@1 # just auth w/ internal feeds # Creates/touches ~/.m2/settings.xml diff --git a/tools/ci_build/github/linux/build_cuda_c_api_package.sh b/tools/ci_build/github/linux/build_cuda_c_api_package.sh index a1a2d347414c7..12a03e686d5b1 100755 --- a/tools/ci_build/github/linux/build_cuda_c_api_package.sh +++ b/tools/ci_build/github/linux/build_cuda_c_api_package.sh @@ -12,6 +12,10 @@ fi docker run -e SYSTEM_COLLECTIONURI --rm --volume \ $BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build \ +-e NPM_CONFIG_USERCONFIG=/tmp/.npmrc \ +--volume "${NPM_CONFIG_USERCONFIG}:/tmp/.npmrc:ro" \ +--volume $HOME/.m2:/home/onnxruntimedev/.m2:ro \ +--volume $HOME/.gradle:/home/onnxruntimedev/.gradle \ -e NIGHTLY_BUILD onnxruntimecuda${CUDA_VERSION_MAJOR}build \ /bin/bash -c "/usr/bin/python3 /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs \ --build_dir /build --config Release --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib \ diff --git a/tools/ci_build/github/linux/build_nodejs_package.sh b/tools/ci_build/github/linux/build_nodejs_package.sh index fac12a4342f60..3d7a7d855ef10 100755 --- a/tools/ci_build/github/linux/build_nodejs_package.sh +++ b/tools/ci_build/github/linux/build_nodejs_package.sh @@ -13,6 +13,10 @@ fi mkdir -p $HOME/.onnx docker run -e SYSTEM_COLLECTIONURI --rm --volume /data/onnx:/data/onnx:ro --volume $BUILD_SOURCESDIRECTORY:/onnxruntime_src \ --volume $BUILD_BINARIESDIRECTORY:/build --volume /data/models:/build/models:ro \ +-e NPM_CONFIG_USERCONFIG=/tmp/.npmrc \ +--volume "${NPM_CONFIG_USERCONFIG}:/tmp/.npmrc:ro" \ +--volume $HOME/.m2:/home/onnxruntimedev/.m2:ro \ +--volume $HOME/.gradle:/home/onnxruntimedev/.gradle \ --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda${CUDA_VERSION_MAJOR}xtrt86build \ /bin/bash -c "/usr/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \ --skip_tests --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --build_nodejs \ diff --git a/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh b/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh index 8c2866bd37833..b6358aa775a24 100755 --- a/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh +++ b/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh @@ -13,6 +13,10 @@ fi mkdir -p $HOME/.onnx docker run -e SYSTEM_COLLECTIONURI --rm --volume /data/onnx:/data/onnx:ro --volume $BUILD_SOURCESDIRECTORY:/onnxruntime_src \ --volume $BUILD_BINARIESDIRECTORY:/build --volume /data/models:/build/models:ro \ + -e NPM_CONFIG_USERCONFIG=/tmp/.npmrc \ + --volume "${NPM_CONFIG_USERCONFIG}:/tmp/.npmrc:ro" \ + --volume $HOME/.m2:/home/onnxruntimedev/.m2:ro \ + --volume $HOME/.gradle:/home/onnxruntimedev/.gradle \ --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda${CUDA_VERSION_MAJOR}xtrt86build \ /bin/bash -c "/usr/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_tests \ --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --build_java --build_nodejs \ diff --git a/tools/ci_build/github/linux/build_webgpu_plugin_package.sh b/tools/ci_build/github/linux/build_webgpu_plugin_package.sh index a3583bf9d17a5..142f6b86e7443 100755 --- a/tools/ci_build/github/linux/build_webgpu_plugin_package.sh +++ b/tools/ci_build/github/linux/build_webgpu_plugin_package.sh @@ -25,6 +25,10 @@ docker run --rm \ --volume "${BUILD_BINARIESDIRECTORY}:/build" \ --volume /data/models:/build/models:ro \ --volume "${HOME}/.onnx:/home/onnxruntimedev/.onnx" \ + -e NPM_CONFIG_USERCONFIG=/tmp/.npmrc \ + --volume "${NPM_CONFIG_USERCONFIG}:/tmp/.npmrc:ro" \ + --volume $HOME/.m2:/home/onnxruntimedev/.m2:ro \ + --volume $HOME/.gradle:/home/onnxruntimedev/.gradle \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ -e SYSTEM_COLLECTIONURI \ diff --git a/tools/ci_build/github/linux/run_python_dockerbuild.sh b/tools/ci_build/github/linux/run_python_dockerbuild.sh index 915f8727f12cb..cc79c30969102 100755 --- a/tools/ci_build/github/linux/run_python_dockerbuild.sh +++ b/tools/ci_build/github/linux/run_python_dockerbuild.sh @@ -27,12 +27,17 @@ if [ "${BUILD_EXTR_PAR}" != "" ] ; then DOCKER_SCRIPT_OPTIONS+=("-x" "${BUILD_EXTR_PAR}") fi +# HACK: `ADDITIONAL_DOCKER_PARAMETER` is passed in via env in some pipelines docker run -e SYSTEM_COLLECTIONURI --rm \ --volume /data/onnx:/data/onnx:ro \ --volume "${BUILD_SOURCESDIRECTORY}:/onnxruntime_src" \ --volume "${BUILD_BINARIESDIRECTORY}:/build" \ --volume /data/models:/build/models:ro \ --volume "${HOME}/.onnx:/home/onnxruntimedev/.onnx" \ + -e NPM_CONFIG_USERCONFIG=/tmp/.npmrc \ + --volume "${NPM_CONFIG_USERCONFIG}:/tmp/.npmrc:ro" \ + --volume "$HOME/.m2:/home/onnxruntimedev/.m2:ro" \ + --volume "$HOME/.gradle:/home/onnxruntimedev/.gradle" \ -w /onnxruntime_src \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ diff --git a/tools/ci_build/github/linux/run_python_dockertest.sh b/tools/ci_build/github/linux/run_python_dockertest.sh index 1523824f8d05e..993ea3b20d18f 100755 --- a/tools/ci_build/github/linux/run_python_dockertest.sh +++ b/tools/ci_build/github/linux/run_python_dockertest.sh @@ -1,29 +1,37 @@ #!/bin/bash -set -e -x +set -xeuo pipefail BUILD_CONFIG="Release" -while getopts "i:d:x:c:" parameter_Option -do case "${parameter_Option}" -in -i) DOCKER_IMAGE=${OPTARG};; -d) DEVICE=${OPTARG};; -c) BUILD_CONFIG=${OPTARG};; -esac +while getopts "i:d:c:" parameter_Option; do + case "${parameter_Option}" in + i) DOCKER_IMAGE=${OPTARG} ;; + d) DEVICE=${OPTARG} ;; + c) BUILD_CONFIG=${OPTARG} ;; + *) + echo "Usage: $0 -i -d [-c ]" + exit 1 + ;; + esac done -if [ $DEVICE = "GPU" ]; then - ADDITIONAL_DOCKER_PARAMETER="--gpus all" +ADDITIONAL_DOCKER_PARAMETERS=() +if [ "$DEVICE" = "GPU" ]; then + ADDITIONAL_DOCKER_PARAMETERS+=("--gpus" "all") fi -mkdir -p $HOME/.onnx +mkdir -p "$HOME/.onnx" docker run -e SYSTEM_COLLECTIONURI --rm \ - --volume /data/onnx:/data/onnx:ro \ - --volume $BUILD_SOURCESDIRECTORY:/onnxruntime_src \ - --volume $BUILD_BINARIESDIRECTORY:/build \ - --volume /data/models:/build/models:ro \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ - -w /onnxruntime_src \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - $ADDITIONAL_DOCKER_PARAMETER \ - $DOCKER_IMAGE tools/ci_build/github/linux/run_python_tests.sh -d $DEVICE -c $BUILD_CONFIG + --volume /data/onnx:/data/onnx:ro \ + --volume "$BUILD_SOURCESDIRECTORY:/onnxruntime_src" \ + --volume "$BUILD_BINARIESDIRECTORY:/build" \ + --volume /data/models:/build/models:ro \ + --volume "$HOME/.onnx:/home/onnxruntimedev/.onnx" \ + -e NPM_CONFIG_USERCONFIG=/tmp/.npmrc \ + --volume "${NPM_CONFIG_USERCONFIG}:/tmp/.npmrc:ro" \ + --volume "$HOME/.m2:/home/onnxruntimedev/.m2:ro" \ + --volume "$HOME/.gradle:/home/onnxruntimedev/.gradle" \ + -w /onnxruntime_src \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + "${ADDITIONAL_DOCKER_PARAMETERS[@]}" \ + "$DOCKER_IMAGE" tools/ci_build/github/linux/run_python_tests.sh -d "$DEVICE" -c "$BUILD_CONFIG"