Skip to content

Commit b0896bb

Browse files
xiaoyu-workxiaoyu-work
authored
Fix ort nightly pipeline (#1554)
## Describe your changes Fix ort nightly pipeline - Upgrade to cuda 12.6 + TensorRT 10.5. - Fix python 3.8 example test. - Fix & skip some tests. - Fix yaml format. ## Checklist before requesting a review - [ ] Add unit tests for this change. - [ ] Make sure all tests can pass. - [ ] Update documents if necessary. - [ ] Lint and apply fixes to your code by running `lintrunner -a` - [ ] Is this a user-facing change? If yes, give a description of this change to be included in the release notes. - [ ] Is this PR including examples changes? If yes, please remember to update [example documentation](https://github.com/microsoft/Olive/blob/main/docs/source/examples.md) in a follow-up PR. ## (Optional) Issue link
1 parent 3d6e731 commit b0896bb

17 files changed

+206
-158
lines changed

‎.azure_pipelines/dockerfiles/linux-cpu.dockerfile

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22
# Copyright (c) Microsoft Corporation. All rights reserved.
33
# Licensed under the MIT License.
44
# --------------------------------------------------------------------------
5-
FROM ubuntu:22.04
5+
ARG BASE_IMAGE
6+
FROM ${BASE_IMAGE}
67

78
ARG PYTHON_VERSION
89

‎.azure_pipelines/dockerfiles/linux-gpu.dockerfile

+3-2
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@
22
# Copyright (c) Microsoft Corporation. All rights reserved.
33
# Licensed under the MIT License.
44
# --------------------------------------------------------------------------
5-
FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04
5+
ARG BASE_IMAGE
6+
FROM ${BASE_IMAGE}
67

78
ARG PYTHON_VERSION
8-
ARG TENSORRT_VERSION=10.0.1.6-1+cuda12.4
9+
ARG TENSORRT_VERSION
910

1011
RUN apt-get update && \
1112
apt-get install -y \

‎.azure_pipelines/job_templates/build-docker-image-template.yaml

+8-1
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,18 @@ parameters:
44
dockerfile: ''
55
python_version: ''
66
docker_image: ''
7+
base_image: ''
8+
trt_version: ''
79

810
steps:
911
- script: |
1012
docker login -u $(docker-username) -p $(docker-password)
11-
docker build --build-arg PYTHON_VERSION=${{ parameters.python_version }} -t ${{ parameters.docker_image }} -f $(Build.SourcesDirectory)/${{ parameters.dockerfile }} .
13+
docker build \
14+
--build-arg BASE_IMAGE=${{ parameters.base_image }} \
15+
--build-arg TENSORRT_VERSION=${{ parameters.trt_version }} \
16+
--build-arg PYTHON_VERSION=${{ parameters.python_version }} \
17+
-t ${{ parameters.docker_image }} \
18+
-f $(Build.SourcesDirectory)/${{ parameters.dockerfile }} .
1219
displayName: Build Docker Image
1320

1421
- script: |

‎.azure_pipelines/job_templates/olive-build-doc-template.yaml

+35-35
Original file line numberDiff line numberDiff line change
@@ -8,42 +8,42 @@ parameters:
88
doc_version: 'latest'
99

1010
jobs:
11-
- job: ${{parameters.job_name}}
12-
displayName: ${{parameters.display_name}}
13-
pool: ${{ parameters.pool}}
14-
variables:
15-
PIP_CACHE_DIR: $(Pipeline.Workspace)/.cache/pip
11+
- job: ${{parameters.job_name}}
12+
displayName: ${{parameters.display_name}}
13+
pool: ${{ parameters.pool}}
14+
variables:
15+
PIP_CACHE_DIR: $(Pipeline.Workspace)/.cache/pip
1616

17-
steps:
18-
- template: olive-setup-template.yaml
19-
parameters:
20-
python_version: '3.9'
21-
onnxruntime: 'onnxruntime'
22-
torch: 'torch'
17+
steps:
18+
- template: olive-setup-template.yaml
19+
parameters:
20+
python_version: '3.9'
21+
onnxruntime: 'onnxruntime'
22+
torch: 'torch'
2323

24-
# checkout release branch if doc_version provided
25-
- script: |
26-
git config --global user.email "[email protected]"
27-
git config --global user.name "olivedevteam"
28-
git fetch origin rel-${{parameters.doc_version}}
29-
git checkout rel-${{parameters.doc_version}}
30-
displayName: Checkout release branch
31-
condition: ne('${{parameters.doc_version}}', 'latest')
24+
# checkout release branch if doc_version provided
25+
- script: |
26+
git config --global user.email "[email protected]"
27+
git config --global user.name "olivedevteam"
28+
git fetch origin rel-${{parameters.doc_version}}
29+
git checkout rel-${{parameters.doc_version}}
30+
displayName: Checkout release branch
31+
condition: ne('${{parameters.doc_version}}', 'latest')
3232
33-
- script: |
34-
# set -e, otherwise make html fails but the build continues
35-
set -e
36-
cd docs
37-
python -m pip install -r requirements.txt
38-
make html
39-
make linkcheck
40-
make schema
41-
displayName: Make Docs
33+
- script: |
34+
# set -e, otherwise make html fails but the build continues
35+
set -e
36+
cd docs
37+
python -m pip install -r requirements.txt
38+
make html
39+
make linkcheck
40+
make schema
41+
displayName: Make Docs
4242
43-
- task: PublishPipelineArtifact@1
44-
inputs:
45-
path: $(Build.SourcesDirectory)/docs/build/html
46-
artifactName: olive_doc_src
47-
artifactType: pipeline
48-
displayName: Publish Docs to Pipeline Artifact
49-
condition: and(succeeded(), eq('${{ parameters.publish_docs }}', 'true'))
43+
- task: PublishPipelineArtifact@1
44+
inputs:
45+
path: $(Build.SourcesDirectory)/docs/build/html
46+
artifactName: olive_doc_src
47+
artifactType: pipeline
48+
displayName: Publish Docs to Pipeline Artifact
49+
condition: and(succeeded(), eq('${{ parameters.publish_docs }}', 'true'))

‎.azure_pipelines/job_templates/olive-example-linux-template.yaml

+66-57
Original file line numberDiff line numberDiff line change
@@ -7,72 +7,81 @@ parameters:
77
device: 'cpu'
88
dockerfile: '.azure_pipelines/dockerfiles/linux-cpu.dockerfile'
99
docker_image: 'olive-pipeline:latest'
10+
base_image: 'ubuntu:22.04'
11+
trt_version: ''
1012
onnxruntime: 'onnxruntime'
1113
subfolder: 'local'
1214
torch: 'torch'
1315
test_script: 'run_test.sh'
1416
onnxruntime_nightly: false
1517

1618
jobs:
17-
- job: ${{ parameters.name }}_Test_Examples
18-
timeoutInMinutes: 300
19-
pool:
20-
name: ${{ parameters.pool }}
21-
strategy:
22-
matrix:
23-
${{ insert }}: ${{ parameters.examples }}
24-
variables:
25-
PIP_CACHE_DIR: $(Pipeline.Workspace)/.cache/pip
26-
HF_HOME: $(Pipeline.Workspace)/.cache/huggingface
27-
OLIVE_TEMPDIR: $(Pipeline.Workspace)/.olive_tempdir
19+
- job: ${{ parameters.name }}_Test_Examples
20+
timeoutInMinutes: 300
21+
pool:
22+
name: ${{ parameters.pool }}
23+
strategy:
24+
matrix:
25+
${{ insert }}: ${{ parameters.examples }}
26+
variables:
27+
PIP_CACHE_DIR: $(Pipeline.Workspace)/.cache/pip
28+
HF_HOME: $(Pipeline.Workspace)/.cache/huggingface
29+
OLIVE_TEMPDIR: $(Pipeline.Workspace)/.olive_tempdir
2830

29-
steps:
30-
- template: build-docker-image-template.yaml
31-
parameters:
32-
python_version: ${{ parameters.python_version }}
33-
dockerfile: ${{ parameters.dockerfile }}
34-
docker_image: ${{ parameters.docker_image }}
31+
steps:
32+
- script: docker system df && docker system prune -a -f && docker system df
33+
displayName: Clean docker images
34+
continueOnError: true
3535

36-
# set exampleRequirements to requirements.txt if user does not specify
37-
- script:
38-
echo "##vso[task.setvariable variable=exampleRequirements]requirements.txt"
39-
displayName: Set exampleRequirements
40-
condition: eq(variables['exampleRequirements'], '')
36+
- template: build-docker-image-template.yaml
37+
parameters:
38+
python_version: ${{ parameters.python_version }}
39+
dockerfile: ${{ parameters.dockerfile }}
40+
docker_image: ${{ parameters.docker_image }}
41+
base_image: ${{ parameters.base_image }}
42+
trt_version: ${{ parameters.trt_version }}
4143

42-
- script: |
43-
GPU_OPTION=""
44-
if [ "${{ parameters.device }}" = "gpu" ]; then
45-
GPU_OPTION="--gpus=all"
46-
fi
47-
docker run \
48-
$GPU_OPTION \
49-
-v $(Build.SourcesDirectory)/logs:/logs \
50-
-e WORKSPACE_SUBSCRIPTION_ID=$(workspace-subscription-id) \
51-
-e WORKSPACE_RESOURCE_GROUP=$(workspace-resource-group) \
52-
-e WORKSPACE_NAME=$(workspace-name) \
53-
-e MANAGED_IDENTITY_CLIENT_ID=$(olive-1es-identity-client-id) \
54-
-e PIPELINE_TEST_ACCOUNT_NAME=$(pipeline-test-account-name) \
55-
-e PIPELINE_TEST_CONTAINER_NAME=$(pipeline-test-container-name) \
56-
-e KEYVAULT_NAME=$(keyvault-name) \
57-
-e HF_TOKEN=$(hf_token) \
58-
${{ parameters.docker_image }} \
59-
bash .azure_pipelines/scripts/${{ parameters.test_script }} \
60-
${{ parameters.torch }} \
61-
${{ parameters.onnxruntime }} \
62-
${{ parameters.onnxruntime_nightly }} \
63-
examples/$(exampleFolder)/$(exampleRequirements) \
64-
examples/test/${{ parameters.subfolder }}/test_$(exampleName).py
65-
displayName: Run Tests in Docker
44+
# set exampleRequirements to requirements.txt if user does not specify
45+
- script:
46+
echo "##vso[task.setvariable variable=exampleRequirements]requirements.txt"
47+
displayName: Set exampleRequirements
48+
condition: eq(variables['exampleRequirements'], '')
6649

67-
# Step 3: Publish test results
68-
- task: PublishTestResults@2
69-
condition: succeededOrFailed()
70-
inputs:
71-
testResultsFiles: '**/logs/test_examples-TestOlive.xml'
72-
testRunTitle: '$(Build.BuildNumber)[$(Agent.JobName)]'
73-
failTaskOnFailedTests: true
74-
displayName: Publish Test Results
50+
- script: |
51+
GPU_OPTION=""
52+
if [ "${{ parameters.device }}" = "gpu" ]; then
53+
GPU_OPTION="--gpus=all"
54+
fi
55+
docker run \
56+
--shm-size=4g \
57+
$GPU_OPTION \
58+
-v $(Build.SourcesDirectory)/logs:/logs \
59+
-e WORKSPACE_SUBSCRIPTION_ID=$(workspace-subscription-id) \
60+
-e WORKSPACE_RESOURCE_GROUP=$(workspace-resource-group) \
61+
-e WORKSPACE_NAME=$(workspace-name) \
62+
-e MANAGED_IDENTITY_CLIENT_ID=$(olive-1es-identity-client-id) \
63+
-e PIPELINE_TEST_ACCOUNT_NAME=$(pipeline-test-account-name) \
64+
-e PIPELINE_TEST_CONTAINER_NAME=$(pipeline-test-container-name) \
65+
-e KEYVAULT_NAME=$(keyvault-name) \
66+
-e HF_TOKEN=$(hf_token) \
67+
${{ parameters.docker_image }} \
68+
bash .azure_pipelines/scripts/${{ parameters.test_script }} \
69+
${{ parameters.torch }} \
70+
${{ parameters.onnxruntime }} \
71+
${{ parameters.onnxruntime_nightly }} \
72+
examples/$(exampleFolder)/$(exampleRequirements) \
73+
examples/test/${{ parameters.subfolder }}/test_$(exampleName).py
74+
displayName: Run Tests in Docker
7575
76-
- script: sudo git clean -dfX
77-
condition: always()
78-
displayName: Clean remaining artifacts
76+
# Step 3: Publish test results
77+
- task: PublishTestResults@2
78+
condition: succeededOrFailed()
79+
inputs:
80+
testResultsFiles: '**/logs/test_examples-TestOlive.xml'
81+
testRunTitle: '$(Build.BuildNumber)[$(Agent.JobName)]'
82+
failTaskOnFailedTests: true
83+
displayName: Publish Test Results
84+
85+
- script: sudo git clean -dfX
86+
condition: always()
87+
displayName: Clean remaining artifacts

‎.azure_pipelines/job_templates/olive-example-win-template.yaml

+50-48
Original file line numberDiff line numberDiff line change
@@ -7,60 +7,62 @@ parameters:
77
device: 'cpu'
88
python_version: '3.10'
99
onnxruntime: 'onnxruntime'
10+
onnxruntime_nightly: false
1011
subfolder: 'local'
1112
torch: 'torch'
1213

1314
jobs:
14-
- job: ${{ parameters.name }}_Test_Examples
15-
timeoutInMinutes: 300
16-
pool:
17-
name: ${{ parameters.pool }}
18-
strategy:
19-
matrix:
20-
${{ insert }}: ${{ parameters.examples }}
21-
variables:
22-
PIP_CACHE_DIR: $(Pipeline.Workspace)/.cache/pip
23-
HF_HOME: $(Pipeline.Workspace)/.cache/huggingface
24-
OLIVE_TEMPDIR: $(Pipeline.Workspace)/.olive_tempdir
25-
PYTEST_BASETEMP: $(Pipeline.Workspace)/.pytest_basetemp
15+
- job: ${{ parameters.name }}_Test_Examples
16+
timeoutInMinutes: 300
17+
pool:
18+
name: ${{ parameters.pool }}
19+
strategy:
20+
matrix:
21+
${{ insert }}: ${{ parameters.examples }}
22+
variables:
23+
PIP_CACHE_DIR: $(Pipeline.Workspace)/.cache/pip
24+
HF_HOME: $(Pipeline.Workspace)/.cache/huggingface
25+
OLIVE_TEMPDIR: $(Pipeline.Workspace)/.olive_tempdir
26+
PYTEST_BASETEMP: $(Pipeline.Workspace)/.pytest_basetemp
2627

27-
steps:
28-
- template: olive-setup-template.yaml
29-
parameters:
30-
python_version: ${{ parameters.python_version }}
31-
onnxruntime: ${{ parameters.onnxruntime }}
32-
torch: ${{ parameters.torch }}
28+
steps:
29+
- template: olive-setup-template.yaml
30+
parameters:
31+
python_version: ${{ parameters.python_version }}
32+
onnxruntime: ${{ parameters.onnxruntime }}
33+
onnxruntime_nightly: ${{ parameters.onnxruntime_nightly }}
34+
torch: ${{ parameters.torch }}
3335

34-
# set exampleRequirements to requirements.txt if user does not specify
35-
- script:
36-
echo "##vso[task.setvariable variable=exampleRequirements]requirements.txt"
37-
displayName: Set exampleRequirements
38-
condition: eq(variables['exampleRequirements'], '')
36+
# set exampleRequirements to requirements.txt if user does not specify
37+
- script:
38+
echo "##vso[task.setvariable variable=exampleRequirements]requirements.txt"
39+
displayName: Set exampleRequirements
40+
condition: eq(variables['exampleRequirements'], '')
3941

40-
- script: |
41-
python -m pip install pytest
42-
python -m pip install azure-identity azure-storage-blob tabulate
43-
python -m pip install -r $(Build.SourcesDirectory)/examples/$(exampleFolder)/$(exampleRequirements)
44-
python -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test_examples-TestOlive.xml $(Build.SourcesDirectory)/examples/test/${{ parameters.subfolder }}/test_$(exampleName).py --basetemp $(PYTEST_BASETEMP)
45-
displayName: Test Examples
46-
env:
47-
WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id)
48-
WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group)
49-
WORKSPACE_NAME: $(workspace-name)
50-
MANAGED_IDENTITY_CLIENT_ID: $(olive-1es-identity-client-id)
51-
PIPELINE_TEST_ACCOUNT_NAME: $(pipeline-test-account-name)
52-
PIPELINE_TEST_CONTAINER_NAME: $(pipeline-test-container-name)
53-
KEYVAULT_NAME: $(keyvault-name)
54-
HF_TOKEN: $(hf_token)
42+
- script: |
43+
python -m pip install pytest
44+
python -m pip install azure-identity azure-storage-blob tabulate
45+
python -m pip install -r $(Build.SourcesDirectory)/examples/$(exampleFolder)/$(exampleRequirements)
46+
python -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test_examples-TestOlive.xml $(Build.SourcesDirectory)/examples/test/${{ parameters.subfolder }}/test_$(exampleName).py --basetemp $(PYTEST_BASETEMP)
47+
displayName: Test Examples
48+
env:
49+
WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id)
50+
WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group)
51+
WORKSPACE_NAME: $(workspace-name)
52+
MANAGED_IDENTITY_CLIENT_ID: $(olive-1es-identity-client-id)
53+
PIPELINE_TEST_ACCOUNT_NAME: $(pipeline-test-account-name)
54+
PIPELINE_TEST_CONTAINER_NAME: $(pipeline-test-container-name)
55+
KEYVAULT_NAME: $(keyvault-name)
56+
HF_TOKEN: $(hf_token)
5557
56-
- task: PublishTestResults@2
57-
condition: succeededOrFailed()
58-
inputs:
59-
testResultsFiles: '**/*TestOlive*.xml'
60-
testRunTitle: '$(Build.BuildNumber)[$(Agent.JobName)]'
61-
failTaskOnFailedTests: true
62-
displayName: Upload pipeline run test results
58+
- task: PublishTestResults@2
59+
condition: succeededOrFailed()
60+
inputs:
61+
testResultsFiles: '**/*TestOlive*.xml'
62+
testRunTitle: '$(Build.BuildNumber)[$(Agent.JobName)]'
63+
failTaskOnFailedTests: true
64+
displayName: Upload pipeline run test results
6365

64-
- script: git clean -dfX
65-
condition: always()
66-
displayName: Clean remaining artifacts
66+
- script: git clean -dfX
67+
condition: always()
68+
displayName: Clean remaining artifacts

‎.azure_pipelines/job_templates/olive-setup-template.yaml

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
parameters:
22
python_version: '3.10'
33
onnxruntime: 'onnxruntime'
4+
onnxruntime_nightly: false
45
torch: torch
56

67
steps:
@@ -15,11 +16,10 @@ steps:
1516
- script: python -m pip install .
1617
displayName: Install Olive
1718

18-
- ${{ if startsWith(parameters.onnxruntime, 'ort-nightly') }}:
19+
- ${{ if eq(parameters.onnxruntime_nightly, true) }}:
1920
- script: |
20-
pip install onnxruntime
21-
pip uninstall -y onnxruntime
22-
pip install ${{ parameters.onnxruntime }} --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/
21+
pip install -r https://raw.githubusercontent.com/microsoft/onnxruntime/refs/heads/main/requirements.txt
22+
pip install ${{ parameters.onnxruntime }} --pre --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/ --disable-pip-version-check
2323
displayName: Install ${{ parameters.onnxruntime }}
2424
- ${{ else }}:
2525
- script: |

0 commit comments

Comments
 (0)