Repository navigation
feat(ci): wire the amd-repo-setup package into native package CI #7280
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
b503af0
58a3688
f03e791
2d60e4e
de7e7b4
e112b77
3d18eee
02cc544
88c8f04
6ec918a
314da96
4b5e805
c5b094c
b838e56
ba940f0
dce4f4f
459dcbb
a0bfec8
f693784
80a0040
79ad88f
f11c8d9
c7500eb
a4189da
834c3c6
ae70235
65d5fe9
2d4ca70
5700c95
bca691b
49fc1a4
5a6c168
19bd8b3
1b0af2e
3892158
c97cfd8
97aff39
e4aea6a
6dfa00e
2aed73e
6cab81e
07d9b27
56fb52e
acd3cf6
d7ccb2a
119adea
605c41a
f709e51
dc016b4
e4e5433
98e65cc
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -61,6 +61,11 @@ on: | |
| built_packages.txt exist under the packages directory. | ||
| type: boolean | ||
| default: true | ||
| enable_repo_setup_package: | ||
| description: "Build, verify and upload the amd-repo-setup package" | ||
| type: boolean | ||
| required: false | ||
| default: false | ||
| outputs: | ||
| package_repository_url: | ||
| description: "Public repository URL for package installation" | ||
|
|
@@ -72,6 +77,75 @@ permissions: | |
| run-name: Build ${{ inputs.native_package_type }} packages (${{ inputs.rocm_version }}, ${{ inputs.dist_amdgpu_families }}${{ inputs.release_type && format(', {0}', inputs.release_type) || '' }}) | ||
|
|
||
| jobs: | ||
| # Resolve the parameters used to build and upload the amd-repo-setup package: | ||
| # the OS-profile build matrix for this package type, each profile's container | ||
| # image, and the repo.amd.com stream the build line configures, with its | ||
| # tier, packages base URL, signing-key URL and build sub-folder. | ||
| # | ||
| # The build line and the stream are different vocabularies: prerelease | ||
| # configures rc. release_type stays the workflow input because it also selects | ||
| # the artifact bucket; the stream is derived from it. | ||
| resolve_repo_setup_params: | ||
| name: Resolve amd-repo-setup parameters | ||
| # nightly and prerelease are the lines that upload the package. release | ||
| # configures a stream too, but has no artifacts bucket to upload into. | ||
| # Sanitizer variants skip: the resolved repository URL serves neither ASAN | ||
| # packages (promoted under packages-asan) nor TSAN ones (not promoted). | ||
| if: inputs.enable_repo_setup_package && contains(fromJSON('["nightly", "prerelease"]'), inputs.release_type) && !contains(inputs.build_variant, 'asan') && !contains(inputs.build_variant, 'tsan') | ||
| runs-on: ${{ github.repository_owner == 'ROCm' && inputs.build_runs_on || 'ubuntu-26.04' }} | ||
| container: | ||
| image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:15241d0e87bf81fe0986006ab832a36ea53a2cda0f9a31b4a5dc7826bc2396b6 | ||
| # Resolves parameters only; it needs no cloud credentials. | ||
| permissions: | ||
| contents: read | ||
| env: | ||
| NATIVE_PACKAGE_TYPE: ${{ inputs.native_package_type }} | ||
| RELEASE_TYPE: ${{ inputs.release_type }} | ||
| ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}" | ||
| outputs: | ||
| os_profiles: ${{ steps.derive.outputs.os_profiles }} | ||
| images: ${{ steps.derive.outputs.images }} | ||
| stream: ${{ steps.derive.outputs.stream }} | ||
| tier: ${{ steps.derive.outputs.tier }} | ||
| repo_base_url: ${{ steps.derive.outputs.repo_base_url }} | ||
| gpg_key_url: ${{ steps.derive.outputs.gpg_key_url }} | ||
| repo_sub_folder: ${{ steps.derive.outputs.repo_sub_folder }} | ||
| steps: | ||
| - name: "Quartz - started - build amd-repo-setup" | ||
| continue-on-error: true | ||
| if: ${{ vars.NOTIFY_QUARTZ_ENABLED == 'true' }} | ||
| uses: ROCm/Quartz/.github/actions/notify_quartz@f386a9756620938616af0b4d5d04b24ae6e0353f # notify_quartz/v1.1.1 | ||
| with: | ||
| run_phase: started | ||
| reporting_workflow: multi_arch_build_native_linux_packages.yml | ||
| workflow_inputs: ${{ toJSON(inputs) }} | ||
| gh_app_client_id: ${{ secrets.GH_APP_HAULY_CID }} | ||
| gh_app_private_key: ${{ secrets.GH_APP_HAULY_PRIVATE_KEY }} | ||
|
|
||
| - name: Checking out repository | ||
| timeout-minutes: 15 | ||
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | ||
| with: | ||
| persist-credentials: false | ||
| repository: ${{ inputs.repository || github.repository }} | ||
| ref: ${{ inputs.ref || '' }} | ||
|
|
||
| - name: Set up Python | ||
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | ||
| with: | ||
| python-version: '3.12' | ||
|
|
||
| - name: Install Python requirements | ||
| run: pip install jinja2 | ||
|
|
||
| - name: Derive amd-repo-setup parameters | ||
| id: derive | ||
| run: | | ||
| python ./build_tools/packaging/linux/get_url_repo_params.py get-repo-setup-params \ | ||
| --pkg-type "${NATIVE_PACKAGE_TYPE}" \ | ||
| --release-type "${RELEASE_TYPE}" \ | ||
| --run-id "${ARTIFACT_RUN_ID}" | ||
|
|
||
| build_native_packages: | ||
| permissions: | ||
| contents: read | ||
|
|
@@ -218,3 +292,133 @@ jobs: | |
| workflow_step_outputs: ${{ toJSON(steps) }} | ||
| gh_app_client_id: ${{ secrets.GH_APP_HAULY_CID }} | ||
| gh_app_private_key: ${{ secrets.GH_APP_HAULY_PRIVATE_KEY }} | ||
|
|
||
| # Build the amd-repo-setup package for each OS profile of this package type, | ||
| # check its contents and upload it to this run's artifacts bucket, under | ||
| # amd-repo-setup/<os_profile>/. | ||
| stage_repo_setup_package: | ||
| name: Stage amd-repo-setup (${{ matrix.os_profile }}) | ||
| needs: [resolve_repo_setup_params] | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we need native packages build step also in dependency?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Shouldn't be needed. The package only uses the setup job's parameters, nothing from the native build, and in the release workflow |
||
| if: inputs.enable_repo_setup_package && needs.resolve_repo_setup_params.outputs.repo_base_url != '' | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| # resolve_repo_setup_params does not run on the lines that do not upload, and a | ||
| # skipped job's outputs are empty strings, which fromJSON cannot parse. | ||
| # Fall back to an empty matrix so this job skips cleanly. | ||
| os_profile: ${{ fromJSON(needs.resolve_repo_setup_params.outputs.os_profiles || '[]') }} | ||
| runs-on: ubuntu-26.04 | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| permissions: | ||
| contents: read | ||
| id-token: write # Authenticate to AWS to upload the amd-repo-setup package. | ||
| container: | ||
| # get_url_repo_params.py manages image references for each OS profile. | ||
| # zizmor cannot inspect the dynamically selected image. | ||
| image: ${{ fromJSON(needs.resolve_repo_setup_params.outputs.images || '{}')[matrix.os_profile] }} # zizmor: ignore[unpinned-images] | ||
| env: | ||
| ROCM_VERSION: ${{ inputs.rocm_version }} | ||
| NATIVE_PACKAGE_TYPE: ${{ inputs.native_package_type }} | ||
| RELEASE_TYPE: ${{ inputs.release_type }} | ||
| ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}" | ||
| OS_PROFILE: ${{ matrix.os_profile }} | ||
| STREAM: ${{ needs.resolve_repo_setup_params.outputs.stream }} | ||
| TIER: ${{ needs.resolve_repo_setup_params.outputs.tier }} | ||
| REPO_BASE_URL: ${{ needs.resolve_repo_setup_params.outputs.repo_base_url }} | ||
| GPG_KEY_URL: ${{ needs.resolve_repo_setup_params.outputs.gpg_key_url }} | ||
| REPO_SUB_FOLDER: ${{ needs.resolve_repo_setup_params.outputs.repo_sub_folder }} | ||
| steps: | ||
| - name: Checkout | ||
| timeout-minutes: 15 | ||
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | ||
| with: | ||
| persist-credentials: false | ||
| repository: ${{ inputs.repository || github.repository }} | ||
| ref: ${{ inputs.ref || '' }} | ||
|
|
||
| - name: Install system prerequisites | ||
| run: | | ||
| bash build_tools/packaging/linux/setup_repo_build_deps.sh \ | ||
| --os-profile "${OS_PROFILE}" | ||
|
|
||
| - name: Setup Python | ||
| run: | | ||
| bash build_tools/packaging/linux/setup_python_cmd.sh \ | ||
| --os-profile "${OS_PROFILE}" \ | ||
| --install-runtime \ | ||
| --output-format github | ||
|
|
||
| - name: Install Python dependencies | ||
| run: | | ||
| $PYTHON_CMD -m venv /tmp/repo-venv | ||
| /tmp/repo-venv/bin/python -m pip install --upgrade pip | ||
| /tmp/repo-venv/bin/python -m pip install jinja2 boto3 | ||
| echo "PYTHON_CMD=/tmp/repo-venv/bin/python" >> "$GITHUB_ENV" | ||
| # configure_aws_artifacts_credentials runs bare `python`, which these | ||
| # per-distro images do not provide. A venv always does, so putting its | ||
| # bin/ on PATH lets the credentials action run here and the package | ||
| # upload from the job that built it. | ||
| echo "/tmp/repo-venv/bin" >> "$GITHUB_PATH" | ||
|
|
||
| - name: Build amd-repo-setup | ||
| run: | | ||
| # The build sub-folder is empty except on a per-build stream, which is | ||
| # the only kind the builder accepts one for. The key URL is empty on an | ||
| # unsigned stream, where the builder needs none. | ||
| $PYTHON_CMD build_tools/packaging/linux/build_repo_setup_package.py \ | ||
| --os-profile "${OS_PROFILE}" \ | ||
| --stream "${STREAM}" \ | ||
| --repo-base-url "${REPO_BASE_URL}" \ | ||
| --gpg-key-url "${GPG_KEY_URL}" \ | ||
| --repo-sub-folder "${REPO_SUB_FOLDER}" \ | ||
| --rocm-version "${ROCM_VERSION}" \ | ||
| --dest-dir repo-setup-package-out \ | ||
| --verify-repo-url | ||
|
|
||
| # Check the package before it is uploaded: the repository file for this | ||
| # stream (and, on a signed stream, the key) at the expected paths, nothing | ||
| # it must not ship, and everything owned by root. | ||
| - name: Inspect amd-repo-setup | ||
| run: | | ||
| $PYTHON_CMD build_tools/packaging/linux/inspect_repo_setup_package.py \ | ||
| --package-dir repo-setup-package-out \ | ||
| --os-profile "${OS_PROFILE}" \ | ||
| --stream "${STREAM}" | ||
|
|
||
| # Configured after the build so the key fetch, the repository check and | ||
| # the package build run without artifacts-bucket credentials. | ||
| - name: Configure AWS credentials for artifact uploads | ||
| uses: ./.github/actions/configure_aws_artifacts_credentials | ||
| with: | ||
| release_type: ${{ inputs.release_type }} | ||
|
|
||
| - name: Upload amd-repo-setup | ||
| id: upload-repo-setup-package | ||
| run: | | ||
| $PYTHON_CMD build_tools/packaging/linux/upload_repo_setup_package.py \ | ||
| --dest-dir repo-setup-package-out \ | ||
| --tier "${TIER}" \ | ||
| --run-id "${ARTIFACT_RUN_ID}" \ | ||
| --os-profile "${OS_PROFILE}" \ | ||
| --pkg-type "${NATIVE_PACKAGE_TYPE}" | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No quartz integration for repo package creation
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done, I've added Quartz integration. The setup job reports the start, and a new job reports completion, the same way |
||
|
|
||
| # Report the amd-repo-setup jobs' result to Quartz. resolve_repo_setup_params reports | ||
| # their start, so this runs only when that job ran. | ||
| notify_quartz_completed: | ||
| name: "Quartz - completed - build amd-repo-setup" | ||
| if: ${{ always() && vars.NOTIFY_QUARTZ_ENABLED == 'true' && needs.resolve_repo_setup_params.result != 'skipped' }} | ||
| needs: [resolve_repo_setup_params, stage_repo_setup_package] | ||
| runs-on: ubuntu-26.04 | ||
| timeout-minutes: 10 | ||
| continue-on-error: true | ||
| permissions: | ||
| contents: read | ||
| steps: | ||
| - name: "Quartz - completed - build amd-repo-setup" | ||
| uses: ROCm/Quartz/.github/actions/notify_quartz@f386a9756620938616af0b4d5d04b24ae6e0353f # notify_quartz/v1.1.1 | ||
| with: | ||
| run_phase: completed | ||
| reporting_workflow: multi_arch_build_native_linux_packages.yml | ||
| workflow_inputs: ${{ toJSON(inputs) }} | ||
| workflow_captured_outputs: ${{ toJSON(needs) }} | ||
| gh_app_client_id: ${{ secrets.GH_APP_HAULY_CID }} | ||
| gh_app_private_key: ${{ secrets.GH_APP_HAULY_PRIVATE_KEY }} | ||
This file was deleted.
Uh oh!
There was an error while loading. Please reload this page.