diff --git a/.azuredevops/hipblas-common.yml b/.azuredevops/hipblas-common.yml index 1e0dc5d4a24..1e45376f79e 100644 --- a/.azuredevops/hipblas-common.yml +++ b/.azuredevops/hipblas-common.yml @@ -32,8 +32,11 @@ trigger: pr: none -jobs: +stages: +- stage: hipblas-common + jobs: - template: ${{ variables.CI_COMPONENT_PATH }}/hipBLAS-common.yml@pipelines_repo parameters: sparseCheckoutDir: projects/hipblas-common triggerDownstreamJobs: ${{ parameters.triggerDownstreamJobs }} +- template: templates/report-summary-check-wrapper.yml diff --git a/.azuredevops/hipblaslt.yml b/.azuredevops/hipblaslt.yml index c82d65ba197..c2476b1d2d6 100644 --- a/.azuredevops/hipblaslt.yml +++ b/.azuredevops/hipblaslt.yml @@ -37,8 +37,11 @@ trigger: pr: none -jobs: +stages: +- stage: hipblaslt + jobs: - template: ${{ variables.CI_COMPONENT_PATH }}/hipBLASLt.yml@pipelines_repo parameters: sparseCheckoutDir: projects/hipblaslt triggerDownstreamJobs: ${{ parameters.triggerDownstreamJobs }} +- template: templates/report-summary-check-wrapper.yml diff --git a/.azuredevops/hipcub.yml b/.azuredevops/hipcub.yml index 8a20cd0015d..96d7ab11188 100644 --- a/.azuredevops/hipcub.yml +++ b/.azuredevops/hipcub.yml @@ -40,8 +40,11 @@ trigger: pr: none -jobs: +stages: +- stage: hipcub + jobs: - template: ${{ variables.CI_COMPONENT_PATH }}/hipCUB.yml@pipelines_repo parameters: sparseCheckoutDir: projects/hipcub triggerDownstreamJobs: ${{ parameters.triggerDownstreamJobs }} +- template: templates/report-summary-check-wrapper.yml diff --git a/.azuredevops/hipfft.yml b/.azuredevops/hipfft.yml index 81dcf13de99..c81fdc9098e 100644 --- a/.azuredevops/hipfft.yml +++ b/.azuredevops/hipfft.yml @@ -37,8 +37,11 @@ trigger: pr: none -jobs: +stages: +- stage: hipfft + jobs: - template: ${{ variables.CI_COMPONENT_PATH }}/hipFFT.yml@pipelines_repo parameters: sparseCheckoutDir: projects/hipfft triggerDownstreamJobs: ${{ parameters.triggerDownstreamJobs }} +- template: templates/report-summary-check-wrapper.yml diff --git a/.azuredevops/hiprand.yml b/.azuredevops/hiprand.yml index cb73c204d86..c8c573cd8fc 100644 --- a/.azuredevops/hiprand.yml +++ b/.azuredevops/hiprand.yml @@ -38,8 +38,11 @@ trigger: pr: none -jobs: +stages: +- stage: hiprand + jobs: - template: ${{ variables.CI_COMPONENT_PATH }}/hipRAND.yml@pipelines_repo parameters: sparseCheckoutDir: projects/hiprand triggerDownstreamJobs: ${{ parameters.triggerDownstreamJobs }} +- template: templates/report-summary-check-wrapper.yml diff --git a/.azuredevops/rocblas.yml b/.azuredevops/rocblas.yml index 7a2a9648194..9e8e50e5c1c 100644 --- a/.azuredevops/rocblas.yml +++ b/.azuredevops/rocblas.yml @@ -38,8 +38,11 @@ trigger: pr: none -jobs: +stages: +- stage: rocblas + jobs: - template: ${{ variables.CI_COMPONENT_PATH }}/rocBLAS.yml@pipelines_repo parameters: sparseCheckoutDir: projects/rocblas triggerDownstreamJobs: ${{ parameters.triggerDownstreamJobs }} +- template: templates/report-summary-check-wrapper.yml diff --git a/.azuredevops/rocfft.yml b/.azuredevops/rocfft.yml index 709fec551ee..1b0ca6881c7 100644 --- a/.azuredevops/rocfft.yml +++ b/.azuredevops/rocfft.yml @@ -37,8 +37,11 @@ trigger: pr: none -jobs: +stages: +- stage: rocfft + jobs: - template: ${{ variables.CI_COMPONENT_PATH }}/rocFFT.yml@pipelines_repo parameters: sparseCheckoutDir: projects/rocfft triggerDownstreamJobs: ${{ parameters.triggerDownstreamJobs }} +- template: templates/report-summary-check-wrapper.yml diff --git a/.azuredevops/rocprim.yml b/.azuredevops/rocprim.yml index ef421aafc36..1b91de933ad 100644 --- a/.azuredevops/rocprim.yml +++ b/.azuredevops/rocprim.yml @@ -40,8 +40,11 @@ trigger: pr: none -jobs: +stages: +- stage: rocprim + jobs: - template: ${{ variables.CI_COMPONENT_PATH }}/rocPRIM.yml@pipelines_repo parameters: sparseCheckoutDir: projects/rocprim triggerDownstreamJobs: ${{ parameters.triggerDownstreamJobs }} +- template: templates/report-summary-check-wrapper.yml diff --git a/.azuredevops/rocrand.yml b/.azuredevops/rocrand.yml index edabd72327e..25c31d13b60 100644 --- a/.azuredevops/rocrand.yml +++ b/.azuredevops/rocrand.yml @@ -38,8 +38,11 @@ trigger: pr: none -jobs: +stages: +- stage: rocrand + jobs: - template: ${{ variables.CI_COMPONENT_PATH }}/rocRAND.yml@pipelines_repo parameters: sparseCheckoutDir: projects/rocrand triggerDownstreamJobs: ${{ parameters.triggerDownstreamJobs }} +- template: templates/report-summary-check-wrapper.yml diff --git a/.azuredevops/rocsolver.yml b/.azuredevops/rocsolver.yml index 0614f11012b..6dbf426e412 100644 --- a/.azuredevops/rocsolver.yml +++ b/.azuredevops/rocsolver.yml @@ -36,8 +36,11 @@ trigger: pr: none -jobs: +stages: +- stage: rocsolver + jobs: - template: ${{ variables.CI_COMPONENT_PATH }}/rocSOLVER.yml@pipelines_repo parameters: sparseCheckoutDir: projects/rocsolver triggerDownstreamJobs: ${{ parameters.triggerDownstreamJobs }} +- template: templates/report-summary-check-wrapper.yml diff --git a/.azuredevops/rocthrust.yml b/.azuredevops/rocthrust.yml index 3643b3f4535..13dab7bc56e 100644 --- a/.azuredevops/rocthrust.yml +++ b/.azuredevops/rocthrust.yml @@ -40,8 +40,11 @@ trigger: pr: none -jobs: +stages: +- stage: rocthrust + jobs: - template: ${{ variables.CI_COMPONENT_PATH }}/rocThrust.yml@pipelines_repo parameters: sparseCheckoutDir: projects/rocthrust triggerDownstreamJobs: ${{ parameters.triggerDownstreamJobs }} +- template: templates/report-summary-check-wrapper.yml diff --git a/.azuredevops/templates/report-summary-check-wrapper.yml b/.azuredevops/templates/report-summary-check-wrapper.yml new file mode 100644 index 00000000000..f89dbaf6f12 --- /dev/null +++ b/.azuredevops/templates/report-summary-check-wrapper.yml @@ -0,0 +1,19 @@ +stages: +- stage: report_successful + condition: succeeded() + jobs: + - template: report-summary-check.yml + parameters: + checkConclusion: success +- stage: report_failed + condition: failed() + jobs: + - template: report-summary-check.yml + parameters: + checkConclusion: failure +- stage: report_cancelled + condition: canceled() # note: only 1 L in canceled() + jobs: + - template: report-summary-check.yml + parameters: + checkConclusion: cancelled diff --git a/.azuredevops/templates/report-summary-check.yml b/.azuredevops/templates/report-summary-check.yml new file mode 100644 index 00000000000..512227b8683 --- /dev/null +++ b/.azuredevops/templates/report-summary-check.yml @@ -0,0 +1,112 @@ +parameters: +- name: checkConclusion + type: string + default: success + values: + - success + - failure + - cancelled + +jobs: +- job: report_summary_check + displayName: 'Report check status: ${{ parameters.checkConclusion }}' + variables: + - group: assistant-librarian + pool: + vmImage: ubuntu-latest + steps: + - checkout: none + - task: Bash@3 + displayName: Install GitHub CLI + condition: always() + inputs: + targetType: 'inline' + script: | + (type -p wget >/dev/null || (sudo apt update && sudo apt install wget -y)) \ + && sudo mkdir -p -m 755 /etc/apt/keyrings \ + && out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \ + && cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && sudo mkdir -p -m 755 /etc/apt/sources.list.d \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && sudo apt update \ + && sudo apt install gh jq -y + - task: Bash@3 + displayName: Report CI status to GitHub PR Summary Check + condition: always() + env: + APP_ID: $(APP_ID) + APP_INSTALLATION_ID: $(APP_INSTALLATION_ID) + APP_PRIVATE_KEY: $(APP_PRIVATE_KEY) + inputs: + targetType: 'inline' + script: | + if [[ ! "$(Build.SourceBranch)" =~ ^refs/pull/ ]]; then + echo "This is not a PR build. Exiting." + exit 0 + fi + + # APP_PRIVATE_KEY is generated with `base64 app_private_key.pem | tr -d '\n'` + echo "$APP_PRIVATE_KEY" | base64 -d > app_private_key.pem + chmod 600 app_private_key.pem + create_jwt() { + local header=$(echo -n '{"alg":"RS256","typ":"JWT"}' | openssl base64 -e | tr -d '=' | tr '/+' '_-' | tr -d '\n') + local payload=$(echo -n "{\"iat\":$(date +%s),\"exp\":$(($(date +%s) + 600)),\"iss\":\"$APP_ID\"}" | openssl base64 -e | tr -d '=' | tr '/+' '_-' | tr -d '\n') + local unsigned_token="${header}.${payload}" + local signature=$(echo -n "$unsigned_token" | openssl dgst -sha256 -sign "app_private_key.pem" | openssl base64 -e | tr -d '=' | tr '/+' '_-' | tr -d '\n') + echo "${unsigned_token}.${signature}" + } + JWT=$(create_jwt) + export GH_TOKEN=$(curl -sSX POST \ + -H "Authorization: Bearer $JWT" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/app/installations/$APP_INSTALLATION_ID/access_tokens" | jq -r .token) + + PR_NUMBER=$(echo "$(Build.SourceBranch)" | sed 's|refs/pull/\([0-9]*\)/.*|\1|') + PR_HEAD_SHA=$(curl -s "https://api.github.com/repos/ROCm/rocm-libraries/pulls/$PR_NUMBER" | jq -r '.head.sha') + CHECK=$(curl -s "https://api.github.com/repos/ROCm/rocm-libraries/commits/$PR_HEAD_SHA/check-runs" | jq -r '.check_runs[] | select(.name == "Azure CI Summary")') + CHECK_ID=$(echo "$CHECK" | jq -r '.id') + CHECK_SUMMARY=$(echo "$CHECK" | jq -r '.output.summary') + CHECK_TEXT=$(echo "$CHECK" | jq -r '.output.text') + + if [[ -z "$CHECK_ID" ]]; then + echo "No Azure CI Summary check found for commit $PR_HEAD_SHA" + exit 0 + fi + if [[ "$CHECK_SUMMARY" == *"$(Build.BuildId)"* ]]; then + CHECK_SUMMARY=$(echo "$CHECK_SUMMARY" | sed "s/buildId=$(Build.BuildId)[^|]*|[^|]*|/buildId=$(Build.BuildId)) | ${{ parameters.checkConclusion }} |/") + fi + if [[ "$CHECK_TEXT" == *"$(Build.BuildId)="* ]]; then + CHECK_TEXT=$(echo "$CHECK_TEXT" | sed "s/$(Build.BuildId)=[^;]*;/$(Build.BuildId)=${{ parameters.checkConclusion }};/") + fi + + CHECK_STATUS=$(echo "$CHECK_TEXT" | grep -q "pending" && echo "in_progress" || echo "completed") + CHECK_CONCLUSION=$(echo "$CHECK_TEXT" | grep -q -e "cancelled" -e "failure" && echo "failure" || echo "success") + + if [[ "$CHECK_STATUS" == "completed" ]]; then + gh_output=$(gh api repos/ROCm/rocm-libraries/check-runs/$CHECK_ID \ + -X PATCH \ + -f "name=Azure CI Summary" \ + -f "head_sha=$PR_HEAD_SHA" \ + -f "status=$CHECK_STATUS" \ + -f "conclusion=$CHECK_CONCLUSION" \ + -f "output[title]=Azure CI Summary" \ + -f "output[summary]=$CHECK_SUMMARY" \ + -f "output[text]=$CHECK_TEXT") + else + gh_output=$(gh api repos/ROCm/rocm-libraries/check-runs/$CHECK_ID \ + -X PATCH \ + -f "name=Azure CI Summary" \ + -f "head_sha=$PR_HEAD_SHA" \ + -f "output[title]=Azure CI Summary" \ + -f "output[summary]=$CHECK_SUMMARY" \ + -f "output[text]=$CHECK_TEXT") + fi + + echo "Reported status '${{ parameters.checkConclusion }}' to summary check: $(echo "$gh_output" | jq -r '.id')" + if [[ "$CHECK_STATUS" == "completed" ]]; then + echo "All checks completed with overall conclusion: $CHECK_CONCLUSION" + else + echo "Some checks are still in progress: $CHECK_STATUS" + fi + echo "Summary check URL: $(echo "$gh_output" | jq -r '.html_url')" diff --git a/.azuredevops/tensile.yml b/.azuredevops/tensile.yml index 63dd6e2304c..aba78889990 100644 --- a/.azuredevops/tensile.yml +++ b/.azuredevops/tensile.yml @@ -37,8 +37,11 @@ trigger: pr: none -jobs: +stages: +- stage: tensile + jobs: - template: ${{ variables.CI_COMPONENT_PATH }}/Tensile.yml@pipelines_repo parameters: sparseCheckoutDir: shared/tensile triggerDownstreamJobs: ${{ parameters.triggerDownstreamJobs }} +- template: templates/report-summary-check-wrapper.yml diff --git a/.github/scripts/pr_category_label.py b/.github/scripts/pr_category_label.py index 13efa9cab30..827ab6056fa 100644 --- a/.github/scripts/pr_category_label.py +++ b/.github/scripts/pr_category_label.py @@ -92,7 +92,6 @@ def main(argv=None) -> None: client = GitHubCLIClient() changed_files = [file for file in client.get_changed_files(args.repo, int(args.pr))] - if not changed_files: logger.warning("REST API failed or returned no changed files. Falling back to Git CLI...") try: diff --git a/.github/scripts/pr_detect_changed_subtrees.py b/.github/scripts/pr_detect_changed_subtrees.py index caeb0817c66..8570843dc96 100644 --- a/.github/scripts/pr_detect_changed_subtrees.py +++ b/.github/scripts/pr_detect_changed_subtrees.py @@ -113,6 +113,22 @@ def main(argv=None) -> None: client = GitHubCLIClient() config = load_repo_config(args.config) changed_files = client.get_changed_files(args.repo, int(args.pr)) + + if not changed_files: + logger.warning("REST API failed or returned no changed files. Falling back to Git CLI...") + try: + # Ensure fetch is safe + os.system("git fetch origin +refs/pull/*/merge:refs/remotes/origin/pr/*") + # Get merge commit ref for this PR + base_ref = f"origin/{os.getenv('GITHUB_BASE_REF', 'main')}" + head_ref = "HEAD" # Assumes checkout to PR merge ref + result = os.popen(f"git diff --name-only {base_ref}...{head_ref}").read() + changed_files = result.strip().splitlines() + logger.info(f"Fallback changed files: {changed_files}") + except Exception as e: + logger.error(f"Git CLI fallback failed: {e}") + sys.exit(1) + valid_prefixes = get_valid_prefixes(config, args.require_auto_pull, args.require_auto_push) matched_subtrees = find_matched_subtrees(changed_files, valid_prefixes) output_subtrees(matched_subtrees, args.dry_run) diff --git a/.github/workflows/azure-ci-dispatcher.yml b/.github/workflows/azure-ci-dispatcher.yml index b7afe23e720..a6014f21dda 100644 --- a/.github/workflows/azure-ci-dispatcher.yml +++ b/.github/workflows/azure-ci-dispatcher.yml @@ -12,7 +12,7 @@ # Requires an Azure Personal Access Token with permissions to manage builds. # The token should be stored in the repository secrets as `AZ_PAT`. -name: Dispatch Azure CI +name: Trigger Azure CI on: pull_request_target: @@ -26,10 +26,6 @@ on: - staging - main - release-staging/rocm-rel-7.* - paths-ignore: - - '.github/**' - - 'docs/**' - - '*.md' concurrency: group: azure-ci-dispatcher-${{ github.event.pull_request.number || github.ref }} @@ -37,9 +33,9 @@ concurrency: jobs: dispatch-azure-ci: - name: Dispatch Azure CI + name: Trigger Azure CI + if: github.repository == 'ROCm/rocm-libraries' runs-on: ubuntu-latest - timeout-minutes: 720 steps: - name: Generate a token id: generate-token @@ -48,6 +44,35 @@ jobs: app-id: ${{ secrets.APP_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} + repositories: | + rocm-libraries + + - name: Wait until refs/pull/${{ github.event.pull_request.number }}/merge exists + run: | + merge_ref="refs/pull/${{ github.event.pull_request.number }}/merge" + check_merge_ref() { + git ls-remote "https://github.com/ROCm/rocm-libraries" "$merge_ref" | grep -q "$merge_ref" + } + + max_attempts=10 + attempt=0 + + while [ $attempt -lt $max_attempts ]; do + if check_merge_ref; then + echo "$merge_ref found." + break + else + retry_delay=60 + echo "$merge_ref not found. Retrying in $retry_delay seconds..." + sleep $retry_delay + fi + attempt=$((attempt + 1)) + done + + if [ $attempt -ge $max_attempts ]; then + echo "$merge_ref not found. Maximum attempts reached." + exit 1 + fi - name: Checkout code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -73,9 +98,102 @@ jobs: --config ".github/repos-config.json" \ --require-auto-push - - name: Dispatch Azure CI runs - id: dispatch + - name: Cancel in-progress/not-started runs for current PR + id: cancel-in-progress if: steps.detect.outputs.subtrees + run: | + pr_number=${{ github.event.pull_request.number }} + pr_filter_query="branchName=refs/pull/$pr_number/merge&repositoryType=GitHub&repositoryId=ROCm/rocm-libraries&api-version=7.1" + + res=$(curl -sSX GET "https://dev.azure.com/ROCm-CI/ROCm-CI/_apis/build/builds?$pr_filter_query" \ + -H "Content-Type: application/json") + + runs=$(echo "$res" | jq -r ".value[] | select(.status == \"inProgress\" or .status == \"notStarted\") | .id") + + if [ -z "$runs" ]; then + echo "No in-progress/not-started runs found for ROCm/rocm-libraries PR #$pr_number" + echo "status=false" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Found in-progress/not-started runs for ROCm/rocm-libraries PR #$pr_number: $runs" + echo "status=true" >> $GITHUB_OUTPUT + + for run_id in $runs; do + echo "Cancelling run ID: $run_id" + echo "Run URL: https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=$run_id" + response=$(curl -sSX PATCH "https://dev.azure.com/ROCm-CI/ROCm-CI/_apis/build/builds/$run_id?api-version=7.1" \ + -u ":${{ secrets.AZ_PAT }}" \ + -H "Content-Type: application/json" \ + -d '{"status": "cancelling"}') + + if [ $? -ne 0 ]; then + echo "Failed to cancel run ID: $run_id" + else + echo "Cancelled run ID: $run_id" + fi + done + + - name: Rerun previous failed/cancelled runs for current PR merge commit + id: rerun-failed + if: steps.detect.outputs.subtrees && steps.cancel-in-progress.outputs.status == 'false' + run: | + pr_number=${{ github.event.pull_request.number }} + pr_merge_sha=$(curl -sSX GET "https://api.github.com/repos/ROCm/rocm-libraries/git/ref/pull/${pr_number}/merge" | jq -r '.object.sha') + pr_filter_query="branchName=refs/pull/$pr_number/merge&repositoryType=GitHub&repositoryId=ROCm/rocm-libraries&api-version=7.1" + + res=$(curl -sSX GET "https://dev.azure.com/ROCm-CI/ROCm-CI/_apis/build/builds?$pr_filter_query" \ + -H "Content-Type: application/json") + + failed_runs_info=$(echo "$res" | jq -r ".value[] | + select((.result == \"failed\" or .result == \"canceled\") + and (.sourceVersion | contains(\"$pr_merge_sha\"))) + | {id: .id, name: .definition.name}") + success_runs_info=$(echo "$res" | jq -r ".value[] | + select((.result == \"succeeded\") + and (.sourceVersion | contains(\"$pr_merge_sha\"))) + | {id: .id, name: .definition.name}") + + failed_run_ids=$(echo "$failed_runs_info" | jq -r '.id') + failed_project_names=$(echo "$failed_runs_info" | jq -r '.name') + success_run_ids=$(echo "$success_runs_info" | jq -r '.id') + success_project_names=$(echo "$success_runs_info" | jq -r '.name') + + if [ -z "$failed_run_ids" ]; then + echo "No failed/cancelled runs found for ROCm/rocm-libraries PR #$pr_number at merge commit $pr_merge_sha" + echo "status=false" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Found failed/cancelled runs for ROCm/rocm-libraries PR #$pr_number at merge commit $pr_merge_sha: ${failed_run_ids[*]}" + echo "Found successful runs for ROCm/rocm-libraries PR #$pr_number at merge commit $pr_merge_sha: ${success_run_ids[*]}" + echo "status=true" >> $GITHUB_OUTPUT + + new_run_ids=() + + for run_id in $failed_run_ids; do + echo "Rerunning failed run ID: $run_id" + echo "Run URL: https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=$run_id" + response=$(curl -sSX PATCH "https://dev.azure.com/ROCm-CI/ROCm-CI/_apis/build/builds/$run_id?retry=true&api-version=7.1" \ + -u ":${{ secrets.AZ_PAT }}" \ + -H "Content-Type: application/json") + + if [ $? -ne 0 ]; then + echo "Failed to rerun run ID: $run_id" + else + echo "Rerun requested for run ID: $run_id" + new_run_ids+=("$run_id") + fi + done + + echo "run_ids=${new_run_ids[*]}" >> $GITHUB_OUTPUT + echo "project_names=${failed_project_names[*]}" >> $GITHUB_OUTPUT + echo "success_run_ids=${success_run_ids[*]}" >> $GITHUB_OUTPUT + echo "success_project_names=${success_project_names[*]}" >> $GITHUB_OUTPUT + + - name: Start new Azure CI runs + id: dispatch + if: steps.detect.outputs.subtrees && (steps.cancel-in-progress.outputs.status == 'true' || steps.rerun-failed.outputs.status == 'false') env: GH_TOKEN: ${{ steps.generate-token.outputs.token }} run: | @@ -86,6 +204,7 @@ jobs: > resolved_subtrees.txt run_ids=() + project_names=() while IFS= read -r line; do IFS='=' read -r project_name definition_id <<< "$line" @@ -124,8 +243,9 @@ jobs: run_id=$(echo "$response" | jq -r '.id' || echo "null") if [ "$run_id" != "null" ]; then echo "Run ID for $project_name: $run_id" - echo "https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=$run_id" + echo "Run URL: https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=$run_id" run_ids+=("$run_id") + project_names+=("$project_name") else echo "Failed to request run for $project_name" fi @@ -137,82 +257,74 @@ jobs: done < resolved_subtrees.txt echo "run_ids=${run_ids[*]}" >> $GITHUB_OUTPUT + echo "project_names=${project_names[*]}" >> $GITHUB_OUTPUT - - name: Wait for and report Azure CI status - if: steps.dispatch.outputs.run_ids - timeout-minutes: 720 + - name: Create summary check + env: + GH_TOKEN: ${{ steps.generate-token.outputs.token }} run: | - run_ids=(${{ steps.dispatch.outputs.run_ids }}) - echo "Waiting on runs: ${run_ids[*]}" - - all_finished=0 - exit_status=0 - - while [[ $all_finished -eq 0 ]]; do - sleep 120 - all_finished=1 - echo "==================================================================" - - for run_id in "${run_ids[@]}"; do - max_attempts=3 - retry_delay=5 - attempt=1 - success=false - - while [ $attempt -le $max_attempts ]; do - response=$(curl -sSX GET https://dev.azure.com/ROCm-CI/ROCm-CI/_apis/build/builds/$run_id?api-version=7.1 \ - -u ":${{ secrets.AZ_PAT }}" \ - -H "Content-Type: application/json") - - if [[ $? -eq 0 ]]; then - success=true - break - fi - - echo "Attempt $attempt failed. Retrying in $retry_delay seconds..." - sleep $retry_delay - attempt=$((attempt + 1)) - done - - if [[ "$success" == true ]]; then - pipeline_name=$(echo "$response" | jq -r '.definition.name') - status=$(echo "$response" | jq -r '.status') - result=$(echo "$response" | jq -r '.result') - - if [[ "$status" == "completed" ]]; then - echo "$pipeline_name - run $run_id is finished with result: $result" - echo "https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=$run_id" - if [[ "$result" != "succeeded" ]]; then - exit_status=1 - fi - else - all_finished=0 - echo "$pipeline_name - run $run_id is in progress..." - echo "https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=$run_id" - fi - else - echo "Failed to fetch status for run $run_id after $max_attempts attempts" - echo "https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=$run_id" - exit_status=1 - continue - fi + if [[ -n "${{ steps.dispatch.outputs.run_ids }}" && -n "${{ steps.dispatch.outputs.project_names }}" ]]; then # If new runs were started + run_ids=(${{ steps.dispatch.outputs.run_ids }}) + project_names=(${{ steps.dispatch.outputs.project_names }}) + elif [[ -n "${{ steps.rerun-failed.outputs.run_ids }}" && -n "${{ steps.rerun-failed.outputs.project_names }}" ]]; then # If reruns were requested + run_ids=(${{ steps.rerun-failed.outputs.run_ids }}) + project_names=(${{ steps.rerun-failed.outputs.project_names }}) + success_run_ids=(${{ steps.rerun-failed.outputs.success_run_ids }}) + success_project_names=(${{ steps.rerun-failed.outputs.success_project_names }}) + else + echo "No run IDs or project names found, skipping summary check creation." + exit 0 + fi + + pr_number=${{ github.event.pull_request.number }} + pr_head_sha=${{ github.event.pull_request.head.sha }} + pr_merge_sha=$(curl -sSX GET "https://api.github.com/repos/ROCm/rocm-libraries/git/ref/pull/${pr_number}/merge" | jq -r '.object.sha') + + newline=$'\n' + + summary="PR: [${{ github.event.pull_request.title }} #$pr_number](${{ github.event.pull_request.html_url }})${newline}${newline}" + summary+="HEAD: [$pr_head_sha](https://github.com/ROCm/rocm-libraries/commit/$pr_head_sha)${newline}${newline}" + summary+="MERGE: [$pr_merge_sha](https://github.com/ROCm/rocm-libraries/commit/$pr_merge_sha)${newline}${newline}" + summary+="### Pipelines triggered for this PR${newline}${newline}" + summary+="| Project | Run ID | Status |${newline}" + summary+="|--------------|--------|--------|${newline}" + + if [[ -n "${success_run_ids[*]}" ]]; then + for i in "${!success_project_names[@]}"; do + summary+="| ${success_project_names[i]} | [${success_run_ids[i]}](https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=${success_run_ids[i]}) | success |${newline}" done + fi + + for i in "${!project_names[@]}"; do + summary+="| ${project_names[i]} | [${run_ids[i]}](https://dev.azure.com/ROCm-CI/ROCm-CI/_build/results?buildId=${run_ids[i]}) | pending |${newline}" done - if [[ $exit_status -ne 0 ]] then - exit $exit_status - fi + summary+="${newline}${newline}" + summary+="### Rerun instructions${newline}${newline}" + summary+="To request Azure to rerun jobs, click the \`Re-run all jobs\` button on the [corresponding \`Trigger Azure CI\` run](https://github.com/ROCm/rocm-libraries/actions/runs/${{ github.run_id }}).${newline}${newline}" + summary+="If there are any pending runs for this PR, they will be cancelled, and new runs will be started.${newline}${newline}" + summary+="If there are no pending runs, but there are existing failed or cancelled runs for this PR and merge SHA, the existing runs will be rerun.${newline}${newline}" + summary+="Otherwise, new runs will be started.${newline}${newline}" - - name: Cancel Azure CI runs - if: ${{ cancelled() }} - run: | - run_ids=(${{ steps.dispatch.outputs.run_ids }}) - echo "Cancelling runs: ${run_ids[*]}" + text="" - for run_id in "${run_ids[@]}"; do - response=$(curl -sSX PATCH https://dev.azure.com/ROCm-CI/ROCm-CI/_apis/build/builds/$run_id?api-version=7.1 \ - -u ":${{ secrets.AZ_PAT }}" \ - -H "Content-Type: application/json" \ - -d '{"status": "cancelling"}') - echo $response | jq -r '.status' || echo "Failed to cancel run $run_id" + if [[ -n "${success_run_ids[*]}" ]]; then + for i in "${!success_run_ids[@]}"; do + text+="${success_run_ids[i]}=success;" + done + fi + + for i in "${!project_names[@]}"; do + text+="${run_ids[i]}=pending;" done + + gh_output=$(gh api repos/ROCm/rocm-libraries/check-runs \ + -f "name=Azure CI Summary" \ + -f "head_sha=$pr_head_sha" \ + -f "status=in_progress" \ + -f "output[title]=Azure CI Summary" \ + -f "output[summary]=$summary" \ + -f "output[text]=$text") + + echo "Created summary check with ID: $(echo "$gh_output" | jq -r '.id')" + echo "Summary check URL: https://github.com/ROCm/rocm-libraries/pull/561/checks?check_run_id=$(echo "$gh_output" | jq -r '.id')"