diff --git a/.github/workflows/rocjitsu-benchmarks.yml b/.github/workflows/rocjitsu-benchmarks.yml index 1cce5266ee75..85f90ab0207c 100644 --- a/.github/workflows/rocjitsu-benchmarks.yml +++ b/.github/workflows/rocjitsu-benchmarks.yml @@ -8,6 +8,11 @@ on: - "emulation/rocjitsu/**" - ".github/workflows/rocjitsu-benchmarks.yml" workflow_dispatch: + inputs: + commit: + description: "Rocjitsu commit SHA to benchmark" + required: true + type: string # Each job grants only the access it needs. permissions: {} @@ -17,22 +22,28 @@ defaults: shell: bash env: - CORPUS_SHA: e1bac822f3fd2e8c29e3a08ae6c326db1692a757 # rocjitsu-test-corpus#44 + ROCJITSU_SHA: ${{ inputs.commit || github.sha }} + CORPUS_SHA: b83ff7c47cedd7c0fac3b9c0d015b7322fbd53e7 + TENSILE_SHA: dd77374194a3ea1a7258cd54f87af6747b6362a5 ROCJITSU_SOURCE_DIR: ${{ github.workspace }}/rocm-systems/emulation/rocjitsu CORPUS_SOURCE_DIR: ${{ github.workspace }}/rocjitsu-test-corpus + TENSILE_SOURCE: ${{ github.workspace }}/tensile-source RESULTS_BRANCH: gh-pages-rocjitsu RESULTS_ARTIFACT: rocjitsu-benchmark-${{ github.run_id }}-${{ github.run_attempt }} + RESULTS_DOWNLOAD_DIR: ${{ github.workspace }}/benchmark-results jobs: benchmark: - name: Run nightly benchmark + if: ${{ github.ref_type == 'branch' }} + name: Run performance benchmark runs-on: rocjitsu-benchmark timeout-minutes: 210 permissions: contents: read outputs: - machine: ${{ steps.machine.outputs.name }} - ran: ${{ steps.benchmark.outcome == 'success' || steps.benchmark.outcome == 'failure' }} + run_attempt: ${{ steps.metadata.outputs.run_attempt }} + raw_ready: ${{ steps.upload_raw.outcome == 'success' }} + dashboard_ready: ${{ steps.upload_dashboard.outcome == 'success' }} container: image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:4150afe4759d14822f0e3f8930e1124f26e11f68b5c7b91ec9a02b20b1ebbb98 options: --user 0:0 @@ -44,8 +55,10 @@ jobs: run: | BENCHMARK_DIR="${RUNNER_TEMP}/rocjitsu-benchmark-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" { + echo "BENCHMARK_DIR=${BENCHMARK_DIR}" echo "ROCJITSU_BUILD_DIR=${BENCHMARK_DIR}/build" echo "ROCJITSU_VENV=${BENCHMARK_DIR}/venv" + echo "TENSILE_VENV=${BENCHMARK_DIR}/tensile-venv" echo "ROCJITSU_RESULT_DIR=${BENCHMARK_DIR}/results" } >> "${GITHUB_ENV}" @@ -60,14 +73,48 @@ jobs: git \ libclang-rt-dev \ libdrm-dev \ - ninja-build + libomp-dev \ + libzstd-dev \ + ninja-build \ + python3-dev \ + zlib1g-dev - name: Checkout rocm-systems uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: path: rocm-systems + ref: ${{ env.ROCJITSU_SHA }} + fetch-depth: 0 persist-credentials: false + - name: Require a commit from develop + working-directory: rocm-systems + run: | + if ! git merge-base --is-ancestor HEAD origin/develop; then + echo "The selected commit must be in develop's history." >&2 + exit 1 + fi + + # Use the workflow's converter even when benchmarking an older Rocjitsu commit. + - name: Checkout workflow tools + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + path: workflow-source + ref: ${{ github.sha }} + persist-credentials: false + sparse-checkout: emulation/rocjitsu/website/dashboard/scripts/dashboard-data + + - name: Record benchmark metadata + id: metadata + working-directory: rocm-systems + run: | + COMMIT_MESSAGE="$(git show -s --format=%s HEAD)" + { + printf 'commit_message=%s\n' "${COMMIT_MESSAGE}" + printf 'machine=%s\n' "${RUNNER_NAME}" + printf 'run_attempt=%s\n' "${GITHUB_RUN_ATTEMPT}" + } >> "${GITHUB_OUTPUT}" + - name: Checkout benchmark corpus uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: @@ -76,9 +123,13 @@ jobs: path: rocjitsu-test-corpus persist-credentials: false - - name: Record benchmark machine - id: machine - run: echo "name=${RUNNER_NAME}" >> "${GITHUB_OUTPUT}" + - name: Checkout pinned Tensile source + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + repository: ROCm/rocm-libraries + ref: ${{ env.TENSILE_SHA }} + path: tensile-source + persist-credentials: false - name: Set up Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -92,9 +143,18 @@ jobs: --disable-pip-version-check \ -r "${CORPUS_SOURCE_DIR}/benchmarks/requirements.txt" + python -m venv "${TENSILE_VENV}" + "${TENSILE_VENV}/bin/python" -m pip install \ + -r "${TENSILE_SOURCE}/projects/hipblaslt/tensilelite/requirements.txt" + ROCM_PATH="$("${ROCJITSU_VENV}/bin/rocm-sdk" path --root)" - echo "ROCM_PATH=${ROCM_PATH}" >> "${GITHUB_ENV}" - echo "LD_LIBRARY_PATH=${ROCM_PATH}/lib:${LD_LIBRARY_PATH:-}" >> "${GITHUB_ENV}" + { + echo "ROCM_PATH=${ROCM_PATH}" + echo "HIP_PATH=${ROCM_PATH}" + echo "PYTHON=${ROCJITSU_VENV}/bin/python" + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${ROCM_PATH}/lib:${ROCM_PATH}/lib/rocm_sysdeps/lib:${ROCM_PATH}/lib/llvm/lib:${ROCM_PATH}/lib/host-math/lib" + echo "PATH=${ROCM_PATH}/bin:${ROCM_PATH}/lib/llvm/bin:${ROCJITSU_VENV}/bin:/usr/local/bin:/usr/bin:/bin" + } >> "${GITHUB_ENV}" - name: Build rocjitsu run: | @@ -109,150 +169,263 @@ jobs: -DCMAKE_C_COMPILER="${ROCM_PATH}/lib/llvm/bin/amdclang" \ -DCMAKE_CXX_COMPILER="${ROCM_PATH}/lib/llvm/bin/amdclang++" cmake --build "${ROCJITSU_BUILD_DIR}" \ + --parallel "$(($(nproc) / 2))" \ --target rocjitsu_bin rocjitsu_shared - - name: Run nightly suite + - name: Install ROCISA and build native adapter + timeout-minutes: 45 + run: | + export ROCISA_BACKEND=native + export PYTHON="${TENSILE_VENV}/bin/python" + export PATH="${TENSILE_VENV}/bin:${PATH}" + "${TENSILE_VENV}/bin/python" -m pip install --no-deps \ + --config-settings="build-dir=${BENCHMARK_DIR}/rocisa-build" \ + --config-settings="cmake.define.ROCM_PATH=${ROCM_PATH}" \ + --config-settings="cmake.define.CMAKE_CXX_COMPILER=${ROCM_PATH}/bin/amdclang++" \ + "${TENSILE_SOURCE}/projects/hipblaslt/tensilelite/rocisa" + cmake \ + -S "${CORPUS_SOURCE_DIR}/corpus/benchmarks/tensile_candidates" \ + -B "${BENCHMARK_DIR}/tensile-native" \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DROCM_PATH="${ROCM_PATH}" \ + -DCMAKE_PREFIX_PATH="${ROCM_PATH}" \ + -DCMAKE_CXX_COMPILER="${ROCM_PATH}/bin/amdclang++" \ + -DROCmCMakeBuildTools_DIR="${ROCM_PATH}/share/rocmcmakebuildtools/cmake" \ + -DLLVM_DIR="${ROCM_PATH}/lib/llvm/lib/cmake/llvm" \ + -DTENSILE_SOURCE="${TENSILE_SOURCE}" \ + -DPython_EXECUTABLE="${TENSILE_VENV}/bin/python" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 + cmake --build "${BENCHMARK_DIR}/tensile-native" \ + --parallel "$(($(nproc) / 2))" \ + --target tensile_candidate tensile_candidate_host_tests + "${BENCHMARK_DIR}/tensile-native/tensile_candidate_host_tests" + + - name: Generate pinned Tensile artifacts + id: prepare + timeout-minutes: 45 + run: | + export PYTHON="${TENSILE_VENV}/bin/python" + export PATH="${TENSILE_VENV}/bin:${PATH}" + "${TENSILE_VENV}/bin/python" "${CORPUS_SOURCE_DIR}/corpus/benchmarks/tensile_candidates/generate.py" \ + --source-root "${TENSILE_SOURCE}" \ + --output "${BENCHMARK_DIR}/tensile-artifacts" + { + echo "TENSILE_CANDIDATE_ARTIFACTS=${BENCHMARK_DIR}/tensile-artifacts" + echo "TENSILE_CANDIDATE_RUNNER=${BENCHMARK_DIR}/tensile-native/tensile_candidate" + } >> "${GITHUB_ENV}" + + - name: Run performance suite + if: ${{ success() && steps.prepare.outcome == 'success' }} timeout-minutes: 60 id: benchmark working-directory: ${{ env.CORPUS_SOURCE_DIR }} run: | + # Capture only this invocation; keep console output visible. + set +e "${ROCJITSU_VENV}/bin/python" -m benchmarks.runner \ --rocjitsu-source-dir "${ROCJITSU_SOURCE_DIR}" \ --build-dir "${ROCJITSU_BUILD_DIR}" \ --target-config "gfx950=${ROCJITSU_SOURCE_DIR}/configs/gfx950_mi355x_kmd.json" \ --target-config "gfx1250=${ROCJITSU_SOURCE_DIR}/configs/gfx1250_mi455x.json" \ --run-wrapper "\"${ROCJITSU_BUILD_DIR}/tools/rocjitsu/rocjitsu\" --config {config} --" \ - --manifest benchmarks/suites/nightly.toml \ - --output "${ROCJITSU_RESULT_DIR}" + --manifest benchmarks/suites/performance-v1.toml \ + --output "${ROCJITSU_RESULT_DIR}" | tee "${BENCHMARK_DIR}/runner-output.txt" + RUNNER_PIPE_STATUS=("${PIPESTATUS[@]}") + set -e + RUNNER_EXIT="${RUNNER_PIPE_STATUS[0]}" + if [[ "${RUNNER_PIPE_STATUS[1]}" -ne 0 ]]; then exit "${RUNNER_PIPE_STATUS[1]}"; fi + case "${RUNNER_EXIT}" in + 0) SUITE_STATUS=completed ;; + 1) SUITE_STATUS=failed ;; + *) exit "${RUNNER_EXIT}" ;; + esac + # Only main's final line follows a normal run_suite return. An abort + # can also exit 1 and set finished_at, so neither alone is sufficient. + EXPECTED_ARTIFACT="$(realpath -- "${ROCJITSU_RESULT_DIR}")/run.json" + if [[ "$(tail -n 1 "${BENCHMARK_DIR}/runner-output.txt")" != "run ${SUITE_STATUS}: ${EXPECTED_ARTIFACT}" ]]; then + echo "Runner did not report normal suite completion." >&2 + exit 1 + fi + if ! python -m json.tool "${ROCJITSU_RESULT_DIR}/run.json" > /dev/null; then exit 1; fi + if [[ "${RUNNER_EXIT}" -eq 1 ]]; then + echo "case_failures=true" >> "${GITHUB_OUTPUT}" + echo "::warning::Suite completed with benchmark failures; preserving results." + fi + + - name: Upload workload failure diagnostics + if: ${{ !cancelled() && (steps.benchmark.outcome == 'failure' || steps.benchmark.outputs.case_failures == 'true') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ env.RESULTS_ARTIFACT }}-diagnostics + path: | + ${{ env.ROCJITSU_RESULT_DIR }}/*/*/cases/*/stdout.txt + ${{ env.ROCJITSU_RESULT_DIR }}/*/*/cases/*/stderr.txt + if-no-files-found: ignore + retention-days: 14 + + - name: Upload original raw benchmark JSON + id: upload_raw + if: ${{ !cancelled() && (steps.benchmark.outcome == 'success' || steps.benchmark.outcome == 'failure') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ env.RESULTS_ARTIFACT }}-raw + path: ${{ env.ROCJITSU_RESULT_DIR }}/run.json + if-no-files-found: error + retention-days: 14 + + # The new dashboard requires both default and single thread-policy data. + # This workflow publishes only default-policy dashboard data to match the + # existing dashboard. Normal completion may include failed/timeout cases; + # runner errors and aborted suites still fail before conversion. + - name: Prepare dashboard benchmark result + id: dashboard + if: ${{ !cancelled() && steps.benchmark.outcome == 'success' }} + env: + BENCHMARK_COMMIT_MESSAGE: ${{ steps.metadata.outputs.commit_message }} + BENCHMARK_MACHINE: ${{ steps.metadata.outputs.machine }} + BENCHMARK_WORKFLOW_SHA: ${{ github.sha }} + run: | + python "${GITHUB_WORKSPACE}/workflow-source/emulation/rocjitsu/website/dashboard/scripts/dashboard-data/prepare-benchmark-result.py" \ + --raw-run "${ROCJITSU_RESULT_DIR}/run.json" \ + --output "${BENCHMARK_DIR}/dashboard/github-${GITHUB_RUN_ID}-attempt-${GITHUB_RUN_ATTEMPT}-default.json" \ + --run-id "${GITHUB_RUN_ID}" --run-attempt "${GITHUB_RUN_ATTEMPT}" \ + --repository "${GITHUB_REPOSITORY}" --event "${GITHUB_EVENT_NAME}" \ + --commit-message "${BENCHMARK_COMMIT_MESSAGE}" --machine "${BENCHMARK_MACHINE}" \ + --workflow-sha "${BENCHMARK_WORKFLOW_SHA}" --tensile-sha "${TENSILE_SHA}" - - name: Upload benchmark diagnostics - if: always() + - name: Upload dashboard benchmark JSON + id: upload_dashboard + if: ${{ !cancelled() && steps.dashboard.outcome == 'success' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: ${{ env.RESULTS_ARTIFACT }} - path: ${{ env.ROCJITSU_RESULT_DIR }} - if-no-files-found: warn + name: ${{ env.RESULTS_ARTIFACT }}-dashboard + path: ${{ env.BENCHMARK_DIR }}/dashboard/*.json + if-no-files-found: error retention-days: 14 publish: - name: Publish dashboard data + name: Publish benchmark results needs: benchmark - if: ${{ !cancelled() && github.ref == 'refs/heads/develop' && needs.benchmark.outputs.ran == 'true' }} + # Failed-job reruns reuse artifacts from the successful benchmark attempt. + env: + BENCHMARK_RUN_ATTEMPT: ${{ needs.benchmark.outputs.run_attempt }} + RESULTS_ARTIFACT: rocjitsu-benchmark-${{ github.run_id }}-${{ needs.benchmark.outputs.run_attempt }} + if: ${{ !cancelled() && github.ref == 'refs/heads/develop' && needs.benchmark.outputs.raw_ready == 'true' }} runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 10 + concurrency: + group: rocjitsu-benchmark-publication + cancel-in-progress: false permissions: contents: write - env: - BENCHMARK_MACHINE: ${{ needs.benchmark.outputs.machine }} - steps: - - name: Download benchmark diagnostics + - name: Download original raw JSON uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: ${{ env.RESULTS_ARTIFACT }} - path: incoming + name: ${{ env.RESULTS_ARTIFACT }}-raw + path: ${{ env.RESULTS_DOWNLOAD_DIR }}/raw - - name: Checkout publisher corpus - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - name: Download dashboard JSON + if: ${{ needs.benchmark.outputs.dashboard_ready == 'true' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - repository: ROCm/rocjitsu-test-corpus - ref: ${{ env.CORPUS_SHA }} - path: rocjitsu-test-corpus - persist-credentials: false + name: ${{ env.RESULTS_ARTIFACT }}-dashboard + path: ${{ env.RESULTS_DOWNLOAD_DIR }}/dashboard - - name: Checkout repository for publication + - name: Checkout publication validator + if: ${{ needs.benchmark.outputs.dashboard_ready == 'true' }} uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: - path: publication + path: workflow-source + ref: ${{ github.sha }} + persist-credentials: false + sparse-checkout: | + emulation/rocjitsu/website/dashboard/scripts + emulation/rocjitsu/website/dashboard/src/data - - name: Checkout dashboard validator - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - name: Set up Node.js + if: ${{ needs.benchmark.outputs.dashboard_ready == 'true' }} + uses: actions/setup-node@v4 with: - repository: ROCm/rocm-systems - path: dashboard - persist-credentials: false - sparse-checkout: emulation/rocjitsu/website/dashboard + node-version: "22.13.0" - - name: Set up publisher Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - name: Checkout publication branch + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: - python-version: "3.12" + path: publication + ref: ${{ env.RESULTS_BRANCH }} - - name: Publish dashboard-ready result + - name: Publish benchmark files working-directory: publication + env: + DASHBOARD_READY: ${{ needs.benchmark.outputs.dashboard_ready }} + DASHBOARD_VALIDATOR: ${{ github.workspace }}/workflow-source/emulation/rocjitsu/website/dashboard/scripts/validate-dashboard-data.mjs run: | git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - RUN_ID="github-${GITHUB_RUN_ID}-attempt-${GITHUB_RUN_ATTEMPT}" - TRIGGER=auto - if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then - TRIGGER=manual - fi - # Read metadata before switching this checkout to the results branch. - COMMIT_MESSAGE="$(git show -s --format=%s "${GITHUB_SHA}")" - - MAX_ATTEMPTS=20 - for ATTEMPT in $(seq 1 "${MAX_ATTEMPTS}"); do - # Reconstruct every attempt from the latest results branch instead - # of rebasing a commit containing a stale generated index. - git reset --hard HEAD - git clean -ffd - - set +e - git ls-remote --exit-code --heads origin \ - "refs/heads/${RESULTS_BRANCH}" >/dev/null - REMOTE_STATUS=$? - set -e - if [[ "${REMOTE_STATUS}" == 0 ]]; then - git fetch --no-tags origin \ - "+refs/heads/${RESULTS_BRANCH}:refs/remotes/origin/${RESULTS_BRANCH}" - git switch --detach "refs/remotes/origin/${RESULTS_BRANCH}" - elif [[ "${REMOTE_STATUS}" == 2 ]]; then - git switch --orphan \ - "publish-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${ATTEMPT}" - git rm -rf --ignore-unmatch . - else - exit "${REMOTE_STATUS}" + DESTINATION="raw-benchmarks/github-${GITHUB_RUN_ID}-attempt-${BENCHMARK_RUN_ATTEMPT}" + DATA=rocjitsu-dashboard/data + NAME="github-${GITHUB_RUN_ID}-attempt-${BENCHMARK_RUN_ATTEMPT}-default.json" + CATALOG_REFERENCE_FAILED=0 + if [[ "${DASHBOARD_READY}" == true ]]; then + if ! CATALOG="$(jq -er \ + '.testCatalog | select(type == "string") | select(test("^test-catalogs/[A-Za-z0-9._-]+\\.json\\z"))' \ + "${RESULTS_DOWNLOAD_DIR}/dashboard/${NAME}")"; then + echo "Invalid dashboard test catalog reference; publishing raw only." >&2 + CATALOG_REFERENCE_FAILED=1 fi + fi - ( - cd "${CORPUS_SOURCE_DIR}" - python -m benchmarks.dashboard_publish \ - --raw-run "${GITHUB_WORKSPACE}/incoming/run.json" \ - --data-dir "${GITHUB_WORKSPACE}/publication/rocjitsu-dashboard/data" \ - --run-id "${RUN_ID}" \ - --repository "https://github.com/${GITHUB_REPOSITORY}" \ - --environment-id "rocjitsu-benchmark-u2404-4150afe4-rocm-10.1.0a20260812" \ - --expected-sha "${GITHUB_SHA}" \ - --expected-corpus-sha "${CORPUS_SHA}" \ - --machine-id "${BENCHMARK_MACHINE}" \ - --trigger "${TRIGGER}" \ - --branch develop \ - --commit-message "${COMMIT_MESSAGE}" \ - --source-ref "${GITHUB_REF_NAME}" \ - --is-beta - ) - - node "${GITHUB_WORKSPACE}/dashboard/emulation/rocjitsu/website/dashboard/scripts/validate-dashboard-data.mjs" \ - "${GITHUB_WORKSPACE}/publication/rocjitsu-dashboard/data" - - git add rocjitsu-dashboard/data - if git diff --cached --quiet; then - echo "Dashboard run ${RUN_ID} is already published" - exit 0 - fi - git commit --signoff \ - -m "Publish rocjitsu benchmark run ${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}" - if git push origin "HEAD:refs/heads/${RESULTS_BRANCH}"; then - exit 0 - fi - if [[ "${ATTEMPT}" == "${MAX_ATTEMPTS}" ]]; then - exit 1 + # A concurrent writer may advance the publication branch before our push. + # Retry from the latest branch with three attempts, reapplying files and merging the index. + for ATTEMPT in 1 2 3; do + git fetch origin "${RESULTS_BRANCH}" + git checkout --detach FETCH_HEAD + + # Publish the raw benchmark JSON. + mkdir -p "${DESTINATION}" + cp "${RESULTS_DOWNLOAD_DIR}/raw/run.json" "${DESTINATION}/raw.json" + git add -- "${DESTINATION}/raw.json" + + # Recheck the catalog against the latest publication branch. + CATALOG_FAILED="${CATALOG_REFERENCE_FAILED}" + if [[ "${DASHBOARD_READY}" == true && "${CATALOG_FAILED}" -eq 0 ]]; then + CATALOG_PATH="$(realpath -m -- "${DATA}/${CATALOG}")" + if [[ "${CATALOG_PATH}" != "$(realpath -m -- "${DATA}")/"* || ! -f "${CATALOG_PATH}" ]]; then + echo "Required dashboard test catalog is missing or outside the data directory; publishing raw only: ${CATALOG}" >&2 + CATALOG_FAILED=1 + fi + # Publish the dashboard benchmark JSON and update the index.json. + if [[ "${CATALOG_FAILED}" -eq 0 ]]; then + mkdir -p "${DATA}/runs" + cp "${RESULTS_DOWNLOAD_DIR}/dashboard/${NAME}" "${DATA}/runs/${NAME}" + if [[ ! -f "${DATA}/index.json" ]]; then + printf '{"runFiles":[]}\n' > "${DATA}/index.json" + fi + jq --arg file "runs/${NAME}" --arg now "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '.runFiles = ((.runFiles // []) + [$file] | unique) | .generatedAt = $now' \ + "${DATA}/index.json" > "${DATA}/index.json.tmp" + mv "${DATA}/index.json.tmp" "${DATA}/index.json" + git add -- "${DATA}/runs/${NAME}" "${DATA}/index.json" + fi fi - DELAY=$((ATTEMPT * 2)) - if ((DELAY > 30)); then - DELAY=30 + + if [[ "${DASHBOARD_READY}" == true && "${CATALOG_FAILED}" -eq 0 ]] && + ! node "${DASHBOARD_VALIDATOR}" "${DATA}"; then + echo "Dashboard dataset failed validation; publishing raw only." >&2 + # Discard only this attempt's dashboard edits, keeping the raw archive staged. + git restore --source=HEAD --staged --worktree -- "${DATA}" + CATALOG_FAILED=1 fi - sleep "$((DELAY + RANDOM % 5))" + + # Even after a successful raw-only push (or an unchanged archive), + # report catalog or dataset failure so the skipped dashboard update stays visible. + if git diff --cached --quiet; then exit "${CATALOG_FAILED}"; fi + git commit -m "Publish benchmark ${GITHUB_RUN_ID}.${BENCHMARK_RUN_ATTEMPT}" + if git push origin "HEAD:${RESULTS_BRANCH}"; then exit "${CATALOG_FAILED}"; fi + if [[ "${ATTEMPT}" -lt 3 ]]; then sleep 2; fi done + echo "Publication failed after 3 attempts" >&2 + exit 1 diff --git a/.github/workflows/rocjitsu-website-test.yml b/.github/workflows/rocjitsu-website-test.yml index e7316ec8a9af..dc458e713d61 100644 --- a/.github/workflows/rocjitsu-website-test.yml +++ b/.github/workflows/rocjitsu-website-test.yml @@ -6,6 +6,7 @@ on: - "emulation/rocjitsu/website/dashboard/**" - ".github/workflows/rocjitsu-website-test.yml" - ".github/workflows/rocjitsu-publish-website.yml" + - ".github/workflows/rocjitsu-benchmarks.yml" push: branches: - develop @@ -13,6 +14,7 @@ on: - "emulation/rocjitsu/website/dashboard/**" - ".github/workflows/rocjitsu-website-test.yml" - ".github/workflows/rocjitsu-publish-website.yml" + - ".github/workflows/rocjitsu-benchmarks.yml" workflow_dispatch: permissions: @@ -23,6 +25,40 @@ concurrency: cancel-in-progress: true jobs: + dashboard-data: + name: Verify dashboard data tools + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: emulation/rocjitsu/website/dashboard + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: emulation/rocjitsu/website/dashboard/scripts/dashboard-data/requirements-test.txt + + - name: Install test tools + run: | + sudo apt-get update + sudo apt-get install -y bash git jq + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "22.13.0" + + - name: Install Python test dependencies + run: python -m pip install -r scripts/dashboard-data/requirements-test.txt + + - name: Test dashboard data tools + run: python -m pytest scripts/dashboard-data/tests + verify: name: Verify website runs-on: ubuntu-latest diff --git a/emulation/rocjitsu/website/dashboard/docs/build-and-test.md b/emulation/rocjitsu/website/dashboard/docs/build-and-test.md index 29e0f0a117a0..4426ea2035e6 100644 --- a/emulation/rocjitsu/website/dashboard/docs/build-and-test.md +++ b/emulation/rocjitsu/website/dashboard/docs/build-and-test.md @@ -71,6 +71,20 @@ If Chromium's system libraries are already installed, `npx playwright install ch installs just the browser without changing OS packages. Rerun browser installation after updating Playwright if its required browser version changes. +## Dashboard-data Python tests + +Use a Python 3.12 environment with Node.js (as above), Bash, Git, jq, and coreutils +(including `realpath`) on `PATH`. Publication tests use the real dashboard +validator; no npm dependencies are required. From `emulation/rocjitsu/website/dashboard`: + +```bash +python -m pip install -r scripts/dashboard-data/requirements-test.txt +python -m pytest scripts/dashboard-data/tests +``` + +The `dashboard-data` CI job uses the same requirements file. These tests are +separate from `npm run verify` and do not require the npm dependencies. + ## Run locally ```bash diff --git a/emulation/rocjitsu/website/dashboard/docs/website-data-contract.md b/emulation/rocjitsu/website/dashboard/docs/website-data-contract.md index 46faf83c3362..127ff017181a 100644 --- a/emulation/rocjitsu/website/dashboard/docs/website-data-contract.md +++ b/emulation/rocjitsu/website/dashboard/docs/website-data-contract.md @@ -284,6 +284,14 @@ Environments may change between independent historical runs. Runs sharing a `comparisonId` must have the same normalized environment because plugin overhead is valid only when the compared executions use the same environment. +Benchmark workflow runs include `workflow.commit_sha` (label **Workflow commit**) +and `tensile.commit_sha` (label **Tensile commit**) for the workflow-tools revision +and pinned Tensile source revision. These are separate from the measured Rocjitsu +`source.commit`. Their key/value pairs participate in the existing environment +identity: different workflow or Tensile revisions intentionally distinguish +comparison environments. Older runs may omit these details; do not rewrite +historical run files to add them. + ### `targets` and results `targets` must contain exactly one group for every target defined by the referenced catalog. Each group contains: @@ -299,6 +307,12 @@ Every result contains: - `durationSeconds`: positive numeric seconds for a completed result; `null` for failed or timed-out results. - `status`: required; `completed`, `failed`, or `timeout`. - `error`: failure description or `null`. +- `timing_results_s`: optional array of individual timing samples in seconds, in + input measurement order (not sorted). The benchmark converter preserves these + samples and uses their median for a completed result's `durationSeconds`. Failed + and timed-out results keep a `null` duration, even if samples are present. The + current validator accepts this extra field, but normalization omits it from the + UI model; the dashboard uses `durationSeconds`, not the sample vector. Failed and timed-out tests must remain in `results`. Omitting a catalog result invalidates the run instead of silently changing its denominator. diff --git a/emulation/rocjitsu/website/dashboard/scripts/dashboard-data/prepare-benchmark-result.py b/emulation/rocjitsu/website/dashboard/scripts/dashboard-data/prepare-benchmark-result.py new file mode 100644 index 000000000000..b5746099f9d5 --- /dev/null +++ b/emulation/rocjitsu/website/dashboard/scripts/dashboard-data/prepare-benchmark-result.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +"""Convert raw benchmark JSON to one default-policy dashboard run JSON. + +Existing dashboard catalogs are referenced, not created or published here. +""" + +import argparse +from datetime import datetime +import hashlib +import json +import math +from pathlib import Path +import statistics + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def encode_json_bytes(value): + return (json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + '\n').encode( + 'utf-8' + ) + + +def parse_timestamp(value): + return datetime.fromisoformat(value.replace('Z', '+00:00')) + + +def build_published_run(raw, workflow): + github = { + 'run_id': workflow['run_id'], + 'run_attempt': workflow['run_attempt'], + 'trigger': workflow['trigger'], + } + source = { + 'commit': workflow['source_sha'], + 'commit_subject': workflow['source_subject'], + 'repository_url': 'https://github.com/' + workflow['repository'], + } + summary = raw['execution_summary'] + require(summary['finished_at'] is not None, 'Cannot convert unfinished checkpoint') + require( + parse_timestamp(summary['finished_at']) + >= parse_timestamp(summary['timestamp']), + 'Finish precedes start', + ) + require( + type(summary['wall_time_s']) in (int, float) + and math.isfinite(summary['wall_time_s']) + and summary['wall_time_s'] >= 0, + 'Invalid wall time', + ) + committed = raw['provenance']['rocjitsu']['rocjitsu_commit_timestamp'] + require( + parse_timestamp(committed) <= parse_timestamp(summary['timestamp']), + 'Source commit is later than benchmark start', + ) + samples = summary['samples'] + require( + type(samples) is int and samples > 0 and samples % 2 == 1, + 'Invalid sample count', + ) + selected = [ + result + for result in raw['benchmark_results'] + if result.get('thread_policy') == 'default' + ] + identities = [(result['id'], result['target']) for result in selected] + require( + identities and len(identities) == len(set(identities)), + 'Missing or duplicate default-policy results', + ) + definitions, targets = {}, {} + for result in selected: + problem = dict(result['problem']) + require( + 'operation' not in problem or problem['operation'] == result['operation'], + 'Operation metadata collision', + ) + problem['operation'] = result['operation'] + # The dashboard data contract requires scalar problem values. Serialize + # containers and null as JSON strings to preserve their contents. + problem = { + key: ( + value + if type(value) in (str, int, float, bool) + else json.dumps(value, sort_keys=True, allow_nan=False) + ) + for key, value in problem.items() + } + definition = { + 'id': result['id'], + 'suite': result['suite'], + 'name': result['name'], + 'problem': problem, + } + require( + result['id'] not in definitions or definitions[result['id']] == definition, + 'Conflicting workload identity', + ) + definitions[result['id']] = definition + timings = result['timing_results_s'] + require( + isinstance(timings, list) + and all( + type(t) in (int, float) and math.isfinite(t) and t > 0 for t in timings + ), + 'Invalid timing samples', + ) + status = result['status'] + require(status in ('completed', 'failed', 'timeout'), 'Invalid result status') + require( + result['error'] is None + or isinstance(result['error'], str) + and result['error'].strip(), + 'Invalid result error', + ) + if status == 'completed': + require( + len(timings) == samples + and result['error'] is None + and result['exit_code'] in (0, None), + 'Invalid completed measurement', + ) + targets.setdefault(result['target'], []).append( + { + 'testId': result['id'], + 'status': status, + 'durationSeconds': ( + statistics.median(timings) if status == 'completed' else None + ), + 'error': result['error'], + 'timing_results_s': timings, + } + ) + catalog = { + 'tests': [definitions[key] for key in sorted(definitions)], + 'targets': { + target: sorted(result['testId'] for result in results) + for target, results in sorted(targets.items()) + }, + } + catalog['id'] = 'default-' + hashlib.sha256(encode_json_bytes(catalog)).hexdigest() + run_id = f"github-{github['run_id']}-attempt-{github['run_attempt']}-default" + details = {'thread_policy': 'default'} + for section, values in raw['provenance'].items(): + for key, value in values.items(): + if section == 'rocjitsu' and key in ( + 'rocjitsu_commit_sha', + 'rocjitsu_commit_timestamp', + ): + continue + details[f'{section}.{key}'] = ( + value + if type(value) in (str, int, float, bool) and value != '' + else json.dumps(value, sort_keys=True, allow_nan=False) + ) + details['workflow.commit_sha'] = workflow['workflow_sha'] + details['tensile.commit_sha'] = workflow['tensile_sha'] + for key in ('warmups', 'samples', 'timeout_seconds', 'benchmark_suite'): + details['execution.' + key] = summary[key] + labels = { + 'thread_policy': 'Thread policy', + 'workflow.commit_sha': 'Workflow commit', + 'tensile.commit_sha': 'Tensile commit', + 'machine.hostname': 'Hostname', + 'machine.platform': 'Platform', + 'machine.kernel': 'Kernel', + 'machine.cpu': 'CPU', + 'rocjitsu.rocm_sdk_version': 'ROCm SDK', + 'corpus.corpus_commit_sha': 'Corpus commit', + 'corpus.corpus_commit_timestamp': 'Corpus commit timestamp', + 'auxiliary.python': 'Python', + 'execution.warmups': 'Warmups', + 'execution.samples': 'Samples', + 'execution.timeout_seconds': 'Timeout (seconds)', + 'execution.benchmark_suite': 'Benchmark suite', + } + run = { + 'id': run_id, + 'comparisonId': f"github-{github['run_id']}-attempt-{github['run_attempt']}", + 'testCatalog': f"test-catalogs/{catalog['id']}.json", + 'plugin': {'id': 'vanilla', 'name': 'Vanilla'}, + 'source': { + # The workflow requires the measured commit to be in develop history. + # This is its source branch, not the workflow dispatch branch. + 'branch': 'develop', + 'commit': source['commit'], + 'committedAt': committed, + 'message': source['commit_subject'], + }, + 'execution': { + 'completedAt': summary['finished_at'], + 'trigger': github['trigger'], + 'machine': workflow['runner'], + }, + 'environment': [ + { + 'key': key, + 'label': labels.get( + key, + key.replace('.', ' ').replace('_', ' ').replace('-', ' ').title(), + ), + 'value': value, + } + for key, value in sorted(details.items()) + ], + 'targets': [ + {'id': target, 'results': sorted(results, key=lambda r: r['testId'])} + for target, results in sorted(targets.items()) + ], + } + return run + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--raw-run', required=True, type=Path) + parser.add_argument('--output', required=True, type=Path) + parser.add_argument('--run-id', required=True) + parser.add_argument('--run-attempt', required=True) + parser.add_argument('--repository', required=True) + parser.add_argument('--commit-message', required=True) + parser.add_argument('--machine', required=True) + parser.add_argument('--workflow-sha', required=True) + parser.add_argument('--tensile-sha', required=True) + parser.add_argument('--event', choices=('push', 'workflow_dispatch'), required=True) + args = parser.parse_args() + raw = json.loads(args.raw_run.read_text()) + workflow = { + 'run_id': args.run_id, + 'run_attempt': args.run_attempt, + 'repository': args.repository, + 'source_subject': args.commit_message, + 'source_sha': raw['provenance']['rocjitsu']['rocjitsu_commit_sha'], + 'runner': args.machine, + 'workflow_sha': args.workflow_sha, + 'tensile_sha': args.tensile_sha, + 'trigger': 'manual' if args.event == 'workflow_dispatch' else 'auto', + } + result = build_published_run(raw, workflow) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_bytes(encode_json_bytes(result)) + + +if __name__ == '__main__': + main() diff --git a/emulation/rocjitsu/website/dashboard/scripts/dashboard-data/requirements-test.txt b/emulation/rocjitsu/website/dashboard/scripts/dashboard-data/requirements-test.txt new file mode 100644 index 000000000000..f162d3925084 --- /dev/null +++ b/emulation/rocjitsu/website/dashboard/scripts/dashboard-data/requirements-test.txt @@ -0,0 +1,2 @@ +pytest==9.1.1 +PyYAML==6.0.3 diff --git a/emulation/rocjitsu/website/dashboard/scripts/dashboard-data/tests/test_benchmark_result.py b/emulation/rocjitsu/website/dashboard/scripts/dashboard-data/tests/test_benchmark_result.py new file mode 100644 index 000000000000..eb641f63431d --- /dev/null +++ b/emulation/rocjitsu/website/dashboard/scripts/dashboard-data/tests/test_benchmark_result.py @@ -0,0 +1,976 @@ +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT +"""Fixture-based conversion and local file publication tests; no remote writes.""" + +import json +import os +from pathlib import Path +import shlex +import subprocess +import sys + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[7] +WORKFLOW = REPO_ROOT / '.github/workflows/rocjitsu-benchmarks.yml' +HELPER = Path(__file__).resolve().parents[1] / 'prepare-benchmark-result.py' +SOURCE_SHA = 'b' * 40 +WORKFLOW_SHA = 'c' * 40 +TENSILE_SHA = 'd' * 40 +CORPUS_SHA = '6186d5c8bcd5e9bc93f7406e7cb50ed2a7698621' +VALIDATOR = HELPER.parents[1] / 'validate-dashboard-data.mjs' +SOURCE_SUBJECT = 'Source "subject" with spaces = literal $(touch SHOULD_NOT_EXIST)' + + +def raw_run(): + result = { + 'id': 'fixture.case', + 'suite': 'Fixture', + 'name': 'Fixture case', + 'operation': 'copy', + 'target': 'gfx950', + 'thread_policy': 'default', + 'problem': {'dtype': 'fp32', 'n': 64}, + 'status': 'completed', + 'exit_code': 0, + 'error': None, + 'timing_results_s': [0.3, 0.1, 0.2], + } + return { + 'execution_summary': { + 'timestamp': '2026-10-01T00:00:00Z', + 'finished_at': '2026-10-01T00:01:00Z', + 'wall_time_s': 59.25, + 'warmups': 1, + 'samples': 3, + 'timeout_seconds': 60, + 'benchmark_suite': 'fixture-suite', + }, + 'provenance': { + 'rocjitsu': { + 'rocjitsu_commit_sha': SOURCE_SHA, + 'rocjitsu_commit_timestamp': '2026-09-30T00:00:00Z', + }, + 'corpus': {'corpus_commit_sha': CORPUS_SHA}, + 'machine': {'hostname': 'fixture-host'}, + 'auxiliary': {'optional': None}, + }, + 'benchmark_results': [ + result, + {**result, 'thread_policy': 'single', 'timing_results_s': [9.0, 8.0, 7.0]}, + ], + } + + +def convert( + tmp_path, attempt='1', raw=None, workflow_sha=WORKFLOW_SHA, tensile_sha=TENSILE_SHA +): + source = tmp_path / 'raw.json' + # Unusual indentation and trailing whitespace detect raw JSON reserialization. + # Publication must archive these exact bytes, not just equivalent JSON. + original = ( + json.dumps(raw_run() if raw is None else raw, indent=1) + '\n ' + ).encode() + source.write_bytes(original) + output = tmp_path / f'github-123-attempt-{attempt}-default.json' + command = [ + sys.executable, + str(HELPER), + '--raw-run', + str(source), + '--output', + str(output), + '--run-id', + '123', + '--run-attempt', + attempt, + '--repository', + 'ROCm/rocm-systems', + '--event', + 'workflow_dispatch', + '--commit-message', + SOURCE_SUBJECT, + '--machine', + 'benchmark machine', + '--workflow-sha', + workflow_sha, + '--tensile-sha', + tensile_sha, + ] + result = subprocess.run(command, capture_output=True, text=True) + assert source.read_bytes() == original + return result, output, original + + +@pytest.fixture +def publication_catalog(): + return { + # Independently provisioned for raw_run: default- plus SHA256 of the + # catalog body without id, using the converter's JSON encoding. + 'id': 'default-67daf36efa1cd84a9f2beb1f0fa30860edbecf572f2b733c86642cdbbabea8cd', + 'tests': [ + { + 'id': 'fixture.case', + 'suite': 'Fixture', + 'name': 'Fixture case', + 'problem': {'dtype': 'fp32', 'n': 64, 'operation': 'copy'}, + } + ], + 'targets': {'gfx950': ['fixture.case']}, + } + + +def test_conversion(tmp_path): + result, output, _ = convert(tmp_path) + assert result.returncode == 0, result.stderr + run = json.loads(output.read_text()) + assert output.stem == run['id'] + assert run['targets'][0]['results'][0]['durationSeconds'] == 0.2 + assert len(run['targets'][0]['results']) == 1 + assert run['source']['message'] == SOURCE_SUBJECT + assert ( + run['execution']['completedAt'] == raw_run()['execution_summary']['finished_at'] + ) + assert run['testCatalog'].startswith('test-catalogs/default-') + assert {p.name for p in tmp_path.iterdir()} == {'raw.json', output.name} + + +def test_environment_labels_in_converter_output(tmp_path): + raw = raw_run() + raw['provenance']['auxiliary']['custom_option'] = {'nested': [1, None]} + result, output, _ = convert(tmp_path, raw=raw) + assert result.returncode == 0, result.stderr + run = json.loads(output.read_text()) + details = {detail['key']: detail for detail in run['environment']} + assert details['thread_policy']['label'] == 'Thread policy' + assert details['corpus.corpus_commit_sha']['label'] == 'Corpus commit' + assert details['execution.warmups']['label'] == 'Warmups' + assert details['machine.hostname']['label'] == 'Hostname' + assert details['auxiliary.custom_option']['label'] == 'Auxiliary Custom Option' + assert details['auxiliary.custom_option']['value'] == '{"nested": [1, null]}' + assert list(details) == sorted(details) + + assert [(detail['key'], detail['value']) for detail in run['environment']] == [ + ('auxiliary.custom_option', '{"nested": [1, null]}'), + ('auxiliary.optional', 'null'), + ('corpus.corpus_commit_sha', CORPUS_SHA), + ('execution.benchmark_suite', 'fixture-suite'), + ('execution.samples', 3), + ('execution.timeout_seconds', 60), + ('execution.warmups', 1), + ('machine.hostname', 'fixture-host'), + ('tensile.commit_sha', TENSILE_SHA), + ('thread_policy', 'default'), + ('workflow.commit_sha', WORKFLOW_SHA), + ] + assert run['targets'][0]['results'][0]['timing_results_s'] == [0.3, 0.1, 0.2] + + +def test_converter_revision_metadata(tmp_path): + first, output, original = convert(tmp_path) + assert first.returncode == 0, first.stderr + run = json.loads(output.read_text()) + details = {detail['key']: detail for detail in run['environment']} + assert details['workflow.commit_sha'] == { + 'key': 'workflow.commit_sha', + 'label': 'Workflow commit', + 'value': WORKFLOW_SHA, + } + assert details['tensile.commit_sha'] == { + 'key': 'tensile.commit_sha', + 'label': 'Tensile commit', + 'value': TENSILE_SHA, + } + assert len({run['source']['commit'], WORKFLOW_SHA, TENSILE_SHA}) == 3 + assert (tmp_path / 'raw.json').read_bytes() == original + second, changed_output, _ = convert( + tmp_path, + workflow_sha='e' * 40, + tensile_sha='f' * 40, + ) + assert second.returncode == 0, second.stderr + changed = json.loads(changed_output.read_text()) + for key in ('id', 'comparisonId', 'testCatalog', 'source', 'execution', 'targets'): + assert changed[key] == run[key] + changed_details = {detail['key']: detail for detail in changed['environment']} + assert changed_details['workflow.commit_sha']['value'] == 'e' * 40 + assert changed_details['tensile.commit_sha']['value'] == 'f' * 40 + for key in details.keys() - {'workflow.commit_sha', 'tensile.commit_sha'}: + assert changed_details[key] == details[key] + + +def test_attempt_identity(tmp_path): + first, a, _ = convert(tmp_path) + second, b, _ = convert(tmp_path, '2') + assert first.returncode == second.returncode == 0 + assert json.loads(a.read_text())['id'] != json.loads(b.read_text())['id'] + + +def test_invalid_input(tmp_path): + raw = raw_run() + raw['benchmark_results'][0]['timing_results_s'] = [-1] + result, output, _ = convert(tmp_path, raw=raw) + assert result.returncode != 0 + assert not output.exists() + + +def test_dashboard_data_ci(): + doc = yaml.safe_load( + (REPO_ROOT / '.github/workflows/rocjitsu-website-test.yml').read_text() + ) + # Covers path filters and ordinary pytest wiring, not workflow execution. + # PyYAML safe_load treats the unquoted YAML key on as boolean True. + triggers = doc['on' if 'on' in doc else True] + for event in ('pull_request', 'push'): + assert '.github/workflows/rocjitsu-benchmarks.yml' in triggers[event]['paths'] + assert any( + shlex.split(step.get('run', '')) + == ['python', '-m', 'pytest', 'scripts/dashboard-data/tests'] + for step in doc['jobs']['dashboard-data']['steps'] + ) + + +@pytest.mark.parametrize( + 'status,timings,duration', + [ + ('completed', [0.3, 0.1, 0.2], 0.2), + ('failed', [0.3], None), + ('timeout', [], None), + ], +) +def test_converter_result_status(tmp_path, status, timings, duration): + raw = raw_run() + measurement = raw['benchmark_results'][0] + measurement.update( + status=status, + timing_results_s=timings, + error=None if status == 'completed' else 'fixture workload failure', + exit_code=0 if status == 'completed' else None, + ) + result, output, _ = convert(tmp_path, raw=raw) + assert result.returncode == 0, result.stderr + published = json.loads(output.read_text())['targets'][0]['results'][0] + assert published['status'] == status + assert published['durationSeconds'] == duration + assert published['error'] == measurement['error'] + assert published['timing_results_s'] == timings + + +def test_workload_failure_diagnostics(tmp_path): + benchmark = yaml.safe_load(WORKFLOW.read_text())['jobs']['benchmark'] + logs = next( + step + for step in benchmark['steps'] + if step['name'] == 'Upload workload failure diagnostics' + ) + assert logs['if'] == ( + "${{ !cancelled() && (steps.benchmark.outcome == 'failure' " + "|| steps.benchmark.outputs.case_failures == 'true') }}" + ) + assert logs['uses'].startswith('actions/upload-artifact@') + assert logs['with'] == { + 'name': '${{ env.RESULTS_ARTIFACT }}-diagnostics', + 'path': ( + '${{ env.ROCJITSU_RESULT_DIR }}/*/*/cases/*/stdout.txt\n' + '${{ env.ROCJITSU_RESULT_DIR }}/*/*/cases/*/stderr.txt\n' + ), + 'if-no-files-found': 'ignore', + 'retention-days': 14, + } + # Match only current-run workload streams, not setup logs or other runs. + paths = [ + 'current/gfx950/default/cases/fixture.case/stdout.txt', + 'current/gfx1250/single/cases/fixture.case/stderr.txt', + 'current/gfx950/default/cases/fixture.case/config.json', + 'current/setup/stderr.txt', + 'older/gfx950/default/cases/fixture.case/stderr.txt', + ] + for path in paths: + file = tmp_path / path + file.parent.mkdir(parents=True, exist_ok=True) + file.touch() + matched = { + str(file.relative_to(tmp_path)) + for pattern in logs['with']['path'].splitlines() + for file in (tmp_path / 'current').glob( + pattern.removeprefix('${{ env.ROCJITSU_RESULT_DIR }}/') + ) + } + assert matched == set(paths[:2]) + + +def steps(): + return yaml.safe_load(WORKFLOW.read_text())['jobs']['publish']['steps'] + + +@pytest.mark.parametrize('existing', [False, True]) +def test_copy_and_index_preserve_history(tmp_path, existing, publication_catalog): + result, output, original = convert(tmp_path) + assert result.returncode == 0, result.stderr + download = tmp_path / 'download' + (download / 'raw').mkdir(parents=True) + (download / 'dashboard').mkdir() + (download / 'raw/run.json').write_bytes(original) + (download / 'dashboard' / output.name).write_bytes(output.read_bytes()) + publication = tmp_path / 'publication' + publication.mkdir() + subprocess.run(['git', 'init', '-q', str(publication)], check=True) + (publication / 'unrelated.txt').write_text('preserved') + data = publication / 'rocjitsu-dashboard/data' + catalog = data / 'test-catalogs' / (publication_catalog['id'] + '.json') + catalog.parent.mkdir(parents=True) + catalog.write_text(json.dumps(publication_catalog)) + assert json.loads(output.read_text())['testCatalog'] == str( + catalog.relative_to(data) + ) + if existing: + (data / 'index.json').write_text( + json.dumps({'runFiles': ['runs/older.json'], 'custom': 'keep'}) + ) + env = { + **os.environ, + 'RESULTS_DOWNLOAD_DIR': str(download), + 'GITHUB_RUN_ID': '123', + 'GITHUB_RUN_ATTEMPT': '1', + 'BENCHMARK_RUN_ATTEMPT': '1', + } + body = next( + step['run'] for step in steps() if step['name'] == 'Publish benchmark files' + ) + setup_body = body[body.index('DESTINATION=') : body.index('for ATTEMPT in 1 2 3')] + copy_body = ( + setup_body + + body[ + body.index(' mkdir -p "${DESTINATION}"') : body.index( + ' if [[ "${DASHBOARD_READY}" == true && "${CATALOG_FAILED}" -eq 0 ]] &&' + ) + ] + ) + copies = [{'run': copy_body}] + env['DASHBOARD_READY'] = 'true' + for _ in range(2): + for step in copies: + run = subprocess.run( + ['bash', '-euo', 'pipefail', '-c', step['run']], + cwd=publication, + env=env, + capture_output=True, + text=True, + ) + assert run.returncode == 0, run.stderr + index = json.loads((data / 'index.json').read_text()) + assert index['runFiles'].count('runs/' + output.name) == 1 + if existing: + assert 'runs/older.json' in index['runFiles'] + assert index['custom'] == 'keep' + assert (data / 'runs' / output.name).read_bytes() == output.read_bytes() + assert ( + publication / 'raw-benchmarks/github-123-attempt-1/raw.json' + ).read_bytes() == original + assert (publication / 'unrelated.txt').read_text() == 'preserved' + assert json.loads(catalog.read_text()) == publication_catalog + assert not (data / 'metadata.json').exists() + + +@pytest.mark.parametrize( + 'catalog_case', + [ + 'present', + 'missing', + 'traversal', + 'absolute', + 'symlink', + 'directory', + 'not-string', + 'newline', + ], +) +def test_publication_catalog_path(tmp_path, publication_catalog, catalog_case): + result, output, _ = convert(tmp_path) + assert result.returncode == 0, result.stderr + run = json.loads(output.read_text()) + publication = tmp_path / 'publication' + publication.mkdir() + subprocess.run(['git', 'init', '-q', str(publication)], check=True) + data = publication / 'rocjitsu-dashboard/data' + catalog = data / run['testCatalog'] + catalog.parent.mkdir(parents=True) + outside = publication / 'rocjitsu-dashboard/outside.json' + outside.write_text(json.dumps(publication_catalog)) + if catalog_case == 'directory': + catalog.mkdir() + elif catalog_case == 'symlink': + catalog.symlink_to(outside) + elif catalog_case != 'missing': + catalog.write_text(json.dumps(publication_catalog)) + if catalog_case == 'traversal': + run['testCatalog'] = '../outside.json' + elif catalog_case == 'absolute': + run['testCatalog'] = str(outside) + elif catalog_case == 'not-string': + run['testCatalog'] = 123 + (data / '123').write_text(json.dumps(publication_catalog)) + elif catalog_case == 'newline': + run['testCatalog'] += '\n' + original_index = b'{"runFiles":["runs/older.json"],"custom":"keep"}\n' + (data / 'index.json').write_bytes(original_index) + download = tmp_path / 'download/dashboard' + download.mkdir(parents=True) + (download / output.name).write_text(json.dumps(run)) + body = next( + step['run'] for step in steps() if step['name'] == 'Publish benchmark files' + ) + dashboard_setup = body[ + body.index('DATA=rocjitsu-dashboard/data') : body.index('for ATTEMPT in 1 2 3') + ] + dashboard_body = ( + dashboard_setup + + body[ + body.index(' CATALOG_FAILED=') : body.index( + ' if [[ "${DASHBOARD_READY}" == true && "${CATALOG_FAILED}" -eq 0 ]] &&' + ) + ] + ) + result = subprocess.run( + [ + 'bash', + '-euo', + 'pipefail', + '-c', + dashboard_body + '\nexit "${CATALOG_FAILED}"\n', + ], + cwd=publication, + env={ + **os.environ, + 'DASHBOARD_READY': 'true', + 'RESULTS_DOWNLOAD_DIR': str(download.parent), + 'GITHUB_RUN_ID': '123', + 'GITHUB_RUN_ATTEMPT': '1', + 'BENCHMARK_RUN_ATTEMPT': '1', + }, + capture_output=True, + text=True, + ) + if catalog_case == 'present': + assert result.returncode == 0, result.stderr + assert ( + 'runs/' + output.name + in json.loads((data / 'index.json').read_text())['runFiles'] + ) + else: + assert result.returncode != 0, result.stdout + result.stderr + assert (data / 'index.json').read_bytes() == original_index + assert not (data / 'runs').exists() + + +def test_benchmark_artifact_contract(): + doc = yaml.safe_load(WORKFLOW.read_text()) + benchmark = {step['name']: step for step in doc['jobs']['benchmark']['steps']} + uploads = { + step['name'] + for step in benchmark.values() + if step.get('uses', '').startswith('actions/upload-artifact@') + } + assert uploads == { + 'Upload original raw benchmark JSON', + 'Upload dashboard benchmark JSON', + 'Upload workload failure diagnostics', + } + raw = benchmark['Upload original raw benchmark JSON'] + assert raw['if'] == ( + "${{ !cancelled() && (steps.benchmark.outcome == 'success' " + "|| steps.benchmark.outcome == 'failure') }}" + ) + assert raw['with'] == { + 'name': '${{ env.RESULTS_ARTIFACT }}-raw', + 'path': '${{ env.ROCJITSU_RESULT_DIR }}/run.json', + 'if-no-files-found': 'error', + 'retention-days': 14, + } + dashboard = benchmark['Upload dashboard benchmark JSON'] + assert ( + dashboard['if'] == "${{ !cancelled() && steps.dashboard.outcome == 'success' }}" + ) + assert dashboard['with'] == { + 'name': '${{ env.RESULTS_ARTIFACT }}-dashboard', + 'path': '${{ env.BENCHMARK_DIR }}/dashboard/*.json', + 'if-no-files-found': 'error', + 'retention-days': 14, + } + converter = benchmark['Prepare dashboard benchmark result'] + assert ( + converter['if'] == "${{ !cancelled() && steps.benchmark.outcome == 'success' }}" + ) + assert converter['env'] == { + 'BENCHMARK_COMMIT_MESSAGE': '${{ steps.metadata.outputs.commit_message }}', + 'BENCHMARK_MACHINE': '${{ steps.metadata.outputs.machine }}', + 'BENCHMARK_WORKFLOW_SHA': '${{ github.sha }}', + } + assert doc['jobs']['benchmark']['outputs'] == { + 'run_attempt': '${{ steps.metadata.outputs.run_attempt }}', + 'raw_ready': "${{ steps.upload_raw.outcome == 'success' }}", + 'dashboard_ready': "${{ steps.upload_dashboard.outcome == 'success' }}", + } + downloads = { + step['name']: step + for step in doc['jobs']['publish']['steps'] + if step.get('uses', '').startswith('actions/download-artifact@') + } + assert set(downloads) == {'Download original raw JSON', 'Download dashboard JSON'} + assert downloads['Download original raw JSON']['with'] == { + 'name': '${{ env.RESULTS_ARTIFACT }}-raw', + 'path': '${{ env.RESULTS_DOWNLOAD_DIR }}/raw', + } + assert downloads['Download dashboard JSON']['with'] == { + 'name': '${{ env.RESULTS_ARTIFACT }}-dashboard', + 'path': '${{ env.RESULTS_DOWNLOAD_DIR }}/dashboard', + } + assert downloads['Download dashboard JSON']['if'] == ( + "${{ needs.benchmark.outputs.dashboard_ready == 'true' }}" + ) + assert doc['jobs']['publish']['env'] == { + 'BENCHMARK_RUN_ATTEMPT': '${{ needs.benchmark.outputs.run_attempt }}', + 'RESULTS_ARTIFACT': ( + 'rocjitsu-benchmark-${{ github.run_id }}-' + '${{ needs.benchmark.outputs.run_attempt }}' + ), + } + metadata = benchmark['Record benchmark metadata']['run'] + assert "printf 'run_attempt=%s\\n' \"${GITHUB_RUN_ATTEMPT}\"" in metadata + publication = {step['name']: step for step in steps()} + validator_checkout = publication['Checkout publication validator'] + assert validator_checkout['with']['ref'] == '${{ github.sha }}' + assert validator_checkout['with']['path'] == 'workflow-source' + assert validator_checkout['with']['persist-credentials'] is False + assert validator_checkout['with']['sparse-checkout'].splitlines() == [ + 'emulation/rocjitsu/website/dashboard/scripts', + 'emulation/rocjitsu/website/dashboard/src/data', + ] + assert validator_checkout['if'] == downloads['Download dashboard JSON']['if'] + assert publication['Set up Node.js']['if'] == validator_checkout['if'] + assert publication['Set up Node.js']['with']['node-version'] == '22.13.0' + assert publication['Publish benchmark files']['env']['DASHBOARD_VALIDATOR'] == ( + '${{ github.workspace }}/workflow-source/emulation/rocjitsu/website/dashboard/' + 'scripts/validate-dashboard-data.mjs' + ) + + +@pytest.mark.parametrize( + 'symlinked,artifact,expected_exit', + [(False, 'run.json', 0), (True, 'run.json', 0), (False, 'wrong.json', 1)], +) +def test_completion_marker_output_path(tmp_path, symlinked, artifact, expected_exit): + workspace = tmp_path / 'real workspace' + results = workspace / 'results' + results.mkdir(parents=True) + (results / 'run.json').write_text('{}') + if symlinked: + link = tmp_path / 'workspace link' + link.symlink_to(workspace, target_is_directory=True) + selected_results = link / 'results' + else: + selected_results = results + # Match the pinned runner's resolved artifact path without running a runner. + (tmp_path / 'runner-output.txt').write_text( + f'run failed: {selected_results.resolve()}/{artifact}\n' + ) + body = next( + step['run'] + for step in yaml.safe_load(WORKFLOW.read_text())['jobs']['benchmark']['steps'] + if step['name'] == 'Run performance suite' + ) + comparison = body.split('esac\n', 1)[1].split('if ! python -m json.tool', 1)[0] + result = subprocess.run( + ['bash', '-euo', 'pipefail', '-c', comparison], + env={ + **os.environ, + 'BENCHMARK_DIR': str(tmp_path), + 'ROCJITSU_RESULT_DIR': str(selected_results), + 'SUITE_STATUS': 'failed', + }, + capture_output=True, + text=True, + ) + assert result.returncode == expected_exit, result.stderr + + +def test_workflow_shell_and_source_contracts(): + doc = yaml.safe_load(WORKFLOW.read_text()) + for job in doc['jobs'].values(): + for step in job['steps']: + if 'run' in step: + result = subprocess.run( + ['bash', '-n'], input=step['run'], text=True, capture_output=True + ) + assert result.returncode == 0, result.stderr + assert doc['jobs']['benchmark']['if'] == "${{ github.ref_type == 'branch' }}" + assert doc['jobs']['publish']['if'] == ( + "${{ !cancelled() && github.ref == 'refs/heads/develop' " + "&& needs.benchmark.outputs.raw_ready == 'true' }}" + ) + # PyYAML safe_load treats the unquoted YAML key on as boolean True. + assert doc['on' if 'on' in doc else True]['workflow_dispatch']['inputs'][ + 'commit' + ] == { + 'description': 'Rocjitsu commit SHA to benchmark', + 'required': True, + 'type': 'string', + } + assert doc['env']['ROCJITSU_SHA'] == '${{ inputs.commit || github.sha }}' + benchmark = {step['name']: step for step in doc['jobs']['benchmark']['steps']} + assert ( + benchmark['Checkout rocm-systems']['with']['ref'] == '${{ env.ROCJITSU_SHA }}' + ) + assert benchmark['Checkout workflow tools']['with']['ref'] == '${{ github.sha }}' + assert '/workflow-source/' in benchmark['Prepare dashboard benchmark result']['run'] + + +@pytest.mark.parametrize('publish_attempt', ['1', '2']) +@pytest.mark.parametrize( + 'always_fail,dashboard_ready,catalog_state', + [ + (False, True, 'present'), + (False, False, 'missing'), + (True, True, 'present'), + (False, True, 'missing'), + (False, True, 'removed-on-race'), + (False, True, 'added-on-race'), + (False, True, 'archived-missing'), + (False, True, 'archived-existing-run'), + (False, True, 'invalid'), + (False, True, 'incompatible'), + (False, True, 'incompatible-on-race'), + (False, True, 'archived-incompatible'), + ], +) +def test_publication_retry( + tmp_path, + always_fail, + dashboard_ready, + catalog_state, + publication_catalog, + publish_attempt, +): + # All Git operations are local. The git wrapper intercepts push before Git + # can execute it and simulates a branch race using local fetch/update-ref. + import shutil + + real_git = shutil.which('git') + assert real_git is not None + seed = tmp_path / 'origin' + publication = tmp_path / 'publication' + + def git(cwd, *args): + return subprocess.run( + [real_git, '-C', str(cwd), *args], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + result, output, original = convert(tmp_path) + assert result.returncode == 0, result.stderr + seed.mkdir() + git(seed, 'init', '-q', '-b', 'gh-pages-rocjitsu') + git(seed, 'config', 'user.name', 'Fixture') + git(seed, 'config', 'user.email', 'fixture@example.invalid') + git(seed, 'config', '--local', 'commit.gpgsign', 'false') + data = seed / 'rocjitsu-dashboard/data' + data.mkdir(parents=True) + (data / 'metadata.json').write_text( + json.dumps( + { + 'schemaVersion': 1, + 'repository': 'https://github.com/ROCm/rocm-systems', + 'isBeta': True, + } + ) + ) + (data / 'runs').mkdir() + for name in ('older', 'concurrent'): + run = json.loads(output.read_text()) + run['id'] = name + run['comparisonId'] = name + if catalog_state in ('incompatible', 'archived-incompatible'): + run['execution']['machine'] = 'another machine' + (data / 'runs' / f'{name}.json').write_text(json.dumps(run)) + catalog = data / 'test-catalogs' / (publication_catalog['id'] + '.json') + if catalog_state in ( + 'present', + 'removed-on-race', + 'incompatible', + 'incompatible-on-race', + 'archived-incompatible', + ): + catalog.parent.mkdir() + catalog.write_text(json.dumps(publication_catalog)) + (data / 'index.json').write_text( + json.dumps( + { + 'runFiles': ['runs/older.json'], + 'custom': 'keep', + 'generatedAt': '2026-10-01T00:01:00Z', + } + ) + ) + if catalog_state.startswith('archived-'): + archive = seed / 'raw-benchmarks/github-123-attempt-1/raw.json' + archive.parent.mkdir(parents=True) + archive.write_bytes(original) + if catalog_state == 'archived-existing-run': + previous_run = data / 'runs' / output.name + previous_run.parent.mkdir(exist_ok=True) + previous_run.write_text('preserve existing dashboard bytes\n') + (data / 'index.json').write_text( + json.dumps( + { + 'runFiles': ['runs/older.json', 'runs/' + output.name], + 'custom': 'keep', + } + ) + ) + git(seed, 'add', '.') + git(seed, 'commit', '-qm', 'Initial fixture') + initial = git(seed, 'rev-parse', 'HEAD') + initial_index = (data / 'index.json').read_bytes() + if catalog_state in ('incompatible', 'archived-incompatible'): + # Each input validates alone; only the combined dataset violates machine policy. + subprocess.run( + ['node', str(VALIDATOR), str(data)], check=True, capture_output=True + ) + isolated = tmp_path / 'isolated' + (isolated / 'runs').mkdir(parents=True) + (isolated / 'test-catalogs').mkdir() + shutil.copy2(data / 'metadata.json', isolated / 'metadata.json') + shutil.copy2(catalog, isolated / 'test-catalogs' / catalog.name) + shutil.copy2(output, isolated / 'runs' / output.name) + (isolated / 'index.json').write_text( + json.dumps( + { + 'generatedAt': '2026-10-01T00:01:00Z', + 'runFiles': ['runs/' + output.name], + } + ) + ) + subprocess.run( + ['node', str(VALIDATOR), str(isolated)], check=True, capture_output=True + ) + subprocess.run([real_git, 'clone', '-q', str(seed), str(publication)], check=True) + git(publication, 'config', '--local', 'commit.gpgsign', 'false') + git(seed, 'checkout', '-qb', 'race') + (data / 'index.json').write_text( + json.dumps( + { + 'runFiles': ['runs/older.json', 'runs/concurrent.json'], + 'custom': 'keep', + 'generatedAt': '2026-10-01T00:01:00Z', + } + ) + ) + (seed / 'concurrent.txt').write_text('preserve concurrent writer') + if catalog_state == 'incompatible-on-race': + run = json.loads((data / 'runs/concurrent.json').read_text()) + run['execution']['machine'] = 'another machine' + (data / 'runs/concurrent.json').write_text(json.dumps(run)) + if catalog_state == 'removed-on-race': + catalog.unlink() + elif catalog_state == 'added-on-race': + catalog.parent.mkdir() + catalog.write_text(json.dumps(publication_catalog)) + race_index = (data / 'index.json').read_bytes() + git(seed, 'add', '.') + git(seed, 'commit', '-qm', 'Concurrent writer fixture') + race = git(seed, 'rev-parse', 'HEAD') + download = tmp_path / 'download' + (download / 'raw').mkdir(parents=True) + (download / 'raw/run.json').write_bytes(original) + if dashboard_ready: + (download / 'dashboard').mkdir() + shutil.copy2(output, download / 'dashboard' / output.name) + if catalog_state == 'invalid': + invalid = json.loads(output.read_text()) + invalid['testCatalog'] = '../outside.json' + (download / 'dashboard' / output.name).write_text(json.dumps(invalid)) + wrapper = tmp_path / 'bin' + wrapper.mkdir() + wrapper_code = r"""import os, pathlib, subprocess, sys +args = sys.argv[1:] +real = os.environ['REAL_GIT'] +if args[0] != 'push': + os.execv(real, [real, *args]) +counter = pathlib.Path(os.environ['PUSH_COUNTER']) +count = int(counter.read_text()) + 1 if counter.exists() else 1 +counter.write_text(str(count)) +origin = os.environ['TEST_ORIGIN'] +ref = 'refs/heads/gh-pages-rocjitsu' +def git(*args): + return subprocess.check_output([real, '-C', origin, *args], text=True).strip() +if count == 1: + git('update-ref', ref, 'refs/heads/race') + sys.exit(1) +if os.environ['ALWAYS_FAIL'] == 'true': + sys.exit(1) +old = git('rev-parse', ref) +git('fetch', os.getcwd(), 'HEAD') +new = git('rev-parse', 'FETCH_HEAD') +subprocess.run([real, '-C', origin, 'merge-base', '--is-ancestor', old, new], check=True) +git('update-ref', ref, new, old) +""" + (wrapper / 'git').write_text('#!' + sys.executable + '\n' + wrapper_code) + (wrapper / 'git').chmod(0o755) + env = { + **os.environ, + 'PATH': str(wrapper) + os.pathsep + os.environ['PATH'], + 'REAL_GIT': real_git, + 'TEST_ORIGIN': str(seed), + 'PUSH_COUNTER': str(tmp_path / 'push-count'), + 'ALWAYS_FAIL': str(always_fail).lower(), + 'DASHBOARD_READY': str(dashboard_ready).lower(), + 'RESULTS_DOWNLOAD_DIR': str(download), + 'RESULTS_BRANCH': 'gh-pages-rocjitsu', + 'GITHUB_RUN_ID': '123', + # Publish attempt 2 reruns only failed jobs; files still come from benchmark attempt 1. + 'GITHUB_RUN_ATTEMPT': publish_attempt, + 'BENCHMARK_RUN_ATTEMPT': '1', + 'DASHBOARD_VALIDATOR': str(VALIDATOR), + } + body = next( + step['run'] for step in steps() if step['name'] == 'Publish benchmark files' + ) + result = subprocess.run( + ['bash', '-euo', 'pipefail', '-c', body], + cwd=publication, + env=env, + capture_output=True, + text=True, + ) + catalog_ok = catalog_state in ('present', 'added-on-race') + catalog_failed = dashboard_ready and not catalog_ok + counter = tmp_path / 'push-count' + if catalog_state.startswith('archived-'): + assert result.returncode == 1, result.stdout + result.stderr + assert ( + 'machine' if catalog_state == 'archived-incompatible' else 'catalog' + ) in result.stderr.lower() + assert not counter.exists() + assert git(publication, 'rev-parse', 'HEAD') == initial + assert git(seed, 'rev-parse', 'gh-pages-rocjitsu') == initial + assert ( + publication / 'rocjitsu-dashboard/data/index.json' + ).read_bytes() == initial_index + if catalog_state == 'archived-existing-run': + assert ( + publication / 'rocjitsu-dashboard/data/runs' / output.name + ).read_text() == 'preserve existing dashboard bytes\n' + else: + assert not ( + publication / 'rocjitsu-dashboard/data/runs' / output.name + ).exists() + else: + assert counter.exists(), result.stdout + result.stderr + assert int(counter.read_text()) == (3 if always_fail else 2) + assert result.returncode == ( + 1 if always_fail or catalog_failed else 0 + ), result.stderr + if always_fail: + assert git(seed, 'rev-parse', 'gh-pages-rocjitsu') == race + assert 'Publication failed after 3 attempts' in result.stderr + elif not catalog_state.startswith('archived-'): + index = json.loads( + git(seed, 'show', 'gh-pages-rocjitsu:rocjitsu-dashboard/data/index.json') + ) + assert ( + 'runs/concurrent.json' in index['runFiles'] + and 'runs/older.json' in index['runFiles'] + ) + assert index['custom'] == 'keep' + assert ('runs/' + output.name in index['runFiles']) == ( + dashboard_ready and catalog_ok + ) + assert ( + git(seed, 'show', 'gh-pages-rocjitsu:concurrent.txt') + == 'preserve concurrent writer' + ) + git(seed, 'merge-base', '--is-ancestor', race, 'gh-pages-rocjitsu') + if catalog_failed: + assert ( + 'machine' if catalog_state.startswith('incompatible') else 'catalog' + ) in result.stderr.lower() + assert not ( + publication / 'rocjitsu-dashboard/data/runs' / output.name + ).exists() + assert ( + publication / 'rocjitsu-dashboard/data/index.json' + ).read_bytes() == race_index + assert ( + git( + seed, + 'diff', + 'refs/heads/race', + 'gh-pages-rocjitsu', + '--', + 'rocjitsu-dashboard/data', + ) + == '' + ) + elif dashboard_ready: + subprocess.run( + ['node', str(VALIDATOR), str(publication / 'rocjitsu-dashboard/data')], + check=True, + capture_output=True, + ) + if catalog_state == 'incompatible-on-race': + assert 'Dashboard data is valid' in result.stdout + if not always_fail: + published = subprocess.check_output( + [ + real_git, + '-C', + str(seed), + 'show', + 'gh-pages-rocjitsu:raw-benchmarks/github-123-attempt-1/raw.json', + ] + ) + assert published == original + + +def test_develop_history_gate(tmp_path): + repository = tmp_path / 'source' + repository.mkdir() + + def git(*args): + return subprocess.run( + ['git', '-C', str(repository), *args], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + git('init', '-q', '-b', 'develop') + git('config', 'user.name', 'Fixture') + git('config', 'user.email', 'fixture@example.invalid') + git('config', '--local', 'commit.gpgsign', 'false') + git('commit', '--allow-empty', '-qm', 'Initial develop') + ancestor = git('rev-parse', 'HEAD') + git('commit', '--allow-empty', '-qm', 'Develop tip') + tip = git('rev-parse', 'HEAD') + git('update-ref', 'refs/remotes/origin/develop', tip) + git('checkout', '-qb', 'feature', ancestor) + git('commit', '--allow-empty', '-qm', 'Unmerged feature') + feature = git('rev-parse', 'HEAD') + doc = yaml.safe_load(WORKFLOW.read_text()) + benchmark = {step['name']: step for step in doc['jobs']['benchmark']['steps']} + assert benchmark['Checkout rocm-systems']['with']['fetch-depth'] == 0 + body = benchmark['Require a commit from develop']['run'] + for revision, expected in [(ancestor, 0), (tip, 0), (feature, 1)]: + git('checkout', '--detach', revision) + result = subprocess.run( + ['bash', '-euo', 'pipefail', '-c', body], + cwd=repository, + capture_output=True, + text=True, + ) + assert result.returncode == expected, result.stderr + if expected: + assert "must be in develop's history" in result.stderr