QVAC-17830 feat: VLM performance metrics across desktop + mobile (CPU & GPU)

.github/workflows/benchmark-performance-qvac-lib-infer-llamacpp-llm.yml

@@ -89,6 +89,7 @@ jobs:
       ref: ${{ needs.context.outputs.ref }}
       qvac_perf_runs: ${{ inputs.qvac_perf_runs }}
       qvac_perf_warmup_runs: ${{ inputs.qvac_perf_warmup_runs }}
+      qvac_perf_only: true
 
   mobile-benchmarks:
     needs: [context, prebuild]
@@ -105,6 +106,7 @@ jobs:
       ref: ${{ needs.context.outputs.ref }}
       qvac_perf_runs: ${{ inputs.qvac_perf_runs }}
       qvac_perf_warmup_runs: ${{ inputs.qvac_perf_warmup_runs }}
+      qvac_perf_only: true
 
   summarize:
     # `if: always()` lets summarize run even when one of the benchmark

.github/workflows/integration-test-qvac-lib-infer-llamacpp-llm.yml

@@ -20,6 +20,11 @@ on:
         type: string
         required: false
         default: ""
+      qvac_perf_only:
+        description: "If true, run only the perf-emitting tests (image-elephant, image-fruit-plate, image-high-res-aurora, bitnet, tool-calling)."
+        type: boolean
+        required: false
+        default: false
 
   workflow_dispatch:
     inputs:
@@ -42,6 +47,11 @@ on:
         type: string
         required: false
         default: ""
+      qvac_perf_only:
+        description: "If true, run only the perf-emitting tests."
+        type: boolean
+        required: false
+        default: false
 
 jobs:
   run-integration-tests:
@@ -61,39 +71,53 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # QVAC-17830: `label` disambiguates matrix entries that share
+        # the same `<platform>-<arch>` tuple so every entry produces a
+        # uniquely-named perf-report artifact. Without this, the two
+        # `linux-x64` entries (CPU-only vs GPU runner) and the two
+        # `linux-arm64` entries (ubuntu 22 vs 24) collided on upload
+        # and `actions/download-artifact` silently dropped one of each,
+        # hiding GPU data in the combined report.
         include:
           - os: ubuntu-22.04
             platform: linux
             arch: x64
             runner: ubuntu-22.04
             no_gpu: 'true'
+            label: linux-x64-cpu
           - os: ubuntu-24.04
             platform: linux
             arch: x64
             runner: ai-run-linux-gpu
             timeout_minutes: 480
+            label: linux-x64-gpu
           - os: ubuntu-24.04-arm
             platform: linux
             arch: arm64
             runner: ubuntu-24.04-arm
             no_gpu: 'true'
+            label: linux-arm64-u24
           - os: ubuntu-22.04-arm
             platform: linux
             arch: arm64
             runner: ubuntu-22.04-arm
             no_gpu: 'true'
+            label: linux-arm64-u22
           - os: macos-15-xlarge
             platform: darwin
             arch: arm64
             runner: macos-15-xlarge
+            label: darwin-arm64
           - os: macos-15-large
             platform: darwin
             arch: x64
             runner: macos-15-large
+            label: darwin-x64
           - os: windows-11
             platform: win32
             arch: x64
             runner: ai-run-windows11-gpu
+            label: win32-x64
 
     steps:
       - name: Setup Node.js
@@ -211,6 +235,17 @@ jobs:
         if: ${{ matrix.platform != 'win32' }}
         working-directory: ${{ env.WORKDIR }}
         run: |
+          if [ "${{ inputs.qvac_perf_only }}" = "true" ]; then
+            echo "qvac_perf_only=true: regenerating test/integration/all.js with perf-emitting tests only"
+            npx brittle -r test/integration/all.js \
+              test/integration/bitnet.test.js \
+              test/integration/tool-calling.test.js \
+              test/integration/image-elephant.test.js \
+              test/integration/image-fruit-plate.test.js \
+              test/integration/image-high-res-aurora.test.js
+            bare test/integration/all.js --exit 2>&1 | tee test-output.log
+            exit ${PIPESTATUS[0]}
+          fi
           npm run test:integration 2>&1 | tee test-output.log
           exit ${PIPESTATUS[0]}
         shell: bash
@@ -223,10 +258,49 @@ jobs:
         if: ${{ matrix.platform == 'win32' }}
         working-directory: ${{ env.WORKDIR }}
         run: |
-          npm run test:integration:generate
+          if ("${{ inputs.qvac_perf_only }}" -eq "true") {
+            Write-Host "qvac_perf_only=true: regenerating test/integration/all.js with perf-emitting tests only"
+            npx brittle -r test/integration/all.js `
+              test/integration/bitnet.test.js `
+              test/integration/tool-calling.test.js `
+              test/integration/image-elephant.test.js `
+              test/integration/image-fruit-plate.test.js `
+              test/integration/image-high-res-aurora.test.js
+          } else {
+            npm run test:integration:generate
+          }
           bare test/integration/all.js --exit | Tee-Object test-output.log
         shell: powershell
         env:
           QASE_API_TOKEN: ${{ secrets.QASE_API_TOKEN }}
           QVAC_PERF_RUNS: ${{ inputs.qvac_perf_runs }}
           QVAC_PERF_WARMUP_RUNS: ${{ inputs.qvac_perf_warmup_runs }}
+
+      - name: Generate HTML performance report
+        if: ${{ always() }}
+        working-directory: ${{ env.WORKDIR }}
+        shell: bash
+        run: |
+          if [ -f test/results/performance-report.json ]; then
+            echo "Found performance-report.json, generating HTML/MD/summary..."
+            node ../../scripts/perf-report/aggregate.js \
+              --dir test/results \
+              --output-html test/results/performance-report.html \
+              --output-json test/results/performance-summary.json \
+              --output test/results/performance-report.md
+          else
+            echo "performance-report.json not found - skipping HTML generation"
+          fi
+
+      - name: Upload performance report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # 7.0.0
+        with:
+          name: perf-report-llamacpp-llm-${{ matrix.label }}-${{ github.run_number }}
+          path: |
+            ${{ env.WORKDIR }}/test/results/performance-report.json
+            ${{ env.WORKDIR }}/test/results/performance-report.html
+            ${{ env.WORKDIR }}/test/results/performance-summary.json
+            ${{ env.WORKDIR }}/test/results/performance-report.md
+          retention-days: 90
+          if-no-files-found: ignore

.github/workflows/on-pr-qvac-lib-infer-llamacpp-llm.yml

@@ -168,6 +168,228 @@
       repository: ${{ github.event.pull_request.head.repo.full_name }}
       ref: ${{ github.event.pull_request.head.ref }}
 
+  # QVAC-17830: per-run joint perf reporter. Lives in the umbrella (not in
+  # either child workflow) so it can block on BOTH `run-integration-tests`
+  # (desktop matrix) and `run-mobile-integration-tests` (Android / iOS)
+  # before scanning for perf-report artifacts. Fixes the race where
+  # test-darwin-x64 (or any slow desktop job) finished AFTER the old
+  # mobile-local combine-reports had already shipped the summary.
+  # SECURITY (CodeQL js/cache-poisoning, alerts 733/734):
+  # This job runs in the privileged `pull_request_target` context (see
+  # `on:` at the top of this file). The integration matrix it depends
+  # on already runs PR-author code on real hardware, so this aggregator
+  # by definition consumes attacker-controllable artifacts. The
+  # mitigations are:
+  #   - `permissions: contents: read` only — no write access, no
+  #     packages / id-token / pull-requests permissions.
+  #   - No use of `actions/cache` here, so there is no shared cache
+  #     for a malicious PR to poison for the default branch.
+  #   - `aggregate.js` does not touch GITHUB_TOKEN, secrets, or any
+  #     external network endpoints; its only outputs are artifact files.
+  #   - `device.name` (the only attacker-controllable string we let
+  #     near a filesystem path) is sanitised through a strict
+  #     `[A-Za-z0-9-]` allowlist + 64-char cap below before use.
+  combine-perf-reports:
+    name: Combined Performance Report
+    needs: [authorize, run-integration-tests, run-mobile-integration-tests]
+    if: always() && needs.authorize.outputs.allowed == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout addon repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2
+        with:
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+          ref: ${{ github.event.pull_request.head.ref || github.ref }}
+          sparse-checkout: |
+            scripts/perf-report
+            packages/qvac-lib-infer-llamacpp-llm/media
+
+      - name: Setup Node.js
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # 4.4.0
+        with:
+          node-version: lts/*
+
+      - name: Download all perf report artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # 8.0.1
+        with:
+          pattern: perf-report-llamacpp-llm-*-${{ github.run_number }}
+          path: combined-reports
+        continue-on-error: true
+
+      - name: Fix desktop device names
+        shell: bash
+        run: |
+          # Every desktop matrix entry uses a unique `label` in its
+          # artifact name (e.g. linux-x64-cpu / linux-x64-gpu). The sed
+          # below strips our fixed prefix + run number; the remainder
+          # IS the raw matrix label.
+          #
+          # QVAC-17830 (combined-report fix): the CI matrix intentionally
+          # runs two legs on the same physical Linux x64 target (one
+          # no_gpu=true for pure CPU data, one on a GPU runner for CPU+GPU
+          # data) and likewise two ubuntu versions for linux-arm64. In the
+          # combined summary those should show up as ONE column per
+          # platform — otherwise the GPU row on the "*-cpu" / "*-u22" /
+          # "*-u24" columns renders as "-" and looks like missing data
+          # even though the data lives in the sibling leg.
+          #
+          # We therefore fold them to the physical platform name here so
+          # `aggregate.js` buckets CPU and GPU measurements under one
+          # device. Test labels already carry [CPU]/[GPU], so the rows
+          # stay distinct.
+          for dir in combined-reports/perf-report-llamacpp-llm-*/; do
+            [ -d "$dir" ] || continue
+            base=$(basename "$dir")
+            platform=$(echo "$base" | sed "s/^perf-report-llamacpp-llm-//" | sed "s/-${{ github.run_number }}$//")
+
+            case "$platform" in Android|iOS) continue ;; esac
+
+            case "$platform" in
+              linux-x64-cpu|linux-x64-gpu) device_name="linux-x64" ;;
+              linux-arm64-u22|linux-arm64-u24) device_name="linux-arm64" ;;
+              *) device_name="$platform" ;;
+            esac
+
+            for json in $(find "$dir" -name "performance-report.json" 2>/dev/null); do
+              if command -v jq >/dev/null 2>&1; then
+                jq --arg name "$device_name" '.device.name = $name' "$json" > "${json}.tmp" && mv "${json}.tmp" "$json"
+                echo "Patched device name in $json -> $device_name (was matrix label $platform)"
+              fi
+            done
+          done
+
+      - name: Generate combined report
+        shell: bash
+        run: |
+          # SECURITY (CodeQL alert 733, actions/cache-poisoning/poisonable-step):
+          # `aggregate.js` is checked out from PR head and runs in the
+          # umbrella's pull_request_target context. The job-level
+          # `permissions: contents: read` (no actions/cache, no secrets,
+          # no GITHUB_TOKEN write usage) bounds the blast radius — see
+          # the block comment on the `combine-perf-reports` job above.
+          # The script's only side effect is writing artifact files to
+          # `combined-output/`; the artifacts themselves are uploaded
+          # via SHA-pinned `actions/upload-artifact` and consumed only
+          # by reviewers from the run page.
+          if ! find combined-reports -name "performance-report.json" -type f 2>/dev/null | grep -q .; then
+            echo "No performance reports found."
+            exit 0
+          fi
+
+          echo "=== Reports found ==="
+          find combined-reports -name "performance-report.json" -type f
+
+          mkdir -p combined-output
+
+          # QVAC-17830: combined PR summary surfaces BOTH the squashed
+          # Mean ± std mini-tables (Total Time / TTFT / TPS, grouped
+          # by scenario) AND the per-device detail tables underneath
+          # them. Detail-table cells now render `mean ±std` themselves
+          # so the std is visible at every metric, not just the
+          # rolled-up ones. HTML keeps the same content.
+          node scripts/perf-report/aggregate.js \
+            --dir combined-reports \
+            --addon-type vision \
+            --device-details \
+            --output-html combined-output/performance-report-combined.html \
+            --output-json combined-output/performance-summary-combined.json \
+            --output combined-output/performance-report-combined.md
+
+      - name: Generate per-device HTML reports
+        if: always()
+        shell: bash
+        run: |
+          # SECURITY: this job runs in pull_request_target context and the
+          # `performance-report.json` files are produced by PR-author code.
+          # `device.name` is therefore attacker-controlled. We do not
+          # interpolate it into shell or pass it as code; we sanitise it to
+          # an alphanumeric+hyphen filename slug capped at 64 chars before
+          # using it as a path suffix, which prevents path traversal and
+          # any kind of shell-meta exposure even if the JSON were crafted.
+          mkdir -p per-device-reports
+          for json in $(find combined-reports -name "performance-report.json" -type f 2>/dev/null); do
+            device_name=$(node -e "const d=JSON.parse(require('fs').readFileSync(process.argv[1],'utf8'));process.stdout.write((d.device&&typeof d.device.name==='string'?d.device.name:'Unknown'))" "$json" 2>/dev/null)
+            [ -z "$device_name" ] || [ "$device_name" = "Unknown" ] && continue
+
+            # Strict allowlist: collapse anything that is not [A-Za-z0-9-]
+            # to '_', then truncate to 64 characters.
+            safe_name=$(printf '%s' "$device_name" | LC_ALL=C tr -c 'A-Za-z0-9-' '_' | cut -c1-64)
+            [ -z "$safe_name" ] && continue
+
+            tmp_dir=$(mktemp -d)
+            mkdir -p "$tmp_dir/device"
+            cp "$json" "$tmp_dir/device/performance-report.json"
+
+            echo "Generating HTML for $device_name (slug: $safe_name)..."
+            node scripts/perf-report/aggregate.js \
+              --dir "$tmp_dir" \
+              --addon-type vision \
+              --device-details \
+              --output-html "per-device-reports/HTML-Report-${safe_name}.html" \
+              2>/dev/null || true
+
+            rm -rf "$tmp_dir"
+          done
+          echo "=== Per-device reports generated ==="
+          ls -la per-device-reports/ 2>/dev/null || echo "No per-device reports"
+
+      - name: Upload combined HTML report
+        if: always()
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # 7.0.0
+        with:
+          name: HTML-Report-All-Platforms-${{ github.run_number }}
+          path: combined-output/performance-report-combined.html
+          retention-days: 90
+          if-no-files-found: ignore
+
+      - name: Upload per-device HTML reports
+        if: always()
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # 7.0.0
+        with:
+          name: HTML-Reports-Per-Device-${{ github.run_number }}
+          path: per-device-reports/
+          retention-days: 90
+          if-no-files-found: ignore
+
+      - name: Write combined summary
+        if: always()
+        shell: bash
+        run: |
+          set +e
+          MD_FILE="combined-output/performance-report-combined.md"
+          echo "## LLM / VLM Performance Report (All Platforms)" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          if [ -f "$MD_FILE" ]; then
+            cat "$MD_FILE" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "No combined performance report available." >> $GITHUB_STEP_SUMMARY
+          fi
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "---" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "### Downloadable HTML Reports" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          if [ -f "combined-output/performance-report-combined.html" ]; then
+            echo "> **Full Combined Report (all platforms)**: download artifact \`HTML-Report-All-Platforms-${{ github.run_number }}\`" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+          fi
+          HAS_DEVICES=0
+          for html in per-device-reports/HTML-Report-*.html; do
+            [ -f "$html" ] || continue
+            if [ "$HAS_DEVICES" -eq 0 ]; then
+              echo "> **Individual Device Reports**: download artifact \`HTML-Reports-Per-Device-${{ github.run_number }}\`" >> $GITHUB_STEP_SUMMARY
+              echo ">" >> $GITHUB_STEP_SUMMARY
+              echo "> Includes:" >> $GITHUB_STEP_SUMMARY
+              HAS_DEVICES=1
+            fi
+            device=$(basename "$html" .html | sed 's/^HTML-Report-//' | tr '-' ' ')
+            echo "> - ${device}" >> $GITHUB_STEP_SUMMARY
+          done
+          echo "" >> $GITHUB_STEP_SUMMARY
+
   merge-guard:
     needs:
       [