perf(state): compact v23 FTS layout — external-content indexes, tool-row-free trigram, opt-in hermes sessions optimize-storage

.github/actions/detect-changes/action.yml

@@ -7,7 +7,7 @@ description: >-
 
 inputs:
   github-token:
-    description: Token for the GitHub API (gh CLI). Pass secrets.AUTOFIX_BOT_PAT from the calling workflow.
+    description: Token for the GitHub API (gh CLI). Pass steps.app-token.outputs.token from the calling workflow.
     required: false
     default: ${{ github.token }}
 
@@ -47,7 +47,12 @@ runs:
       id: classify
       shell: bash
       env:
-        GH_TOKEN: ${{ inputs.github-token }}
+        # Fall back to the built-in read-only token when the caller passes an
+        # empty value. Fork PRs get no repo secrets, so AUTOFIX_BOT_PAT is ""
+        # there, and an input `default:` only applies when the input is omitted,
+        # not when it's passed empty. Without this fallback the compare API
+        # fails on forks and the classifier fails open (every lane forced on).
+        GH_TOKEN: ${{ inputs.github-token || github.token }}
         REPO: ${{ github.repository }}
         EVENT_NAME: ${{ github.event_name }}
         BASE_SHA: ${{ github.event.pull_request.base.sha }}
@@ -67,12 +72,19 @@ runs:
           # Retried: a rate-limit blip or eventual-consistency 404 on a
           # freshly-pushed HEAD would otherwise silently fall open (all lanes
           # run — safe, but wasteful and it masks the API failure).
+          #
+          # `.files[]?` (null-safe): with --paginate, a PR more than 100
+          # commits ahead of its merge-base paginates the compare, and pages
+          # after the first carry `files: null` — bare `.files[]` makes jq
+          # die with "cannot iterate over: null", which fails every retry
+          # and forces the fail-open path (seen on stacked PRs). The full
+          # file list (up to the API's 300-file cap) is on page one.
           CHANGED=""
           for i in 1 2 3; do
             if CHANGED="$(gh api \
               --paginate \
               "repos/${REPO}/compare/${BASE_SHA}...${HEAD_SHA}" \
-              --jq '.files[].filename')"; then
+              --jq '.files[]?.filename')"; then
               break
             fi
             if [ "$i" = 3 ]; then

.github/actions/get-app-token/action.yml

@@ -0,0 +1,59 @@
+name: Get GitHub App Token
+description: >-
+  Mint a short-lived (1-hour) installation access token from the repo's
+  GitHub App, replacing the long-lived AUTOFIX_BOT_PAT. App tokens get
+  5,000 req/hr per installation (vs 1,000 for the default GITHUB_TOKEN)
+  and are scoped to the App's installation permissions, not a user account.
+
+  Falls back to the built-in GITHUB_TOKEN when APP_CLIENT_ID is not set —
+  this happens on fork PRs where repo secrets are unavailable. The fallback
+  ensures classification, timings, and review comments still work on
+  forks (with the lower GITHUB_TOKEN rate limit).
+
+  Composite actions cannot access the secrets context directly, so the
+  calling workflow must pass secrets.APP_CLIENT_ID and secrets.APP_PRIVATE_KEY
+  as inputs. When both are empty (fork PRs), the fallback fires.
+
+inputs:
+  client-id:
+    description: GitHub App Client ID. Pass secrets.APP_CLIENT_ID from the calling workflow.
+    required: false
+    default: ''
+  private-key:
+    description: GitHub App private key PEM. Pass secrets.APP_PRIVATE_KEY from the calling workflow.
+    required: false
+    default: ''
+
+outputs:
+  token:
+    description: A GitHub App installation access token (1-hour TTL), or GITHUB_TOKEN on forks.
+    value: ${{ steps.app-token.outputs.token || steps.fallback.outputs.token }}
+
+runs:
+  using: composite
+  steps:
+    - name: Check if App credentials exist
+      id: check
+      shell: bash
+      env:
+        CLIENT_ID: ${{ inputs.client-id }}
+      run: |
+        if [ -n "$CLIENT_ID" ]; then
+          echo "has_app=true" >> "$GITHUB_OUTPUT"
+        else
+          echo "has_app=false" >> "$GITHUB_OUTPUT"
+        fi
+
+    - name: Create GitHub App token
+      id: app-token
+      if: steps.check.outputs.has_app == 'true'
+      uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+      with:
+        client-id: ${{ inputs.client-id }}
+        private-key: ${{ inputs.private-key }}
+
+    - name: Fall back to GITHUB_TOKEN
+      id: fallback
+      if: steps.check.outputs.has_app != 'true'
+      shell: bash
+      run: echo "token=${{ github.token }}" >> "$GITHUB_OUTPUT"

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ on:
 
 permissions:
   contents: read
-  pull-requests: write # needed by lint (PR comment) + supply-chain (PR comment)
+  pull-requests: write # needed by lint (PR comment) + supply-chain review_status
   actions: read # needed by osv-scanner (SARIF upload)
   security-events: write # needed by osv-scanner (SARIF upload)
   packages: write # needed by docker build
@@ -49,11 +49,19 @@ jobs:
       event_name: ${{ github.event_name }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Get GitHub App token
+        id: app-token
+        uses: ./.github/actions/get-app-token
+        with:
+          client-id: ${{ secrets.APP_CLIENT_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
       - name: Detect affected areas
         id: classify
         uses: ./.github/actions/detect-changes
         with:
-          github-token: ${{ secrets.AUTOFIX_BOT_PAT }}
+          # The get-app-token composite action falls back to GITHUB_TOKEN
+          # on fork PRs where APP_ID is unavailable.
+          github-token: ${{ steps.app-token.outputs.token }}
 
   # ─────────────────────────────────────────────────────────────────────
   # Lane-gated sub-workflows. Each runs in parallel after detect finishes.
@@ -71,11 +79,10 @@ jobs:
   lint:
     name: Python lints
     needs: detect
-    if: needs.detect.outputs.python == 'true' || needs.detect.outputs.ci_review == 'true'
+    if: needs.detect.outputs.python == 'true'
     uses: ./.github/workflows/lint.yml
     with:
       event_name: ${{ needs.detect.outputs.event_name }}
-      ci_review: ${{ needs.detect.outputs.ci_review == 'true' }}
     secrets: inherit
 
   js-tests:
@@ -85,6 +92,12 @@ jobs:
     uses: ./.github/workflows/js-tests.yml
     secrets: inherit
 
+  e2e-desktop:
+    name: Desktop E2E
+    needs: detect
+    if: needs.detect.outputs.python == 'true' || needs.detect.outputs.frontend == 'true'
+    uses: ./.github/workflows/e2e-desktop.yml
+
   docs-site:
     name: Docs Site
     needs: detect
@@ -136,54 +149,153 @@ jobs:
   supply-chain:
     name: Supply-chain scan
     needs: detect
-    if: needs.detect.outputs.event_name == 'pull_request' && (needs.detect.outputs.scan == 'true' || needs.detect.outputs.deps == 'true' || needs.detect.outputs.mcp_catalog == 'true')
+    if: needs.detect.outputs.event_name == 'pull_request' && (needs.detect.outputs.scan == 'true' || needs.detect.outputs.deps == 'true')
     uses: ./.github/workflows/supply-chain-audit.yml
     with:
       event_name: ${{ needs.detect.outputs.event_name }}
       scan: ${{ needs.detect.outputs.scan == 'true' }}
       deps: ${{ needs.detect.outputs.deps == 'true' }}
+
+  review-labels:
+    name: Review label gate
+    needs: [detect, supply-chain]
+    if: always() && needs.detect.outputs.event_name == 'pull_request' && (needs.detect.outputs.ci_review == 'true' || needs.detect.outputs.mcp_catalog == 'true' || needs.supply-chain.outputs.critical_findings == 'true')
+    uses: ./.github/workflows/review-labels.yml
+    with:
+      ci_review: ${{ needs.detect.outputs.ci_review == 'true' }}
       mcp_catalog: ${{ needs.detect.outputs.mcp_catalog == 'true' }}
+      supply_chain: ${{ needs.supply-chain.outputs.critical_findings == 'true' }}
     secrets: inherit
 
   osv-scanner:
     name: OSV scan
     uses: ./.github/workflows/osv-scanner.yml
     secrets: inherit
 
+  # ─────────────────────────────────────────────────────────────────────
+  # Live-updating PR review comment.
+  #
+  # A single ``comment-live`` job polls the GitHub Actions API every 15s
+  # for job statuses in this run, re-assembles the review comment from
+  # whatever results are available, and upserts it via the
+  # ``<!-- hermes-ci-review-bot -->`` marker.
+  #
+  # The poller exits when all non-infra jobs are completed (or on
+  # timeout). ci-timings' review_status is picked up automatically when
+  # its artifact becomes available — the poller downloads and merges it.
+  # ─────────────────────────────────────────────────────────────────────
+  comment-live:
+    name: CI review comment (live)
+    needs: [detect, review-labels, lockfile-diff, supply-chain, osv-scanner, uv-lockfile, history-check, contributor-check]
+    if: always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork != true
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Run live comment poller
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          # Commit info for the review comment header.
+          COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
+          COMMIT_MESSAGE: ${{ github.event.pull_request.head.commit.message }}
+          COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/commits/${{ github.event.pull_request.head.sha }}
+          # Structured review statuses from workflow_call jobs.
+          # Each job outputs a JSON array of {source, results: [...]} objects
+          # that the assembler renders directly — no hardcoded job-name
+          # matching. We merge all available outputs into one array.
+          REVIEW_STATUSES: ${{ toJSON(needs.*.outputs.review_status) }}
+        run: |
+          set -uo pipefail
+
+          # REVIEW_STATUSES is a JSON array of strings (some may be empty
+          # when a job was skipped). Parse each string and merge into one
+          # flat array for the assembler.
+          python3 - <<'PYEOF'
+          import json, os, sys
+
+          raw = os.environ.get("REVIEW_STATUSES", "")
+          merged = []
+          if raw:
+              try:
+                  arr = json.loads(raw)
+              except (json.JSONDecodeError, TypeError):
+                  arr = []
+              for item in arr:
+                  if not item:
+                      continue
+                  try:
+                      statuses = json.loads(item)
+                  except (json.JSONDecodeError, TypeError):
+                      continue
+                  if isinstance(statuses, list):
+                      merged.extend(statuses)
+
+          # Write merged array to a temp file the poller reads.
+          with open("/tmp/review_statuses.json", "w") as f:
+              json.dump(merged, f)
+          print(f"Merged {len(merged)} review status entries")
+          PYEOF
+
+          python3 scripts/ci/live_comment.py \
+            --interval 15 \
+            --timeout 2100 \
+            --review-statuses-file /tmp/review_statuses.json
+
   # ─────────────────────────────────────────────────────────────────────
   # Gate: runs after everything. ``if: always()`` ensures it reports a
   # status even when some deps were skipped. Only actual ``failure``
   # results cause it to fail; ``skipped`` is treated as success.
   #
   # Branch protection should require ONLY this check.
+  #
+  # Outputs ``needs-json`` — a compact ``{job_name: result}`` dict — so
+  # the live comment poller can list failed jobs in the PR comment.
   # ─────────────────────────────────────────────────────────────────────
   all-checks-pass:
     name: All required checks pass
     needs:
+      - detect
       - tests
       - lint
       - js-tests
+      - e2e-desktop
       - docs-site
       - history-check
       - contributor-check
       - uv-lockfile
       - lockfile-diff
       - docker-lint
       - supply-chain
+      - review-labels
       - osv-scanner
+      # comment-live is a polling job — it doesn't block the gate.
       # we don't require docker to pass rn because it's so slow lol
       # - docker
     if: always()
     runs-on: ubuntu-latest
     timeout-minutes: 10
+    outputs:
+      needs-json: ${{ steps.evaluate.outputs.needs-json }}
     steps:
       - name: Evaluate job results
+        id: evaluate
         env:
           NEEDS: ${{ toJSON(needs) }}
         run: |
           echo "$NEEDS" | python3 -c "
           import json, sys
           needs = json.load(sys.stdin)
+          # Emit compact {job_name: result} for the comment assembler.
+          compact = {name: info['result'] for name, info in needs.items()}
+          print(f'needs-json={json.dumps(compact)}')
+          with open('$GITHUB_OUTPUT', 'a') as f:
+              f.write(f'needs-json={json.dumps(compact)}\n')
           failed = [name for name, info in needs.items() if info['result'] == 'failure']
           for name, info in sorted(needs.items()):
               result = info['result']
@@ -200,6 +312,9 @@ jobs:
   # cache them on main (as a baseline), and on PRs generate an HTML diff
   # report with a gantt chart + per-step breakdown. The report is uploaded
   # as an artifact and a markdown summary is written to $GITHUB_STEP_SUMMARY.
+  #
+  # The live comment poller picks up ci-timings' completion automatically —
+  # it reads review-status.json from the artifact when the job finishes.
   # ─────────────────────────────────────────────────────────────────────
   ci-timings:
     name: CI timing report
@@ -211,6 +326,13 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
+      - name: Get GitHub App token
+        id: app-token
+        uses: ./.github/actions/get-app-token
+        with:
+          client-id: ${{ secrets.APP_CLIENT_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+
       - name: Restore baseline cache (PR only)
         if: github.event_name == 'pull_request'
         uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
@@ -224,24 +346,31 @@ jobs:
 
       - name: Collect timings and generate report
         env:
-          GITHUB_TOKEN: ${{ secrets.AUTOFIX_BOT_PAT }}
+          # Forks get no repo secrets (AUTOFIX_BOT_PAT is empty); fall back to
+          # the built-in read-only token so the timings API read still works
+          # there instead of hard-failing this advisory job on every fork PR.
+          # The get-app-token composite action falls back to GITHUB_TOKEN
+          # on fork PRs where APP_ID is unavailable.
+          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
         run: |
           python3 scripts/ci/timings_report.py \
             --baseline ci-timings-baseline.json \
             --output ci-timings-report.html \
             --json-out ci-timings.json \
-            --summary-out ci-timings-summary.md
+            --summary-out ci-timings-summary.md \
+            --review-status-out review-status.json
 
-      - name: Upload HTML report
+      - name: Upload HTML report + review status
         # Advisory report — artifact-service blips must not fail the job.
         continue-on-error: true
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         id: ci-timings-artifact
         with:
           name: ci-timings-report
-          path: ci-timings-report.html
+          path: |
+            ci-timings-report.html
+            review-status.json
           retention-days: 14
-          archive: false
 
       - name: Output summary
         env: