diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000000..beda11e674c2 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,18 @@ +# actionlint configuration. +# +# Declares the self-hosted runner labels served by the GKE ARC scale sets +# (see the hermes-agent-ci-infra repo). Without this, actionlint only knows +# GitHub-hosted labels and reports every `runs-on:` in the repo as unknown — +# 40 warnings that bury real findings. +# +# Keep in sync with the scale sets deployed in hermes-agent-ci-infra. +self-hosted-runner: + labels: + # Default set: general-purpose jobs. + - arc-runner-set + # Short gate jobs (detect, lint, small checks) — no docker sidecar. + - arc-runner-small + # Jobs that need a docker daemon (image build/test). + - arc-runner-docker + # arm64 image builds. + - arc-runner-arm64 diff --git a/.github/actions/merge-base/action.yml b/.github/actions/merge-base/action.yml new file mode 100644 index 000000000000..0afff667b79a --- /dev/null +++ b/.github/actions/merge-base/action.yml @@ -0,0 +1,86 @@ +name: Ensure merge base is present +description: >- + Guarantee that the local (shallow) clone contains the merge base of two + commits, so `git diff base...head` is meaningful. Three-dot diff is defined + as "since the merge base", so having both endpoint commits is NOT enough — + git reports "fatal: no merge base" until the histories actually connect. + A caller that swallows that error silently diffs nothing, which for a + security scanner means reporting clean without having looked at anything. + +inputs: + base: + description: Base commit SHA (e.g. github.event.pull_request.base.sha). + required: true + head: + description: Head commit SHA (e.g. github.event.pull_request.head.sha). + required: true + fail-on-missing: + description: >- + Fail the step when no merge base exists. Default true: a caller about to + run a three-dot diff must never continue, or it silently diffs nothing. + Set false when absence is the thing you are measuring (history-check), + and branch on the `found` output instead. + default: 'true' + +outputs: + sha: + description: The resolved merge-base commit SHA (empty when none exists). + value: ${{ steps.resolve.outputs.sha }} + found: + description: '"true" when a merge base exists, "false" otherwise.' + value: ${{ steps.resolve.outputs.found }} + +runs: + using: composite + steps: + - id: resolve + shell: bash + env: + BASE: ${{ inputs.base }} + HEAD: ${{ inputs.head }} + FAIL_ON_MISSING: ${{ inputs.fail-on-missing }} + run: | + set -euo pipefail + + # Make sure both endpoints exist locally before deepening. + git cat-file -e "${BASE}^{commit}" 2>/dev/null || git fetch --depth=1 origin "$BASE" -q + git cat-file -e "${HEAD}^{commit}" 2>/dev/null || git fetch --depth=1 origin "$HEAD" -q + + # Escalating deepen. A fixed budget is a correctness bug, not just a + # slow path: main lands ~270 commits/day, so a flat --deepen=100 stops + # connecting histories for any branch more than ~9 hours old. + for depth in 200 1000 5000; do + if git merge-base "$BASE" "$HEAD" >/dev/null 2>&1; then break; fi + git fetch --deepen="$depth" origin "$BASE" -q 2>/dev/null || true + git fetch --deepen="$depth" -q 2>/dev/null || true + done + + # Full history is the only way to PROVE absence, so a genuinely + # unrelated branch always lands here. Rare (that PR is rejected + # anyway) and still cheaper than every job unshallowing by default. + if ! git merge-base "$BASE" "$HEAD" >/dev/null 2>&1; then + if [ "$(git rev-parse --is-shallow-repository)" = "true" ]; then + echo "::warning::deepen did not connect the histories; unshallowing (slow path)" + git fetch --unshallow -q 2>/dev/null || true + git fetch --depth=2147483647 origin "$BASE" -q 2>/dev/null || true + fi + fi + + if MB=$(git merge-base "$BASE" "$HEAD" 2>/dev/null) && [ -n "$MB" ]; then + echo "sha=$MB" >> "$GITHUB_OUTPUT" + echo "found=true" >> "$GITHUB_OUTPUT" + echo "::notice::merge base: $MB" + exit 0 + fi + + # Never fail open. Callers running a three-dot diff must stop here; + # history-check opts out to report the failure in its own words. + echo "sha=" >> "$GITHUB_OUTPUT" + echo "found=false" >> "$GITHUB_OUTPUT" + if [ "$FAIL_ON_MISSING" = "true" ]; then + echo "::error::No merge base between $BASE and $HEAD after deepening." \ + "Refusing to continue: a three-dot diff would silently produce" \ + "an empty result and report a vacuous pass." + exit 1 + fi + echo "::notice::no merge base between $BASE and $HEAD" diff --git a/.github/actions/profile/action.yml b/.github/actions/profile/action.yml new file mode 100644 index 000000000000..0cd5ea9d0b0a --- /dev/null +++ b/.github/actions/profile/action.yml @@ -0,0 +1,91 @@ +name: Profile a command (CPU/RAM/Disk) +description: >- + Run a shell command while sampling CPU, RAM, and disk IO every second. + Produces a resource-profile.json artifact per job so the CI timing + report can show per-job resource usage and identify bottlenecks. + +inputs: + command: + description: Shell command to run (and profile). + required: true + label: + description: Label for this profile (e.g. "tests slice 1/8"). + required: true + working-directory: + description: Directory to run in. + default: '.' + +runs: + using: composite + steps: + - name: Start resource profiler + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + # Start profiler in background. It writes to resource-profile.json + # on SIGTERM (or when the command finishes and we signal it). + python3 scripts/ci/resource_profile.py \ + --output resource-profile.json \ + --label "$PROFILE_LABEL" & + echo $! > "$RUNNER_TEMP/profiler.pid" + env: + PROFILE_LABEL: ${{ inputs.label }} + + - name: Run command + shell: bash + working-directory: ${{ inputs.working-directory }} + env: + _CMD: ${{ inputs.command }} + run: | + # -e / pipefail: match the semantics of a normal `run:` step + # (bash -e {0}) so a failing early line (e.g. `source .venv/...`) + # fails the step instead of silently running the rest. + bash -eo pipefail -c "$_CMD" + + - name: Stop profiler and collect results + id: stop-profiler + if: always() + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + if [ -f "$RUNNER_TEMP/profiler.pid" ]; then + PID=$(cat "$RUNNER_TEMP/profiler.pid") + if kill -0 "$PID" 2>/dev/null; then + kill -TERM "$PID" + # Give it a moment to write the JSON + for i in 1 2 3 4 5; do + if kill -0 "$PID" 2>/dev/null; then + sleep 0.2 + else + break + fi + done + kill -KILL "$PID" 2>/dev/null || true + fi + fi + # hashFiles() only matches inside the workspace, so surface file + # existence as a step output instead for the upload condition. + if [ -s resource-profile.json ]; then + echo "profile_written=true" >> "$GITHUB_OUTPUT" + else + echo "profile_written=false" >> "$GITHUB_OUTPUT" + fi + + - name: Sanitize resource profile label + id: sanitize + if: always() + shell: bash + env: + PROFILE_LABEL: ${{ inputs.label }} + run: | + SAFE=$(printf '%s' "$PROFILE_LABEL" | sed -E 's/[^a-zA-Z0-9]+/-/g') + echo "safe_label=$SAFE" >> "$GITHUB_OUTPUT" + + - name: Upload resource profile + if: always() && steps.stop-profiler.outputs.profile_written == 'true' + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: resource-profile-${{ steps.sanitize.outputs.safe_label }} + path: ${{ inputs.working-directory }}/resource-profile.json + retention-days: 14 diff --git a/.github/actions/uv-cache/action.yml b/.github/actions/uv-cache/action.yml new file mode 100644 index 000000000000..b9ee805a190e --- /dev/null +++ b/.github/actions/uv-cache/action.yml @@ -0,0 +1,24 @@ +name: Cache uv downloads +description: >- + Persist uv's download/wheel cache (~/.cache/uv) across runs, keyed on the + dependency manifests. This is the half of astral-sh/setup-uv we still need: + uv itself and CPython 3.11 are baked into the nousresearch/nous-gke-runner + image (see hermes-agent-ci-infra runner/Dockerfile), but the wheel cache is + per-workspace and must still be restored. Without it `uv sync` re-downloads + and re-builds every wheel on every job — the toolchain would be faster to + set up and the sync dramatically slower, a net loss. + +runs: + using: composite + steps: + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/uv + # runner.arch in the key: the cache holds built wheels, which are + # arch-specific — the docker workflow runs this on arm64 too. + key: uv-cache-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('pyproject.toml', 'uv.lock') }} + # Fall back to any older cache for this arch: a stale wheel set still + # saves most of the download, and `uv sync --locked` re-resolves from + # uv.lock regardless, so a partial hit can't produce a wrong env. + restore-keys: | + uv-cache-${{ runner.os }}-${{ runner.arch }}- diff --git a/.github/workflows/newci-ci.yml b/.github/workflows/newci-ci.yml new file mode 100644 index 000000000000..133b39754d96 --- /dev/null +++ b/.github/workflows/newci-ci.yml @@ -0,0 +1,468 @@ +name: '[newci] CI' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production ci.yml, running on the GKE self-hosted +# (ARC) runners so the migration can be observed for a few days without +# touching the workflows that gate merges. Production CI in this branch is +# byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - cache keys are newci-prefixed, so production caches stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - the PR review comment runs --dry-run (prints, never posts) +# - the gate job is renamed; it does NOT gate merges +# +# To retire: delete .github/workflows/newci-*.yml. + +# Orchestrator workflow. Runs ``detect-changes`` once, then conditionally +# calls the sub-workflows that a PR can actually affect. A final +# ``all-checks-pass`` gate job aggregates results so branch protection only +# needs to require a single check. +# +# Sub-workflows are triggered via ``workflow_call`` and keep their own job +# definitions, matrices, and concurrency settings. They no longer have +# ``push:`` / ``pull_request:`` triggers of their own — everything flows +# through this file. +# +# SECURITY: this workflow runs PR-controlled actions, workflows, and code. +# Do not add ``secrets: inherit`` or GitHub App credentials here. Trusted +# main-only automation uses protected environments in its own workflows. + +on: + pull_request: + +permissions: + contents: read + pull-requests: write # needed by lint (PR comment) + supply-chain review_status + actions: read # needed by osv-scanner (SARIF upload) + security-events: write # needed by osv-scanner (SARIF upload) + id-token: write # needed by docker.yml (WIF -> Artifact Registry buildx cache) + packages: write # needed by docker build + +concurrency: + group: newci-ci-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + # ───────────────────────────────────────────────────────────────────── + # detect: run the classifier once. Every downstream job reads its outputs + # to decide whether to run. On push/dispatch the classifier fails open + # (all lanes true) so post-merge validation is never weakened. + # ───────────────────────────────────────────────────────────────────── + detect: + name: Detect affected areas + # Small runner: gates 19 downstream jobs, so the warm pod matters most here. + runs-on: arc-runner-small + timeout-minutes: 10 + outputs: + python: ${{ steps.classify.outputs.python }} + python_prod: ${{ steps.classify.outputs.python_prod }} + frontend: ${{ steps.classify.outputs.frontend }} + site: ${{ steps.classify.outputs.site }} + scan: ${{ steps.classify.outputs.scan }} + deps: ${{ steps.classify.outputs.deps }} + npm_lock: ${{ steps.classify.outputs.npm_lock }} + docker_meta: ${{ steps.classify.outputs.docker_meta }} + mcp_catalog: ${{ steps.classify.outputs.mcp_catalog }} + ci_review: ${{ steps.classify.outputs.ci_review }} + ci_review_files: ${{ steps.classify.outputs.ci_review_files }} + event_name: ${{ github.event_name }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: | + scripts/ci/classify_changes.py + .github/actions/detect-changes/action.yml + sparse-checkout-cone-mode: false + - name: Detect affected areas + id: classify + uses: ./.github/actions/detect-changes + with: + github-token: ${{ github.token }} + + # ───────────────────────────────────────────────────────────────────── + # Lane-gated sub-workflows. Each runs in parallel after detect finishes. + # Skipped workflows (if condition is false) don't spin up runners. + # ───────────────────────────────────────────────────────────────────── + tests: + name: Python tests + needs: detect + if: needs.detect.outputs.python == 'true' + uses: ./.github/workflows/newci-tests.yml + with: + slice_count: 8 + + lint: + name: Python lints + needs: detect + if: needs.detect.outputs.python == 'true' + uses: ./.github/workflows/newci-lint.yml + with: + event_name: ${{ needs.detect.outputs.event_name }} + + js-tests: + name: JS & TS checks + needs: detect + if: needs.detect.outputs.frontend == 'true' + uses: ./.github/workflows/newci-js-tests.yml + + e2e-desktop: + name: Desktop E2E + needs: detect + # python_prod (not python): the Playwright suite exercises the built app + # + `hermes serve` backend, which never import anything under tests/. + # Tests-only PRs (~17% of commits) skip this 5-minute job — the longest + # single job in the workflow — while still running the full pytest lanes. + # + # ⛔ TEMPORARILY DISABLED (Aug 2, 2026, Teknium) — the suite is red on + # every PR and on main itself since the Aug 1 night engines/npm churn + # (#76499 → #76562 → #76575): the mock-backend Electron window never + # gets a title, so boot/chat/setup/interim specs all fail identically + # regardless of the PR's diff (verified on #76573 and the docs-only + # #76582). Tracking issue: #76627 (assigned: Ari). To re-enable, + # delete the `false &&` below — nothing else changed. + if: ${{ false && (needs.detect.outputs.python_prod == 'true' || needs.detect.outputs.frontend == 'true') }} + uses: ./.github/workflows/newci-e2e-desktop.yml + + docs-site: + name: Docs Site + needs: detect + if: needs.detect.outputs.site == 'true' + uses: ./.github/workflows/newci-docs-site-checks.yml + + history-check: + name: Deny unrelated histories + needs: detect + if: needs.detect.outputs.event_name == 'pull_request' + uses: ./.github/workflows/newci-history-check.yml + + contributor-check: + name: Check contributors + needs: detect + if: needs.detect.outputs.python == 'true' + uses: ./.github/workflows/newci-contributor-check.yml + + uv-lockfile: + name: Check uv.lock + needs: detect + uses: ./.github/workflows/newci-uv-lockfile-check.yml + + infographic-check: + name: Check no committed infographics + needs: detect + uses: ./.github/workflows/newci-infographic-check.yml + + lockfile-diff: + name: package-lock.json diff + needs: detect + if: needs.detect.outputs.event_name == 'pull_request' && needs.detect.outputs.npm_lock == 'true' + uses: ./.github/workflows/newci-lockfile-diff.yml + + docker-lint: + name: Lint Docker scripts + needs: detect + if: needs.detect.outputs.docker_meta == 'true' + uses: ./.github/workflows/newci-docker-lint.yml + + docker: + name: Build&Test Docker image + needs: detect + # Trusted main pushes run docker.yml directly so its container-publish + # environment secrets never cross this reusable-workflow call. PR runs + # remain build/test-only and secret-free. Gated on python_prod (not + # python): the image copies installed code, never tests/ — tests-only + # PRs skip the build. + if: needs.detect.outputs.event_name == 'pull_request' && (needs.detect.outputs.python_prod == 'true' || needs.detect.outputs.frontend == 'true' || needs.detect.outputs.docker_meta == 'true') + uses: ./.github/workflows/newci-docker.yml + + supply-chain: + name: Supply-chain scan + needs: detect + if: needs.detect.outputs.event_name == 'pull_request' && (needs.detect.outputs.scan == 'true' || needs.detect.outputs.deps == 'true') + uses: ./.github/workflows/newci-supply-chain-audit.yml + with: + event_name: ${{ needs.detect.outputs.event_name }} + scan: ${{ needs.detect.outputs.scan == 'true' }} + deps: ${{ needs.detect.outputs.deps == 'true' }} + + review-labels: + name: Review label gate + needs: [detect, supply-chain] + if: always() && needs.detect.outputs.event_name == 'pull_request' && (needs.detect.outputs.ci_review == 'true' || needs.detect.outputs.mcp_catalog == 'true' || needs.supply-chain.outputs.critical_findings == 'true') + uses: ./.github/workflows/newci-review-labels.yml + with: + ci_review: ${{ needs.detect.outputs.ci_review == 'true' }} + ci_review_files: ${{ needs.detect.outputs.ci_review_files }} + mcp_catalog: ${{ needs.detect.outputs.mcp_catalog == 'true' }} + supply_chain: ${{ needs.supply-chain.outputs.critical_findings == 'true' }} + + osv-scanner: + name: OSV scan + uses: ./.github/workflows/newci-osv-scanner.yml + + # ───────────────────────────────────────────────────────────────────── + # Live-updating PR review comment. + # + # A single ``comment-live`` job polls the GitHub Actions API every 15s + # for job statuses in this run, re-assembles the review comment from + # whatever results are available, and upserts it via the + # ```` marker. + # + # When the visible job set goes quiet, the poller waits 10 seconds and polls + # once more so downstream jobs created by an aggregate gate get included. + # ───────────────────────────────────────────────────────────────────── + comment-live: + name: CI review comment (live) + needs: + [ + detect, + review-labels, + lockfile-diff, + supply-chain, + osv-scanner, + uv-lockfile, + history-check, + contributor-check, + e2e-desktop + ] + if: always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork != true + runs-on: arc-runner-set + timeout-minutes: 40 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + - name: Run live comment poller + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + PR_NUMBER: ${{ github.event.pull_request.number }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + # Commit info for the review comment header. + COMMIT_SHA: ${{ github.event.pull_request.head.sha }} + COMMIT_MESSAGE: ${{ github.event.pull_request.head.commit.message }} + COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/commits/${{ github.event.pull_request.head.sha }} + # Structured review statuses from workflow_call jobs. + # Each job outputs a JSON array of {source, results: [...]} objects + # that the assembler renders directly — no hardcoded job-name + # matching. We merge all available outputs into one array. + REVIEW_STATUSES: ${{ toJSON(needs.*.outputs.review_status) }} + run: | + set -uo pipefail + + # REVIEW_STATUSES is a JSON array of strings (some may be empty + # when a job was skipped). Parse each string and merge into one + # flat array for the assembler. + python3 - <<'PYEOF' + import json, os, sys + + raw = os.environ.get("REVIEW_STATUSES", "") + merged = [] + if raw: + try: + arr = json.loads(raw) + except (json.JSONDecodeError, TypeError): + arr = [] + for item in arr: + if not item: + continue + try: + statuses = json.loads(item) + except (json.JSONDecodeError, TypeError): + continue + if isinstance(statuses, list): + merged.extend(statuses) + + # Write merged array to a temp file the poller reads. + with open("/tmp/review_statuses.json", "w") as f: + json.dump(merged, f) + print(f"Merged {len(merged)} review status entries") + PYEOF + + python3 scripts/ci/live_comment.py \ + --interval 15 \ + --timeout 2100 \ + --review-statuses-file /tmp/review_statuses.json \ + --dry-run # shadow: print the body, never post it + + # ───────────────────────────────────────────────────────────────────── + # Gate: runs after everything. ``if: always()`` ensures it reports a + # status even when some deps were skipped. Only actual ``failure`` + # results cause it to fail; ``skipped`` is treated as success. + # + # Branch protection should require ONLY this check. + # + # Outputs ``needs-json`` — a compact ``{job_name: result}`` dict — so + # the live comment poller can list failed jobs in the PR comment. + # ───────────────────────────────────────────────────────────────────── + all-checks-pass: + name: '[newci] All checks pass (informational — does NOT gate merges)' + needs: + - detect + - tests + - lint + - js-tests + - e2e-desktop + - docs-site + - history-check + - contributor-check + - uv-lockfile + - lockfile-diff + - docker-lint + - supply-chain + - review-labels + - osv-scanner + # comment-live is a polling job — it doesn't block the gate. + # we don't require docker to pass rn because it's so slow lol + # - docker + if: always() + # Small runner: one script step, no checkout. + runs-on: arc-runner-small + timeout-minutes: 10 + outputs: + needs-json: ${{ steps.evaluate.outputs.needs-json }} + steps: + - name: Evaluate job results + id: evaluate + env: + NEEDS: ${{ toJSON(needs) }} + run: | + echo "$NEEDS" | python3 -c " + import json, sys + needs = json.load(sys.stdin) + # Emit compact {job_name: result} for the comment assembler. + compact = {name: info['result'] for name, info in needs.items()} + print(f'needs-json={json.dumps(compact)}') + with open('$GITHUB_OUTPUT', 'a') as f: + f.write(f'needs-json={json.dumps(compact)}\n') + failed = [name for name, info in needs.items() if info['result'] == 'failure'] + for name, info in sorted(needs.items()): + result = info['result'] + icon = '✅' if result in ('success', 'skipped') else '❌' + print(f'{icon} {name}: {result}') + if failed: + print(f'::error::{len(failed)} job(s) failed: {\", \".join(failed)}') + sys.exit(1) + print('All checks passed (or were skipped)') + " + + # ───────────────────────────────────────────────────────────────────── + # CI timing report: collect per-job/step durations from the GitHub API, + # cache them on main (as a baseline), and on PRs generate an HTML diff + # report with a gantt chart + per-step breakdown. The report is uploaded + # as an artifact and a markdown summary is written to $GITHUB_STEP_SUMMARY. + # + # The live comment poller can read the standalone review-status artifact + # after the HTML report is uploaded, so its link points straight at that report. + # ───────────────────────────────────────────────────────────────────── + ci-timings: + name: CI timing report + needs: [all-checks-pass, docker] + if: always() + # Small runner: checkout plus a python report script. + runs-on: arc-runner-small + timeout-minutes: 10 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Restore baseline cache (PR only) + if: github.event_name == 'pull_request' + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ci-timings-baseline.json + # Prefix-match: exact key will never hit (run_id differs), so + # restore-keys finds the most recent baseline from main. + key: newci-ci-timings-baseline-never-exact + restore-keys: | + newci-ci-timings-baseline- + + - name: Download resource profiles + # Advisory — if no profiles were uploaded (e.g. sub-workflows + # didn't run), this step finds nothing and the report still works. + # NOTE: no merge-multiple — every artifact contains a file named + # resource-profile.json, so merging would clobber all but one. + # Per-artifact subdirs are exactly what load_resource_profiles walks. + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: resource-profile-* + path: resource-profiles + + - name: Collect timings and generate report + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + python3 scripts/ci/timings_report.py \ + --baseline ci-timings-baseline.json \ + --output ci-timings-report.html \ + --json-out ci-timings.json \ + --summary-out ci-timings-summary.md \ + --profiles-dir resource-profiles + + - name: Upload HTML report + # Advisory report — artifact-service blips must not fail the job. + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + id: ci-timings-html + with: + name: ci-timings-report + path: ci-timings-report.html + archive: false + retention-days: 14 + + - name: Build linked review status + if: hashFiles('ci-timings.json') != '' + env: + CI_TIMINGS_REPORT_URL: ${{ steps.ci-timings-html.outputs.artifact-url }} + run: | + python3 scripts/ci/timings_report.py \ + --from-json ci-timings.json \ + --baseline ci-timings-baseline.json \ + --profiles-dir resource-profiles \ + --review-status-out review-status.json \ + --review-status-only + + - name: Upload review status + if: hashFiles('review-status.json') != '' + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ci-timings-review-status + path: review-status.json + retention-days: 14 + + - name: Output summary + env: + REPORT_URL: ${{ steps.ci-timings-html.outputs.artifact-url}} + run: | + { + echo "# CI Timing report" + echo "[View the full interactive report]($REPORT_URL)" + } >> "$GITHUB_STEP_SUMMARY" + cat ci-timings-summary.md >> "$GITHUB_STEP_SUMMARY" + + - name: Save baseline cache (main only) + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + run: | + # Degraded runs (API rate-limited) produce no ci-timings.json — + # skip rather than fail, and never cache an empty baseline. + if [ -f ci-timings.json ]; then + cp ci-timings.json ci-timings-baseline.json + else + echo "No timings JSON this run — skipping baseline update" + fi + + - name: Upload baseline to cache (main only) + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && hashFiles('ci-timings-baseline.json') != '' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ci-timings-baseline.json + key: newci-ci-timings-baseline-${{ github.run_id }} diff --git a/.github/workflows/newci-contributor-check.yml b/.github/workflows/newci-contributor-check.yml new file mode 100644 index 000000000000..c55ee60a774f --- /dev/null +++ b/.github/workflows/newci-contributor-check.yml @@ -0,0 +1,112 @@ +name: '[newci] Contributor Attribution Check' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production contributor-check.yml, running on the GKE self-hosted +# (ARC) runners so the migration can be observed for a few days without +# touching the workflows that gate merges. Production CI in this branch is +# byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - cache keys are newci-prefixed, so production caches stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - the PR review comment runs --dry-run (prints, never posts) +# - the gate job is renamed; it does NOT gate merges +# +# To retire: delete .github/workflows/newci-*.yml. + +on: + workflow_call: + outputs: + review_status: + description: "JSON array of review status objects" + value: ${{ jobs.check-attribution.outputs.review_status }} + +permissions: + contents: read + +jobs: + check-attribution: + runs-on: arc-runner-set + timeout-minutes: 10 + outputs: + review_status: ${{ steps.check-emails.outputs.review_status }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 # Full history needed for git log + + - name: Check for unmapped contributor emails + id: check-emails + run: | + # Commits this PR adds on top of main. `origin/main..HEAD` already + # means "reachable from HEAD, not from main", which is exactly what + # resolving the merge base and using `..HEAD` computes — the two + # are equivalent (verified, including when the branch has merged main + # into itself). Dropping the extra git call also drops an unquoted + # expansion of the resulting SHA. + NEW_EMAILS=$(git log origin/main..HEAD --format='%ae' --no-merges | sort -u) + + if [ -z "$NEW_EMAILS" ]; then + echo "No new commits to check." + echo "review_status=[]" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # An email is mapped if it has a file in contributors/emails/ + # (one file per email — conflict-free) or an entry in the frozen + # legacy AUTHOR_MAP in scripts/release.py. + MISSING="" + while IFS= read -r email; do + # Skip teknium and bot emails + case "$email" in + *teknium*|*noreply@github.com*|*dependabot*|*github-actions*|*anthropic.com*|*cursor.com*) + continue ;; + esac + + if echo "$email" | grep -qP '\+.*@users\.noreply\.github\.com'; then + continue # GitHub id+login noreply emails auto-resolve + fi + + if [ -f "contributors/emails/${email}" ]; then + continue # mapped via the contributors directory + fi + + if ! grep -qF "\"${email}\"" scripts/release.py 2>/dev/null; then + AUTHOR=$(git log --author="$email" --format='%an' -1) + MISSING="${MISSING}\n ${email} (${AUTHOR})" + fi + done <<< "$NEW_EMAILS" + + if [ -n "$MISSING" ]; then + echo "" + echo "⚠️ New contributor email(s) without a mapping:" + echo -e "$MISSING" + echo "" + echo "Add a mapping file (do NOT edit AUTHOR_MAP in release.py):" + echo " python3 scripts/audit_pr_attribution.py --fix # auto-resolve + create files" + echo "or manually:" + echo -e "$MISSING" | while read -r line; do + email=$(echo "$line" | sed 's/^ *//' | cut -d' ' -f1) + [ -z "$email" ] && continue + echo " python3 scripts/add_contributor.py ${email} " + done + echo "" + echo "To find the GitHub username for an email:" + echo " gh api 'search/users?q=EMAIL+in:email' --jq '.items[0].login'" + + # Emit review_status for unmapped emails + DETAIL=$(echo -e "$MISSING" | sed '/^$/d; s/^ //') + HOW_TO_FIX=$'Run from the PR branch:\n```\npython3 scripts/audit_pr_attribution.py --fix\ngit add contributors && git commit -m "chore: map contributor emails" && git push\n```\nOr map one email manually (do NOT edit AUTHOR_MAP in release.py):\n```\npython3 scripts/add_contributor.py \n```\nTo find the GitHub username for an email:\n```\ngh api \'search/users?q=EMAIL+in:email\' --jq \'.items[0].login\'\n```\n' + REVIEW_STATUS=$(jq -nc \ + --arg detail "$DETAIL" \ + --arg how_to_fix "$HOW_TO_FIX" \ + '[{"source":"contributor attribution","results":[{"kind":"action_required","title":"Unmapped contributor email(s)","summary":"New contributor email(s) are not in AUTHOR_MAP.","detail":$detail,"how_to_fix":$how_to_fix}]}]') + echo "review_status=$REVIEW_STATUS" >> "$GITHUB_OUTPUT" + + exit 1 + else + echo "✅ All contributor emails are mapped." + fi diff --git a/.github/workflows/newci-docker-lint.yml b/.github/workflows/newci-docker-lint.yml new file mode 100644 index 000000000000..9bd09dd0826c --- /dev/null +++ b/.github/workflows/newci-docker-lint.yml @@ -0,0 +1,78 @@ +name: '[newci] Docker / shell lint' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production docker-lint.yml, running on the GKE self-hosted +# (ARC) runners so the migration can be observed for a few days without +# touching the workflows that gate merges. Production CI in this branch is +# byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - cache keys are newci-prefixed, so production caches stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - the PR review comment runs --dry-run (prints, never posts) +# - the gate job is renamed; it does NOT gate merges +# +# To retire: delete .github/workflows/newci-*.yml. + +# Lints the container build inputs: Dockerfile (via hadolint) and any shell +# scripts under docker/ (via shellcheck). These catch the class of regression +# the behavioral docker smoke test can't — unquoted variable +# expansions, silently-failing RUN commands, etc. +# +# Rules and ignores are documented in .hadolint.yaml at the repo root. +# shellcheck severity is pinned to `error` so SC1091-style "can't follow +# sourced script" info-level warnings don't fail the job — the .venv +# activate script doesn't exist at lint time. + +on: + workflow_call: + +permissions: + contents: read + +concurrency: + group: newci-docker-lint-${{ github.ref }} + cancel-in-progress: true + +jobs: + hadolint: + name: Lint Dockerfile (hadolint) + # arc-runner-docker, NOT arc-runner-small: hadolint-action declares + # `runs: using: docker`, so the runner builds and runs it as a container + # and needs a real daemon — even though no step here shells out to + # docker. Grepping job bodies for docker commands misses this; check + # each action's `runs.using` before routing a job to a dind-less set. + runs-on: arc-runner-docker + timeout-minutes: 5 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: hadolint + uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf # v3.1.0 + with: + dockerfile: Dockerfile + config: .hadolint.yaml + failure-threshold: warning + + shellcheck: + name: Lint docker/ shell scripts (shellcheck) + # Short gate job: small runner, no dind (see hermes-agent-ci-infra). + runs-on: arc-runner-small + timeout-minutes: 5 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: shellcheck + uses: ludeeus/action-shellcheck@00cae500b08a931fb5698e11e79bfbd38e612a38 # v2.0.0 + env: + # Severity = error: SC1091 (can't follow sourced script) is info- + # level and would otherwise fail when the venv activate script + # doesn't exist at lint time. + SHELLCHECK_OPTS: --severity=error + with: + scandir: ./docker diff --git a/.github/workflows/newci-docker.yml b/.github/workflows/newci-docker.yml new file mode 100644 index 000000000000..2e0426d7a4d2 --- /dev/null +++ b/.github/workflows/newci-docker.yml @@ -0,0 +1,374 @@ +name: '[newci] Docker Build, Test, and Publish' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production docker.yml, running on the GKE self-hosted +# (ARC) runners so the migration can be observed for a few days without +# touching the workflows that gate merges. Production CI in this branch is +# byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - cache keys are newci-prefixed, so production caches stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - the PR review comment runs --dry-run (prints, never posts) +# - the gate job is renamed; it does NOT gate merges +# +# To retire: delete .github/workflows/newci-*.yml. + +on: + # Trusted main pushes run this workflow directly so environment-scoped + # Docker Hub secrets are resolved by the top-level workflow, never across + # a reusable-workflow boundary. + push: + branches: [main] + release: + types: [published] + # CI calls this only for untrusted PR build/test coverage. Those runs never + # reach the protected publish or merge jobs below. + workflow_call: + +permissions: + contents: read + +# Concurrency: push/release runs are NEVER cancelled so every merge gets +# its own image. PR runs reuse a PR-scoped group with +# cancel-in-progress: true so rapid pushes to the same PR collapse to +# the latest commit. +concurrency: + group: newci-docker-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + IMAGE_NAME: nousresearch/hermes-agent + +jobs: + # Build and test the image for each architecture. This job runs PR code, + # so it must remain secret-free. Publishing happens in the separate, + # protected publish job after these tests pass. + # + # Buildx layer cache lives in Artifact Registry (us-central1, same region + # as the ARC runners) instead of GitHub's cache CDN. Reads are keyless via + # GKE Workload Identity on the runner pods; writes are keyless via GitHub + # OIDC -> GCP WIF (google-github-actions/auth) and happen ONLY on trusted + # main-push/release contexts — PR builds of any origin are read-only so + # PR-controlled code can never write cache layers the publish job reads. + build: + if: github.repository == 'NousResearch/hermes-agent' + permissions: + contents: read + # OIDC token for WIF cache writes — only minted on non-PR events + # (see the gcp-auth step); PR runs stay secret-free. + id-token: write + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + # dind lives only on arc-runner-docker now; the general amd64 + # set dropped it so the ~20 workflows that never touch a daemon + # stop paying for a privileged sidecar on every pod. + runner: arc-runner-docker + platform: linux/amd64 + - arch: arm64 + runner: arc-runner-arm64 + platform: linux/arm64 + + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # Retry once on transient Docker Hub / buildkit pull failures + # (connection reset, auth token timeout, rate limiting). The action + # generates a unique builder name per invocation so the retry doesn't + # collide with the failed first attempt. A genuine persistent failure + # still fails the job — only the first attempt has continue-on-error. + # Refs: docker/setup-buildx-action#510 + - name: Set up Docker Buildx + id: buildx + continue-on-error: true + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + - name: Set up Docker Buildx (retry) + if: steps.buildx.outcome == 'failure' + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + # Keyless GCP auth via GitHub OIDC -> WIF. PR builds (fork or + # same-repo) get NO write token: this job runs PR-controlled code + # and the publish job reads the same cache ref, so a PR-writable + # cache would be a layer-poisoning vector. PRs read the cache via + # the pod's GKE Workload Identity; writes happen only on trusted + # main-push/release contexts. + - name: Authenticate to GCP (WIF, cache writes) + id: gcp-auth + if: github.event_name != 'pull_request' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + project_id: hermes-agent-github-actions + workload_identity_provider: projects/1067970703723/locations/global/workloadIdentityPools/github-actions/providers/github-oidc + service_account: gha-buildx-cache@hermes-agent-github-actions.iam.gserviceaccount.com + token_format: access_token + + # PR builds mint a READ-ONLY registry token from the runner pod's GKE + # Workload Identity instead. The pod's KSA (arc-runners/buildx-cache- + # reader) impersonates gha-buildx-cache-ro@, which holds only + # artifactregistry.reader on the ci-cache repo — so this token cannot + # write cache layers, and the layer-poisoning boundary above holds. + # + # This step is what actually turns pod WI into a docker credential: + # buildx only forwards auth that's present in the docker cred store, + # and there is no ambient cred helper for *.pkg.dev in the runner + # image — without an explicit login, cache-from falls back to an + # anonymous pull, gets 403, and every PR build runs cache-cold + # (~15 min instead of ~3). + - name: Mint read-only cache token (pod Workload Identity) + id: wi-token + if: github.event_name == 'pull_request' + run: | + set -euo pipefail + token=$(curl -sSf -H "Metadata-Flavor: Google" \ + "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" \ + | jq -r .access_token) + echo "::add-mask::$token" + echo "token=$token" >> "$GITHUB_OUTPUT" + + - name: Log in to Artifact Registry + if: steps.gcp-auth.outputs.access_token != '' || steps.wi-token.outputs.token != '' + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + registry: us-central1-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.gcp-auth.outputs.access_token || steps.wi-token.outputs.token }} + + # Build once, load into the local daemon for testing. Cached + # per-arch; the push step below reuses every layer from this build. + # Cache lives in same-region Artifact Registry: reads always work + # (pod Workload Identity); writes only when the WIF auth step ran. + - name: Build image (${{ matrix.arch }}) + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 + with: + context: . + file: Dockerfile + load: true + platforms: ${{ matrix.platform }} + tags: ${{ env.IMAGE_NAME }}:test + build-args: | + HERMES_GIT_SHA=${{ github.sha }} + cache-from: type=registry,ref=us-central1-docker.pkg.dev/hermes-agent-github-actions/ci-cache/hermes-agent:buildcache-${{ matrix.arch }} + cache-to: ${{ steps.gcp-auth.outputs.access_token != '' && format('type=registry,ref=us-central1-docker.pkg.dev/hermes-agent-github-actions/ci-cache/hermes-agent:buildcache-{0},mode=max,image-manifest=true', matrix.arch) || '' }} + + # Run the docker-integration test suite against the freshly-built + # image already loaded into the local daemon (`:test`). + # + # Piggybacking here avoids a second image build: the build step + # already loaded the image into the daemon under + # `${IMAGE_NAME}:test`, so we just point ``HERMES_TEST_IMAGE`` at + # that. The fixture's ``HERMES_TEST_IMAGE`` branch (see + # tests/docker/conftest.py:62-63) short-circuits the rebuild. + # + # Why this job and not a standalone one: the image is 5GB+; passing + # it between jobs via ``docker save``/``upload-artifact`` is slower + # than the build itself. Reusing the existing daemon state is the + # cheapest path to coverage on every PR that touches docker code. + # --------------------------------------------------------------------- + - name: Restore uv cache (for docker tests) + uses: ./.github/actions/uv-cache + + - name: Install Python dependencies (for docker tests) + # ``dev`` extra pulls in pytest, pytest-asyncio — + # everything tests/docker/ needs. We deliberately avoid ``all`` + # here because the docker tests only drive the container via + # subprocess and don't import hermes_agent's optional deps. + uses: ./.github/actions/retry + with: + command: uv sync --locked --python 3.11 --extra dev + + - name: Run docker integration tests + # HERMES_TEST_WORKERS=8: this suite shares ONE dockerd, so width + # is daemon-bound, not CPU-bound. Width sweep with prewarmed image + # + split files (2026-07): -j4 58-62s, -j8 39s, -j12 35s but with + # ~2x per-file contention inflation at 12. 8 is the knee. + # run_tests.sh forwards HERMES_TEST_WORKERS through its hermetic + # env -i into the parallel runner. + uses: ./.github/actions/profile + with: + label: docker-tests-${{ matrix.arch }} + command: HERMES_TEST_WORKERS=8 HERMES_TEST_IMAGE="${{ env.IMAGE_NAME }}:test" scripts/run_tests.sh tests/docker/ --file-timeout 600 + + # --------------------------------------------------------------------------- + # Rebuild and push each architecture only after the unprivileged build/test + # matrix passes. This job is the sole Docker Hub credential boundary. + # --------------------------------------------------------------------------- + publish: + if: github.repository == 'NousResearch/hermes-agent' && (github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'release') + needs: [build] + environment: container-publish + permissions: + contents: read + id-token: write + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + # See the build job: dind lives only on arc-runner-docker. + runner: arc-runner-docker + platform: linux/amd64 + - arch: arm64 + runner: arc-runner-arm64 + platform: linux/arm64 + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + steps: + - name: Checkout trusted source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # Retry once on transient Docker Hub / buildkit pull failures. + # See build job for rationale; same pattern. + - name: Set up Docker Buildx + id: buildx + continue-on-error: true + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + - name: Set up Docker Buildx (retry) + if: steps.buildx.outcome == 'failure' + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + # Same keyless AR cache as the build job. Publish only runs on + # trusted main/release contexts, so WIF auth is unconditional here. + - name: Authenticate to GCP (WIF, cache) + id: gcp-auth + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + project_id: hermes-agent-github-actions + workload_identity_provider: projects/1067970703723/locations/global/workloadIdentityPools/github-actions/providers/github-oidc + service_account: gha-buildx-cache@hermes-agent-github-actions.iam.gserviceaccount.com + token_format: access_token + + - name: Log in to Artifact Registry + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + registry: us-central1-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.gcp-auth.outputs.access_token }} + + - name: Log in to Docker Hub + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + # Push by digest only (no tag). The merge job assembles the tagged + # manifest list after both architecture publishers complete. + - name: Push ${{ matrix.arch }} by digest + id: push + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 + with: + context: . + file: Dockerfile + platforms: ${{ matrix.platform }} + labels: | + org.opencontainers.image.revision=${{ github.sha }} + build-args: | + HERMES_GIT_SHA=${{ github.sha }} + outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + cache-from: type=registry,ref=us-central1-docker.pkg.dev/hermes-agent-github-actions/ci-cache/hermes-agent:buildcache-${{ matrix.arch }} + cache-to: type=registry,ref=us-central1-docker.pkg.dev/hermes-agent-github-actions/ci-cache/hermes-agent:buildcache-${{ matrix.arch }},mode=max,image-manifest=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.push.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: digest-${{ matrix.arch }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + # --------------------------------------------------------------------------- + # Stitch both per-arch digests into a single tagged multi-arch manifest. + # This is a registry-side operation — no building, no layer re-push — + # so it runs in ~30 seconds. + # + # On main pushes: tags both :main and :latest. + # On releases: tags :. + # --------------------------------------------------------------------------- + merge: + if: github.repository == 'NousResearch/hermes-agent' && (github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'release') + # Needs buildx to assemble the manifest list — see the build job. + runs-on: arc-runner-docker + needs: [publish] + timeout-minutes: 10 + environment: container-publish + steps: + - name: Download digests + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: /tmp/digests + pattern: digest-* + merge-multiple: true + + # Retry once on transient Docker Hub / buildkit pull failures. + # See build job for rationale; same pattern. + - name: Set up Docker Buildx + id: buildx + continue-on-error: true + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + - name: Set up Docker Buildx (retry) + if: steps.buildx.outcome == 'failure' + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + - name: Log in to Docker Hub + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Create manifest list and push + working-directory: /tmp/digests + env: + IMAGE_NAME: ${{ env.IMAGE_NAME }} + RELEASE_TAG: ${{ github.event.release.tag_name }} + run: | + set -euo pipefail + args=() + for digest_file in *; do + args+=("${IMAGE_NAME}@sha256:${digest_file}") + done + if [ "${{ github.event_name }}" = "release" ]; then + tags=(-t "${IMAGE_NAME}:${RELEASE_TAG}") + else + tags=(-t "${IMAGE_NAME}:main" -t "${IMAGE_NAME}:latest") + fi + # Retry: Docker Hub API + just-pushed digest eventual consistency + # can transiently fail the create; the operation is idempotent. + for i in 1 2 3; do + if docker buildx imagetools create "${tags[@]}" "${args[@]}"; then + break + fi + if [ "$i" = 3 ]; then + echo "::error::imagetools create failed after 3 attempts" + exit 1 + fi + echo "::warning::imagetools create failed (attempt $i); retrying in 20s" + sleep 20 + done + + - name: Inspect image + env: + IMAGE_NAME: ${{ env.IMAGE_NAME }} + RELEASE_TAG: ${{ github.event.release.tag_name }} + run: | + if [ "${{ github.event_name }}" = "release" ]; then + docker buildx imagetools inspect "${IMAGE_NAME}:${RELEASE_TAG}" + else + docker buildx imagetools inspect "${IMAGE_NAME}:main" + fi diff --git a/.github/workflows/newci-docs-site-checks.yml b/.github/workflows/newci-docs-site-checks.yml new file mode 100644 index 000000000000..6657e79bc883 --- /dev/null +++ b/.github/workflows/newci-docs-site-checks.yml @@ -0,0 +1,86 @@ +name: '[newci] Docs Site Checks' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production docs-site-checks.yml, running on the GKE self-hosted +# (ARC) runners so the migration can be observed for a few days without +# touching the workflows that gate merges. Production CI in this branch is +# byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - cache keys are newci-prefixed, so production caches stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - the PR review comment runs --dry-run (prints, never posts) +# - the gate job is renamed; it does NOT gate merges +# +# To retire: delete .github/workflows/newci-*.yml. + +on: + workflow_call: + +permissions: + contents: read + +jobs: + docs-site-checks: + runs-on: arc-runner-set + timeout-minutes: 20 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + id: npm-cache + with: + path: website/node_modules + # node26: native builds + engine checks are node-major-specific; + # keep in sync with NODE_MAJOR in the runner image (see + # hermes-agent-ci-infra runner/Dockerfile). + key: newci-node-modules-cache-node26-${{ hashFiles('website/package-lock.json') }} + + - name: Install website dependencies + uses: ./.github/actions/retry + with: + command: npm ci + working-directory: website + if: steps.npm-cache.outputs.cache-hit != 'true' + + - name: Install ascii-guard + # Production uses setup-python + `python -m pip install`, which leaves + # the interpreter AND the ascii-guard console script on PATH. The ARC + # image has no such ambient env: `uv pip install` with no venv errors + # out ("No virtual environment found"), so create one explicitly. + # + # Deliberately NOT `uv run`: this repo has a pyproject.toml, so `uv run` + # is a *project* run — it would sync hermes-agent's full locked + # dependency set into .venv and prune ascii-guard/pyyaml back out as + # extraneous. This job only needs those two packages. + uses: ./.github/actions/retry + with: + command: | + uv venv --python 3.11 .venv + uv pip install --python .venv ascii-guard==2.3.0 pyyaml==6.0.3 + + - name: Put the docs venv on PATH + # `npm run lint:diagrams` shells out to the `ascii-guard` binary, and + # the extract/generate scripts need `yaml` importable — both come from + # the venv, so it has to lead PATH for the rest of the job. + run: echo "${GITHUB_WORKSPACE}/.venv/bin" >> "$GITHUB_PATH" + + - name: Extract skill metadata for dashboard + run: python3 website/scripts/extract-skills.py + + - name: Regenerate per-skill docs pages + catalogs + run: python3 website/scripts/generate-skill-docs.py + + - name: Lint docs diagrams + run: npm run lint:diagrams + working-directory: website + + - name: Build Docusaurus + uses: ./.github/actions/profile + with: + label: docs-site-build + working-directory: website + command: npm run build diff --git a/.github/workflows/newci-e2e-desktop.yml b/.github/workflows/newci-e2e-desktop.yml new file mode 100644 index 000000000000..0219adc19fc3 --- /dev/null +++ b/.github/workflows/newci-e2e-desktop.yml @@ -0,0 +1,277 @@ +name: '[newci] E2E Desktop' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production e2e-desktop.yml, running on the GKE self-hosted +# (ARC) runners so the migration can be observed for a few days without +# touching the workflows that gate merges. Production CI in this branch is +# byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - cache keys are newci-prefixed, so production caches stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - the PR review comment runs --dry-run (prints, never posts) +# - the gate job is renamed; it does NOT gate merges +# +# To retire: delete .github/workflows/newci-*.yml. + +on: + workflow_call: + outputs: + review_status: + description: Screenshot and visual-diff status for the CI review comment. + value: ${{ jobs.e2e.outputs.review_status }} + +permissions: + contents: read + +concurrency: + group: newci-e2e-desktop-${{ github.ref }} + cancel-in-progress: true + +jobs: + e2e: + name: Playwright E2E (Linux) + runs-on: arc-runner-set + timeout-minutes: 20 + outputs: + review_status: ${{ steps.review-status.outputs.review_status }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # ── System deps for Electron on headless Ubuntu ─────────────────── + # xvfb and Electron's GTK/NSS/audio libraries are baked into + # nousresearch/nous-gke-runner so this works in the ARC runner pod + # without requiring passwordless sudo. + + # ── Node ─────────────────────────────────────────────────────────── + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + id: npm-cache + with: + # All workspace node_modules; `full` key = installs that ran + # lifecycle scripts (electron binary download, node-pty build). + path: | + node_modules + apps/*/node_modules + ui-tui/node_modules + ui-tui/packages/*/node_modules + web/node_modules + tests-js/node_modules + # node26 in the key: node-pty/electron native artifacts are + # node-major-specific; sync with NODE_MAJOR in the runner image + # (hermes-agent-ci-infra runner/Dockerfile). + key: newci-node-modules-full-node26-${{ runner.arch }}-${{ hashFiles('package-lock.json') }} + + + # Full npm ci (not --ignore-scripts): electron's postinstall + # downloads the binary we launch, and node-pty's native build is + # needed for the terminal pane. + - uses: ./.github/actions/retry + with: + command: npm ci + if: steps.npm-cache.outputs.cache-hit != 'true' + + # ── Python (for the hermes serve backend) ────────────────────────── + - name: Restore uv cache + uses: ./.github/actions/uv-cache + + - name: Install Python dependencies + uses: ./.github/actions/retry + with: + command: uv sync --locked --python 3.11 --extra all --extra dev + + # ── Build desktop app ───────────────────────────────────────────── + # The Playwright step below runs `npm run build` before testing so + # dist/ is always fresh — no separate build step needed here. + + # ── Restore visual baseline screenshots from main ────────────────── + # Baselines are generated on main (via --update-snapshots) and cached. + # On PRs, we restore them so toHaveScreenshot has something to compare + # against. The cache key is keyed on the desktop source files so a + # UI change naturally invalidates it — but we fall back to the main + # cache to avoid cold starts on unrelated PRs. + - name: Restore visual baseline screenshots + id: restore-baselines + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: apps/desktop/e2e/*-snapshots + key: newci-visual-baselines-${{ github.ref_name }} + restore-keys: | + newci-visual-baselines-main + + # ── Run Playwright E2E under xvfb ───────────────────────────────── + # xvfb runs at a fixed 1280x1024 screen so the 1220x800 Electron + # window always has a consistent viewport for screenshot comparison. + # On main, we run with --update-snapshots to generate baselines. + # `npm run test:e2e` builds dist/ as a pretest hook so the renderer + # is always fresh — no separate build step needed. + - name: Run Playwright E2E tests + uses: ./.github/actions/profile + with: + label: e2e-desktop + working-directory: apps/desktop + command: | + if [ "${{ github.ref_name }}" = "main" ]; then + echo "On main — generating/updating baseline screenshots" + npm run build && xvfb-run -a --server-args="-screen 0 1280x1024x24" \ + npx playwright test --reporter=list --update-snapshots + else + echo "On PR — comparing against cached baselines" + npm run build && xvfb-run -a --server-args="-screen 0 1280x1024x24" \ + npx playwright test --reporter=list + fi + env: + CI: 'true' + # Ensure no real API keys leak into the test env. + OPENROUTER_API_KEY: '' + OPENAI_API_KEY: '' + NOUS_API_KEY: '' + + # ── Save updated baselines to cache (main only) ─────────────────── + - name: Save updated baselines to cache + if: github.ref_name == 'main' && always() + uses: actions/cache/save@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + with: + path: apps/desktop/e2e/*-snapshots + key: newci-visual-baselines-main + + # ── Upload Playwright report (HTML + traces) ────────────────────── + - name: Upload Playwright report + id: upload-report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: playwright-report-${{ github.sha }} + path: apps/desktop/playwright-report + retention-days: 14 + overwrite: true + + # ── Upload test results (screenshots, traces, diffs) ─────────────── + - name: Upload test results + id: upload-results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: playwright-test-results-${{ github.sha }} + path: apps/desktop/test-results + retention-days: 14 + overwrite: true + + # ── Upload just the visual diffs (small, fast to review) ────────── + - name: Upload visual diffs + id: upload-diffs + if: always() && github.ref_name != 'main' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: visual-diffs-${{ github.sha }} + path: | + apps/desktop/test-results/**/*-diff.png + apps/desktop/test-results/**/*-actual.png + apps/desktop/test-results/**/*-expected.png + retention-days: 14 + overwrite: true + if-no-files-found: ignore + + - name: Build screenshot review status + id: review-status + if: always() + working-directory: apps/desktop + env: + RESULTS_URL: ${{ steps.upload-results.outputs.artifact-url }} + run: | + python3 ../../scripts/ci/e2e_screenshot_status.py \ + --results-dir test-results \ + --manifest-output /tmp/e2e-screenshot-manifest.json \ + --evidence-dir /tmp/e2e-evidence \ + --artifact-url "$RESULTS_URL" \ + --output /tmp/e2e-review-status.json + { + echo 'review_status<<__E2E_REVIEW_STATUS__' + cat /tmp/e2e-review-status.json + echo '__E2E_REVIEW_STATUS__' + } >> "$GITHUB_OUTPUT" + + # The trusted workflow_run publisher consumes only this flat, bounded + # artifact. It turns selected images into GitHub attachment URLs; it + # never checks out or runs this PR's code. + - name: Upload inline E2E evidence + if: always() && github.ref_name != 'main' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-evidence-${{ github.sha }} + path: /tmp/e2e-evidence + retention-days: 14 + overwrite: true + if-no-files-found: error + + # ── Generate step summary with visual diff info ─────────────────── + # Parse the JSON report + scan for diff images, then post a summary + # to the GitHub Actions step output so reviewers can see what changed + # without downloading artifacts. Runs AFTER uploads so it can link + # the artifact download URLs from their step outputs. + - name: Generate visual diff summary + if: always() + working-directory: apps/desktop + env: + REPORT_URL: ${{ steps.upload-report.outputs.artifact-url }} + RESULTS_URL: ${{ steps.upload-results.outputs.artifact-url }} + DIFFS_URL: ${{ steps.upload-diffs.outputs.artifact-url }} + run: | + { + echo "## Desktop E2E — Visual Diff Report" + echo "" + + # Count diff images (playwright writes *-diff.png on mismatch) + DIFF_COUNT=$(find test-results -name '*-diff.png' 2>/dev/null | wc -l) + ACTUAL_COUNT=$(find test-results -name '*-actual.png' 2>/dev/null | wc -l) + + if [ "$DIFF_COUNT" -eq 0 ]; then + echo "✅ All $ACTUAL_COUNT screenshot(s) matched their baselines (or no baselines existed yet)." + else + echo "📸 **$DIFF_COUNT of $ACTUAL_COUNT screenshot(s) differ from baseline:**" + echo "" + echo "| Test | Diff | Actual | Expected |" + echo "|------|------|--------|----------|" + + # List each diff image with a link to the artifact + for diff in $(find test-results -name '*-diff.png' 2>/dev/null | sort); do + base=${diff%-diff.png} + test_name=$(basename "$base") + echo "| $test_name | [diff]($diff) | [actual](${base}-actual.png) | [expected](${base}-expected.png) |" + done + fi + + echo "" + echo "📥 **Artifacts:**" + echo "" + if [ -n "$RESULTS_URL" ]; then + echo "- [playwright-test-results]($RESULTS_URL) — all screenshots (actual + expected + diff) + traces" + fi + if [ -n "$REPORT_URL" ]; then + echo "- [playwright-report]($REPORT_URL) — interactive HTML report" + fi + if [ -n "$DIFFS_URL" ]; then + echo "- [visual-diffs]($DIFFS_URL) — just the diffed screenshots (small, fast to review)" + fi + echo "" + echo "**To update baselines:** merge to main (baselines auto-update on main runs) or run \`npx playwright test --update-snapshots\` locally." + + # Also parse the JSON report for pass/fail counts + if [ -f playwright-report/results.json ]; then + echo "" + echo "### Test Results" + echo "" + node -e " + const r = require('./playwright-report/results.json'); + const stats = r.stats || {}; + console.log('| Status | Count |'); + console.log('|--------|-------|'); + console.log('| ✅ Passed | ' + (stats.expected || 0) + ' |'); + console.log('| ❌ Failed | ' + (stats.unexpected || 0) + ' |'); + console.log('| ⏭️ Skipped | ' + (stats.skipped || 0) + ' |'); + console.log('| 🔄 Flaky | ' + (stats.flaky || 0) + ' |'); + " 2>/dev/null || true + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/newci-history-check.yml b/.github/workflows/newci-history-check.yml new file mode 100644 index 000000000000..2e0efb7b7f69 --- /dev/null +++ b/.github/workflows/newci-history-check.yml @@ -0,0 +1,74 @@ +name: '[newci] History Check' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production history-check.yml, running on the GKE self-hosted +# (ARC) runners so the migration can be observed for a few days without +# touching the workflows that gate merges. Production CI in this branch is +# byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - cache keys are newci-prefixed, so production caches stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - the PR review comment runs --dry-run (prints, never posts) +# - the gate job is renamed; it does NOT gate merges +# +# To retire: delete .github/workflows/newci-*.yml. + +# Rejects PRs whose branch has no common ancestor with main. +# +# GitHub's merge UI does not refuse unrelated-history merges. PR #25045 +# (May 2026) landed from a disconnected branch: its parent-less root commit +# was grafted into main as a second root and `git blame` for ~1500 files +# collapsed onto it. + +on: + workflow_call: + outputs: + review_status: + description: "JSON array of review_status objects for the synthesizer." + value: ${{ jobs.check-common-ancestor.outputs.review_status }} + +permissions: + contents: read + +jobs: + check-common-ancestor: + # Short gate job: small runner, no dind (see hermes-agent-ci-infra). + runs-on: arc-runner-small + timeout-minutes: 10 + outputs: + review_status: ${{ steps.merge-base-check.outputs.review_status }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Resolve merge base + id: mb + uses: ./.github/actions/merge-base + with: + base: ${{ github.event.pull_request.base.sha }} + head: ${{ github.event.pull_request.head.sha }} + # Absence is the thing this job measures — report it below, in our + # own words, instead of dying with the action's generic error. + fail-on-missing: 'false' + + - id: merge-base-check + name: Reject PRs with no common ancestor on main + env: + FOUND: ${{ steps.mb.outputs.found }} + BASE: ${{ steps.mb.outputs.sha }} + run: | + if [ "$FOUND" != "true" ]; then + STATUS='[{"source":"unrelated histories","results":[{"kind":"action_required","title":"Unrelated histories","summary":"This PR has no common ancestor with main.","detail":"","how_to_fix":"Rebase your changes onto current main:\n```\ngit fetch origin main\ngit checkout -b fix-branch origin/main\n# re-apply your changes (cherry-pick, copy files, etc.)\ngit push -f origin fix-branch\n```\n"}]}]' + echo "review_status=${STATUS}" >> "$GITHUB_OUTPUT" + echo "::error::This PR has no common ancestor with main. Merging it would"\ + "graft a parent-less root commit into main and collapse git blame for"\ + "every file in the snapshot. Rebase onto current main:"\ + "git fetch origin main && git checkout -b fix-branch origin/main,"\ + "re-apply your changes, then force-push." + exit 1 + fi + echo "::notice::Common ancestor with main: $BASE" + echo "review_status=[]" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/newci-infographic-check.yml b/.github/workflows/newci-infographic-check.yml new file mode 100644 index 000000000000..608955d83f78 --- /dev/null +++ b/.github/workflows/newci-infographic-check.yml @@ -0,0 +1,96 @@ +name: '[newci] Infographic Check' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production infographic-check.yml, running on the GKE self-hosted +# (ARC) runners so the migration can be observed for a few days without +# touching the workflows that gate merges. Production CI in this branch is +# byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - cache keys are newci-prefixed, so production caches stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - the PR review comment runs --dry-run (prints, never posts) +# - the gate job is renamed; it does NOT gate merges +# +# To retire: delete .github/workflows/newci-*.yml. + +# Rejects PRs that commit PR-infographic images into the repo. +# +# PR infographics are rendered to an image-provider URL (fal.media) and +# embedded in the PR *description*. The PR body is the archive; the binary +# never belongs in git history. +# +# This has now leaked twice. PR #48261 removed the first batch, PR #54564 +# removed a second batch and added `infographic/` to `.gitignore` — but +# `.gitignore` only stops *accidental* `git add`. It does nothing against +# `git add -f`, and it does nothing for a path that does not literally match +# the ignore pattern. Nine more PNGs (~14MB) were committed in the four +# weeks AFTER that rule landed, plus PR #70552 caught an `infograficos/` +# spelling that sidestepped the pattern entirely. +# +# A passive ignore rule cannot enforce a policy. This check can. + +on: + workflow_call: + outputs: + review_status: + description: "JSON array of review_status objects for the synthesizer." + value: ${{ jobs.check-no-committed-infographics.outputs.review_status }} + +permissions: + contents: read + +jobs: + check-no-committed-infographics: + # Short gate job: small runner, no dind (see hermes-agent-ci-infra). + runs-on: arc-runner-small + timeout-minutes: 10 + outputs: + review_status: ${{ steps.infographic-check.outputs.review_status }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - id: infographic-check + name: Reject committed PR-infographic images + run: | + # Match on the IMAGE, not on a directory name. Keying this to + # `infographic/` is what let `infograficos/` through in #70552 — + # any localized or typo'd directory would sidestep it again. + # Instead: find tracked raster images whose path contains an + # infographic-ish segment, in any spelling, at any depth. + # + # `docs/assets` and `website/` legitimately hold product imagery + # and are excluded; those are referenced from shipped docs pages. + OFFENDERS=$(git ls-files -z \ + | tr '\0' '\n' \ + | grep -iE '(^|/)(infograph|infograf)[^/]*/' \ + | grep -iE '\.(png|jpe?g|webp|gif)$' \ + || true) + + if [ -n "$OFFENDERS" ]; then + COUNT=$(printf '%s\n' "$OFFENDERS" | wc -l | tr -d ' ') + STATUS='[{"source":"committed infographics","results":[{"kind":"action_required","title":"PR infographic committed to the repo","summary":"Infographic images belong in the PR description, never in git.","detail":"","how_to_fix":"Untrack the image and reference the provider URL from the PR body instead:\n```\ngit rm --cached \n```\nThen put it in the PR description:\n```\n## Infographic\n\n![slug](https://)\n```\n"}]}]' + echo "review_status=${STATUS}" >> "$GITHUB_OUTPUT" + echo "" + echo "::error::${COUNT} PR-infographic image(s) are tracked in git." + echo "" + printf '%s\n' "$OFFENDERS" | sed 's/^/ /' + echo "" + echo "PR infographics are rendered to an image-provider URL and" + echo "embedded in the PR DESCRIPTION. The PR body is the archive —" + echo "the binary never enters git history." + echo "" + echo "This rule has been re-established twice already (#48261," + echo "#54564) and leaked both times, because .gitignore cannot stop" + echo "'git add -f' or a differently-spelled directory (#70552)." + echo "" + echo "To fix:" + echo " git rm --cached # keeps your local copy" + echo " # then embed the provider URL in the PR description" + exit 1 + fi + echo "::notice::No committed PR-infographic images." + echo "review_status=[]" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/newci-install-e2e-run.yml b/.github/workflows/newci-install-e2e-run.yml new file mode 100644 index 000000000000..c72931996486 --- /dev/null +++ b/.github/workflows/newci-install-e2e-run.yml @@ -0,0 +1,246 @@ +name: '[newci] Install & Update E2E (reusable)' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production install-e2e-run.yml, running on the GKE +# self-hosted (ARC) runners so the migration can be observed without touching +# the workflows that gate merges. Production CI in this branch is +# byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - artifact names are newci-prefixed, so production artifacts stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - this workflow is never called by production install-e2e.yml, so its +# result cannot fail the production job +# +# To retire: delete .github/workflows/newci-*.yml. +# +# ───────────────────────────────────────────────────────────────────────── +# Why this shadow is expected to SKIP its legs today +# ───────────────────────────────────────────────────────────────────────── +# +# tests/install/install-update-e2e.sh runs inside scripts/dev-sandbox.sh, +# which is built on bubblewrap. bwrap needs to remount / as slave and mount +# a fresh /proc, and in a stock ARC pod both are denied. Measured in-cluster +# on nousresearch/nous-gke-runner (2026-08-04), installing the same deps the +# production job apt-installs: +# +# pod securityContext bwrap result +# ---------------------------------------- ---------------------------- +# default (what arc-runner-set gives you) Failed to make / slave: EPERM +# capabilities.add: [SYS_ADMIN] Can't mount proc: EPERM +# SYS_ADMIN + apparmor/seccomp Unconfined Can't mount proc: EPERM +# privileged: true OK (all probes pass) +# +# So this needs a scale set whose *runner* container is privileged. +# arc-runner-docker does NOT qualify — only its dind sidecar is privileged, +# the runner container is not. Until such a set exists, the preflight job +# below detects the missing capability and skips the legs with a warning +# rather than burning ~11 minutes per leg to fail at the same place. +# +# When infra adds the set: pass its label as `runner` from the caller and +# the legs start running with no other change here. + +on: + workflow_call: + inputs: + route: + description: 'Update path to exercise: update (hermes update) or installer (re-run install.sh).' + required: true + type: string + install-ref: + description: 'What to install before updating: a branch, a tag (v2026.7.7), or a SHA reachable from main.' + required: false + type: string + default: refs/heads/main + runner: + description: 'Runner label. Needs a privileged runner container for bubblewrap — see the header.' + required: false + type: string + default: arc-runner-set + timeout-minutes: + description: 'Job timeout. A cold run installs real toolchains twice.' + required: false + type: number + default: 45 + +permissions: + contents: read + +jobs: + # Probe bwrap before paying for a full install. This is the whole point of + # the shadow: it answers "can ARC run the install E2E yet?" in ~30 seconds + # instead of ~11 minutes, and says exactly what is missing when it cannot. + preflight: + name: Probe sandbox capability + runs-on: ${{ inputs.runner }} + timeout-minutes: 10 + outputs: + supported: ${{ steps.probe.outputs.supported }} + steps: + - name: Probe bubblewrap + id: probe + run: | + set -uo pipefail + + # The production job apt-installs these on ubuntu-latest; do the same + # here so the probe measures the sandbox, not a missing package. + sudo apt-get update -qq + sudo apt-get install -y -qq bubblewrap slirp4netns uidmap util-linux + + supported=true + note="" + + if ! bwrap --ro-bind / / --dev /dev --proc /proc --unshare-all true 2>/tmp/bwrap-err; then + supported=false + note="$(cat /tmp/bwrap-err)" + fi + + echo "supported=$supported" >> "$GITHUB_OUTPUT" + + if [ "$supported" = true ]; then + echo "✅ bubblewrap works on this runner — running the real legs." + { + echo "## Sandbox preflight: supported ✅" + echo "" + echo "\`bwrap\` can create its namespaces on \`${{ inputs.runner }}\`." + } >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + echo "::warning::bubblewrap cannot run on ${{ inputs.runner }}; skipping the install E2E legs. ${note}" + { + echo "## Sandbox preflight: unsupported ⚠️" + echo "" + echo "\`bwrap\` failed on \`${{ inputs.runner }}\`:" + echo "" + echo '```' + echo "${note}" + echo '```' + echo "" + echo "The install E2E runs inside \`scripts/dev-sandbox.sh\` (bubblewrap)," + echo "which needs to remount \`/\` as slave and mount a fresh \`/proc\`." + echo "A stock ARC pod denies both." + echo "" + echo "| pod securityContext | bwrap |" + echo "|---|---|" + echo "| default | \`Failed to make / slave: EPERM\` |" + echo "| \`capabilities.add: [SYS_ADMIN]\` | \`Can't mount proc: EPERM\` |" + echo "| SYS_ADMIN + apparmor/seccomp Unconfined | \`Can't mount proc: EPERM\` |" + echo "| \`privileged: true\` | works |" + echo "" + echo "This needs a scale set whose **runner** container is privileged." + echo "\`arc-runner-docker\` does not qualify — only its dind sidecar is." + echo "" + echo "**This is a skip, not a regression.** Production \`install-e2e.yml\`" + echo "on GitHub-hosted runners is unaffected." + } >> "$GITHUB_STEP_SUMMARY" + + e2e: + name: ${{ inputs.route }} from ${{ inputs.install-ref }} + needs: preflight + if: needs.preflight.outputs.supported == 'true' + runs-on: ${{ inputs.runner }} + timeout-minutes: ${{ inputs.timeout-minutes }} + + steps: + # Full history: the sandbox fetches the starting commit and the test + # compares against this commit, so a shallow clone is not enough. + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + # bubblewrap + slirp4netns are what the sandbox is built on; util-linux + # supplies the `unshare` that builds the multi-uid userns for the + # user-level (non-root) install. + - name: Install sandbox dependencies + run: | + set -euo pipefail + sudo apt-get update -qq + sudo apt-get install -y -qq bubblewrap slirp4netns uidmap util-linux + + # The production job flips kernel.apparmor_restrict_unprivileged_userns + # on its Ubuntu 24.04 VM. In an ARC pod that sysctl is not present in + # /proc at all (verified in-cluster), and a pod cannot set node-level + # sysctls anyway — so report the state and move on rather than failing + # on a `sysctl -w` that was never going to apply here. + - name: Report user-namespace state + run: | + set -euo pipefail + echo "--- kernel userns settings" + sysctl kernel.unprivileged_userns_clone 2>/dev/null || echo " (sysctl absent — expected in a pod)" + sysctl kernel.apparmor_restrict_unprivileged_userns 2>/dev/null || echo " (sysctl absent — expected in a pod)" + echo "--- subuid/subgid for $(id -un)" + grep "^$(id -un):" /etc/subuid /etc/subgid || echo " (none — sandbox will say so)" + + - name: Run install + update E2E + id: run + # Step-level continue-on-error IS supported (job-level is not, for jobs + # that call a reusable workflow). This is what makes the shadow + # non-blocking: the script's exit code is captured as an outcome and + # reported below, but never fails the job — so the shadow workflow's + # own check stays green and nobody has to triage a red X that only + # means "the migration isn't ready yet". + continue-on-error: true + run: | + set -euo pipefail + tests/install/install-update-e2e.sh \ + --route '${{ inputs.route }}' \ + --install-ref '${{ inputs.install-ref }}' + env: + # Outside the workspace on purpose: the script creates this directory + # up front, and an untracked dir inside the repo makes the worktree + # dirty -- which dev-sandbox reacts to by snapshotting the working + # copy into a fresh fake-main commit on every invocation, moving the + # update target mid-run. + HERMES_E2E_LOG_DIR: ${{ runner.temp }}/e2e-logs + + - name: Report leg outcome + if: always() + run: | + set -uo pipefail + outcome='${{ steps.run.outcome }}' + if [ "$outcome" = "success" ]; then + icon="✅" + else + icon="❌" + echo "::warning::[newci shadow] ${{ inputs.route }} from ${{ inputs.install-ref }} failed ($outcome). Not blocking — production install-e2e.yml is unaffected." + fi + { + echo "## ${icon} ${{ inputs.route }} from ${{ inputs.install-ref }} — ${outcome}" + echo "" + echo "Runner: \`${{ inputs.runner }}\`" + echo "" + echo "Shadow leg. A failure here is migration data, not a broken build;" + echo "production \`Install & Update E2E\` on GitHub-hosted runners is" + echo "unaffected by this result." + } >> "$GITHUB_STEP_SUMMARY" + + + # Artifact names cannot contain '/', and install-ref may be a full ref + # like refs/heads/main. GitHub Actions expressions have no string-replace + # function, so build the safe name here. Runs even on failure -- that is + # exactly when the logs are wanted. + - name: Build artifact name + if: always() + id: artifact + run: | + set -euo pipefail + safe_ref='${{ inputs.install-ref }}' + safe_ref="${safe_ref//\//-}" + echo "name=newci-install-e2e-${{ inputs.route }}-${safe_ref}" >> "$GITHUB_OUTPUT" + + # The installer's own transcripts say far more than the assertion that + # tripped when a real install breaks. + - name: Upload installer logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + # Unique per leg: a matrix over releases runs this workflow several + # times per route, and same-named artifacts collide. + name: ${{ steps.artifact.outputs.name }}-${{ github.sha }} + path: ${{ runner.temp }}/e2e-logs + retention-days: 14 + if-no-files-found: ignore diff --git a/.github/workflows/newci-install-e2e.yml b/.github/workflows/newci-install-e2e.yml new file mode 100644 index 000000000000..670543b83066 --- /dev/null +++ b/.github/workflows/newci-install-e2e.yml @@ -0,0 +1,199 @@ +name: '[newci] Install & Update E2E' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production install-e2e.yml, running on the GKE self-hosted +# (ARC) runners so the migration can be observed without touching the workflows +# that gate merges. Production CI in this branch is byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - artifact names are newci-prefixed, so production artifacts stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - the E2E step is continue-on-error, so a failing leg never reddens a check +# +# To retire: delete .github/workflows/newci-*.yml. +# +# ───────────────────────────────────────────────────────────────────────── +# This workflow CANNOT fail the production Install & Update E2E +# ───────────────────────────────────────────────────────────────────────── +# +# Two independent reasons, both deliberate: +# +# 1. It is a separate workflow with its own triggers. GitHub reports it as +# its own check run; production install-e2e.yml never calls it and never +# reads its result. Nothing about this file can turn that one red. +# 2. Belt and braces: inside newci-install-e2e-run.yml the E2E step carries +# `continue-on-error: true`, so a failing leg is recorded as an outcome +# and surfaced in the summary rather than failing its job. Even this +# shadow's own check stays green — a migration probe should produce +# information, not a red X someone has to triage. +# +# NOTE on (2): job-level `continue-on-error` is NOT valid on a job that calls +# a reusable workflow (GitHub allows only name/uses/with/secrets/strategy/ +# needs/if/concurrency/permissions there), which is why the tolerance lives on +# the step inside the reusable workflow instead. Do not "fix" this by adding +# continue-on-error to the matrix jobs below — it will not parse. +# +# Read the results in the run summary, not in the check status. +# +# Triggers mirror production install-e2e.yml so the shadow fires whenever the +# real one does: every 12 hours, on release tags, and manually. The cron is +# offset by 5 minutes so the two runs do not contend for the same runner pool +# at the same instant. +# +# NOTE: `schedule` only fires from the default branch. Until this branch is +# merged, the scheduled leg will not run on its own — use workflow_dispatch +# (pick this branch in the Run workflow dropdown) to exercise it meanwhile. + +on: + workflow_dispatch: + inputs: + route: + description: 'Which update route to exercise.' + required: false + type: choice + default: both + options: [both, update, installer] + tag-count: + description: 'How many release tags to sample (newest, oldest, and a spread between).' + required: false + type: string + default: '5' + runner: + description: 'Runner label to shadow on. Needs a privileged runner container for bubblewrap.' + required: false + type: string + default: arc-runner-set + schedule: + # Production runs at :20; offset so the shadow does not contend with it. + - cron: '25 7,19 * * *' + push: + tags: + # Release tags only: the repo also carries backup/* and one-off tags. + - 'v[0-9]+.[0-9]+.[0-9]+' + - 'v[0-9]+.[0-9]+.[0-9]+.[0-9]+' + +permissions: + contents: read + +concurrency: + group: newci-install-e2e-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Which released versions do we test updating FROM? Resolved once and shared + # by both route matrices, so the two routes cover the same set. + pick-releases: + name: Pick release tags + # Small runner: reads tag names and runs one script. + runs-on: arc-runner-small + timeout-minutes: 5 + outputs: + tags: ${{ steps.pick.outputs.tags }} + steps: + # This job only reads tag names and runs one script, so take the cheap + # checkout: no blobs (filter), no other files (sparse), but DO fetch tags + # -- they are the whole input, and the default shallow checkout has none. + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + filter: blob:none + fetch-tags: true + sparse-checkout: scripts/sandbox/pick-release-tags.sh + sparse-checkout-cone-mode: false + - id: pick + run: | + set -euo pipefail + tags="$(scripts/sandbox/pick-release-tags.sh --count '${{ inputs.tag-count || 5 }}')" + echo "Testing updates from: $tags" + echo "tags=$tags" >> "$GITHUB_OUTPUT" + + # `hermes update` -- the route most users take. + update: + if: github.event_name != 'workflow_dispatch' || inputs.route != 'installer' + needs: pick-releases + strategy: + # One release breaking is worth knowing about even if another already + # failed, so let every leg report. + fail-fast: false + matrix: + install-ref: ${{ fromJSON(needs.pick-releases.outputs.tags) }} + uses: ./.github/workflows/newci-install-e2e-run.yml + with: + route: update + install-ref: ${{ matrix.install-ref }} + runner: ${{ inputs.runner || 'arc-runner-set' }} + + # Re-running the curl one-liner over an existing checkout: autostash + pull + # rather than the updater's own git handling. + installer: + if: github.event_name != 'workflow_dispatch' || inputs.route != 'update' + needs: pick-releases + strategy: + fail-fast: false + max-parallel: 3 + matrix: + install-ref: ${{ fromJSON(needs.pick-releases.outputs.tags) }} + uses: ./.github/workflows/newci-install-e2e-run.yml + with: + route: installer + install-ref: ${{ matrix.install-ref }} + runner: ${{ inputs.runner || 'arc-runner-set' }} + + # Report what the shadow saw WITHOUT ever failing. This is the job to read; + # it converts leg results into a summary table instead of a red X. It has no + # `exit 1` path on purpose — see the header. + shadow-summary: + name: '[newci] Shadow result (informational — never fails)' + needs: [pick-releases, update, installer] + if: always() + # Small runner: one script step, no checkout. + runs-on: arc-runner-small + timeout-minutes: 10 + steps: + - name: Summarize + env: + NEEDS: ${{ toJSON(needs) }} + shell: python + run: | + import json, os + + needs = json.loads(os.environ["NEEDS"]) + icons = { + "success": "✅", + "skipped": "⏭️", + "cancelled": "⚪", + "failure": "❌", + } + + lines = [ + "## [newci] Install & Update E2E — shadow result", + "", + "Observation only. This never fails, and it cannot affect the", + "production `Install & Update E2E` check.", + "", + "| Job | Result |", + "|---|---|", + ] + for name, info in sorted(needs.items()): + result = info.get("result", "unknown") + lines.append(f"| {name} | {icons.get(result, '❓')} {result} |") + + failed = sorted(n for n, i in needs.items() if i.get("result") == "failure") + lines.append("") + if failed: + lines += [ + f"⚠️ {len(failed)} shadow job(s) failed: {', '.join(failed)}", + "", + "If the legs were skipped instead, the runner lacks bubblewrap", + "support — see the preflight summary in", + "`newci-install-e2e-run.yml` for the capability matrix.", + ] + else: + lines.append("No shadow failures.") + + body = "\n".join(lines) + print(body) + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as fh: + fh.write(body + "\n") diff --git a/.github/workflows/newci-js-tests.yml b/.github/workflows/newci-js-tests.yml new file mode 100644 index 000000000000..f0223eadd11b --- /dev/null +++ b/.github/workflows/newci-js-tests.yml @@ -0,0 +1,126 @@ +# .github/workflows/js-tests.yml +name: '[newci] JS Tests' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production js-tests.yml, running on the GKE self-hosted +# (ARC) runners so the migration can be observed for a few days without +# touching the workflows that gate merges. Production CI in this branch is +# byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - cache keys are newci-prefixed, so production caches stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - the PR review comment runs --dry-run (prints, never posts) +# - the gate job is renamed; it does NOT gate merges +# +# To retire: delete .github/workflows/newci-*.yml. + +on: + workflow_call: + +jobs: + workspaces: + name: List npm workspaces + # Small runner: checkout plus an npm workspace query. + runs-on: arc-runner-small + timeout-minutes: 20 + outputs: + checks: ${{ steps.set-matrix.outputs.checks }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - id: set-matrix + run: | + node -e ' + const fs = require("fs"); + const path = require("path"); + + const rootPkg = JSON.parse(fs.readFileSync("package.json", "utf-8")); + const patterns = rootPkg.workspaces || []; + + // minimal glob support: exact dirs ("packages/foo") and single-level wildcards ("packages/*") + function expandPattern(pattern) { + if (!pattern.includes("*")) { + return fs.existsSync(pattern) ? [pattern] : []; + } + const [base] = pattern.split("*"); + const parentDir = base.replace(/\/$/, ""); + if (!fs.existsSync(parentDir)) return []; + return fs.readdirSync(parentDir, { withFileTypes: true }) + .filter(d => d.isDirectory()) + .map(d => path.join(parentDir, d.name)); + } + + const dirs = [...new Set(patterns.flatMap(expandPattern))]; + + const pkgs = dirs + .map(dir => { + const pkgPath = path.join(dir, "package.json"); + if (!fs.existsSync(pkgPath)) return null; + const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf-8")); + return { location: dir, name: pkg.name, scripts: pkg.scripts || {} }; + }) + .filter(Boolean); + + if (pkgs.length === 0) { + console.error("::error::Workspace discovery produced an empty package list — refusing to emit a zero-length matrix (would skip all JS/TS checks silently)."); + process.exit(1); + } + + const checks = []; + for (const pkg of pkgs) { + const subs = Object.keys(pkg.scripts).filter(s => /^check:.+$/.test(s)); + if (subs.length > 0) { + for (const script of subs) { + checks.push({ package: pkg.location, script }); + } + } else if (pkg.scripts.check) { + checks.push({ package: pkg.location, script: "check" }); + } + } + + if (checks.length === 0) { + console.error("::error::No check scripts found in any workspace package."); + process.exit(1); + } + + process.stdout.write("checks=" + JSON.stringify(checks) + "\n"); + ' >> "$GITHUB_OUTPUT" + + check: + name: ${{ matrix.package }} / ${{ matrix.script }} + needs: workspaces + runs-on: arc-runner-set + timeout-minutes: 20 + strategy: + matrix: + include: ${{ fromJson(needs.workspaces.outputs.checks) }} + fail-fast: false # report all failures, not just the first one + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + id: npm-cache + with: + path: | + node_modules + apps/*/node_modules + ui-tui/node_modules + ui-tui/packages/*/node_modules + web/node_modules + tests-js/node_modules + # node26 in the key: node-pty/electron native artifacts are + # node-major-specific; sync with NODE_MAJOR in the runner image + # (hermes-agent-ci-infra runner/Dockerfile). + key: newci-node-modules-full-node26-${{ runner.arch }}-${{ hashFiles('package-lock.json') }} + + - uses: ./.github/actions/retry + with: + command: npm ci + if: steps.npm-cache.outputs.cache-hit != 'true' + + - uses: ./.github/actions/profile + with: + label: js-${{ matrix.package }}-${{ matrix.script }} + command: npm run --prefix ${{ matrix.package }} ${{ matrix.script }} diff --git a/.github/workflows/newci-lint.yml b/.github/workflows/newci-lint.yml new file mode 100644 index 000000000000..e96001bab746 --- /dev/null +++ b/.github/workflows/newci-lint.yml @@ -0,0 +1,179 @@ +name: '[newci] Lint (ruff + ty)' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production lint.yml, running on the GKE self-hosted +# (ARC) runners so the migration can be observed for a few days without +# touching the workflows that gate merges. Production CI in this branch is +# byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - cache keys are newci-prefixed, so production caches stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - the PR review comment runs --dry-run (prints, never posts) +# - the gate job is renamed; it does NOT gate merges +# +# To retire: delete .github/workflows/newci-*.yml. + +# Two things here: +# 1. Advisory diff — ruff + ty diagnostics as a diff vs the target branch. +# Writes a Markdown summary to the run page. Exit zero always. +# 2. Blocking ``ruff check .`` — enforces the explicit rules in +# ``[tool.ruff.lint.select]`` (currently PLW1514). Failure blocks merge. +# Separate job so the advisory diff still runs even when enforcement +# fails. +# +# CI-sensitive file review was previously here as a ``ci-review`` job but +# has moved to ``review-labels.yml`` so it can be rerun independently. + +on: + workflow_call: + inputs: + event_name: + description: The event name from the calling orchestrator (pull_request or push). + type: string + required: true + +permissions: + contents: read + +concurrency: + group: newci-lint-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint-diff: + name: ruff + ty diff + if: inputs.event_name == 'pull_request' + # Small runner: ruff/ty are cheap; the job is mostly checkout. + runs-on: arc-runner-small + timeout-minutes: 10 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 1 + + - name: Fetch the base commit + id: base + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + set -euo pipefail + # This job is PR-only (see the job-level `if`) and checks out the PR + # merge ref, so HEAD already contains main merged in and base.sha is + # the correct comparison point — `git merge-base base.sha HEAD` + # returns base.sha itself. Resolving one would be a no-op at best; + # comparing against the branch point instead would drag diagnostics + # main introduced after the branch point into this PR's diff. + # + # So there is no merge base to deepen for: fetch that one commit. + git fetch --depth=1 origin "$BASE_SHA" -q + echo "sha=${BASE_SHA}" >> "$GITHUB_OUTPUT" + echo "ref=${{ github.base_ref }}" >> "$GITHUB_OUTPUT" + echo "Base SHA: ${BASE_SHA}" + + - name: Install ruff + ty + uses: ./.github/actions/retry + with: + command: uv tool install ruff && uv tool install ty + + - name: Run ruff + ty on HEAD + run: | + mkdir -p .lint-reports/head + ruff check --output-format json --exit-zero \ + > .lint-reports/head/ruff.json || true + ty check --output-format gitlab --exit-zero \ + > .lint-reports/head/ty.json || true + echo "HEAD ruff: $(wc -c < .lint-reports/head/ruff.json) bytes" + echo "HEAD ty: $(wc -c < .lint-reports/head/ty.json) bytes" + + - name: Run ruff + ty on base (via git worktree) + run: | + mkdir -p .lint-reports/base + # Use a worktree so we don't clobber the main checkout. If the basex + # SHA is identical to HEAD (e.g. first commit), skip and leave the + # base reports empty — the diff script handles missing files. + HEAD_SHA=$(git rev-parse HEAD) + BASE_SHA="${{ steps.base.outputs.sha }}" + if [ "$BASE_SHA" = "$HEAD_SHA" ]; then + echo "Base SHA == HEAD SHA, skipping base scan." + echo '[]' > .lint-reports/base/ruff.json + echo '[]' > .lint-reports/base/ty.json + else + git worktree add --detach /tmp/lint-base "$BASE_SHA" + ( + cd /tmp/lint-base + ruff check --output-format json --exit-zero \ + > "$GITHUB_WORKSPACE/.lint-reports/base/ruff.json" || true + ty check --output-format gitlab --exit-zero \ + > "$GITHUB_WORKSPACE/.lint-reports/base/ty.json" || true + ) + git worktree remove --force /tmp/lint-base + fi + echo "base ruff: $(wc -c < .lint-reports/base/ruff.json) bytes" + echo "base ty: $(wc -c < .lint-reports/base/ty.json) bytes" + + - name: Generate diff summary + env: + HEAD_REF: ${{ inputs.event_name == 'pull_request' && github.head_ref || github.ref_name }} + run: | + python3 scripts/lint_diff.py \ + --base-ruff .lint-reports/base/ruff.json \ + --head-ruff .lint-reports/head/ruff.json \ + --base-ty .lint-reports/base/ty.json \ + --head-ty .lint-reports/head/ty.json \ + --base-ref "${{ steps.base.outputs.ref }}" \ + --head-ref "$HEAD_REF" \ + --output .lint-reports/summary.md + cat .lint-reports/summary.md >> "$GITHUB_STEP_SUMMARY" + + ruff-blocking: + # Enforce the rules in pyproject.toml [tool.ruff.lint.select]. Currently + # PLW1514 (unspecified-encoding) — catches bare ``open()`` / + # ``read_text()`` / ``write_text()`` calls that default to locale + # encoding on Windows. Failure here blocks merge; the advisory + # ``lint-diff`` job above runs independently so reviewers still get + # the diff comment even when enforcement fails. + name: ruff enforcement (blocking) + # Small runner: ruff/ty are cheap; the job is mostly checkout. + runs-on: arc-runner-small + timeout-minutes: 5 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install ruff + uses: ./.github/actions/retry + with: + command: uv tool install ruff + + - name: ruff check . + # No --exit-zero, no || true. Exit code propagates to the job, + # which propagates to the required-check gate. + uses: ./.github/actions/profile + with: + label: ruff-blocking + command: ruff check . + + windows-footguns: + # Static guardrails on Windows-unsafe Python primitives — os.kill(pid, 0), + # os.killpg, os.setsid, signal.SIGKILL without getattr fallback, + # shebang scripts via subprocess, bare open() without encoding=, etc. + # See scripts/check-windows-footguns.py for the full rule list. + name: Windows footguns (blocking) + # Small runner: checkout plus a small python checker. + runs-on: arc-runner-small + timeout-minutes: 5 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Run footgun checker + # Plain python3, not `uv run`: the checker is stdlib-only, and inside a + # project dir `uv run` would sync the whole locked dependency set just + # to execute it. python3 3.11 is baked into the ARC runner image (same + # assumption the generate job in newci-tests.yml makes). + run: python3 scripts/check-windows-footguns.py --all diff --git a/.github/workflows/newci-lockfile-diff.yml b/.github/workflows/newci-lockfile-diff.yml new file mode 100644 index 000000000000..44738739a507 --- /dev/null +++ b/.github/workflows/newci-lockfile-diff.yml @@ -0,0 +1,111 @@ +name: '[newci] Lockfile diff' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production lockfile-diff.yml, running on the GKE self-hosted +# (ARC) runners so the migration can be observed for a few days without +# touching the workflows that gate merges. Production CI in this branch is +# byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - cache keys are newci-prefixed, so production caches stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - the PR review comment runs --dry-run (prints, never posts) +# - the gate job is renamed; it does NOT gate merges +# +# To retire: delete .github/workflows/newci-*.yml. + +# Advisory PR comment showing the *semantic* diff of package-lock.json +# changes — which packages were added/removed/updated and their versions. +# The raw textual diff of a lockfile is unreadable (npm reorders entries +# and rewrites integrity hashes), so scripts/ci/lockfile_diff.py parses +# the ``packages`` map at the merge base and at HEAD and set-diffs the +# {install path: version} maps instead. +# +# The semantic diff is exposed as a workflow_call output ``review_status`` +# (a JSON array in the unified status format) and an artifact +# (``lockfile-diff`` containing the markdown fragment) for the step +# summary. +# +# Never blocking — this is review signal, not enforcement. + +on: + workflow_call: + outputs: + changed: + description: Whether package-lock.json changed relative to the target branch. + value: ${{ jobs.diff.outputs.changed }} + review_status: + description: JSON array of review status objects for the unified PR comment. + value: ${{ jobs.diff.outputs.review_status }} + +permissions: + contents: read + +concurrency: + group: newci-lockfile-diff-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + diff: + name: package-lock.json semantic diff + # Small runner: checkout plus a diff script. + runs-on: arc-runner-small + timeout-minutes: 5 + outputs: + changed: ${{ steps.diff.outputs.changed }} + review_status: ${{ steps.emit-status.outputs.review_status }} + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Generate semantic lockfile diff + id: diff + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + set -euo pipefail + # Fetch just the base commit. + git fetch --depth=1 origin "${BASE_SHA}" + echo "Base commit: ${BASE_SHA}" + python3 scripts/ci/lockfile_diff.py \ + --base "$BASE_SHA" \ + --head HEAD \ + --output /tmp/lockfile-diff.md + if [ -s /tmp/lockfile-diff.md ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + { + echo "## package-lock.json semantic diff" + echo "" + cat /tmp/lockfile-diff.md + } >> "$GITHUB_STEP_SUMMARY" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + : > /tmp/lockfile-diff.md + fi + + - name: Emit review_status + id: emit-status + run: | + set -euo pipefail + CHANGED="${{ steps.diff.outputs.changed }}" + STATUS="[]" + + if [ "$CHANGED" = "true" ]; then + CONTENT=$(cat /tmp/lockfile-diff.md | python3 -c "import sys,json; print(json.dumps(sys.stdin.read()))") + STATUS="[{\"source\":\"lockfile-diff\",\"results\":[{\"kind\":\"action_required\",\"title\":\"package-lock.json\",\"summary\":\"Locked npm dependency versions changed.\",\"detail\":${CONTENT},\"how_to_fix\":\"Add the \`ci-reviewed\` label after verifying the version changes are expected.\"}]}" + else + STATUS="[]" + fi + + echo "review_status=${STATUS}" >> "$GITHUB_OUTPUT" + + - name: Upload diff artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: lockfile-diff + path: /tmp/lockfile-diff.md + retention-days: 1 + overwrite: true diff --git a/.github/workflows/newci-osv-scanner.yml b/.github/workflows/newci-osv-scanner.yml new file mode 100644 index 000000000000..dd6dbc49fdcc --- /dev/null +++ b/.github/workflows/newci-osv-scanner.yml @@ -0,0 +1,147 @@ +name: '[newci] OSV-Scanner' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production osv-scanner.yml, running on the GKE self-hosted +# (ARC) runners so the migration can be observed for a few days without +# touching the workflows that gate merges. Production CI in this branch is +# byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - cache keys are newci-prefixed, so production caches stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - the PR review comment runs --dry-run (prints, never posts) +# - the gate job is renamed; it does NOT gate merges +# +# To retire: delete .github/workflows/newci-*.yml. + +# Scans lockfiles (uv.lock, package-lock.json) against the OSV vulnerability +# database. Runs on every PR/push (via the ci.yml orchestrator's workflow_call) +# and on a weekly schedule against main. +# +# This is detection-only — OSV-Scanner does NOT open PRs or modify pins. +# It reports known CVEs in currently-pinned dependency versions so we can +# decide when and how to patch on our own schedule. Our pinning strategy +# (full SHA / exact version) is preserved; only the notification signal +# is added. +# +# Complements the supply-chain-audit.yml workflow (which scans for malicious +# code patterns in PR diffs) by covering the orthogonal "currently-pinned +# dep became known-vulnerable" case. +# +# Uses Google's officially-recommended reusable workflow, pinned by SHA. +# Findings land in the repo's Security tab (Code Scanning > OSV-Scanner). +# fail-on-vuln is disabled so the job does not block merges on pre-existing +# vulnerabilities in pinned deps that we may need to patch deliberately. +# +# The reusable workflow can't emit custom outputs, so a wrapper job +# downloads the SARIF result and summarizes the vulnerability count into +# a review_status for the unified PR comment. + +on: + workflow_call: + schedule: + # Weekly scan against main — catches CVEs published after merge for + # deps that haven't changed since. + - cron: '0 9 * * 1' + workflow_dispatch: + +permissions: + # Required to upload SARIF file to CodeQL. See: https://github.com/github/codeql-action/issues/2117 + actions: read + contents: read + security-events: write + +jobs: + scan: + name: Scan lockfiles + uses: google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@9a498708959aeaef5ef730655706c5a1df1edbc2 # v2.3.8 + with: + # Scan explicit lockfiles rather than recursing, so we only look at + # the five sources of truth and skip vendored / test / worktree dirs. + scan-args: |- + --lockfile=uv.lock + --lockfile=package-lock.json + --lockfile=website/package-lock.json + --lockfile=plugins/platforms/photon/sidecar/package-lock.json + --lockfile=scripts/whatsapp-bridge/package-lock.json + # The upstream reusable workflow uploads this exact file under its + # fixed artifact name, which the wrapper downloads below. + results-file-name: osv-results.sarif + fail-on-vuln: false + + emit-status: + name: Emit review status + # Short gate job: small runner, no dind (see hermes-agent-ci-infra). + runs-on: arc-runner-small + needs: scan + if: always() + outputs: + review_status: ${{ steps.emit.outputs.review_status }} + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Download SARIF result + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: OSV Scanner SARIF file + path: /tmp/osv-results + continue-on-error: true + + - name: Emit review_status + id: emit + run: | + set -euo pipefail + STATUS="[]" + + if [ -f /tmp/osv-results/osv-results.sarif ]; then + # Count vulnerabilities from the SARIF file + VULN_COUNT=$(python3 -c " + import json, sys + try: + with open('/tmp/osv-results/osv-results.sarif') as f: + data = json.load(f) + count = 0 + vulns = [] + for run in data.get('runs', []): + for result in run.get('results', []): + count += 1 + rule_id = result.get('ruleId', 'unknown') + message = result.get('message', {}).get('text', '') + loc = result.get('locations', [{}])[0].get('physicalLocation', {}).get('artifactLocation', {}).get('uri', '') + vulns.append(f'- {rule_id} in {loc}: {message}') + print(count) + if vulns: + print('\n'.join(vulns[:20]), file=sys.stderr) + except Exception: + print(0) + ") + + VULN_DETAIL="" + if [ "$VULN_COUNT" -gt 0 ] 2>/dev/null; then + VULN_PLURAL=$([ "$VULN_COUNT" -eq 1 ] && echo "y" || echo "ies") + VULN_DETAIL=$(python3 -c " + import json, sys + try: + with open('/tmp/osv-results/osv-results.sarif') as f: + data = json.load(f) + vulns = [] + for run in data.get('runs', []): + for result in run.get('results', []): + rule_id = result.get('ruleId', 'unknown') + loc = result.get('locations', [{}])[0].get('physicalLocation', {}).get('artifactLocation', {}).get('uri', '') + vulns.append(f'- {rule_id} in {loc}') + print(json.dumps('\n'.join(vulns[:20]))) + except Exception: + print(json.dumps('')) + ") + STATUS="[{\"source\":\"osv scan\",\"results\":[{\"kind\":\"warning\",\"title\":\"OSV vulnerability scan\",\"summary\":\"${VULN_COUNT} known vulnerabilit${VULN_PLURAL} found in pinned dependencies.\",\"detail\":${VULN_DETAIL},\"how_to_fix\":\"Review the findings in the [Security tab](../../security/code-scanning). Update the affected dependencies if a patched version is available.\"}]}]" + else + STATUS="[]" + fi + fi + + echo "review_status=${STATUS}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/newci-review-labels.yml b/.github/workflows/newci-review-labels.yml new file mode 100644 index 000000000000..75c48900fe69 --- /dev/null +++ b/.github/workflows/newci-review-labels.yml @@ -0,0 +1,127 @@ +name: '[newci] Review labels' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production review-labels.yml, running on the GKE self-hosted +# (ARC) runners so the migration can be observed for a few days without +# touching the workflows that gate merges. Production CI in this branch is +# byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - cache keys are newci-prefixed, so production caches stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - the PR review comment runs --dry-run (prints, never posts) +# - the gate job is renamed; it does NOT gate merges +# +# To retire: delete .github/workflows/newci-*.yml. + +# Require explicit maintainer review when CI-sensitive files or the MCP +# catalog change. Previously this was split across two jobs in two +# workflows: ``ci-review`` in lint.yml (gated on ``ci_review``) and +# ``mcp-catalog-review`` in supply-chain-audit.yml (gated on +# ``mcp_catalog``). Both checked for their own label. +# +# Now consolidated: a single ``ci-reviewed`` label covers both. The +# comment sections tell the reviewer exactly what to verify per area, +# so one label is enough — the human reads the comment, not the label +# name. +# +# Outputs: +# ci_reviewed — "true" / "false" / "" (empty when neither lane ran) +# review_status — JSON array of status objects consumed by the review +# comment assembler. See scripts/ci/emit_review_status.py. + +on: + workflow_call: + inputs: + ci_review: + description: Whether CI-sensitive files (eslint config, workflows, actions) changed. + type: boolean + default: false + ci_review_files: + description: JSON list of CI-sensitive files changed by the pull request. + type: string + default: '[]' + mcp_catalog: + description: Whether the MCP catalog / installer changed. + type: boolean + default: false + supply_chain: + description: Whether the critical supply-chain scan found a risk requiring review. + type: boolean + default: false + outputs: + ci_reviewed: + description: Whether the ci-reviewed label is present. Empty when neither input was true. + value: ${{ jobs.check.outputs.ci_reviewed }} + review_status: + description: JSON array of status objects for the review comment assembler. + value: ${{ jobs.check.outputs.review_status }} + +permissions: + contents: read + pull-requests: read # read PR labels + +jobs: + check: + name: Review label gate + if: inputs.ci_review || inputs.mcp_catalog || inputs.supply_chain + # Short gate job: small runner, no dind (see hermes-agent-ci-infra). + runs-on: arc-runner-small + timeout-minutes: 2 + outputs: + ci_reviewed: ${{ steps.label-check.outputs.ci_reviewed }} + review_status: ${{ steps.build-status.outputs.review_status }} + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Check ci-reviewed label + id: label-check + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + PR="${{ github.event.pull_request.number }}" + LABELS=$(gh pr view "$PR" --repo "$REPO" --json labels --jq '.labels[].name' || true) + + if echo "$LABELS" | grep -Fxq 'ci-reviewed'; then + echo "ci-reviewed label present." + echo "ci_reviewed=true" >> "$GITHUB_OUTPUT" + else + echo "ci-reviewed label missing." + echo "ci_reviewed=false" >> "$GITHUB_OUTPUT" + fi + + - name: Build review_status JSON + id: build-status + env: + CI_REVIEW: ${{ inputs.ci_review }} + CI_REVIEW_FILES: ${{ inputs.ci_review_files }} + MCP_CATALOG: ${{ inputs.mcp_catalog }} + SUPPLY_CHAIN: ${{ inputs.supply_chain }} + LABEL_PRESENT: ${{ steps.label-check.outputs.ci_reviewed }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + args=() + if [ "$CI_REVIEW" = "true" ]; then args+=(--ci-review); fi + args+=(--ci-review-files "$CI_REVIEW_FILES") + if [ "$MCP_CATALOG" = "true" ]; then args+=(--mcp-catalog); fi + if [ "$SUPPLY_CHAIN" = "true" ]; then args+=(--supply-chain); fi + if [ "$LABEL_PRESENT" = "true" ]; then args+=(--label-present); fi + + python3 scripts/ci/emit_review_status.py "${args[@]}" \ + --repo-url "$REPO_URL" --base-sha "$BASE_SHA" --head-sha "$HEAD_SHA" \ + --output "$GITHUB_OUTPUT" + + - name: Fail on missing label + if: steps.label-check.outputs.ci_reviewed != 'true' + run: | + echo "::error::CI-sensitive changes require the ci-reviewed label. Add the label and re-run this check." + exit 1 diff --git a/.github/workflows/newci-supply-chain-audit.yml b/.github/workflows/newci-supply-chain-audit.yml new file mode 100644 index 000000000000..de5111f31cc2 --- /dev/null +++ b/.github/workflows/newci-supply-chain-audit.yml @@ -0,0 +1,325 @@ +name: '[newci] Supply Chain Audit' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production supply-chain-audit.yml, running on the GKE self-hosted +# (ARC) runners so the migration can be observed for a few days without +# touching the workflows that gate merges. Production CI in this branch is +# byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - cache keys are newci-prefixed, so production caches stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - the PR review comment runs --dry-run (prints, never posts) +# - the gate job is renamed; it does NOT gate merges +# +# To retire: delete .github/workflows/newci-*.yml. + +# Narrow, high-signal scanner. Only fires on critical indicators of supply +# chain attacks (e.g. the litellm-style payloads). Low-signal heuristics +# (plain base64, plain exec/eval, dependency/Dockerfile/workflow edits, +# Actions version unpinning, outbound POST/PUT) were intentionally +# removed — they fired on nearly every PR and trained reviewers to ignore +# the scanner. Keep this file's checks ruthlessly narrow: if you find +# yourself adding WARNING-tier patterns here again, make a separate +# advisory-only workflow instead. +# +# Path-gating is handled centrally by the ``ci.yml`` orchestrator's +# ``detect`` job. The orchestrator passes ``scan`` / ``deps`` booleans as +# inputs; this workflow's jobs gate on those inputs instead of re-computing +# the diff. MCP catalog review was previously here but has moved to +# ``review-labels.yml`` so it can be rerun independently. +# +# Outputs: +# review_status — JSON array of status objects consumed by the review +# comment assembler (scripts/ci/assemble_review_comment.py). +# critical_findings — "true" when the narrow critical-pattern scan found +# something. The review-label gate consumes this and +# owns the action-required result, so adding +# ``ci-reviewed`` can heal the run on rerun. + +on: + workflow_call: + inputs: + event_name: + description: The event name from the calling orchestrator. + type: string + required: true + scan: + description: Whether supply-chain-relevant files changed. + type: boolean + required: true + deps: + description: Whether pyproject.toml changed. + type: boolean + required: true + outputs: + review_status: + description: JSON array of review status objects for the review comment assembler. + value: ${{ jobs.aggregate.outputs.review_status }} + critical_findings: + description: Whether the critical-pattern scan found a risk requiring maintainer review. + value: ${{ jobs.aggregate.outputs.critical_findings }} + +permissions: + pull-requests: write + contents: read + +jobs: + scan: + name: Scan PR for critical supply chain risks + if: inputs.scan + # Small runner: checkout plus a pattern scan over the diff. + runs-on: arc-runner-small + timeout-minutes: 15 + outputs: + review_status: ${{ steps.emit-status.outputs.review_status }} + critical_findings: ${{ steps.scan.outputs.found }} + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Ensure merge base is present + uses: ./.github/actions/merge-base + with: + base: ${{ github.event.pull_request.base.sha }} + head: ${{ github.event.pull_request.head.sha }} + + - name: Scan diff for critical patterns + id: scan + env: + GH_TOKEN: ${{ github.token }} + CI_REVIEWED: ${{ contains(github.event.pull_request.labels.*.name, 'ci-reviewed') }} + run: | + set -euo pipefail + + BASE="${{ github.event.pull_request.base.sha }}" + HEAD="${{ github.event.pull_request.head.sha }}" + + # Added lines only, excluding lockfiles. Three-dot diff = "since the + # merge base", so changes that landed on main after the branch point + # are excluded. + # + # No `|| true`: the merge-base step guarantees the histories connect, + # so a git failure here is real and must not degrade into an empty + # DIFF that scans nothing and reports clean. + DIFF=$(git diff "$BASE"..."$HEAD" -- . ':!uv.lock' ':!*.lock' ':!package-lock.json' ':!yarn.lock') + + FINDINGS="" + + # --- .pth files (auto-execute on Python startup) --- + # The exact mechanism used in the litellm supply chain attack: + # https://github.com/BerriAI/litellm/issues/24512 + PTH_FILES=$(git diff --diff-filter=d --name-only "$BASE"..."$HEAD" | grep '\.pth$' || true) + if [ -n "$PTH_FILES" ]; then + FINDINGS="${FINDINGS} + ### 🚨 CRITICAL: .pth file added or modified + Python \`.pth\` files in \`site-packages/\` execute automatically when the interpreter starts — no import required. + + **Files:** + \`\`\` + ${PTH_FILES} + \`\`\` + " + fi + + # --- base64 decode + exec/eval on the same line (the litellm attack pattern) --- + B64_EXEC_HITS=$(echo "$DIFF" | grep -n '^+' | grep -iE 'base64\.(b64decode|decodebytes|urlsafe_b64decode)' | grep -iE 'exec\(|eval\(' | head -10 || true) + if [ -n "$B64_EXEC_HITS" ]; then + FINDINGS="${FINDINGS} + ### 🚨 CRITICAL: base64 decode + exec/eval combo + Base64-decoded strings passed directly to exec/eval — the signature of hidden credential-stealing payloads. + + **Matches:** + \`\`\` + ${B64_EXEC_HITS} + \`\`\` + " + fi + + # --- subprocess with encoded/obfuscated command argument --- + PROC_HITS=$(echo "$DIFF" | grep -n '^+' | grep -E 'subprocess\.(Popen|call|run)\s*\(' | grep -iE 'base64|\\x[0-9a-f]{2}|chr\(' | head -10 || true) + if [ -n "$PROC_HITS" ]; then + FINDINGS="${FINDINGS} + ### 🚨 CRITICAL: subprocess with encoded/obfuscated command + Subprocess calls whose command strings are base64- or hex-encoded are a strong indicator of payload execution. + + **Matches:** + \`\`\` + ${PROC_HITS} + \`\`\` + " + fi + + # --- Install-hook files (setup.py/sitecustomize/usercustomize/__init__.pth) --- + # These execute during pip install or interpreter startup. + # Anchored at repo root: only the top-level setup.py/setup.cfg run during + # `pip install`, and only top-level sitecustomize.py/usercustomize.py are + # auto-loaded by the interpreter via site.py. Any nested file with the + # same name (e.g. hermes_cli/setup.py — the CLI setup wizard) is unrelated + # and produced false positives that trained reviewers to ignore the scanner. + SETUP_HITS=$(git diff --diff-filter=d --name-only "$BASE"..."$HEAD" | grep -E '^(setup\.py|setup\.cfg|sitecustomize\.py|usercustomize\.py|__init__\.pth)$' || true) + # A maintainer-applied ci-reviewed label records the manual review + # required for intentional changes to an install hook. The scanner + # still blocks every unreviewed addition or modification. + if [ -n "$SETUP_HITS" ] && [ "$CI_REVIEWED" != "true" ]; then + FINDINGS="${FINDINGS} + ### 🚨 CRITICAL: Install-hook file added or modified + These files can execute code during package installation or interpreter startup. + + **Files:** + \`\`\` + ${SETUP_HITS} + \`\`\` + " + fi + + if [ -n "$FINDINGS" ]; then + echo "found=true" >> "$GITHUB_OUTPUT" + echo "$FINDINGS" > /tmp/findings.md + else + echo "found=false" >> "$GITHUB_OUTPUT" + fi + + - name: Emit review_status + id: emit-status + if: always() + env: + FOUND: ${{ steps.scan.outputs.found }} + run: | + python3 - <<'PYEOF' + import json, os + + # The review-label gate renders and blocks critical findings. Keep + # this scan a fact-finder so adding ci-reviewed can rerun the gate + # without requiring the scanner itself to fail again. + status = [] + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + f.write(f"review_status={json.dumps(status)}\n") + PYEOF + + dep-bounds: + name: Check PyPI dependency upper bounds + if: inputs.deps + # Small runner: checkout plus a pyproject scan. + runs-on: arc-runner-small + timeout-minutes: 15 + outputs: + review_status: ${{ steps.emit-status.outputs.review_status }} + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Ensure merge base is present + uses: ./.github/actions/merge-base + with: + base: ${{ github.event.pull_request.base.sha }} + head: ${{ github.event.pull_request.head.sha }} + + - name: Check for unbounded PyPI deps + id: bounds + run: | + set -euo pipefail + + BASE="${{ github.event.pull_request.base.sha }}" + HEAD="${{ github.event.pull_request.head.sha }}" + + # Only check added lines in pyproject.toml. `|| true` absorbs grep's + # exit 1 on no-match, not a git failure — pipefail catches that, so + # this can't silently degrade into "no unbounded deps". + ADDED=$(git diff "$BASE"..."$HEAD" -- pyproject.toml | grep '^+' | grep -v '^+++' || true) + + if [ -z "$ADDED" ]; then + echo "found=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Match PyPI dep specs that have >= and no < ceiling. + # Pattern: "package>=version" without a following ",<" bound. + # Excludes git+ URLs (which use commit SHAs) and comments. + UNBOUNDED=$(echo "$ADDED" | grep -oE '"[a-zA-Z0-9_-]+(\[[^\]]*\])?>=[ 0-9.]+"' | grep -v ',<' || true) + + if [ -n "$UNBOUNDED" ]; then + echo "found=true" >> "$GITHUB_OUTPUT" + echo "$UNBOUNDED" > /tmp/unbounded.txt + else + echo "found=false" >> "$GITHUB_OUTPUT" + fi + + - name: Emit review_status + id: emit-status + if: always() + env: + FOUND: ${{ steps.bounds.outputs.found }} + run: | + python3 - <<'PYEOF' + import json, os + + found = os.environ.get("FOUND", "") == "true" + + if found: + with open("/tmp/unbounded.txt", encoding="utf-8") as f: + detail = f.read() + status = [{ + "source": "supply chain", + "results": [{ + "kind": "action_required", + "title": "Unbounded PyPI dependencies", + "summary": "This PR adds PyPI dependencies without upper bounds.", + "detail": detail, + "how_to_fix": 'Add a `=1.2.0,<2"`. See CONTRIBUTING.md dependency pinning policy.' + }] + }] + else: + status = [] + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + f.write(f"review_status={json.dumps(status)}\n") + PYEOF + + - name: Fail on unbounded deps + if: steps.bounds.outputs.found == 'true' + run: | + echo "::error::PyPI dependencies without upper bounds detected. Add ~3MB) and materialize only the script + # itself. `--discover-from-git` below lists the test paths from + # the git index, which a sparse checkout leaves fully populated. + # Checkout was ~90% of this job's wall time, and every test slice + # waits on it. + filter: blob:none + sparse-checkout: scripts/run_tests_parallel.py + sparse-checkout-cone-mode: false + + - name: Restore duration cache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: test_durations.json + key: newci-test-durations + # Saves use test-durations-${run_id}, so the exact key above never + # matches — without this prefix fallback the cache ALWAYS missed, + # LPT slicing ran on no data, and unbalanced slices pushed heavy + # files toward the per-file timeout under load. + restore-keys: | + newci-test-durations- + + - name: Generate test slices + id: matrix + run: | + MATRIX=$(python3 scripts/run_tests_parallel.py --generate-slices ${{ inputs.slice_count }} --discover-from-git) + echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" + + test: + name: Run tests slice ${{ matrix.slice.index }}/${{ inputs.slice_count }} + needs: generate + runs-on: arc-runner-set + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.generate.outputs.matrix) }} + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Restore uv cache + uses: ./.github/actions/uv-cache + + - name: Install dependencies + # `uv sync --locked` installs the exact pinned set from uv.lock (and + # fails if the lock is out of sync with pyproject.toml), giving a + # reproducible env. It also creates .venv itself, so no separate + # `uv venv` step is needed. + # + # The trailing extras beyond all/dev are the lazy-install features + # (tools/lazy_deps.py) that tests exercise for real: provider.anthropic, + # stt/tts.mistral, image.fal, terminal.modal, terminal.daytona, + # memory.hindsight, search.parallel. The hermetic test env forbids + # mid-run pip installs (HERMES_DISABLE_LAZY_INSTALLS=1 in + # tests/conftest.py), so the SDKs those tests need must be in the + # venv up front — resolved from uv.lock like everything else, which + # also honors the exact supply-chain pins these extras carry. + uses: ./.github/actions/retry + with: + command: uv sync --locked --python 3.11 --extra all --extra dev --extra anthropic --extra mistral --extra fal --extra modal --extra daytona --extra hindsight --extra parallel-web + + - name: Minimize uv cache + # Optimized for CI: prunes pre-built wheels that are cheap to + # re-download, keeping the persisted cache small and fast to restore. + run: uv cache prune --ci + + - name: Run tests (slice ${{ matrix.slice.index }}/${{ inputs.slice_count }}) + # Per-file isolation via scripts/run_tests.sh: each test file runs + # in its own freshly-spawned `python -m pytest ` subprocess + # with bounded parallelism. No xdist, no shared workers, no + # module-level state leakage between files. + # + # File list is pre-computed by the generate job (--generate-slices) + # which runs LPT distribution once and passes the file list to each + # matrix job via --files. Previously each job re-discovered files + # and re-ran LPT independently — redundant N times. + uses: ./.github/actions/profile + with: + label: tests-slice-${{ matrix.slice.index }} + command: | + source .venv/bin/activate + scripts/run_tests.sh --files '${{ matrix.slice.files }}' + env: + # Ensure tests don't accidentally call real APIs + OPENROUTER_API_KEY: "" + OPENAI_API_KEY: "" + NOUS_API_KEY: "" + + - name: Upload per-slice durations + # Advisory artifact (feeds slice balancing) — a transient artifact- + # service blip must not fail an otherwise-green test slice. + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-durations-slice-${{ matrix.slice.index }} + path: test_durations.json + retention-days: 1 + + # Merge per-slice duration data into a single cache, so future runs + # (including PRs) get balanced slicing. + save-durations: + needs: test + if: needs.test.result == 'success' && github.ref == 'refs/heads/main' + # Small runner: downloads artifacts and merges JSON. + runs-on: arc-runner-small + timeout-minutes: 10 + steps: + - name: Download all slice durations + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: test-durations-slice-* + path: durations + merge-multiple: true + + - name: Merge into single durations file + run: | + python3 -c " + import json, glob, os + merged = {} + for f in glob.glob('durations/*test_durations.json'): + with open(f) as fh: + merged.update(json.load(fh)) + with open('test_durations.json', 'w') as fh: + json.dump(merged, fh, indent=2, sort_keys=True) + print(f'Merged {len(merged)} file durations') + " + + - name: Save merged duration cache + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: test_durations.json + key: newci-test-durations-${{ github.run_id }} + + e2e: + runs-on: arc-runner-set + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Restore uv cache + uses: ./.github/actions/uv-cache + + - name: Install dependencies + # `uv sync --locked` installs the exact pinned set from uv.lock (and + # fails if the lock is out of sync with pyproject.toml), giving a + # reproducible env. It also creates .venv itself, so no separate + # `uv venv` step is needed. + # + # Same extras as the test job's sync above: the hermetic test env + # forbids mid-run pip installs (HERMES_DISABLE_LAZY_INSTALLS=1 in + # tests/conftest.py), so lazy-install SDKs exercised by tests must be + # in the venv up front. + uses: ./.github/actions/retry + with: + command: uv sync --locked --python 3.11 --extra all --extra dev --extra anthropic --extra mistral --extra fal --extra modal --extra daytona --extra hindsight --extra parallel-web + + - name: Minimize uv cache + # Optimized for CI: prunes pre-built wheels that are cheap to + # re-download, keeping the persisted cache small and fast to restore. + run: uv cache prune --ci + + - name: Run e2e tests + uses: ./.github/actions/profile + with: + label: tests-e2e + command: | + uv run --no-sync pytest tests/e2e/ -v --tb=short + env: + OPENROUTER_API_KEY: "" + OPENAI_API_KEY: "" + NOUS_API_KEY: "" diff --git a/.github/workflows/newci-uv-lockfile-check.yml b/.github/workflows/newci-uv-lockfile-check.yml new file mode 100644 index 000000000000..4943d1344467 --- /dev/null +++ b/.github/workflows/newci-uv-lockfile-check.yml @@ -0,0 +1,147 @@ +name: '[newci] uv.lock check' + +# ⚠️ SHADOW WORKFLOW — temporary, for the ARC runner migration. +# +# A duplicate of the production uv-lockfile-check.yml, running on the GKE self-hosted +# (ARC) runners so the migration can be observed for a few days without +# touching the workflows that gate merges. Production CI in this branch is +# byte-identical to main. +# +# Safety properties (keep these when editing): +# - concurrency groups are newci-prefixed, so a shadow run can never +# cancel the production run it shadows +# - cache keys are newci-prefixed, so production caches stay clean +# - reusable-workflow calls point only at other newci-* workflows +# - the PR review comment runs --dry-run (prints, never posts) +# - the gate job is renamed; it does NOT gate merges +# +# To retire: delete .github/workflows/newci-*.yml. + +# Verify uv.lock is in sync with pyproject.toml. Blocking check — PRs +# that modify pyproject.toml without regenerating uv.lock (or vice versa) +# must not merge, because the Docker build's `uv sync --frozen` step will +# fail on a stale lockfile and we'd rather catch it here than in the +# docker workflow on main. +# +# ───────────────────────────────────────────────────────────────────────── +# IMPORTANT: this check runs against the MERGED state, not just your branch +# ───────────────────────────────────────────────────────────────────────── +# +# For `pull_request` events, GitHub checks out `refs/pull//merge` by +# default — a synthetic commit that merges your PR branch into the CURRENT +# state of `main`. That means the pyproject.toml evaluated here is +# `main's pyproject.toml + your PR's changes to pyproject.toml`, not just +# what's on your branch. +# +# Failure mode this creates: if `main` has advanced since you branched +# (e.g. someone merged a PR that added a dep to pyproject.toml + its +# corresponding uv.lock entries), your branch's uv.lock is missing those +# new entries. `uv lock --check` resolves against the merged pyproject +# and sees a lockfile that doesn't cover all the current deps → fails +# with "The lockfile at uv.lock needs to be updated." +# +# This can be confusing: `uv lock --check` passes locally (your branch +# is internally consistent) but fails in CI (merged state isn't). +# +# Fix is to sync your branch with main and regenerate the lockfile: +# +# git fetch origin main +# git rebase origin/main # or merge, whatever the repo prefers +# uv lock # regenerates uv.lock against new pyproject.toml +# git add uv.lock +# git commit -m "chore: refresh uv.lock after rebase onto main" +# git push --force-with-lease # if you rebased +# +# If you also changed pyproject.toml in your PR, `uv lock` handles that +# at the same time — one regeneration covers both your changes and the +# drift from main. +# +# This is the correct behavior! The check is protecting main's Docker +# build: a post-merge build would see the same merged state and fail +# the same way. Better to catch it here than after merge. + +on: + workflow_call: + outputs: + review_status: + description: "JSON review status for the review-status aggregator" + value: ${{ jobs.check.outputs.review_status }} + +permissions: + contents: read + +concurrency: + group: newci-uv-lockfile-check-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + check: + name: uv lock --check + # Small runner: checkout plus `uv lock --check`. + runs-on: arc-runner-small + timeout-minutes: 5 + outputs: + review_status: ${{ steps.verify.outputs.review_status }} + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # `uv lock --check` re-resolves the project from pyproject.toml and + # compares the result to uv.lock, exiting non-zero if they disagree. + # No network writes, no file modifications. + # + # On PRs this runs against the merge commit (see comment at the top + # of this file) — failures often mean "your branch is behind main, + # rebase and regenerate uv.lock." + - name: Verify uv.lock is up-to-date + id: verify + run: | + # uv lock --check re-resolves against PyPI (network). Retry so a + # registry blip doesn't read as "lockfile stale". A genuinely stale + # lockfile fails all attempts (deterministic), costing only seconds. + ok=false + for i in 1 2 3; do + if uv lock --check; then + ok=true + break + fi + [ "$i" = 3 ] && break + echo "::warning::uv lock --check failed (attempt $i); retrying in 10s" + sleep 10 + done + if [ "$ok" != true ]; then + cat <<'EOF' >> "$GITHUB_STEP_SUMMARY" + ## ❌ uv.lock is out of sync with pyproject.toml + + **If this is a PR:** this check runs against the merged state + (your branch + current `main`), not just your branch. If + `uv lock --check` passes locally, your branch is likely behind + `main` — recent changes to `pyproject.toml` on `main` aren't + reflected in your branch's `uv.lock` yet. + + To fix, sync with main and regenerate the lockfile: + + ```bash + git fetch origin main + git rebase origin/main # or `git merge origin/main` + uv lock # regenerate against new pyproject.toml + git add uv.lock + git commit -m "chore: refresh uv.lock after syncing with main" + git push --force-with-lease # drop --force-with-lease if you merged + ``` + + **If you only changed pyproject.toml:** run `uv lock` locally + and commit the result. + + This check is blocking because the Docker image build uses + `uv sync --frozen --extra all`, which rejects stale lockfiles + — catching it here avoids a ~15 min failed docker run + on `main` post-merge. + EOF + echo "::error title=uv.lock out of sync::Run \`uv lock\` locally and commit the result. If on a PR, sync with main first." + review_status='[{"source":"uv.lock check","results":[{"kind":"action_required","title":"uv.lock out of sync","summary":"uv.lock is out of sync with pyproject.toml.","how_to_fix":"Run `uv lock` locally and commit the result. If on a PR, sync with main first:\n```\ngit fetch origin main\ngit rebase origin/main\nuv lock\ngit add uv.lock\ngit commit -m \"chore: refresh uv.lock\"\n```\n"}]}]' + echo "review_status=${review_status}" >> "$GITHUB_OUTPUT" + exit 1 + fi + review_status='[]' + echo "review_status=${review_status}" >> "$GITHUB_OUTPUT" diff --git a/agent/skill_utils.py b/agent/skill_utils.py index a302c6981a47..c901ee30d9da 100644 --- a/agent/skill_utils.py +++ b/agent/skill_utils.py @@ -471,7 +471,7 @@ def _normalize_string_set(values) -> Set[str]: # which becomes the dominant cost of ``hermes`` startup when ~120 skills # each trigger a category lookup during banner construction (10+ seconds # of pure waste). -_EXTERNAL_DIRS_CACHE: Dict[Tuple[str, int], List[Path]] = {} +_EXTERNAL_DIRS_CACHE: Dict[Tuple[str, int, int], List[Path]] = {} def _external_dirs_cache_clear() -> None: @@ -496,11 +496,13 @@ def get_external_skills_dirs() -> List[Path]: if not config_path.exists(): return [] - # Cache key: (absolute path, mtime_ns). stat() is ~2us vs ~85ms for - # the full YAML parse, so the fast path is nearly free. + # Cache key: (absolute path, mtime_ns, size). stat() is ~2us vs ~85ms + # for the full YAML parse, so the fast path is nearly free. Size is in + # the key because overlayfs (CI runner pods) can coalesce rapid writes + # into a single mtime_ns tick. try: stat = config_path.stat() - cache_key: Tuple[str, int] = (str(config_path), stat.st_mtime_ns) + cache_key: Tuple[str, int, int] = (str(config_path), stat.st_mtime_ns, stat.st_size) except OSError: cache_key = None # type: ignore[assignment] diff --git a/gateway/run.py b/gateway/run.py index 24d501b5b752..8f0c51a1311f 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -22269,7 +22269,7 @@ async def _run_process_watcher(self, watcher: dict) -> None: "honcho.runtime_peer_prefix", "honcho.user_peer_aliases", ) - _HONCHO_CACHE_BUSTING_MEMO: dict[tuple[str, int | None], dict[str, Any]] = {} + _HONCHO_CACHE_BUSTING_MEMO: dict[tuple[str, tuple[int, int] | None], dict[str, Any]] = {} @classmethod def _empty_honcho_cache_busting_config(cls) -> dict[str, Any]: @@ -22283,10 +22283,14 @@ def _extract_honcho_cache_busting_config(cls) -> dict[str, Any]: path = resolve_config_path() try: - mtime_ns = path.stat().st_mtime_ns + st = path.stat() + # mtime alone is not enough: overlayfs (CI runner pods) + # coalesces rapid writes into one mtime_ns tick. Size joins + # the key so a same-tick content change still re-parses. + stat_sig = (st.st_mtime_ns, st.st_size) except OSError: - mtime_ns = None - memo_key = (str(path), mtime_ns) + stat_sig = None + memo_key = (str(path), stat_sig) cached = cls._HONCHO_CACHE_BUSTING_MEMO.get(memo_key) if cached is not None: return dict(cached) diff --git a/hermes_cli/doctor.py b/hermes_cli/doctor.py index 18ea422f3e24..339d6f6bc801 100644 --- a/hermes_cli/doctor.py +++ b/hermes_cli/doctor.py @@ -1727,13 +1727,14 @@ def run_doctor(args): # not found" warning. If the user has explicitly chosen # TERMINAL_ENV=docker inside the container they likely mounted # /var/run/docker.sock, so fall through to the normal check. - if terminal_env != "docker": + # Only the implicit local-vs-docker choice is affected: remote + # backends (ssh, daytona, vercel_sandbox, ...) work fine from + # inside a container and must keep their real diagnostics. + if terminal_env == "local": check_info( "Running inside a container — using local terminal backend " "(docker-in-docker is not configured by default)" ) - # Skip to next section; Docker isn't relevant here. - terminal_env = "local" if terminal_env == "docker": if _safe_which("docker"): # Check if docker daemon is running @@ -1757,7 +1758,10 @@ def run_doctor(args): elif _is_termux(): check_info("Docker backend is not available inside Termux (expected on Android)") elif running_in_container: - pass # already explained above + # In-container with a non-docker backend: the info line above only + # prints for the implicit `local` case, but either way a missing + # docker binary inside the container is expected — stay quiet. + pass else: check_warn("docker not found", "(optional)") diff --git a/scripts/ci/resource_profile.py b/scripts/ci/resource_profile.py new file mode 100644 index 000000000000..0d42376981c1 --- /dev/null +++ b/scripts/ci/resource_profile.py @@ -0,0 +1,368 @@ +#!/usr/bin/env python3 +"""CPU / RAM / disk-IO profiler for CI jobs. + +Runs as a background daemon: samples /proc every second, accumulates +stats, and on SIGTERM (or timeout) writes a JSON summary to the output +path. Pure stdlib — runs on the bare runner Python with zero deps. + +Usage: + python3 scripts/ci/resource_profile.py \\ + --output resource-profile.json \\ + --label "tests slice 1/8" + +The composite action (.github/actions/profile) starts this as a +background process, runs the real command, then signals it to stop. + +Output JSON shape: + { + "label": "tests slice 1/8", + "duration_s": 42.3, + "started_at": "2026-01-01T00:01:00Z", # UTC bounds of the sample + "completed_at": "2026-01-01T00:01:42Z", # window, for report placement + "cpu": { + "avg_usage_pct": 55.2, + "peak_usage_pct": 89.1, + "samples": 42 + }, + "memory": { # USED memory (MemTotal - MemAvailable) + "avg_mb": 512.0, + "peak_mb": 684.3, + "samples": 42 + }, + "disk": { + "total_mb": 12.4, # sectors read+written, whole devices only + "avg_ops_per_s": 5.2, # completed read+write IOs per second + "peak_ops_per_s": 20.1, + "avg_util_pct": 31.0, # busy time (iostat %util), whole devices + "peak_util_pct": 96.0, + "samples": 42 + }, + "series": { # downsampled timeline for the report overlay + "interval_s": 1.0, # seconds per point AFTER downsampling + "points": 42, + "cpu_pct": [12, 40, ...], # 0-100 ints, one per point + "mem_pct": [30, 31, ...], # used/total, 0-100 ints + "disk_pct": [5, 80, ...] # busy-time util, 0-100 ints + } + } + +The series is capped at ``_MAX_SERIES_POINTS`` points by mean-bucketing, +so a 40-minute job costs the same handful of KB as a 40-second one — the +CI timing report embeds every profile inline in a single HTML file. + +Caveat: /proc/stat, /proc/meminfo, and /proc/diskstats are NODE-wide. +Inside a Kubernetes pod these numbers include neighbor pods sharing the +node — treat them as indicative, not exact per-job attribution. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import signal +import sys +import time +from datetime import datetime, timezone + +_SAMPLE_INTERVAL_S = 1.0 +_CLK_TICK = os.sysconf("SC_CLK_TCK") if hasattr(os, "sysconf") else 100 +_PROC_STAT = "/proc/stat" +_PROC_MEMINFO = "/proc/meminfo" +_PROC_DISKSTATS = "/proc/diskstats" + +# Upper bound on points in the emitted ``series``. The CI timing report +# inlines every profile into one self-contained HTML file, so an +# unbounded 1Hz series from a 20-minute job would dominate its size. +# 180 points is ~1 point per 7s on a 20-minute job — finer than the +# overlay strip can resolve on screen. +_MAX_SERIES_POINTS = 180 + + +def _read_cpu_usage(prev: dict | None) -> tuple[float, dict]: + """Return (usage_pct since prev sample, current_jiffies_dict). + + Reads /proc/stat line 1 (aggregate CPU). usage_pct = non-idle / total. + """ + try: + with open(_PROC_STAT, encoding="ascii") as f: + first_line = f.readline() + except OSError: + return (0.0, prev or {}) + + parts = first_line.split() + if len(parts) < 5: + return (0.0, prev or {}) + + # user, nice, system, idle, iowait, irq, softirq, steal, ... + vals = [int(x) for x in parts[1:]] + idle = vals[3] + (vals[4] if len(vals) > 4 else 0) + total = sum(vals) + cur = {"total": total, "idle": idle} + + if prev and cur["total"] != prev["total"]: + d_total = cur["total"] - prev["total"] + d_idle = cur["idle"] - prev["idle"] + if d_total > 0: + return (max(0.0, (1.0 - d_idle / d_total) * 100.0), cur) + + return (0.0, cur) + + +def _read_mem_mb() -> float: + """Return *used* memory in MB (MemTotal - MemAvailable) from /proc/meminfo. + + Falls back to MemTotal - MemFree on old kernels without MemAvailable. + Note: in a container this is the host/node view, not the cgroup view — + numbers can include neighbor pods on shared nodes. + """ + try: + with open(_PROC_MEMINFO, encoding="ascii") as f: + text = f.read() + except OSError: + return 0.0 + + total_kb = 0 + available_kb = -1 + free_kb = -1 + for line in text.splitlines(): + if line.startswith("MemTotal:"): + total_kb = int(line.split()[1]) + elif line.startswith("MemAvailable:"): + available_kb = int(line.split()[1]) + elif line.startswith("MemFree:"): + free_kb = int(line.split()[1]) + + if total_kb <= 0: + return 0.0 + unused_kb = available_kb if available_kb >= 0 else max(free_kb, 0) + return max(0, total_kb - unused_kb) / 1024.0 + + +def _read_diskstats() -> dict[str, tuple[int, int, int]]: + """Return {device: (io_ops_completed, sectors_read_plus_written, busy_ms)}. + + We track whole block devices only (skip partitions — track sda not + sda1, nvme0n1 not nvme0n1p1) so IO isn't double-counted. Each + diskstats line: + major minor name reads_completed reads_merged sectors_read time_reading + writes_completed writes_merged sectors_written time_writing ios_in_flight + io_ticks ... + + ``io_ticks`` (field 13, index 12) is milliseconds during which the + device had IO in flight. Its delta over a sample interval is the + device busy time, i.e. iostat's ``%util`` — the saturation signal + ops/s alone cannot give you (a device can be pinned at 100% busy on + few large IOs, or barely busy on many small cached ones). + """ + try: + with open(_PROC_DISKSTATS, encoding="ascii") as f: + lines = f.readlines() + except OSError: + return {} + + result = {} + for line in lines: + parts = line.split() + if len(parts) < 14: + continue + name = parts[2] + # Skip virtual/removable devices + if name.startswith(("loop", "ram", "sr")): + continue + # Skip partitions: sda1, vda2, mmcblk0p1, nvme0n1p1. For devices + # whose base name ends in a digit (nvme0n1, mmcblk0), partitions + # carry a 'p' suffix; for sdX/vdX a bare trailing digit. + if name.startswith(("nvme", "mmcblk")): + if re.search(r"p\d+$", name): + continue + elif name[-1].isdigit(): + continue + reads_completed = int(parts[3]) + writes_completed = int(parts[7]) + sectors = int(parts[5]) + int(parts[9]) + busy_ms = int(parts[12]) + result[name] = (reads_completed + writes_completed, sectors, busy_ms) + + return result + + +def _downsample(values: list[float], max_points: int = _MAX_SERIES_POINTS) -> list[int]: + """Bucket ``values`` down to at most ``max_points``, as rounded 0-100 ints. + + Uses the bucket MEAN rather than picking every Nth sample: a spike + that survives decimation by luck is misleading, whereas a mean keeps + the area under the curve honest for an overlay whose whole job is to + show "was this saturated". Values are clamped to 0-100 because the + consumer draws them as a percentage-height sparkline. + """ + if not values: + return [] + n = len(values) + if n <= max_points: + return [int(round(min(100.0, max(0.0, v)))) for v in values] + + out: list[int] = [] + for i in range(max_points): + lo = i * n // max_points + hi = (i + 1) * n // max_points + if hi <= lo: + hi = lo + 1 + bucket = values[lo:hi] + avg = sum(bucket) / len(bucket) + out.append(int(round(min(100.0, max(0.0, avg))))) + return out + + +def _read_mem_total_mb() -> float: + """Return MemTotal in MB (0.0 if unreadable).""" + try: + with open(_PROC_MEMINFO, encoding="ascii") as f: + for line in f: + if line.startswith("MemTotal:"): + return int(line.split()[1]) / 1024.0 + except OSError: + pass + return 0.0 + + +def run_profiler(output_path: str, label: str, timeout_s: float = 0) -> None: + """Sample resources until SIGTERM or timeout, then write JSON summary.""" + cpu_samples: list[float] = [] + mem_samples: list[float] = [] + mem_pct_samples: list[float] = [] + disk_prev = _read_diskstats() + disk_total_sectors = 0 + disk_ops_samples: list[float] = [] + disk_util_samples: list[float] = [] + + mem_total_mb = _read_mem_total_mb() + + cpu_prev: dict | None = None + start = time.monotonic() + # Wall-clock anchor for the sample window. The report needs to know WHEN + # these samples happened, not just how long they lasted: the profiler + # wraps one step, so on a job whose other steps (checkout, setup, post) + # dominate, the profiled window is a slice in the middle of the bar. + # Without this the overlay gets stretched across the whole job and the + # x-axis lies. monotonic() drives the sampling (immune to clock steps); + # this is only for placement. + started_at = datetime.now(timezone.utc) + last_sample = start + running = [True] # mutable for signal handler + + def _stop(*_): + running[0] = False + + signal.signal(signal.SIGTERM, _stop) + signal.signal(signal.SIGINT, _stop) + + while running[0]: + cpu_pct, cpu_prev = _read_cpu_usage(cpu_prev) + cpu_samples.append(cpu_pct) + + mem_mb = _read_mem_mb() + mem_samples.append(mem_mb) + mem_pct_samples.append((mem_mb / mem_total_mb * 100.0) if mem_total_mb > 0 else 0.0) + + # Wall time actually elapsed since the previous sample. Under a + # loaded/throttled runner the loop can drift well past the + # nominal interval, and dividing a busy-ms delta by the nominal + # 1.0s would then report >100% util. Measure the real gap. + now = time.monotonic() + gap_s = max(now - last_sample, 1e-6) + last_sample = now + + disk_cur = _read_diskstats() + delta_sectors = 0 + delta_ops = 0 + delta_busy_ms = 0 + for dev, (ops, sectors, busy_ms) in disk_cur.items(): + prev_ops, prev_sectors, prev_busy = disk_prev.get(dev, (ops, sectors, busy_ms)) + delta_sectors += max(0, sectors - prev_sectors) + delta_ops += max(0, ops - prev_ops) + # Busiest single device, not the sum: summing across devices + # exceeds 100% on a multi-disk node and is meaningless as a + # saturation percentage. + delta_busy_ms = max(delta_busy_ms, max(0, busy_ms - prev_busy)) + disk_total_sectors += delta_sectors + disk_ops_samples.append(delta_ops / _SAMPLE_INTERVAL_S) + disk_util_samples.append(min(100.0, delta_busy_ms / (gap_s * 1000.0) * 100.0)) + disk_prev = disk_cur + + elapsed = time.monotonic() - start + if timeout_s > 0 and elapsed >= timeout_s: + break + + time.sleep(_SAMPLE_INTERVAL_S) + + duration_s = time.monotonic() - start + n = len(cpu_samples) or 1 + + # Sectors are 512 bytes + disk_read_written_mb = disk_total_sectors * 512 / (1024 * 1024) + + cpu_series = _downsample(cpu_samples) + mem_series = _downsample(mem_pct_samples) + disk_series = _downsample(disk_util_samples) + n_points = max(len(cpu_series), len(mem_series), len(disk_series)) + + summary = { + "label": label, + "duration_s": round(duration_s, 1), + # ISO-8601 UTC bounds of the sample window, in the same format as + # GitHub's job/step timestamps so the report can place the series + # against a job bar's x-axis instead of stretching it to fill. + "started_at": started_at.isoformat().replace("+00:00", "Z"), + "completed_at": ( + datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + ), + "cpu": { + "avg_usage_pct": round(sum(cpu_samples) / n, 1), + "peak_usage_pct": round(max(cpu_samples, default=0.0), 1), + "samples": len(cpu_samples), + }, + "memory": { + "avg_mb": round(sum(mem_samples) / n, 1), + "peak_mb": round(max(mem_samples, default=0.0), 1), + "total_mb": round(mem_total_mb, 1), + "samples": len(mem_samples), + }, + "disk": { + "total_mb": round(disk_read_written_mb, 1), + "avg_ops_per_s": round(sum(disk_ops_samples) / n, 1), + "peak_ops_per_s": round(max(disk_ops_samples, default=0.0), 1), + "avg_util_pct": round(sum(disk_util_samples) / n, 1), + "peak_util_pct": round(max(disk_util_samples, default=0.0), 1), + "samples": len(disk_ops_samples), + }, + # Downsampled timeline. The timing report overlays this on each + # job's gantt bar; consumers must read interval_s rather than + # assuming 1Hz, since long jobs are bucketed. + "series": { + "interval_s": round(duration_s / n_points, 3) if n_points else 0, + "points": n_points, + "cpu_pct": cpu_series, + "mem_pct": mem_series, + "disk_pct": disk_series, + }, + } + + with open(output_path, "w", encoding="utf-8") as f: + json.dump(summary, f, indent=2) + print(f"resource_profile: wrote {output_path} ({n} samples, {duration_s:.1f}s)", file=sys.stderr) + + +def main(): + parser = argparse.ArgumentParser(description="CI resource profiler") + parser.add_argument("--output", required=True, help="Output JSON path") + parser.add_argument("--label", default="", help="Label for this profile") + parser.add_argument("--timeout", type=float, default=0, + help="Max seconds to run (0 = until SIGTERM)") + args = parser.parse_args() + run_profiler(args.output, args.label, args.timeout) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/timings_report.py b/scripts/ci/timings_report.py index 8a4d1cf669e5..c9eb4c934145 100644 --- a/scripts/ci/timings_report.py +++ b/scripts/ci/timings_report.py @@ -19,8 +19,11 @@ from __future__ import annotations import argparse +import difflib +import glob import json import os +import re import sys import time import urllib.error @@ -198,6 +201,125 @@ def _normalize_job(raw: dict) -> dict: } +# Step categories. "Overhead" is everything a job pays before and after its +# actual work: runner setup, checkout, dependency restore/install, teardown. +# Naming is GitHub's, not ours — a `uses:` step is reported as +# "Run /@" and its cleanup as "Post Run <...>", while a +# `run:` step keeps whatever `name:` the workflow gave it. So the setup +# patterns match action refs and the well-known implicit steps, and anything +# unmatched is treated as work (better to under-report overhead than to +# silently classify a real test step as setup). +STEP_SETUP = "setup" +STEP_WORK = "work" +STEP_TEARDOWN = "teardown" + +_SETUP_PATTERNS = ( + "set up job", + "set up python", + "set up node", + "checkout", + "actions/checkout", + "actions/setup-", + "actions/cache", + "restore ", # "Restore uv cache", "Restore baseline cache", ... + "install ", # "Install dependencies", "Install ruff + ty", ... + "minimize uv cache", + "uv-cache", + "set up docker buildx", + "log in to", + "authenticate to", + "mint read-only cache token", + "pull ghcr.io/", + "get-app-token", + "determine base ref", +) + +_TEARDOWN_PATTERNS = ( + "complete job", + "stop containers", + "upload", + "export results", +) + + +def classify_step(name: str) -> str: + """Bucket a step into setup / work / teardown. + + Any ``Post ...`` step is teardown regardless of what it post-processes — + that's GitHub's own cleanup phase for a ``uses:`` step. + """ + low = (name or "").strip().lower() + if low.startswith("post "): + return STEP_TEARDOWN + for pat in _TEARDOWN_PATTERNS: + if pat in low: + return STEP_TEARDOWN + for pat in _SETUP_PATTERNS: + if pat in low: + return STEP_SETUP + return STEP_WORK + + +def display_step_name(name: str) -> str: + """Strip the pinned SHA from an action ref for display. + + GitHub names a ``uses:`` step after the full pinned ref, e.g. + ``Run actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd``. The + 40-char SHA crowds out the part a reader cares about and makes the + overhead table unreadable, so drop it. Only the DISPLAY changes — the + raw name stays the key for baseline comparison and aggregation, since + two different pins are genuinely two different steps. + """ + if not name: + return "" + return re.sub(r"@[0-9a-f]{7,40}\b", "", name) + + +def compute_overhead(timings: dict) -> dict: + """Aggregate setup/work/teardown seconds across every non-skipped job. + + Returns totals plus the worst individual setup steps, so the report can + answer "how much of CI is spent getting ready to work" with a number + instead of an impression. + """ + totals = {STEP_SETUP: 0.0, STEP_WORK: 0.0, STEP_TEARDOWN: 0.0} + by_step: dict[str, dict] = {} + jobs_with_steps = 0 + + for j in timings.get("jobs", []): + if is_skipped(j) or not j.get("steps"): + continue + jobs_with_steps += 1 + for s in j["steps"]: + dur = s.get("duration_s") + if dur is None or dur < 0: + continue + cat = classify_step(s.get("name", "")) + totals[cat] += dur + if cat != STEP_WORK: + agg = by_step.setdefault( + s.get("name", ""), {"name": s.get("name", ""), + "total_s": 0.0, "count": 0, "category": cat} + ) + agg["total_s"] += dur + agg["count"] += 1 + + accounted = sum(totals.values()) + overhead = totals[STEP_SETUP] + totals[STEP_TEARDOWN] + return { + "setup_s": totals[STEP_SETUP], + "work_s": totals[STEP_WORK], + "teardown_s": totals[STEP_TEARDOWN], + "overhead_s": overhead, + "accounted_s": accounted, + "overhead_pct": (overhead / accounted * 100) if accounted > 0 else 0.0, + "jobs_with_steps": jobs_with_steps, + "top_overhead_steps": sorted( + by_step.values(), key=lambda x: -x["total_s"] + )[:8], + } + + def _annotate_wait_times(jobs: list[dict]) -> None: """Annotate each job with ``wait_s`` — how long it sat idle before starting. @@ -417,6 +539,344 @@ def compute_stats(timings: dict, baseline: dict | None = None) -> dict: } +# --------------------------------------------------------------------------- +# Resource profile loading + bottleneck analysis +# --------------------------------------------------------------------------- + +def load_resource_profiles(directory: str) -> dict[str, dict]: + """Load all resource-profile-*/resource-profile.json artifacts. + + Returns {label: profile_dict}. Labels are derived from the artifact + directory name (resource-profile-