diff --git a/.dockerignore b/.dockerignore index ec3d52f81413..cfd0616efb84 100644 --- a/.dockerignore +++ b/.dockerignore @@ -97,9 +97,6 @@ packaging/ plans/ .plans/ -# ACP registry manifest (icon + agent.json) — not consumed at runtime -acp_registry/ - # Repo-level dotfiles that are git-only or dev-tooling config .env.example .envrc diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml index 7d95ba76c9a2..145d742d5b12 100644 --- a/.github/actions/detect-changes/action.yml +++ b/.github/actions/detect-changes/action.yml @@ -7,7 +7,7 @@ description: >- inputs: github-token: - description: Token for the GitHub API (gh CLI). Pass secrets.AUTOFIX_BOT_PAT from the calling workflow. + description: Token for the GitHub API (gh CLI). Pass steps.app-token.outputs.token from the calling workflow. required: false default: ${{ github.token }} @@ -39,6 +39,9 @@ outputs: ci_review: description: Require CI-sensitive file review label. value: ${{ steps.classify.outputs.ci_review }} + ci_review_files: + description: JSON list of CI-sensitive files changed by the pull request. + value: ${{ steps.classify.outputs.ci_review_files }} runs: using: composite diff --git a/.github/actions/get-app-token/action.yml b/.github/actions/get-app-token/action.yml new file mode 100644 index 000000000000..2aaf303ab2de --- /dev/null +++ b/.github/actions/get-app-token/action.yml @@ -0,0 +1,69 @@ +name: Get GitHub App Token +description: >- + Mint a short-lived (1-hour) installation access token from the repo's + GitHub App, replacing the long-lived AUTOFIX_BOT_PAT. App tokens get + 5,000 req/hr per installation (vs 1,000 for the default GITHUB_TOKEN) + and are scoped to the App's installation permissions, not a user account. + + Callers must source App credentials from a protected, main-only environment. + Never pass an App private key to a pull_request job, a local action, or a + reusable workflow resolved from an untrusted PR ref. The fallback keeps a + trusted caller functional when its protected environment is misconfigured. + + Composite actions cannot access contexts directly, so callers pass the + public vars.APP_CLIENT_ID and protected secrets.APP_PRIVATE_KEY as inputs. + When the private key is empty, the fallback fires. + +inputs: + client-id: + description: GitHub App Client ID. Pass vars.APP_CLIENT_ID from the calling workflow. + required: false + default: '' + private-key: + description: GitHub App private key PEM. Pass secrets.APP_PRIVATE_KEY from the calling workflow. + required: false + default: '' + owner: + description: GitHub App installation owner. Empty scopes the token to the current repository. + required: false + default: '' + repositories: + description: Comma- or newline-separated repositories to scope within the installation owner. + required: false + default: '' + +outputs: + token: + description: A GitHub App installation access token (1-hour TTL), or GITHUB_TOKEN on forks. + value: ${{ steps.app-token.outputs.token || steps.fallback.outputs.token }} + +runs: + using: composite + steps: + - name: Check if App credentials exist + id: check + shell: bash + env: + CLIENT_ID: ${{ inputs.client-id }} + run: | + if [ -n "$CLIENT_ID" ]; then + echo "has_app=true" >> "$GITHUB_OUTPUT" + else + echo "has_app=false" >> "$GITHUB_OUTPUT" + fi + + - name: Create GitHub App token + id: app-token + if: steps.check.outputs.has_app == 'true' + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ inputs.client-id }} + private-key: ${{ inputs.private-key }} + owner: ${{ inputs.owner }} + repositories: ${{ inputs.repositories }} + + - name: Fall back to GITHUB_TOKEN + id: fallback + if: steps.check.outputs.has_app != 'true' + shell: bash + run: echo "token=${{ github.token }}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index faae3b6f2704..3441d7bb6db3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,10 @@ name: CI # definitions, matrices, and concurrency settings. They no longer have # ``push:`` / ``pull_request:`` triggers of their own — everything flows # through this file. +# +# SECURITY: this workflow runs PR-controlled actions, workflows, and code. +# Do not add ``secrets: inherit`` or GitHub App credentials here. Trusted +# main-only automation uses protected environments in its own workflows. on: pull_request: @@ -17,7 +21,7 @@ on: permissions: contents: read - pull-requests: write # needed by lint (PR comment) + supply-chain (PR comment) + pull-requests: write # needed by lint (PR comment) + supply-chain review_status actions: read # needed by osv-scanner (SARIF upload) security-events: write # needed by osv-scanner (SARIF upload) packages: write # needed by docker build @@ -46,6 +50,7 @@ jobs: docker_meta: ${{ steps.classify.outputs.docker_meta }} mcp_catalog: ${{ steps.classify.outputs.mcp_catalog }} ci_review: ${{ steps.classify.outputs.ci_review }} + ci_review_files: ${{ steps.classify.outputs.ci_review_files }} event_name: ${{ github.event_name }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -53,9 +58,7 @@ jobs: id: classify uses: ./.github/actions/detect-changes with: - # Forks get no repo secrets (AUTOFIX_BOT_PAT is empty); fall back to - # the built-in read-only token so classification still works there. - github-token: ${{ secrets.AUTOFIX_BOT_PAT || github.token }} + github-token: ${{ github.token }} # ───────────────────────────────────────────────────────────────────── # Lane-gated sub-workflows. Each runs in parallel after detect finishes. @@ -68,89 +71,177 @@ jobs: uses: ./.github/workflows/tests.yml with: slice_count: 8 - secrets: inherit lint: name: Python lints needs: detect - if: needs.detect.outputs.python == 'true' || needs.detect.outputs.ci_review == 'true' + if: needs.detect.outputs.python == 'true' uses: ./.github/workflows/lint.yml with: event_name: ${{ needs.detect.outputs.event_name }} - ci_review: ${{ needs.detect.outputs.ci_review == 'true' }} - secrets: inherit js-tests: name: JS & TS checks needs: detect if: needs.detect.outputs.frontend == 'true' uses: ./.github/workflows/js-tests.yml - secrets: inherit + + e2e-desktop: + name: Desktop E2E + needs: detect + if: needs.detect.outputs.python == 'true' || needs.detect.outputs.frontend == 'true' + uses: ./.github/workflows/e2e-desktop.yml docs-site: name: Docs Site needs: detect if: needs.detect.outputs.site == 'true' uses: ./.github/workflows/docs-site-checks.yml - secrets: inherit history-check: name: Deny unrelated histories needs: detect if: needs.detect.outputs.event_name == 'pull_request' uses: ./.github/workflows/history-check.yml - secrets: inherit contributor-check: name: Check contributors needs: detect if: needs.detect.outputs.python == 'true' uses: ./.github/workflows/contributor-check.yml - secrets: inherit uv-lockfile: name: Check uv.lock needs: detect uses: ./.github/workflows/uv-lockfile-check.yml - secrets: inherit + + infographic-check: + name: Check no committed infographics + needs: detect + uses: ./.github/workflows/infographic-check.yml lockfile-diff: name: package-lock.json diff needs: detect if: needs.detect.outputs.event_name == 'pull_request' && needs.detect.outputs.npm_lock == 'true' uses: ./.github/workflows/lockfile-diff.yml - secrets: inherit docker-lint: name: Lint Docker scripts needs: detect if: needs.detect.outputs.docker_meta == 'true' uses: ./.github/workflows/docker-lint.yml - secrets: inherit docker: name: Build&Test Docker image needs: detect - if: needs.detect.outputs.python == 'true' || needs.detect.outputs.frontend == 'true' || needs.detect.outputs.docker_meta == 'true' + # Trusted main pushes run docker.yml directly so its container-publish + # environment secrets never cross this reusable-workflow call. PR runs + # remain build/test-only and secret-free. + if: needs.detect.outputs.event_name == 'pull_request' && (needs.detect.outputs.python == 'true' || needs.detect.outputs.frontend == 'true' || needs.detect.outputs.docker_meta == 'true') uses: ./.github/workflows/docker.yml - secrets: inherit supply-chain: name: Supply-chain scan needs: detect - if: needs.detect.outputs.event_name == 'pull_request' && (needs.detect.outputs.scan == 'true' || needs.detect.outputs.deps == 'true' || needs.detect.outputs.mcp_catalog == 'true') + if: needs.detect.outputs.event_name == 'pull_request' && (needs.detect.outputs.scan == 'true' || needs.detect.outputs.deps == 'true') uses: ./.github/workflows/supply-chain-audit.yml with: event_name: ${{ needs.detect.outputs.event_name }} scan: ${{ needs.detect.outputs.scan == 'true' }} deps: ${{ needs.detect.outputs.deps == 'true' }} + + review-labels: + name: Review label gate + needs: [detect, supply-chain] + if: always() && needs.detect.outputs.event_name == 'pull_request' && (needs.detect.outputs.ci_review == 'true' || needs.detect.outputs.mcp_catalog == 'true' || needs.supply-chain.outputs.critical_findings == 'true') + uses: ./.github/workflows/review-labels.yml + with: + ci_review: ${{ needs.detect.outputs.ci_review == 'true' }} + ci_review_files: ${{ needs.detect.outputs.ci_review_files }} mcp_catalog: ${{ needs.detect.outputs.mcp_catalog == 'true' }} - secrets: inherit + supply_chain: ${{ needs.supply-chain.outputs.critical_findings == 'true' }} osv-scanner: name: OSV scan uses: ./.github/workflows/osv-scanner.yml - secrets: inherit + + # ───────────────────────────────────────────────────────────────────── + # Live-updating PR review comment. + # + # A single ``comment-live`` job polls the GitHub Actions API every 15s + # for job statuses in this run, re-assembles the review comment from + # whatever results are available, and upserts it via the + # ```` marker. + # + # When the visible job set goes quiet, the poller waits 10 seconds and polls + # once more so downstream jobs created by an aggregate gate get included. + # ───────────────────────────────────────────────────────────────────── + comment-live: + name: CI review comment (live) + needs: [detect, review-labels, lockfile-diff, supply-chain, osv-scanner, uv-lockfile, history-check, contributor-check, e2e-desktop] + if: always() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork != true + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + - name: Run live comment poller + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + PR_NUMBER: ${{ github.event.pull_request.number }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + # Commit info for the review comment header. + COMMIT_SHA: ${{ github.event.pull_request.head.sha }} + COMMIT_MESSAGE: ${{ github.event.pull_request.head.commit.message }} + COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/commits/${{ github.event.pull_request.head.sha }} + # Structured review statuses from workflow_call jobs. + # Each job outputs a JSON array of {source, results: [...]} objects + # that the assembler renders directly — no hardcoded job-name + # matching. We merge all available outputs into one array. + REVIEW_STATUSES: ${{ toJSON(needs.*.outputs.review_status) }} + run: | + set -uo pipefail + + # REVIEW_STATUSES is a JSON array of strings (some may be empty + # when a job was skipped). Parse each string and merge into one + # flat array for the assembler. + python3 - <<'PYEOF' + import json, os, sys + + raw = os.environ.get("REVIEW_STATUSES", "") + merged = [] + if raw: + try: + arr = json.loads(raw) + except (json.JSONDecodeError, TypeError): + arr = [] + for item in arr: + if not item: + continue + try: + statuses = json.loads(item) + except (json.JSONDecodeError, TypeError): + continue + if isinstance(statuses, list): + merged.extend(statuses) + + # Write merged array to a temp file the poller reads. + with open("/tmp/review_statuses.json", "w") as f: + json.dump(merged, f) + print(f"Merged {len(merged)} review status entries") + PYEOF + + python3 scripts/ci/live_comment.py \ + --interval 15 \ + --timeout 2100 \ + --review-statuses-file /tmp/review_statuses.json # ───────────────────────────────────────────────────────────────────── # Gate: runs after everything. ``if: always()`` ensures it reports a @@ -158,13 +249,18 @@ jobs: # results cause it to fail; ``skipped`` is treated as success. # # Branch protection should require ONLY this check. + # + # Outputs ``needs-json`` — a compact ``{job_name: result}`` dict — so + # the live comment poller can list failed jobs in the PR comment. # ───────────────────────────────────────────────────────────────────── all-checks-pass: name: All required checks pass needs: + - detect - tests - lint - js-tests + - e2e-desktop - docs-site - history-check - contributor-check @@ -172,20 +268,30 @@ jobs: - lockfile-diff - docker-lint - supply-chain + - review-labels - osv-scanner + # comment-live is a polling job — it doesn't block the gate. # we don't require docker to pass rn because it's so slow lol # - docker if: always() runs-on: ubuntu-latest timeout-minutes: 10 + outputs: + needs-json: ${{ steps.evaluate.outputs.needs-json }} steps: - name: Evaluate job results + id: evaluate env: NEEDS: ${{ toJSON(needs) }} run: | echo "$NEEDS" | python3 -c " import json, sys needs = json.load(sys.stdin) + # Emit compact {job_name: result} for the comment assembler. + compact = {name: info['result'] for name, info in needs.items()} + print(f'needs-json={json.dumps(compact)}') + with open('$GITHUB_OUTPUT', 'a') as f: + f.write(f'needs-json={json.dumps(compact)}\n') failed = [name for name, info in needs.items() if info['result'] == 'failure'] for name, info in sorted(needs.items()): result = info['result'] @@ -202,6 +308,9 @@ jobs: # cache them on main (as a baseline), and on PRs generate an HTML diff # report with a gantt chart + per-step breakdown. The report is uploaded # as an artifact and a markdown summary is written to $GITHUB_STEP_SUMMARY. + # + # The live comment poller can read the standalone review-status artifact + # after the HTML report is uploaded, so its link points straight at that report. # ───────────────────────────────────────────────────────────────────── ci-timings: name: CI timing report @@ -226,10 +335,7 @@ jobs: - name: Collect timings and generate report env: - # Forks get no repo secrets (AUTOFIX_BOT_PAT is empty); fall back to - # the built-in read-only token so the timings API read still works - # there instead of hard-failing this advisory job on every fork PR. - GITHUB_TOKEN: ${{ secrets.AUTOFIX_BOT_PAT || github.token }} + GITHUB_TOKEN: ${{ github.token }} run: | python3 scripts/ci/timings_report.py \ --baseline ci-timings-baseline.json \ @@ -241,19 +347,40 @@ jobs: # Advisory report — artifact-service blips must not fail the job. continue-on-error: true uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - id: ci-timings-artifact + id: ci-timings-html with: name: ci-timings-report path: ci-timings-report.html retention-days: 14 - archive: false + + - name: Build linked review status + if: hashFiles('ci-timings.json') != '' + env: + CI_TIMINGS_REPORT_URL: ${{ steps.ci-timings-html.outputs.artifact-url }} + run: | + python3 scripts/ci/timings_report.py \ + --from-json ci-timings.json \ + --baseline ci-timings-baseline.json \ + --review-status-out review-status.json \ + --review-status-only + + - name: Upload review status + if: hashFiles('review-status.json') != '' + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ci-timings-review-status + path: review-status.json + retention-days: 14 - name: Output summary env: - REPORT_URL: ${{ steps.ci-timings-artifact.outputs.artifact-url}} + REPORT_URL: ${{ steps.ci-timings-html.outputs.artifact-url}} run: | - echo "# CI Timing report" >> "$GITHUB_STEP_SUMMARY" - echo "[View the full interactive report]($REPORT_URL)" >> "$GITHUB_STEP_SUMMARY" + { + echo "# CI Timing report" + echo "[View the full interactive report]($REPORT_URL)" + } >> "$GITHUB_STEP_SUMMARY" cat ci-timings-summary.md >> "$GITHUB_STEP_SUMMARY" - name: Save baseline cache (main only) diff --git a/.github/workflows/contributor-check.yml b/.github/workflows/contributor-check.yml index 2c5db6f311de..014e1e2ff93d 100644 --- a/.github/workflows/contributor-check.yml +++ b/.github/workflows/contributor-check.yml @@ -2,6 +2,10 @@ name: Contributor Attribution Check on: workflow_call: + outputs: + review_status: + description: "JSON array of review status objects" + value: ${{ jobs.check-attribution.outputs.review_status }} permissions: contents: read @@ -10,12 +14,15 @@ jobs: check-attribution: runs-on: ubuntu-latest timeout-minutes: 10 + outputs: + review_status: ${{ steps.check-emails.outputs.review_status }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 # Full history needed for git log - name: Check for unmapped contributor emails + id: check-emails run: | # Get the merge base between this PR and main MERGE_BASE=$(git merge-base origin/main HEAD) @@ -25,6 +32,7 @@ jobs: if [ -z "$NEW_EMAILS" ]; then echo "No new commits to check." + echo "review_status=[]" >> "$GITHUB_OUTPUT" exit 0 fi @@ -67,6 +75,16 @@ jobs: echo "" echo "To find the GitHub username for an email:" echo " gh api 'search/users?q=EMAIL+in:email' --jq '.items[0].login'" + + # Emit review_status for unmapped emails + DETAIL=$(echo -e "$MISSING" | sed '/^$/d; s/^ //') + HOW_TO_FIX=$'Add mappings to scripts/release.py AUTHOR_MAP:\n```\n"": "",\n```\nTo find the GitHub username for an email:\n```\ngh api \'search/users?q=EMAIL+in:email\' --jq \'.items[0].login\'\n```\n' + REVIEW_STATUS=$(jq -nc \ + --arg detail "$DETAIL" \ + --arg how_to_fix "$HOW_TO_FIX" \ + '[{"source":"contributor attribution","results":[{"kind":"action_required","title":"Unmapped contributor email(s)","summary":"New contributor email(s) are not in AUTHOR_MAP.","detail":$detail,"how_to_fix":$how_to_fix}]}]') + echo "review_status=$REVIEW_STATUS" >> "$GITHUB_OUTPUT" + exit 1 else echo "✅ All contributor emails are mapped." diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index e06a0842a633..3ac2c4741f89 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -56,6 +56,13 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Get GitHub App token + id: app-token + uses: ./.github/actions/get-app-token + with: + client-id: ${{ vars.APP_CLIENT_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: 22 @@ -73,8 +80,8 @@ jobs: - name: Prepare skills index (unified multi-source catalog) env: - GH_TOKEN: ${{ secrets.AUTOFIX_BOT_PAT }} - GITHUB_TOKEN: ${{ secrets.AUTOFIX_BOT_PAT }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} + GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} SKILLS_INDEX_RUN_ID: ${{ github.event.inputs.skills_index_run_id || '' }} REBUILD_SKILLS_INDEX: ${{ github.event.inputs.rebuild_skills_index || 'false' }} run: | diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f500aca99537..5e5c19bdf3b6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,8 +1,15 @@ name: Docker Build, Test, and Publish on: + # Trusted main pushes run this workflow directly so environment-scoped + # Docker Hub secrets are resolved by the top-level workflow, never across + # a reusable-workflow boundary. + push: + branches: [main] release: types: [published] + # CI calls this only for untrusted PR build/test coverage. Those runs never + # reach the protected publish or merge jobs below. workflow_call: permissions: @@ -20,7 +27,9 @@ env: IMAGE_NAME: nousresearch/hermes-agent jobs: - # Build, test, and optionally push the image for each architecture. + # Build and test the image for each architecture. This job runs PR code, + # so it must remain secret-free. Publishing happens in the separate, + # protected publish job after these tests pass. build: if: github.repository == 'NousResearch/hermes-agent' strategy: @@ -62,49 +71,6 @@ jobs: cache-from: ${{ matrix.cache-from }} cache-to: ${{ (github.event_name != 'pull_request') && matrix.cache-to || '' }} - - name: Log in to Docker Hub - if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'release' - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - # Push by digest only (no tag). The merge job assembles the - # tagged manifest list. `push-by-digest=true` is docker's recommended - # pattern for multi-runner multi-platform builds. - - name: Push ${{ matrix.arch }} by digest - id: push - if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'release' - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - context: . - file: Dockerfile - platforms: ${{ matrix.platform }} - labels: | - org.opencontainers.image.revision=${{ github.sha }} - build-args: | - HERMES_GIT_SHA=${{ github.sha }} - outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true - cache-from: ${{ matrix.cache-from }} - cache-to: ${{ matrix.cache-to }} - - # Write the digest to a file and upload it as an artifact so the - # merge job can stitch both per-arch digests into a manifest list. - - name: Export digest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'release' - run: | - mkdir -p /tmp/digests - digest="${{ steps.push.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest artifact - if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'release' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: digest-${{ matrix.arch }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 # Run the docker-integration test suite against the freshly-built # image already loaded into the local daemon (`:test`). @@ -147,6 +113,74 @@ jobs: run: | scripts/run_tests.sh tests/docker/ --file-timeout 600 + # --------------------------------------------------------------------------- + # Rebuild and push each architecture only after the unprivileged build/test + # matrix passes. This job is the sole Docker Hub credential boundary. + # --------------------------------------------------------------------------- + publish: + if: github.repository == 'NousResearch/hermes-agent' && (github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'release') + needs: [build] + environment: container-publish + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-latest + platform: linux/amd64 + cache-from: type=gha,scope=docker-amd64 + cache-to: type=gha,mode=max,scope=docker-amd64 + - arch: arm64 + runner: ubuntu-24.04-arm + platform: linux/arm64 + cache-from: type=gha,scope=docker-arm64 + cache-to: type=gha,mode=max,scope=docker-arm64 + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + steps: + - name: Checkout trusted source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Log in to Docker Hub + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + # Push by digest only (no tag). The merge job assembles the tagged + # manifest list after both architecture publishers complete. + - name: Push ${{ matrix.arch }} by digest + id: push + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 + with: + context: . + file: Dockerfile + platforms: ${{ matrix.platform }} + labels: | + org.opencontainers.image.revision=${{ github.sha }} + build-args: | + HERMES_GIT_SHA=${{ github.sha }} + outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + cache-from: ${{ matrix.cache-from }} + cache-to: ${{ matrix.cache-to }} + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.push.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: digest-${{ matrix.arch }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + # --------------------------------------------------------------------------- # Stitch both per-arch digests into a single tagged multi-arch manifest. # This is a registry-side operation — no building, no layer re-push — @@ -158,8 +192,9 @@ jobs: merge: if: github.repository == 'NousResearch/hermes-agent' && (github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'release') runs-on: ubuntu-latest - needs: [build] + needs: [publish] timeout-minutes: 10 + environment: container-publish steps: - name: Download digests uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 diff --git a/.github/workflows/e2e-desktop.yml b/.github/workflows/e2e-desktop.yml new file mode 100644 index 000000000000..e9131c725224 --- /dev/null +++ b/.github/workflows/e2e-desktop.yml @@ -0,0 +1,255 @@ +name: E2E Desktop + +on: + workflow_call: + outputs: + review_status: + description: Screenshot and visual-diff status for the CI review comment. + value: ${{ jobs.e2e.outputs.review_status }} + +permissions: + contents: read + +concurrency: + group: e2e-desktop-${{ github.ref }} + cancel-in-progress: true + +jobs: + e2e: + name: Playwright E2E (Linux) + runs-on: ubuntu-latest + timeout-minutes: 20 + outputs: + review_status: ${{ steps.review-status.outputs.review_status }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # ── System deps for Electron on headless Ubuntu ─────────────────── + # Electron needs GTK, NSS,atk, etc. even under xvfb. Playwright's + # install-deps covers browsers; for Electron we install the apt + # packages directly. + - name: Install system dependencies for Electron + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq \ + xvfb \ + libgtk-3-0 libnotify4 libnss3 libxss1 libxtst6 \ + xdg-utils libatspi2.0-0 libdrm2 libgbm1 libasound2t64 + + # ── Node ─────────────────────────────────────────────────────────── + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22 + cache: npm + # Full npm ci (not --ignore-scripts): electron's postinstall + # downloads the binary we launch, and node-pty's native build is + # needed for the terminal pane. + - uses: ./.github/actions/retry + with: + command: npm ci + + # ── Python (for the hermes serve backend) ────────────────────────── + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + with: + enable-cache: true + cache-dependency-glob: | + pyproject.toml + uv.lock + - name: Set up Python 3.11 + run: uv python install 3.11 + - name: Install Python dependencies + uses: ./.github/actions/retry + with: + command: uv sync --locked --python 3.11 --extra all --extra dev + + # ── Build desktop app ───────────────────────────────────────────── + # The Playwright step below runs `npm run build` before testing so + # dist/ is always fresh — no separate build step needed here. + + # ── Restore visual baseline screenshots from main ────────────────── + # Baselines are generated on main (via --update-snapshots) and cached. + # On PRs, we restore them so toHaveScreenshot has something to compare + # against. The cache key is keyed on the desktop source files so a + # UI change naturally invalidates it — but we fall back to the main + # cache to avoid cold starts on unrelated PRs. + - name: Restore visual baseline screenshots + id: restore-baselines + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + with: + path: apps/desktop/e2e/*-snapshots + key: visual-baselines-${{ github.ref_name }} + restore-keys: | + visual-baselines-main + + # ── Run Playwright E2E under xvfb ───────────────────────────────── + # xvfb runs at a fixed 1280x1024 screen so the 1220x800 Electron + # window always has a consistent viewport for screenshot comparison. + # On main, we run with --update-snapshots to generate baselines. + # `npm run test:e2e` builds dist/ as a pretest hook so the renderer + # is always fresh — no separate build step needed. + - name: Run Playwright E2E tests + working-directory: apps/desktop + run: | + if [ "${{ github.ref_name }}" = "main" ]; then + echo "On main — generating/updating baseline screenshots" + npm run build && xvfb-run -a --server-args="-screen 0 1280x1024x24" \ + npx playwright test --reporter=list --update-snapshots + else + echo "On PR — comparing against cached baselines" + npm run build && xvfb-run -a --server-args="-screen 0 1280x1024x24" \ + npx playwright test --reporter=list + fi + env: + CI: "true" + # Ensure no real API keys leak into the test env. + OPENROUTER_API_KEY: "" + OPENAI_API_KEY: "" + NOUS_API_KEY: "" + + # ── Save updated baselines to cache (main only) ─────────────────── + - name: Save updated baselines to cache + if: github.ref_name == 'main' && always() + uses: actions/cache/save@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + with: + path: apps/desktop/e2e/*-snapshots + key: visual-baselines-main + + # ── Upload Playwright report (HTML + traces) ────────────────────── + - name: Upload Playwright report + id: upload-report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: playwright-report-${{ github.sha }} + path: apps/desktop/playwright-report + retention-days: 14 + overwrite: true + + # ── Upload test results (screenshots, traces, diffs) ─────────────── + - name: Upload test results + id: upload-results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: playwright-test-results-${{ github.sha }} + path: apps/desktop/test-results + retention-days: 14 + overwrite: true + + # ── Upload just the visual diffs (small, fast to review) ────────── + - name: Upload visual diffs + id: upload-diffs + if: always() && github.ref_name != 'main' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: visual-diffs-${{ github.sha }} + path: | + apps/desktop/test-results/**/*-diff.png + apps/desktop/test-results/**/*-actual.png + apps/desktop/test-results/**/*-expected.png + retention-days: 14 + overwrite: true + if-no-files-found: ignore + + - name: Build screenshot review status + id: review-status + if: always() + working-directory: apps/desktop + env: + RESULTS_URL: ${{ steps.upload-results.outputs.artifact-url }} + run: | + python3 ../../scripts/ci/e2e_screenshot_status.py \ + --results-dir test-results \ + --manifest-output /tmp/e2e-screenshot-manifest.json \ + --evidence-dir /tmp/e2e-evidence \ + --artifact-url "$RESULTS_URL" \ + --output /tmp/e2e-review-status.json + { + echo 'review_status<<__E2E_REVIEW_STATUS__' + cat /tmp/e2e-review-status.json + echo '__E2E_REVIEW_STATUS__' + } >> "$GITHUB_OUTPUT" + + # The trusted workflow_run publisher consumes only this flat, bounded + # artifact. It turns selected images into GitHub attachment URLs; it + # never checks out or runs this PR's code. + - name: Upload inline E2E evidence + if: always() && github.ref_name != 'main' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-evidence-${{ github.sha }} + path: /tmp/e2e-evidence + retention-days: 14 + overwrite: true + if-no-files-found: error + + # ── Generate step summary with visual diff info ─────────────────── + # Parse the JSON report + scan for diff images, then post a summary + # to the GitHub Actions step output so reviewers can see what changed + # without downloading artifacts. Runs AFTER uploads so it can link + # the artifact download URLs from their step outputs. + - name: Generate visual diff summary + if: always() + working-directory: apps/desktop + env: + REPORT_URL: ${{ steps.upload-report.outputs.artifact-url }} + RESULTS_URL: ${{ steps.upload-results.outputs.artifact-url }} + DIFFS_URL: ${{ steps.upload-diffs.outputs.artifact-url }} + run: | + { + echo "## Desktop E2E — Visual Diff Report" + echo "" + + # Count diff images (playwright writes *-diff.png on mismatch) + DIFF_COUNT=$(find test-results -name '*-diff.png' 2>/dev/null | wc -l) + ACTUAL_COUNT=$(find test-results -name '*-actual.png' 2>/dev/null | wc -l) + + if [ "$DIFF_COUNT" -eq 0 ]; then + echo "✅ All $ACTUAL_COUNT screenshot(s) matched their baselines (or no baselines existed yet)." + else + echo "📸 **$DIFF_COUNT of $ACTUAL_COUNT screenshot(s) differ from baseline:**" + echo "" + echo "| Test | Diff | Actual | Expected |" + echo "|------|------|--------|----------|" + + # List each diff image with a link to the artifact + for diff in $(find test-results -name '*-diff.png' 2>/dev/null | sort); do + base=${diff%-diff.png} + test_name=$(basename "$base") + echo "| $test_name | [diff]($diff) | [actual](${base}-actual.png) | [expected](${base}-expected.png) |" + done + fi + + echo "" + echo "📥 **Artifacts:**" + echo "" + if [ -n "$RESULTS_URL" ]; then + echo "- [playwright-test-results]($RESULTS_URL) — all screenshots (actual + expected + diff) + traces" + fi + if [ -n "$REPORT_URL" ]; then + echo "- [playwright-report]($REPORT_URL) — interactive HTML report" + fi + if [ -n "$DIFFS_URL" ]; then + echo "- [visual-diffs]($DIFFS_URL) — just the diffed screenshots (small, fast to review)" + fi + echo "" + echo "**To update baselines:** merge to main (baselines auto-update on main runs) or run \`npx playwright test --update-snapshots\` locally." + + # Also parse the JSON report for pass/fail counts + if [ -f playwright-report/results.json ]; then + echo "" + echo "### Test Results" + echo "" + node -e " + const r = require('./playwright-report/results.json'); + const stats = r.stats || {}; + console.log('| Status | Count |'); + console.log('|--------|-------|'); + console.log('| ✅ Passed | ' + (stats.expected || 0) + ' |'); + console.log('| ❌ Failed | ' + (stats.unexpected || 0) + ' |'); + console.log('| ⏭️ Skipped | ' + (stats.skipped || 0) + ' |'); + console.log('| 🔄 Flaky | ' + (stats.flaky || 0) + ' |'); + " 2>/dev/null || true + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/history-check.yml b/.github/workflows/history-check.yml index a48dba8cb8af..668f0f795dd9 100644 --- a/.github/workflows/history-check.yml +++ b/.github/workflows/history-check.yml @@ -15,6 +15,10 @@ name: History Check on: workflow_call: + outputs: + review_status: + description: "JSON array of review_status objects for the synthesizer." + value: ${{ jobs.check-common-ancestor.outputs.review_status }} permissions: contents: read @@ -23,18 +27,23 @@ jobs: check-common-ancestor: runs-on: ubuntu-latest timeout-minutes: 10 + outputs: + review_status: ${{ steps.merge-base-check.outputs.review_status }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 # full history both sides for merge-base - - name: Reject PRs with no common ancestor on main + - id: merge-base-check + name: Reject PRs with no common ancestor on main run: | # `git merge-base` exits non-zero AND prints nothing when the two # commits share no ancestor. We check both conditions explicitly # so the failure message is clear regardless of which signal fires # first. if ! BASE=$(git merge-base origin/main HEAD 2>/dev/null) || [ -z "$BASE" ]; then + STATUS='[{"source":"unrelated histories","results":[{"kind":"action_required","title":"Unrelated histories","summary":"This PR has no common ancestor with main.","detail":"","how_to_fix":"Rebase your changes onto current main:\n```\ngit fetch origin main\ngit checkout -b fix-branch origin/main\n# re-apply your changes (cherry-pick, copy files, etc.)\ngit push -f origin fix-branch\n```\n"}]}]' + echo "review_status=${STATUS}" >> "$GITHUB_OUTPUT" echo "" echo "::error::This PR has no common ancestor with main." echo "" @@ -56,3 +65,4 @@ jobs: exit 1 fi echo "::notice::Common ancestor with main: $BASE" + echo "review_status=[]" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/infographic-check.yml b/.github/workflows/infographic-check.yml new file mode 100644 index 000000000000..288f6f493a05 --- /dev/null +++ b/.github/workflows/infographic-check.yml @@ -0,0 +1,78 @@ +name: Infographic Check + +# Rejects PRs that commit PR-infographic images into the repo. +# +# PR infographics are rendered to an image-provider URL (fal.media) and +# embedded in the PR *description*. The PR body is the archive; the binary +# never belongs in git history. +# +# This has now leaked twice. PR #48261 removed the first batch, PR #54564 +# removed a second batch and added `infographic/` to `.gitignore` — but +# `.gitignore` only stops *accidental* `git add`. It does nothing against +# `git add -f`, and it does nothing for a path that does not literally match +# the ignore pattern. Nine more PNGs (~14MB) were committed in the four +# weeks AFTER that rule landed, plus PR #70552 caught an `infograficos/` +# spelling that sidestepped the pattern entirely. +# +# A passive ignore rule cannot enforce a policy. This check can. + +on: + workflow_call: + outputs: + review_status: + description: "JSON array of review_status objects for the synthesizer." + value: ${{ jobs.check-no-committed-infographics.outputs.review_status }} + +permissions: + contents: read + +jobs: + check-no-committed-infographics: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + review_status: ${{ steps.infographic-check.outputs.review_status }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - id: infographic-check + name: Reject committed PR-infographic images + run: | + # Match on the IMAGE, not on a directory name. Keying this to + # `infographic/` is what let `infograficos/` through in #70552 — + # any localized or typo'd directory would sidestep it again. + # Instead: find tracked raster images whose path contains an + # infographic-ish segment, in any spelling, at any depth. + # + # `docs/assets` and `website/` legitimately hold product imagery + # and are excluded; those are referenced from shipped docs pages. + OFFENDERS=$(git ls-files -z \ + | tr '\0' '\n' \ + | grep -iE '(^|/)(infograph|infograf)[^/]*/' \ + | grep -iE '\.(png|jpe?g|webp|gif)$' \ + || true) + + if [ -n "$OFFENDERS" ]; then + COUNT=$(printf '%s\n' "$OFFENDERS" | wc -l | tr -d ' ') + STATUS='[{"source":"committed infographics","results":[{"kind":"action_required","title":"PR infographic committed to the repo","summary":"Infographic images belong in the PR description, never in git.","detail":"","how_to_fix":"Untrack the image and reference the provider URL from the PR body instead:\n```\ngit rm --cached \n```\nThen put it in the PR description:\n```\n## Infographic\n\n![slug](https://)\n```\n"}]}]' + echo "review_status=${STATUS}" >> "$GITHUB_OUTPUT" + echo "" + echo "::error::${COUNT} PR-infographic image(s) are tracked in git." + echo "" + printf '%s\n' "$OFFENDERS" | sed 's/^/ /' + echo "" + echo "PR infographics are rendered to an image-provider URL and" + echo "embedded in the PR DESCRIPTION. The PR body is the archive —" + echo "the binary never enters git history." + echo "" + echo "This rule has been re-established twice already (#48261," + echo "#54564) and leaked both times, because .gitignore cannot stop" + echo "'git add -f' or a differently-spelled directory (#70552)." + echo "" + echo "To fix:" + echo " git rm --cached # keeps your local copy" + echo " # then embed the provider URL in the PR description" + exit 1 + fi + echo "::notice::No committed PR-infographic images." + echo "review_status=[]" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/js-autofix.yml b/.github/workflows/js-autofix.yml index 38494abd0e38..2dfbe7e0b17a 100644 --- a/.github/workflows/js-autofix.yml +++ b/.github/workflows/js-autofix.yml @@ -7,7 +7,7 @@ name: auto-fix lint issues & formatting # auto-corrected on merge so PRs aren't blocked by them. The PR-time eslint # check in typecheck.yml fails only when un-fixable errors remain. # -# NOTE: AUTOFIX_BOT_PAT pushes DO trigger further workflow runs (unlike +# NOTE: App token pushes DO trigger further workflow runs (unlike # secrets.GITHUB_TOKEN). The concurrency group (ts-autofix-${{ github.ref }}) # with cancel-in-progress: true prevents an infinite loop — a re-triggered # run cancels the in-flight one, and since the second run finds no new fixes @@ -122,12 +122,20 @@ jobs: if: needs.generate-patch.outputs.has-fixes == 'true' runs-on: ubuntu-latest timeout-minutes: 15 + environment: trusted-automation permissions: contents: write # needed to push to bot/js-autofix pull-requests: write # needed for PR creation + auto-merge steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Get GitHub App token + id: app-token + uses: ./.github/actions/get-app-token + with: + client-id: ${{ vars.APP_CLIENT_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + - name: Download patch uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: @@ -170,7 +178,7 @@ jobs: - name: Create/update PR and enable auto-merge env: - GH_TOKEN: ${{ secrets.AUTOFIX_BOT_PAT }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} BOT_BRANCH: bot/js-autofix run: | set -euo pipefail @@ -193,7 +201,7 @@ jobs: - name: Wait for merge, auto-close on failure or stale env: - GH_TOKEN: ${{ secrets.AUTOFIX_BOT_PAT }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} START_SHA: ${{ github.sha }} run: | set -euo pipefail diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 4e4622f72d04..25786b1c95b6 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 outputs: - packages: ${{ steps.set-matrix.outputs.packages }} + checks: ${{ steps.set-matrix.outputs.checks }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -22,21 +22,40 @@ jobs: command: npm ci --ignore-scripts - id: set-matrix run: | - PACKAGES=$(npm query .workspace | jq -c '[.[].location]') - if [ "$PACKAGES" = "[]" ] || [ -z "$PACKAGES" ]; then - echo "::error::Workspace discovery produced an empty package list — refusing to emit a zero-length matrix (would skip all JS/TS checks silently)." - exit 1 - fi - echo "packages=$PACKAGES" >> "$GITHUB_OUTPUT" + node -e ' + const { execSync } = require("child_process"); + const pkgs = JSON.parse(execSync("npm query .workspace", { encoding: "utf-8" })); + if (pkgs.length === 0) { + console.error("::error::Workspace discovery produced an empty package list — refusing to emit a zero-length matrix (would skip all JS/TS checks silently)."); + process.exit(1); + } + const checks = []; + for (const pkg of pkgs) { + const scripts = pkg.scripts || {}; + const subs = Object.keys(scripts).filter(s => /^check:.+$/.test(s)); + if (subs.length > 0) { + for (const script of subs) { + checks.push({ package: pkg.location, script }); + } + } else if (scripts.check) { + checks.push({ package: pkg.location, script: "check" }); + } + } + if (checks.length === 0) { + console.error("::error::No check scripts found in any workspace package."); + process.exit(1); + } + process.stdout.write("checks=" + JSON.stringify(checks) + "\n"); + ' >> "$GITHUB_OUTPUT" check: - name: Typecheck & Test + name: ${{ matrix.package }} / ${{ matrix.script }} needs: workspaces runs-on: ubuntu-latest timeout-minutes: 20 strategy: matrix: - package: ${{ fromJson(needs.workspaces.outputs.packages) }} + include: ${{ fromJson(needs.workspaces.outputs.checks) }} fail-fast: false # report all failures, not just the first one steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -47,5 +66,4 @@ jobs: - uses: ./.github/actions/retry with: command: npm ci - - run: npm run --prefix ${{ matrix.package }} check - - run: npm run --prefix ${{ matrix.package }} fix + - run: npm run --prefix ${{ matrix.package }} ${{ matrix.script }} diff --git a/.github/workflows/label-rerun.yml b/.github/workflows/label-rerun.yml new file mode 100644 index 000000000000..fbd8b3b89327 --- /dev/null +++ b/.github/workflows/label-rerun.yml @@ -0,0 +1,81 @@ +name: Label rerun + +# When the ``ci-reviewed`` label is added to a PR, rerun all failed jobs in +# the latest CI run. This re-evaluates ``review-labels`` (which now sees the +# label) and GitHub automatically reruns dependent jobs (``comment-live``, +# ``all-checks-pass``) — so the review comment gets updated too. +# +# If the CI run is still in progress when the label is added, we wait for it +# to finish before rerunning (``gh run rerun`` only works on completed runs). +# The wait can be long (20+ min for a full CI run), but it's better than +# silently failing and leaving the reviewer stuck. + +on: + pull_request: + types: [labeled] + +permissions: + actions: write + pull-requests: read + +concurrency: + group: label-rerun-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + rerun-review-labels: + name: Rerun review-labels job + if: github.event.label.name == 'ci-reviewed' + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - name: Wait for CI run to finish, then rerun failed jobs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -uo pipefail + + # Find the latest CI run for this PR's head SHA. + RUN_ID=$(gh run list \ + --repo "$REPO" \ + --commit "$HEAD_SHA" \ + --workflow ci.yml \ + --limit 1 \ + --json databaseId,status \ + --jq '.[0] | "\(.databaseId) \(.status)"' 2>/dev/null || true) + + if [ -z "$RUN_ID" ]; then + echo "No CI run found for this PR — nothing to rerun." + exit 0 + fi + + # Split "RUN_ID STATUS" into two vars. + RUN_ID="${RUN_ID%% *}" + STATUS="${RUN_ID##* }" + + echo "Latest CI run: $RUN_ID (status: $STATUS)" + + # If the run is still in progress, wait for it to finish. + # gh run rerun only works on completed runs — if we try while it's + # running, GitHub rejects with "cannot be rerun; This workflow is + # already running". + if [ "$STATUS" != "completed" ]; then + echo "Run is $STATUS — waiting for completion (this may take a while)..." + # gh run watch --exit-status exits non-zero if the run fails, + # which is expected (the label gate fails). Don't let that kill + # the workflow — we WANT to rerun failed jobs. + timeout 2100 gh run watch "$RUN_ID" --repo "$REPO" --interval 15 || true + + # Verify it's actually completed now. + STATUS=$(gh run view "$RUN_ID" --repo "$REPO" --json status --jq '.status' 2>/dev/null || echo "unknown") + if [ "$STATUS" != "completed" ]; then + echo "Run is still $STATUS after wait — giving up." + exit 0 + fi + fi + + echo "Run completed. Rerunning all failed jobs..." + gh run rerun "$RUN_ID" --repo "$REPO" --failed || true + echo "Done. GitHub will rerun review-labels and all dependent jobs." diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 28df38ad3e5d..670b6f2a44a2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,11 +2,14 @@ name: Lint (ruff + ty) # Two things here: # 1. Advisory diff — ruff + ty diagnostics as a diff vs the target branch. -# Posts a Markdown summary and a PR comment. Exit zero always. +# Writes a Markdown summary to the run page. Exit zero always. # 2. Blocking ``ruff check .`` — enforces the explicit rules in # ``[tool.ruff.lint.select]`` (currently PLW1514). Failure blocks merge. -# Separate job so the advisory diff still runs and posts even when -# enforcement fails. +# Separate job so the advisory diff still runs even when enforcement +# fails. +# +# CI-sensitive file review was previously here as a ``ci-review`` job but +# has moved to ``review-labels.yml`` so it can be rerun independently. on: workflow_call: @@ -15,14 +18,9 @@ on: description: The event name from the calling orchestrator (pull_request or push). type: string required: true - ci_review: - description: Whether CI-sensitive files (eslint config, workflows, actions) changed and require a review label. - type: boolean - default: false permissions: contents: read - pull-requests: write # needed to post/update PR comments concurrency: group: lint-${{ github.ref }} @@ -162,115 +160,3 @@ jobs: - name: Run footgun checker run: python scripts/check-windows-footguns.py --all - - ci-review: - # Require explicit maintainer review when CI-sensitive files change: - # eslint config, workflow YAMLs, or composite actions. These files - # influence what code the js-autofix job executes and pushes to - # main, so a malicious PR could inject arbitrary code via a custom eslint - # rule's `fix` function. The label gate ensures a human reviews before - # merge. Mirrors the mcp-catalog-reviewed pattern in supply-chain-audit.yml. - name: CI-sensitive file review - if: inputs.event_name == 'pull_request' && inputs.ci_review - runs-on: ubuntu-latest - timeout-minutes: 2 - steps: - - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Require ci-reviewed label - id: label-check - env: - # Read-only label lookup. Use the built-in GITHUB_TOKEN (present and - # read-only on forks) so the gate works on fork PRs; fall back to it - # when AUTOFIX_BOT_PAT is empty. `|| true` degrades an API blip to - # "label absent" rather than hard-failing the step. - GH_TOKEN: ${{ secrets.AUTOFIX_BOT_PAT || github.token }} - run: | - set -euo pipefail - PR="${{ github.event.pull_request.number }}" - LABELS=$(gh pr view "$PR" --json labels --jq '.labels[].name' || true) - if echo "$LABELS" | grep -Fxq 'ci-reviewed'; then - echo "reviewed=true" >> "$GITHUB_OUTPUT" - echo "ci-reviewed label present." - exit 0 - fi - echo "reviewed=false" >> "$GITHUB_OUTPUT" - - # On failure: find the bot's previous comment and edit it, or create - # a new one if none exists. Using an HTML comment marker so we can - # locate it reliably across runs without parsing the body text. - # Skipped on fork PRs — GITHUB_TOKEN is read-only there, so the API - # call would fail. The label gate still holds via the step below. - - name: Post or update review warning - if: steps.label-check.outputs.reviewed != 'true' && github.event.pull_request.head.repo.fork != true - env: - GH_TOKEN: ${{ secrets.AUTOFIX_BOT_PAT || github.token }} - run: | - set -euo pipefail - PR="${{ github.event.pull_request.number }}" - MARKER="" - BODY="${MARKER} - ## ⚠️ CI-sensitive file review required - - This PR changes CI-sensitive files (eslint config, workflow YAMLs, - or composite actions). These files influence what code the - js-autofix job executes and pushes to main. - - A maintainer should verify: - - no new eslint rules with custom \`fix\` functions that write outside linted paths, - - no workflow changes that widen permissions or remove guards, - - no composite action changes that alter what gets executed. - - After review, add the \`ci-reviewed\` label and re-run this check." - - # Find an existing comment with our marker. - COMMENT_ID=$(gh api \ - "repos/${{ github.repository }}/issues/${PR}/comments" \ - --paginate --jq ".[] | select(.body | contains(\"${MARKER}\")) | .id" \ - | head -1 || true) - - if [ -n "$COMMENT_ID" ]; then - gh api --method PATCH \ - "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \ - -f body="$BODY" - else - gh pr comment "$PR" --body "$BODY" - fi - - # Fail the job when the label is missing — always runs (including - # fork PRs) so the security gate holds even when the comment step - # was skipped above. - - name: Fail on missing label - if: steps.label-check.outputs.reviewed != 'true' - run: | - echo "::error::CI-sensitive changes require the ci-reviewed label." - exit 1 - - # On success: if a previous warning comment exists, edit it to show - # the review passed so the PR doesn't have a stale ⚠️ sitting around. - # Skipped on fork PRs — no comment was ever posted to update. - - name: Update previous warning to passed - if: steps.label-check.outputs.reviewed == 'true' && github.event.pull_request.head.repo.fork != true - env: - GH_TOKEN: ${{ secrets.AUTOFIX_BOT_PAT || github.token }} - run: | - set -euo pipefail - PR="${{ github.event.pull_request.number }}" - MARKER="" - - # Find an existing comment with our marker. - COMMENT_ID=$(gh api \ - "repos/${{ github.repository }}/issues/${PR}/comments" \ - --paginate --jq ".[] | select(.body | contains(\"${MARKER}\")) | .id" \ - | head -1 || true) - - if [ -n "$COMMENT_ID" ]; then - BODY="${MARKER} - ## ✅ CI-sensitive file review passed - - The \`ci-reviewed\` label is present on this PR." - - gh api --method PATCH \ - "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \ - -f body="$BODY" - fi diff --git a/.github/workflows/lockfile-diff.yml b/.github/workflows/lockfile-diff.yml index 2dcf66ea7c55..9d8d59f6da7e 100644 --- a/.github/workflows/lockfile-diff.yml +++ b/.github/workflows/lockfile-diff.yml @@ -7,22 +7,25 @@ name: Lockfile diff # the ``packages`` map at the merge base and at HEAD and set-diffs the # {install path: version} maps instead. # -# The comment is upserted: the script embeds a hidden HTML marker and the -# workflow PATCHes the existing comment when one is found, so a PR gets -# exactly one lockfile-diff comment that tracks the latest push instead -# of a stack of stale ones. When a later push reverts all lockfile -# changes, the comment is updated to say so (deleting it would be more -# surprising than telling the reviewer it's resolved). +# The semantic diff is exposed as a workflow_call output ``review_status`` +# (a JSON array in the unified status format) and an artifact +# (``lockfile-diff`` containing the markdown fragment) for the step +# summary. # -# Never blocking — this is review signal, not enforcement. Exit is 0 even -# when commenting fails (fork PRs get a read-only GITHUB_TOKEN). +# Never blocking — this is review signal, not enforcement. on: workflow_call: + outputs: + changed: + description: Whether package-lock.json changed relative to the target branch. + value: ${{ jobs.diff.outputs.changed }} + review_status: + description: JSON array of review status objects for the unified PR comment. + value: ${{ jobs.diff.outputs.review_status }} permissions: contents: read - pull-requests: write # post/update the diff comment concurrency: group: lockfile-diff-${{ github.event.pull_request.number || github.ref }} @@ -33,6 +36,9 @@ jobs: name: package-lock.json semantic diff runs-on: ubuntu-latest timeout-minutes: 5 + outputs: + changed: ${{ steps.diff.outputs.changed }} + review_status: ${{ steps.emit-status.outputs.review_status }} steps: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -54,45 +60,36 @@ jobs: --output /tmp/lockfile-diff.md if [ -s /tmp/lockfile-diff.md ]; then echo "changed=true" >> "$GITHUB_OUTPUT" - cat /tmp/lockfile-diff.md >> "$GITHUB_STEP_SUMMARY" + { + echo "## package-lock.json semantic diff" + echo "" + cat /tmp/lockfile-diff.md + } >> "$GITHUB_STEP_SUMMARY" else echo "changed=false" >> "$GITHUB_OUTPUT" + : > /tmp/lockfile-diff.md fi - - name: Post or update PR comment - env: - GH_TOKEN: ${{ secrets.AUTOFIX_BOT_PAT }} - REPO: ${{ github.repository }} - PR: ${{ github.event.pull_request.number }} - CHANGED: ${{ steps.diff.outputs.changed }} + - name: Emit review_status + id: emit-status run: | set -euo pipefail - MARKER='' + CHANGED="${{ steps.diff.outputs.changed }}" + STATUS="[]" - # Find our previous comment (paginated — busy PRs exceed one page). - EXISTING=$(gh api --paginate "repos/${REPO}/issues/${PR}/comments" \ - --jq ".[] | select(.body | startswith(\"$MARKER\")) | .id" \ - | head -1 || true) - - if [ "$CHANGED" != "true" ]; then - if [ -n "$EXISTING" ]; then - # A previous push changed the lockfile but the latest one - # doesn't — update the comment rather than leave stale info. - printf '%s\n✅ package-lock.json changes from an earlier push have been reverted — locked versions now match the target branch.\n' "$MARKER" > /tmp/lockfile-diff.md - else - echo "No lockfile changes and no existing comment — nothing to do." - exit 0 - fi - fi - - if [ -n "$EXISTING" ]; then - echo "Updating existing comment ${EXISTING}" - gh api --method PATCH "repos/${REPO}/issues/comments/${EXISTING}" \ - -F body=@/tmp/lockfile-diff.md > /dev/null \ - || echo "::warning::Could not update PR comment (expected for fork PRs — GITHUB_TOKEN is read-only)" + if [ "$CHANGED" = "true" ]; then + CONTENT=$(cat /tmp/lockfile-diff.md | python3 -c "import sys,json; print(json.dumps(sys.stdin.read()))") + STATUS="[{\"source\":\"lockfile-diff\",\"results\":[{\"kind\":\"action_required\",\"title\":\"package-lock.json\",\"summary\":\"Locked npm dependency versions changed.\",\"detail\":${CONTENT},\"how_to_fix\":\"Add the \`ci-reviewed\` label after verifying the version changes are expected.\"}]}" else - echo "Creating new comment" - gh api "repos/${REPO}/issues/${PR}/comments" \ - -F body=@/tmp/lockfile-diff.md > /dev/null \ - || echo "::warning::Could not post PR comment (expected for fork PRs — GITHUB_TOKEN is read-only)" + STATUS="[]" fi + + echo "review_status=${STATUS}" >> "$GITHUB_OUTPUT" + + - name: Upload diff artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: lockfile-diff + path: /tmp/lockfile-diff.md + retention-days: 1 + overwrite: true diff --git a/.github/workflows/osv-scanner.yml b/.github/workflows/osv-scanner.yml index e5a983b1bca2..455ede33dd56 100644 --- a/.github/workflows/osv-scanner.yml +++ b/.github/workflows/osv-scanner.yml @@ -14,14 +14,14 @@ name: OSV-Scanner # code patterns in PR diffs) by covering the orthogonal "currently-pinned # dep became known-vulnerable" case. # -# Steps below are inlined from Google's officially-recommended reusable -# workflow (google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml), -# rather than called via `uses:` so we can set a `timeout-minutes` in the -# degenerate case where this job hangs. - +# Uses Google's officially-recommended reusable workflow, pinned by SHA. # Findings land in the repo's Security tab (Code Scanning > OSV-Scanner). # fail-on-vuln is disabled so the job does not block merges on pre-existing # vulnerabilities in pinned deps that we may need to patch deliberately. +# +# The reusable workflow can't emit custom outputs, so a wrapper job +# downloads the SARIF result and summarizes the vulnerability count into +# a review_status for the unified PR comment. on: workflow_call: @@ -40,62 +40,88 @@ permissions: jobs: scan: name: Scan lockfiles + uses: google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@9a498708959aeaef5ef730655706c5a1df1edbc2 # v2.3.8 + with: + # Scan explicit lockfiles rather than recursing, so we only look at + # the three sources of truth and skip vendored / test / worktree dirs. + scan-args: |- + --lockfile=uv.lock + --lockfile=package-lock.json + --lockfile=website/package-lock.json + # The upstream reusable workflow uploads this exact file under its + # fixed artifact name, which the wrapper downloads below. + results-file-name: osv-results.sarif + fail-on-vuln: false + + emit-status: + name: Emit review status runs-on: ubuntu-latest + needs: scan + if: always() + outputs: + review_status: ${{ steps.emit.outputs.review_status }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: 'Run scanner' - uses: google/osv-scanner-action/osv-scanner-action@9a498708959aeaef5ef730655706c5a1df1edbc2 # v2.3.8 + - name: Download SARIF result + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - # Scan explicit lockfiles rather than recursing, so we only look at - # the three sources of truth and skip vendored / test / worktree dirs. - scan-args: |- - --output=results.json - --format=json - --lockfile=uv.lock - --lockfile=package-lock.json - --lockfile=website/package-lock.json + name: OSV Scanner SARIF file + path: /tmp/osv-results continue-on-error: true - - name: 'Run osv-scanner-reporter' - uses: google/osv-scanner-action/osv-reporter-action@9a498708959aeaef5ef730655706c5a1df1edbc2 # v2.3.8 - with: - scan-args: |- - --output=results.sarif - --new=results.json - --gh-annotations=false - --fail-on-vuln=false - - # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF - # format to the repository Actions tab. - - name: 'Upload artifact' - id: 'upload_artifact' - if: ${{ !cancelled() }} - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: OSV Scanner SARIF file - path: results.sarif - retention-days: 5 + - name: Emit review_status + id: emit + run: | + set -euo pipefail + STATUS="[]" - # Upload the results to GitHub's code scanning dashboard. - - name: 'Upload to code-scanning' - if: ${{ !cancelled() }} - uses: github/codeql-action/upload-sarif@cdefb33c0f6224e58673d9004f47f7cb3e328b89 # v4.31.10 - with: - sarif_file: results.sarif + if [ -f /tmp/osv-results/osv-results.sarif ]; then + # Count vulnerabilities from the SARIF file + VULN_COUNT=$(python3 -c " + import json, sys + try: + with open('/tmp/osv-results/osv-results.sarif') as f: + data = json.load(f) + count = 0 + vulns = [] + for run in data.get('runs', []): + for result in run.get('results', []): + count += 1 + rule_id = result.get('ruleId', 'unknown') + message = result.get('message', {}).get('text', '') + loc = result.get('locations', [{}])[0].get('physicalLocation', {}).get('artifactLocation', {}).get('uri', '') + vulns.append(f'- {rule_id} in {loc}: {message}') + print(count) + if vulns: + print('\n'.join(vulns[:20]), file=sys.stderr) + except Exception: + print(0) + ") - - name: 'Print Code Scanning URL' - if: ${{ !cancelled() }} - run: | - echo "View the OSV-Scanner results in the 'Security' tab, using the following link:" - echo "${{ github.server_url }}/${{ github.repository }}/security/code-scanning?query=is%3Aopen+branch%3A${GITHUB_REF_NAME}+tool%3Aosv-scanner" - env: - GITHUB_REF_NAME: ${{ github.ref_name }} + VULN_DETAIL="" + if [ "$VULN_COUNT" -gt 0 ] 2>/dev/null; then + VULN_PLURAL=$([ "$VULN_COUNT" -eq 1 ] && echo "y" || echo "ies") + VULN_DETAIL=$(python3 -c " + import json, sys + try: + with open('/tmp/osv-results/osv-results.sarif') as f: + data = json.load(f) + vulns = [] + for run in data.get('runs', []): + for result in run.get('results', []): + rule_id = result.get('ruleId', 'unknown') + loc = result.get('locations', [{}])[0].get('physicalLocation', {}).get('artifactLocation', {}).get('uri', '') + vulns.append(f'- {rule_id} in {loc}') + print(json.dumps('\n'.join(vulns[:20]))) + except Exception: + print(json.dumps('')) + ") + STATUS="[{\"source\":\"osv scan\",\"results\":[{\"kind\":\"warning\",\"title\":\"OSV vulnerability scan\",\"summary\":\"${VULN_COUNT} known vulnerabilit${VULN_PLURAL} found in pinned dependencies.\",\"detail\":${VULN_DETAIL},\"how_to_fix\":\"Review the findings in the [Security tab](../../security/code-scanning). Update the affected dependencies if a patched version is available.\"}]}]" + else + STATUS="[]" + fi + fi - - name: 'Error troubleshooter' - if: ${{ always() && steps.upload_artifact.outcome == 'failure' }} - run: | - echo "::error::Artifact upload failed. This is most likely caused by a error during scanning earlier in the workflow." - exit 1 + echo "review_status=${STATUS}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/publish-e2e-evidence.yml b/.github/workflows/publish-e2e-evidence.yml new file mode 100644 index 000000000000..3d7fe32d7806 --- /dev/null +++ b/.github/workflows/publish-e2e-evidence.yml @@ -0,0 +1,71 @@ +name: Publish E2E evidence + +# This runs only from the default branch after CI completes. It intentionally +# checks out main, never the PR ref, and treats the downloaded artifact as +# untrusted input before uploading validated GitHub attachments. +on: + workflow_run: + workflows: [CI] + types: [completed] + +permissions: + actions: read + contents: read + pull-requests: write + +concurrency: + group: publish-e2e-evidence-${{ github.event.workflow_run.id }} + cancel-in-progress: false + +jobs: + publish: + name: Publish inline E2E evidence + if: github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: gh-image + steps: + - name: Check out trusted publisher + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + # v1.2.0 resolves to 44f4b93ecbbe22de6c45fa2f62f519aee564ca8c. + - name: Install gh-image + env: + GH_TOKEN: ${{ github.token }} + run: gh extension install drogers0/gh-image --pin v1.2.0 + + - name: Download and attach evidence + env: + GH_TOKEN: ${{ github.token }} + GITHUB_TOKEN: ${{ github.token }} + GH_SESSION_TOKEN: ${{ secrets.GH_IMAGE_SESSION_TOKEN }} + SOURCE_REPO: ${{ github.repository }} + SOURCE_RUN_ID: ${{ github.event.workflow_run.id }} + run: | + set -euo pipefail + + PR_NUMBER=$(gh api "repos/$SOURCE_REPO/actions/runs/$SOURCE_RUN_ID" --jq '.pull_requests[0].number // empty') + if [ -z "$PR_NUMBER" ]; then + echo "No pull request is associated with CI run $SOURCE_RUN_ID." + exit 0 + fi + + ARTIFACT_NAME=$(gh api "repos/$SOURCE_REPO/actions/runs/$SOURCE_RUN_ID/artifacts" \ + --jq '.artifacts[] | select(.expired == false and (.name | startswith("e2e-evidence-"))) | .name' \ + | python3 -c 'import sys; print(next(iter(sys.stdin), "").strip())') + if [ -z "$ARTIFACT_NAME" ]; then + echo "No E2E evidence artifact was produced for CI run $SOURCE_RUN_ID." + exit 0 + fi + + EVIDENCE_DIR="$RUNNER_TEMP/e2e-evidence" + mkdir -p "$EVIDENCE_DIR" + gh run download "$SOURCE_RUN_ID" --repo "$SOURCE_REPO" --name "$ARTIFACT_NAME" --dir "$EVIDENCE_DIR" + + python3 scripts/ci/publish_e2e_evidence.py \ + --evidence-dir "$EVIDENCE_DIR" \ + --source-repo "$SOURCE_REPO" \ + --pr-number "$PR_NUMBER" diff --git a/.github/workflows/review-labels.yml b/.github/workflows/review-labels.yml new file mode 100644 index 000000000000..c8ea37dbbcaf --- /dev/null +++ b/.github/workflows/review-labels.yml @@ -0,0 +1,109 @@ +name: Review labels + +# Require explicit maintainer review when CI-sensitive files or the MCP +# catalog change. Previously this was split across two jobs in two +# workflows: ``ci-review`` in lint.yml (gated on ``ci_review``) and +# ``mcp-catalog-review`` in supply-chain-audit.yml (gated on +# ``mcp_catalog``). Both checked for their own label. +# +# Now consolidated: a single ``ci-reviewed`` label covers both. The +# comment sections tell the reviewer exactly what to verify per area, +# so one label is enough — the human reads the comment, not the label +# name. +# +# Outputs: +# ci_reviewed — "true" / "false" / "" (empty when neither lane ran) +# review_status — JSON array of status objects consumed by the review +# comment assembler. See scripts/ci/emit_review_status.py. + +on: + workflow_call: + inputs: + ci_review: + description: Whether CI-sensitive files (eslint config, workflows, actions) changed. + type: boolean + default: false + ci_review_files: + description: JSON list of CI-sensitive files changed by the pull request. + type: string + default: '[]' + mcp_catalog: + description: Whether the MCP catalog / installer changed. + type: boolean + default: false + supply_chain: + description: Whether the critical supply-chain scan found a risk requiring review. + type: boolean + default: false + outputs: + ci_reviewed: + description: Whether the ci-reviewed label is present. Empty when neither input was true. + value: ${{ jobs.check.outputs.ci_reviewed }} + review_status: + description: JSON array of status objects for the review comment assembler. + value: ${{ jobs.check.outputs.review_status }} + +permissions: + contents: read + pull-requests: read # read PR labels + +jobs: + check: + name: Review label gate + if: inputs.ci_review || inputs.mcp_catalog || inputs.supply_chain + runs-on: ubuntu-latest + timeout-minutes: 2 + outputs: + ci_reviewed: ${{ steps.label-check.outputs.ci_reviewed }} + review_status: ${{ steps.build-status.outputs.review_status }} + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Check ci-reviewed label + id: label-check + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + PR="${{ github.event.pull_request.number }}" + LABELS=$(gh pr view "$PR" --repo "$REPO" --json labels --jq '.labels[].name' || true) + + if echo "$LABELS" | grep -Fxq 'ci-reviewed'; then + echo "ci-reviewed label present." + echo "ci_reviewed=true" >> "$GITHUB_OUTPUT" + else + echo "ci-reviewed label missing." + echo "ci_reviewed=false" >> "$GITHUB_OUTPUT" + fi + + - name: Build review_status JSON + id: build-status + env: + CI_REVIEW: ${{ inputs.ci_review }} + CI_REVIEW_FILES: ${{ inputs.ci_review_files }} + MCP_CATALOG: ${{ inputs.mcp_catalog }} + SUPPLY_CHAIN: ${{ inputs.supply_chain }} + LABEL_PRESENT: ${{ steps.label-check.outputs.ci_reviewed }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + args=() + if [ "$CI_REVIEW" = "true" ]; then args+=(--ci-review); fi + args+=(--ci-review-files "$CI_REVIEW_FILES") + if [ "$MCP_CATALOG" = "true" ]; then args+=(--mcp-catalog); fi + if [ "$SUPPLY_CHAIN" = "true" ]; then args+=(--supply-chain); fi + if [ "$LABEL_PRESENT" = "true" ]; then args+=(--label-present); fi + + python3 scripts/ci/emit_review_status.py "${args[@]}" \ + --repo-url "$REPO_URL" --base-sha "$BASE_SHA" --head-sha "$HEAD_SHA" \ + --output "$GITHUB_OUTPUT" + + - name: Fail on missing label + if: steps.label-check.outputs.ci_reviewed != 'true' + run: | + echo "::error::CI-sensitive changes require the ci-reviewed label. Add the label and re-run this check." + exit 1 diff --git a/.github/workflows/skills-index-freshness.yml b/.github/workflows/skills-index-freshness.yml index 5a9bf98a0f46..9e4b2767be56 100644 --- a/.github/workflows/skills-index-freshness.yml +++ b/.github/workflows/skills-index-freshness.yml @@ -21,6 +21,7 @@ jobs: if: github.repository == 'NousResearch/hermes-agent' runs-on: ubuntu-latest timeout-minutes: 10 + environment: trusted-automation steps: - name: Probe live index id: probe @@ -108,10 +109,18 @@ jobs: echo "Summary: ${{ steps.probe.outputs.summary }}" fi + - name: Get GitHub App token + if: steps.probe.outputs.status != 'ok' + id: app-token + uses: ./.github/actions/get-app-token + with: + client-id: ${{ vars.APP_CLIENT_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + - name: Open issue on degraded / failed probe if: steps.probe.outputs.status != 'ok' env: - GH_TOKEN: ${{ secrets.AUTOFIX_BOT_PAT }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} STATUS: ${{ steps.probe.outputs.status }} DETAIL: ${{ steps.probe.outputs.detail }} run: | diff --git a/.github/workflows/skills-index.yml b/.github/workflows/skills-index.yml index 8930a636fc08..5415499e0245 100644 --- a/.github/workflows/skills-index.yml +++ b/.github/workflows/skills-index.yml @@ -21,9 +21,17 @@ jobs: if: github.repository == 'NousResearch/hermes-agent' runs-on: ubuntu-latest timeout-minutes: 15 + environment: trusted-automation steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Get GitHub App token + id: app-token + uses: ./.github/actions/get-app-token + with: + client-id: ${{ vars.APP_CLIENT_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.11" @@ -35,7 +43,7 @@ jobs: - name: Build skills index env: - GITHUB_TOKEN: ${{ secrets.AUTOFIX_BOT_PAT }} + GITHUB_TOKEN: ${{ steps.app-token.outputs.token }} run: python scripts/build_skills_index.py - name: Upload index artifact @@ -53,8 +61,15 @@ jobs: if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest timeout-minutes: 15 + environment: trusted-automation steps: + - name: Get GitHub App token + id: app-token + uses: ./.github/actions/get-app-token + with: + client-id: ${{ vars.APP_CLIENT_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} - name: Trigger Deploy Site workflow env: - GH_TOKEN: ${{ secrets.AUTOFIX_BOT_PAT }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} run: gh workflow run deploy-site.yml --repo ${{ github.repository }} -f skills_index_run_id=${{ github.run_id }} diff --git a/.github/workflows/supply-chain-audit.yml b/.github/workflows/supply-chain-audit.yml index 648a1f7c6a63..cca61e03a452 100644 --- a/.github/workflows/supply-chain-audit.yml +++ b/.github/workflows/supply-chain-audit.yml @@ -10,9 +10,18 @@ name: Supply Chain Audit # advisory-only workflow instead. # # Path-gating is handled centrally by the ``ci.yml`` orchestrator's -# ``detect`` job. The orchestrator passes ``scan`` / ``deps`` / -# ``mcp_catalog`` booleans as inputs; this workflow's jobs gate on those -# inputs instead of re-computing the diff. +# ``detect`` job. The orchestrator passes ``scan`` / ``deps`` booleans as +# inputs; this workflow's jobs gate on those inputs instead of re-computing +# the diff. MCP catalog review was previously here but has moved to +# ``review-labels.yml`` so it can be rerun independently. +# +# Outputs: +# review_status — JSON array of status objects consumed by the review +# comment assembler (scripts/ci/assemble_review_comment.py). +# critical_findings — "true" when the narrow critical-pattern scan found +# something. The review-label gate consumes this and +# owns the action-required result, so adding +# ``ci-reviewed`` can heal the run on rerun. on: workflow_call: @@ -29,10 +38,13 @@ on: description: Whether pyproject.toml changed. type: boolean required: true - mcp_catalog: - description: Whether the MCP catalog / installer changed. - type: boolean - required: true + outputs: + review_status: + description: JSON array of review status objects for the review comment assembler. + value: ${{ jobs.aggregate.outputs.review_status }} + critical_findings: + description: Whether the critical-pattern scan found a risk requiring maintainer review. + value: ${{ jobs.aggregate.outputs.critical_findings }} permissions: pull-requests: write @@ -44,6 +56,9 @@ jobs: if: inputs.scan runs-on: ubuntu-latest timeout-minutes: 15 + outputs: + review_status: ${{ steps.emit-status.outputs.review_status }} + critical_findings: ${{ steps.scan.outputs.found }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -53,7 +68,8 @@ jobs: - name: Scan diff for critical patterns id: scan env: - GH_TOKEN: ${{ secrets.AUTOFIX_BOT_PAT }} + GH_TOKEN: ${{ github.token }} + CI_REVIEWED: ${{ contains(github.event.pull_request.labels.*.name, 'ci-reviewed') }} run: | set -euo pipefail @@ -61,7 +77,7 @@ jobs: HEAD="${{ github.event.pull_request.head.sha }}" # Added lines only, excluding lockfiles. - # Three-dot diff (base...head) diffs from the merge base to HEAD, + # Three-point diff (base...head) diffs from the merge base to HEAD, # so only changes introduced by this PR are included — not changes # that landed on main after the PR branched off. DIFF=$(git diff "$BASE"..."$HEAD" -- . ':!uv.lock' ':!*.lock' ':!package-lock.json' ':!yarn.lock' || true) @@ -71,7 +87,7 @@ jobs: # --- .pth files (auto-execute on Python startup) --- # The exact mechanism used in the litellm supply chain attack: # https://github.com/BerriAI/litellm/issues/24512 - PTH_FILES=$(git diff --name-only "$BASE"..."$HEAD" | grep '\.pth$' || true) + PTH_FILES=$(git diff --diff-filter=d --name-only "$BASE"..."$HEAD" | grep '\.pth$' || true) if [ -n "$PTH_FILES" ]; then FINDINGS="${FINDINGS} ### 🚨 CRITICAL: .pth file added or modified @@ -119,8 +135,11 @@ jobs: # auto-loaded by the interpreter via site.py. Any nested file with the # same name (e.g. hermes_cli/setup.py — the CLI setup wizard) is unrelated # and produced false positives that trained reviewers to ignore the scanner. - SETUP_HITS=$(git diff --name-only "$BASE"..."$HEAD" | grep -E '^(setup\.py|setup\.cfg|sitecustomize\.py|usercustomize\.py|__init__\.pth)$' || true) - if [ -n "$SETUP_HITS" ]; then + SETUP_HITS=$(git diff --diff-filter=d --name-only "$BASE"..."$HEAD" | grep -E '^(setup\.py|setup\.cfg|sitecustomize\.py|usercustomize\.py|__init__\.pth)$' || true) + # A maintainer-applied ci-reviewed label records the manual review + # required for intentional changes to an install hook. The scanner + # still blocks every unreviewed addition or modification. + if [ -n "$SETUP_HITS" ] && [ "$CI_REVIEWED" != "true" ]; then FINDINGS="${FINDINGS} ### 🚨 CRITICAL: Install-hook file added or modified These files can execute code during package installation or interpreter startup. @@ -139,33 +158,32 @@ jobs: echo "found=false" >> "$GITHUB_OUTPUT" fi - - name: Post critical finding comment - if: steps.scan.outputs.found == 'true' + - name: Emit review_status + id: emit-status + if: always() env: - GH_TOKEN: ${{ secrets.AUTOFIX_BOT_PAT }} + FOUND: ${{ steps.scan.outputs.found }} run: | - BODY="## 🚨 CRITICAL Supply Chain Risk Detected - - This PR contains a pattern that has been used in real supply chain attacks. A maintainer must review the flagged code carefully before merging. + python3 - <<'PYEOF' + import json, os - $(cat /tmp/findings.md) + # The review-label gate renders and blocks critical findings. Keep + # this scan a fact-finder so adding ci-reviewed can rerun the gate + # without requiring the scanner itself to fail again. + status = [] - --- - *Scanner only fires on high-signal indicators: .pth files, base64+exec/eval combos, subprocess with encoded commands, or install-hook files. Low-signal warnings were removed intentionally — if you're seeing this comment, the finding is worth inspecting.*" + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + f.write(f"review_status={json.dumps(status)}\n") + PYEOF - gh pr comment "${{ github.event.pull_request.number }}" --body "$BODY" || echo "::warning::Could not post PR comment (expected for fork PRs — GITHUB_TOKEN is read-only)" - - - name: Fail on critical findings - if: steps.scan.outputs.found == 'true' - run: | - echo "::error::CRITICAL supply chain risk patterns detected in this PR. See the PR comment for details." - exit 1 dep-bounds: name: Check PyPI dependency upper bounds if: inputs.deps runs-on: ubuntu-latest timeout-minutes: 15 + outputs: + review_status: ${{ steps.emit-status.outputs.review_status }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -188,7 +206,7 @@ jobs: exit 0 fi - # Match PyPI dep specs that have >= but no < ceiling. + # Match PyPI dep specs that have >= and no < ceiling. # Pattern: "package>=version" without a following ",<" bound. # Excludes git+ URLs (which use commit SHAs) and comments. UNBOUNDED=$(echo "$ADDED" | grep -oE '"[a-zA-Z0-9_-]+(\[[^\]]*\])?>=[ 0-9.]+"' | grep -v ',<' || true) @@ -200,26 +218,36 @@ jobs: echo "found=false" >> "$GITHUB_OUTPUT" fi - - name: Post unbounded dep warning - if: steps.bounds.outputs.found == 'true' + - name: Emit review_status + id: emit-status + if: always() env: - GH_TOKEN: ${{ secrets.AUTOFIX_BOT_PAT }} + FOUND: ${{ steps.bounds.outputs.found }} run: | - BODY="## ⚠️ Unbounded PyPI Dependency Detected - - This PR adds PyPI dependencies without a \`=floor,=1.2.0,<2"\` - - --- - *See PR #2810 and CONTRIBUTING.md for the full policy rationale.*" - - gh pr comment "${{ github.event.pull_request.number }}" --body "$BODY" || echo "::warning::Could not post PR comment (expected for fork PRs)" + python3 - <<'PYEOF' + import json, os + + found = os.environ.get("FOUND", "") == "true" + + if found: + with open("/tmp/unbounded.txt", encoding="utf-8") as f: + detail = f.read() + status = [{ + "source": "supply chain", + "results": [{ + "kind": "action_required", + "title": "Unbounded PyPI dependencies", + "summary": "This PR adds PyPI dependencies without upper bounds.", + "detail": detail, + "how_to_fix": 'Add a `=1.2.0,<2"`. See CONTRIBUTING.md dependency pinning policy.' + }] + }] + else: + status = [] + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + f.write(f"review_status={json.dumps(status)}\n") + PYEOF - name: Fail on unbounded deps if: steps.bounds.outputs.found == 'true' @@ -227,45 +255,39 @@ jobs: echo "::error::PyPI dependencies without upper bounds detected. Add /dev/null 2>&1; then - echo "Release $GITHUB_REF_NAME found" - exit 0 - fi - echo "Waiting for release... ($i/30)" - sleep 10 - done - echo "::warning::Release $GITHUB_REF_NAME not found after 5 minutes — skipping signature upload" - echo "skip_sign=true" >> "$GITHUB_ENV" - - - name: Sign with Sigstore - if: env.skip_sign != 'true' - uses: sigstore/gh-action-sigstore-python@04cffa1d795717b140764e8b640de88853c92acc # v3.3.0 - with: - inputs: >- - ./dist/*.tar.gz - ./dist/*.whl - - - name: Attach signed artifacts to GitHub Release - if: env.skip_sign != 'true' - env: - GITHUB_TOKEN: ${{ secrets.AUTOFIX_BOT_PAT }} - # release.py already created the GitHub Release — just upload - # the Sigstore signatures alongside the existing assets. - run: >- - gh release upload - "$GITHUB_REF_NAME" dist/*.sigstore.json - --repo "$GITHUB_REPOSITORY" - --clobber diff --git a/.github/workflows/uv-lockfile-check.yml b/.github/workflows/uv-lockfile-check.yml index 27b072a99410..aff4f0eb8cbd 100644 --- a/.github/workflows/uv-lockfile-check.yml +++ b/.github/workflows/uv-lockfile-check.yml @@ -45,6 +45,10 @@ name: uv.lock check on: workflow_call: + outputs: + review_status: + description: "JSON review status for the review-status aggregator" + value: ${{ jobs.check.outputs.review_status }} permissions: contents: read @@ -58,6 +62,8 @@ jobs: name: uv lock --check runs-on: ubuntu-latest timeout-minutes: 5 + outputs: + review_status: ${{ steps.verify.outputs.review_status }} steps: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -73,6 +79,7 @@ jobs: # of this file) — failures often mean "your branch is behind main, # rebase and regenerate uv.lock." - name: Verify uv.lock is up-to-date + id: verify run: | # uv lock --check re-resolves against PyPI (network). Retry so a # registry blip doesn't read as "lockfile stale". A genuinely stale @@ -117,5 +124,9 @@ jobs: on `main` post-merge. EOF echo "::error title=uv.lock out of sync::Run \`uv lock\` locally and commit the result. If on a PR, sync with main first." + review_status='[{"source":"uv.lock check","results":[{"kind":"action_required","title":"uv.lock out of sync","summary":"uv.lock is out of sync with pyproject.toml.","how_to_fix":"Run `uv lock` locally and commit the result. If on a PR, sync with main first:\n```\ngit fetch origin main\ngit rebase origin/main\nuv lock\ngit add uv.lock\ngit commit -m \"chore: refresh uv.lock\"\n```\n"}]}]' + echo "review_status=${review_status}" >> "$GITHUB_OUTPUT" exit 1 fi + review_status='[]' + echo "review_status=${review_status}" >> "$GITHUB_OUTPUT" diff --git a/.gitignore b/.gitignore index 6f1b3be6d92b..cd05306af00b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,13 @@ .DS_Store /venv/ /venv.old/ +/venv.stale.runtime-*/ +/.hermes-runtime/ /_pycache/ *.pyc* __pycache__/ +act/ +.act-sandbox-agent.* .venv/ .venv .vscode/ @@ -42,7 +46,10 @@ run_datagen_sonnet.sh source-data/* run_datagen_megascience_glm4-6.sh data/* -node_modules/ +# No trailing slash: also matches node_modules SYMLINKS (worktrees often +# symlink node_modules to the main checkout; the dir-only pattern let one +# slip into a commit and break `npm ci` on CI with ENOTDIR). +node_modules browser-use/ agent-browser/ # Private keys @@ -54,6 +61,10 @@ __pycache__/ hermes_agent.egg-info/ wandb/ testlogs +playwright-report/ +test-results/ +# Playwright visual regression baselines — cached from main in CI, not committed +*-snapshots/ # CLI config (may contain sensitive SSH paths) cli-config.yaml @@ -66,6 +77,8 @@ environments/benchmarks/evals/ # Web UI build output hermes_cli/web_dist/ +# Cross-process web UI build lock (flock target, always empty) +.web_ui_build.lock apps/desktop/build/ apps/desktop/dist/ @@ -139,6 +152,16 @@ docs/superpowers/* .update-incomplete .update-incomplete.lock +# Checkout fingerprint the __pycache__ tree was last validated against +# (launch-time stale-bytecode sweep). Runtime state, never a code change. +.bytecode-fingerprint +.bytecode-fingerprint.tmp + +# Installer-written method stamp in the managed checkout root (scripts/install.sh). +# Runtime metadata only — never a code change. Ignore so `git status` stays clean +# and `hermes update`'s untracked autostash does not treat it as a local edit (#66189 / #54855). +/.install_method + # Tool Search live-test harness output — non-deterministic model transcripts, # regenerated by scripts/tool_search_livetest.py. Never an artifact of the repo. scripts/out/ @@ -157,4 +180,13 @@ apps/desktop/demo/ # image-provider (fal.media) URL — they are NEVER committed to the repo. The # PR body is the archive. See the hermes-agent-dev skill's # pr-infographic-workflow reference (storage rule + lapse #8 / #COMMIT-1). +# +# Spelling variants are listed because a single `infographic/` pattern was +# sidestepped by an `infograficos/` directory (#70552). .gitignore is only +# the first line of defence and cannot stop `git add -f` at all — the +# infographic-check CI job is what actually enforces this. infographic/ +infographics/ +infograficos/ +infografico/ +native/fts5_cjk/*.so diff --git a/AGENTS.md b/AGENTS.md index 49596b9b41be..d623ba59bbf1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -325,7 +325,7 @@ class AIAgent: provider: str = None, api_mode: str = None, # "chat_completions" | "codex_responses" | ... model: str = "", # empty → resolved from config/provider later - max_iterations: int = 90, # tool-calling iterations (shared with subagents) + max_iterations: int = 500, # tool-calling iterations (shared with subagents) enabled_toolsets: list = None, disabled_toolsets: list = None, quiet_mode: bool = False, @@ -998,7 +998,8 @@ Two shapes: Roles: - `role="leaf"` (default) — focused worker. Cannot call `delegate_task`, - `clarify`, `memory`, `send_message`, `execute_code`. + `clarify`, `memory`, `send_message`, `cronjob`. Retains `execute_code` + (programmatic tool calling). - `role="orchestrator"` — retains `delegate_task` so it can spawn its own workers. Gated by `delegation.orchestrator_enabled` (default true) and bounded by `delegation.max_spawn_depth` (default 2). diff --git a/Dockerfile b/Dockerfile index 388056faacde..42870d737761 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,45 @@ +# Debian 13 still ships SQLite 3.46.1, which contains the upstream WAL-reset +# corruption bug. Build a pinned shared library for the runtime image instead +# of relying on a distro backport that trixie does not currently provide. +# See #70480 and https://sqlite.org/wal.html#walresetbug. +FROM debian:13.4 AS sqlite_build +ARG SQLITE_AUTOCONF_VERSION=3530400 +ARG SQLITE_SHA256=0e9483900e92cd5de8fd48d16bf9200145a61f7fd5be542a5ac81d8a9516eb9c +RUN apt-get -o Acquire::Retries=3 update && \ + apt-get -o Acquire::Retries=3 install -y --no-install-recommends \ + build-essential ca-certificates curl && \ + rm -rf /var/lib/apt/lists/* && \ + (curl -fsSL --retry 1 --retry-all-errors --connect-timeout 15 --max-time 60 \ + -o /tmp/sqlite.tar.gz \ + "https://sqlite.org/2026/sqlite-autoconf-${SQLITE_AUTOCONF_VERSION}.tar.gz" || \ + curl -fsSL --retry 3 --retry-all-errors --connect-timeout 15 --max-time 120 \ + -o /tmp/sqlite.tar.gz \ + "https://sources.buildroot.net/sqlite/sqlite-autoconf-${SQLITE_AUTOCONF_VERSION}.tar.gz") && \ + printf '%s %s\n' "${SQLITE_SHA256}" /tmp/sqlite.tar.gz > /tmp/sqlite.sha256 && \ + sha256sum -c /tmp/sqlite.sha256 && \ + tar -xzf /tmp/sqlite.tar.gz -C /tmp && \ + cd "/tmp/sqlite-autoconf-${SQLITE_AUTOCONF_VERSION}" && \ + CFLAGS="-O2 \ + -DSQLITE_ENABLE_FTS3 \ + -DSQLITE_ENABLE_FTS3_PARENTHESIS \ + -DSQLITE_ENABLE_FTS4 \ + -DSQLITE_ENABLE_FTS5 \ + -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_GEOPOLY \ + -DSQLITE_ENABLE_COLUMN_METADATA \ + -DSQLITE_ENABLE_UNLOCK_NOTIFY \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_ENABLE_MATH_FUNCTIONS \ + -DSQLITE_ENABLE_PREUPDATE_HOOK \ + -DSQLITE_ENABLE_SESSION \ + -DSQLITE_SECURE_DELETE \ + -DSQLITE_THREADSAFE=1 \ + -DSQLITE_MAX_VARIABLE_NUMBER=250000" \ + ./configure --prefix=/opt/sqlite-fixed --disable-static && \ + make -j"$(nproc)" && \ + make install + FROM ghcr.io/astral-sh/uv:0.11.6-python3.13-trixie@sha256:b3c543b6c4f23a5f2df22866bd7857e5d304b67a564f4feab6ac22044dde719b AS uv_source # Node 22 LTS source stage. Debian trixie's bundled nodejs is pinned to 20.x # which reached EOL in April 2026 — we copy node + npm + corepack from the @@ -31,6 +73,23 @@ RUN apt-get -o Acquire::Retries=3 update && \ ca-certificates curl iputils-ping python3 python-is-python3 ripgrep ffmpeg gcc g++ make cmake python3-dev python3-venv libffi-dev libolm-dev procps git openssh-client docker-cli xz-utils && \ rm -rf /var/lib/apt/lists/* +# Prefer the fixed SQLite over Debian's vulnerable libsqlite3.so.0. Keep the +# public library name stable so both the system interpreter and the uv-created +# venv resolve the replacement without changing Python import paths. +COPY --from=sqlite_build /opt/sqlite-fixed/lib/libsqlite3.so.3.53.4 /usr/local/lib/ +RUN ln -sf libsqlite3.so.3.53.4 /usr/local/lib/libsqlite3.so.0 && \ + ln -sf libsqlite3.so.3.53.4 /usr/local/lib/libsqlite3.so && \ + printf '/usr/local/lib\n' > /etc/ld.so.conf.d/000-sqlite-fixed.conf && \ + ldconfig && \ + python3 -c "import sqlite3, sys; \ +v = sqlite3.sqlite_version_info; \ +sys.exit(f'linked SQLite {sqlite3.sqlite_version} still has the WAL-reset bug') if v < (3, 51, 3) else None; \ +db = sqlite3.connect(':memory:'); \ +db.execute(\"CREATE VIRTUAL TABLE docs USING fts5(content, tokenize='trigram')\"); \ +db.execute(\"INSERT INTO docs VALUES ('hermes')\"); \ +sys.exit('SQLite FTS5 trigram self-test failed') if db.execute(\"SELECT count(*) FROM docs WHERE docs MATCH 'erm'\").fetchone()[0] != 1 else None; \ +db.close()" + # ---------- s6-overlay install ---------- # s6-overlay provides supervision for the main hermes process, the dashboard, # and per-profile gateways. /init becomes PID 1 below — see ENTRYPOINT. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 159c215ff6b0..000000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,13 +0,0 @@ -graft skills -graft optional-skills -graft optional-mcps -graft locales -# Bundled plugin manifests (plugin.yaml / plugin.yml). Without these the -# PluginManager scan (hermes_cli/plugins.py) finds zero plugins on installs -# built from the sdist (e.g. Homebrew, downstream packagers). package-data -# below covers the wheel; this covers the sdist. See #34034 / #28149. -recursive-include plugins plugin.yaml plugin.yml -# Gateway assets include images plus YAML catalogs such as status_phrases.yaml. -recursive-include gateway/assets * -global-exclude __pycache__ -global-exclude *.py[cod] diff --git a/acp_adapter/entry.py b/acp_adapter/entry.py index 5048b7025982..fb9ed95450c3 100644 --- a/acp_adapter/entry.py +++ b/acp_adapter/entry.py @@ -32,6 +32,7 @@ import argparse import asyncio import logging +import os import sys from pathlib import Path from hermes_constants import get_hermes_home @@ -190,7 +191,7 @@ def _run_setup_browser(assume_yes: bool = False) -> int: """Bootstrap agent-browser + Chromium. Routes through dep_ensure -> install.{sh,ps1} --ensure, sharing code - with ``hermes postinstall`` and the runtime lazy installer. + with the runtime lazy installer. Returns 0 on success, 1 on failure. """ @@ -251,11 +252,13 @@ def main(argv: list[str] | None = None) -> None: # MCP servers dynamically via asyncio.to_thread inside the event # loop; that path is unaffected.) Moved from model_tools.py module # scope to avoid freezing the gateway's loop on lazy import (#16856). - try: - from tools.mcp_tool import discover_mcp_tools - discover_mcp_tools() - except Exception: - logger.debug("MCP tool discovery failed at ACP startup", exc_info=True) + # Metadata-only hosts can opt out of unrelated global MCP startup. + if os.environ.get("HERMES_ACP_SKIP_CONFIGURED_MCP", "").strip() != "1": + try: + from tools.mcp_tool import discover_mcp_tools + discover_mcp_tools() + except Exception: + logger.debug("MCP tool discovery failed at ACP startup", exc_info=True) agent = HermesACPAgent() try: diff --git a/acp_adapter/server.py b/acp_adapter/server.py index d86e40651869..7fee2d932f84 100644 --- a/acp_adapter/server.py +++ b/acp_adapter/server.py @@ -74,6 +74,10 @@ from acp_adapter.provenance import session_provenance_meta from acp_adapter.session import SessionManager, SessionState, _expand_acp_enabled_toolsets from acp_adapter.tools import build_tool_complete, build_tool_start +from agent.context_compressor import ( + COMPRESSED_SUMMARY_METADATA_KEY, + ContextCompressor, +) from tools.approval import ( reset_hermes_interactive_context, set_hermes_interactive_context, @@ -81,6 +85,110 @@ logger = logging.getLogger(__name__) + +def _named_custom_provider_catalogs() -> list[tuple[str, str, list[tuple[str, str]]]]: + """Return ``(slug, label, [(model_id, description), ...])`` for named endpoints. + + Covers both the v12 ``providers:`` mapping and the legacy + ``custom_providers:`` list. These endpoints never appear in canonical + provider enumeration, so without this the ACP model selector hides every + named endpoint that the TUI ``/model`` picker already renders (#47039 + implemented named-endpoint rows for the TUI surface only). + + Model lists come from the entry's declared models (``default_model`` + + ``models``), refreshed from the endpoint's live ``/models`` listing when a + credential is available and ``discover_models`` is not disabled. Declared + models are kept even when live discovery fails — some OpenAI-compatible + endpoints (e.g. Bedrock Mantle Responses) expose no ``/models`` route at + all yet serve the declared models fine. + + Slugs use the ``custom:`` shape that ``parse_model_input`` and + ``resolve_runtime_provider`` already resolve, so encoded choice ids + (``custom::``) round-trip through ``set_session_model`` + unchanged. + """ + try: + from hermes_cli.config import ( + get_compatible_custom_providers, + is_provider_enabled, + load_config, + ) + from hermes_cli.models import fetch_api_models + except ImportError: + return [] + + try: + cfg = load_config() + entries = get_compatible_custom_providers(cfg) + except Exception: + logger.debug("Could not load named custom providers", exc_info=True) + return [] + + # ``get_compatible_custom_providers`` drops the ``enabled`` flag during + # normalization, so collect explicitly disabled provider keys from the + # raw config and skip their entries below. + disabled_keys: set[str] = set() + raw_providers = cfg.get("providers") if isinstance(cfg, dict) else None + if isinstance(raw_providers, dict): + for raw_key, raw_entry in raw_providers.items(): + if isinstance(raw_entry, dict) and not is_provider_enabled(raw_entry): + disabled_keys.add(str(raw_key).strip().lower()) + + catalogs: list[tuple[str, str, list[tuple[str, str]]]] = [] + for entry in entries: + if not isinstance(entry, dict): + continue + provider_key = str(entry.get("provider_key", "") or "").strip() + if provider_key.lower() in disabled_keys: + continue + name = str(entry.get("name", "") or "").strip() + base_url = str(entry.get("base_url", "") or "").strip() + if not name or not base_url: + continue + slug_source = provider_key or name + slug = "custom:" + slug_source.strip().lower().replace(" ", "-") + + api_key = str(entry.get("api_key", "") or "").strip() + if not api_key: + key_env = str(entry.get("key_env", "") or "").strip() + api_key = os.environ.get(key_env, "").strip() if key_env else "" + + declared: list[str] = [] + default_model = str(entry.get("model", "") or "").strip() + if default_model: + declared.append(default_model) + models_cfg = entry.get("models") + if isinstance(models_cfg, dict): + for mid in models_cfg: + mid = str(mid or "").strip() + if mid and mid not in declared: + declared.append(mid) + + if not api_key and not declared: + # No credential to discover with and nothing declared: + # not addressable from the selector. + continue + + model_ids = list(declared) + discover = entry.get("discover_models", True) + if isinstance(discover, str): + discover = discover.lower() not in {"false", "no", "0"} + if discover and api_key: + try: + live = fetch_api_models( + api_key, base_url, api_mode=entry.get("api_mode") + ) + except Exception: + live = None + if live: + model_ids = declared + [m for m in live if m not in declared] + + if not model_ids: + continue + catalogs.append((slug, name, [(mid, "") for mid in model_ids])) + + return catalogs + try: from hermes_cli import __version__ as HERMES_VERSION except Exception: @@ -93,6 +201,13 @@ # does not expose a client-side limit, so this is a fixed cap that clients # paginate against using `cursor` / `next_cursor`. _LIST_SESSIONS_PAGE_SIZE = 50 +# Per-provider cap for the ACP model selector. ACP clients (Zed, Buzz) render +# the whole `availableModels` array in one dropdown, so an unbounded +# cross-provider catalog degrades the picker. Mirrors the cap the MoA picker +# already uses (`hermes_cli/moa_cmd.py`). This bounds each provider's row, not +# the total; aggregator providers stay intentionally uncapped inside the shared +# inventory, and the current model is always kept via the fallback insert below. +ACP_MAX_MODELS_PER_PROVIDER = 200 _MAX_ACP_RESOURCE_BYTES = 512 * 1024 _TEXT_RESOURCE_MIME_PREFIXES = ("text/",) _TEXT_RESOURCE_MIME_TYPES = { @@ -456,7 +571,7 @@ class HermesACPAgent(acp.Agent): "tools": "List available tools", "context": "Show conversation context info", "reset": "Clear conversation history", - "compact": "Compress conversation context", + "compress": "Compress conversation context", "steer": "Inject guidance into the currently running agent turn", "queue": "Queue a prompt to run after the current turn finishes", "version": "Show Hermes version", @@ -485,7 +600,7 @@ class HermesACPAgent(acp.Agent): "description": "Clear conversation history", }, { - "name": "compact", + "name": "compress", "description": "Compress conversation context", }, { @@ -581,46 +696,108 @@ def _encode_model_choice(provider: str | None, model: str | None) -> str: return f"{raw_provider}:{raw_model}" def _build_model_state(self, state: SessionState) -> SessionModelState | None: - """Return the ACP model selector payload for editors like Zed.""" + """Return authenticated providers and their models for ACP clients. + + The shared Hermes inventory is also used by ``hermes model``, the TUI, + and the dashboard. Keeping ACP on that substrate prevents its selector + from silently collapsing to the current provider's curated list. + """ model = str(state.model or getattr(state.agent, "model", "") or "").strip() provider = getattr(state.agent, "provider", None) or detect_provider() or "openrouter" try: - from hermes_cli.models import curated_models_for_provider, normalize_provider, provider_label + from hermes_cli.inventory import build_models_payload, load_picker_context + from hermes_cli.models import normalize_provider, provider_label normalized_provider = normalize_provider(provider) - provider_name = provider_label(normalized_provider) + context = load_picker_context().with_overrides( + current_provider=normalized_provider, + current_model=model, + current_base_url=str(getattr(state.agent, "base_url", "") or ""), + ) + payload = build_models_payload( + context, + explicit_only=True, + include_unconfigured=False, + picker_hints=False, + canonical_order=True, + pricing=False, + capabilities=False, + refresh=False, + probe_custom_providers=False, + probe_current_custom_provider=False, + max_models=ACP_MAX_MODELS_PER_PROVIDER, + ) + available_models: list[ModelInfo] = [] seen_ids: set[str] = set() - - for model_id, description in curated_models_for_provider(normalized_provider): - rendered_model = str(model_id or "").strip() - if not rendered_model: - continue - choice_id = self._encode_model_choice(normalized_provider, rendered_model) - if choice_id in seen_ids: + for row in payload.get("providers") or []: + row_provider = normalize_provider(str(row.get("slug") or "").strip()) + if not row_provider: continue - desc_parts = [f"Provider: {provider_name}"] - if description: - desc_parts.append(str(description).strip()) - if rendered_model == model: - desc_parts.append("current") - available_models.append( - ModelInfo( - model_id=choice_id, - name=rendered_model, - description=" • ".join(part for part in desc_parts if part), - ) + provider_name = str(row.get("name") or "").strip() or provider_label( + row_provider ) - seen_ids.add(choice_id) + for model_entry in row.get("models") or []: + if isinstance(model_entry, dict): + rendered_model = str( + model_entry.get("id") + or model_entry.get("model") + or model_entry.get("name") + or "" + ).strip() + else: + rendered_model = str(model_entry or "").strip() + if not rendered_model: + continue + choice_id = self._encode_model_choice(row_provider, rendered_model) + if choice_id in seen_ids: + continue + is_current = ( + row_provider == normalized_provider and rendered_model == model + ) + description = f"Provider: {provider_name}" + if is_current: + description += " • current" + available_models.append( + ModelInfo( + model_id=choice_id, + name=f"{provider_name} · {rendered_model}", + description=description, + ) + ) + seen_ids.add(choice_id) + + # Named user-defined endpoints (providers: / custom_providers:) + # are invisible to canonical provider enumeration — append them + # so editor clients can select them like the TUI /model picker. + for named_slug, named_label, named_catalog in _named_custom_provider_catalogs(): + for named_model, named_desc in named_catalog: + named_choice = self._encode_model_choice(named_slug, named_model) + if not named_choice or named_choice in seen_ids: + continue + named_parts = [f"Provider: {named_label}"] + if named_desc: + named_parts.append(str(named_desc).strip()) + if named_slug == normalized_provider and named_model == model: + named_parts.append("current") + available_models.append( + ModelInfo( + model_id=named_choice, + name=named_model, + description=" • ".join(part for part in named_parts if part), + ) + ) + seen_ids.add(named_choice) current_model_id = self._encode_model_choice(normalized_provider, model) if current_model_id and current_model_id not in seen_ids: + provider_name = provider_label(normalized_provider) available_models.insert( 0, ModelInfo( model_id=current_model_id, - name=model, + name=f"{provider_name} · {model}", description=f"Provider: {provider_name} • current", ), ) @@ -969,11 +1146,49 @@ def _history_reasoning_text(cls, message: dict[str, Any]) -> str: return text return "" + @staticmethod + def _history_summary_meta(message: dict[str, Any], text: str) -> dict[str, Any] | None: + """Build the ``_meta`` payload for a replayed compaction summary. + + Compaction summaries are persisted as ordinary history messages — + standalone handoffs under ``role="user"`` OR ``role="assistant"`` + (the compressor picks whichever role keeps alternation valid), and + merge-into-tail messages where the summary is appended after the + first preserved tail message's real content. Without a wire flag, + ACP frontends render all of these as ordinary turns. + + Two distinct keys under ``_meta.hermes`` (ACP's extensibility + channel), so clients cannot accidentally hide real content: + + * ``compactionSummary: true`` — the entire chunk is the handoff + summary. Safe to restyle or collapse wholesale. + * ``containsCompactionSummary: true`` — a merged-tail message: real + preserved turn content followed by the summary. Clients may style + it, but collapsing the whole chunk would hide the preserved + content, hence the separate key. + + Detection honors the in-process ``_compressed_summary`` flag and + falls back to content classification, so it also works for a + DB-reloaded session that lost the in-memory flag. + """ + kind = ContextCompressor.classify_summary_content(text) + if kind is None and message.get(COMPRESSED_SUMMARY_METADATA_KEY): + # Flagged in-process but content didn't classify (e.g. future + # prefix drift): treat as a standalone summary — the flag is only + # ever set on summary-bearing messages. + kind = "standalone" + if kind == "standalone": + return {"hermes": {"compactionSummary": True}} + if kind == "merged": + return {"hermes": {"containsCompactionSummary": True}} + return None + @staticmethod def _history_message_update( *, role: str, text: str, + field_meta: dict[str, Any] | None = None, ) -> UserMessageChunk | AgentMessageChunk | None: """Build an ACP history replay update for a user/assistant message.""" block = TextContentBlock(type="text", text=text) @@ -981,11 +1196,13 @@ def _history_message_update( return UserMessageChunk( session_update="user_message_chunk", content=block, + field_meta=field_meta, ) if role == "assistant": return AgentMessageChunk( session_update="agent_message_chunk", content=block, + field_meta=field_meta, ) return None @@ -1056,7 +1273,11 @@ async def _send(update: Any) -> bool: if role == "user": text = self._history_message_text(message) if text: - update = self._history_message_update(role=role, text=text) + update = self._history_message_update( + role=role, + text=text, + field_meta=self._history_summary_meta(message, text), + ) if update is not None and not await _send(update): return continue @@ -1068,7 +1289,11 @@ async def _send(update: Any) -> bool: text = self._history_message_text(message) if text: - update = self._history_message_update(role=role, text=text) + update = self._history_message_update( + role=role, + text=text, + field_meta=self._history_summary_meta(message, text), + ) if update is not None and not await _send(update): return @@ -1218,12 +1443,19 @@ async def cancel(self, session_id: str, **kwargs: Any) -> None: with state.runtime_lock: if state.is_running and state.current_prompt_text: state.interrupted_prompt_text = state.current_prompt_text - state.cancel_event.set() - try: - if getattr(state, "agent", None) and hasattr(state.agent, "interrupt"): - state.agent.interrupt() - except Exception: - logger.debug("Failed to interrupt ACP session %s", session_id, exc_info=True) + # Publish cancellation and hard-stop the agent before another + # prompt can acquire this lock and mistake the turn for + # redirectable work. + state.cancel_event.set() + try: + if getattr(state, "agent", None) and hasattr(state.agent, "interrupt"): + state.agent.interrupt() + except Exception: + logger.debug( + "Failed to interrupt ACP session %s", + session_id, + exc_info=True, + ) logger.info("Cancelled session %s", session_id) async def fork_session( @@ -1352,6 +1584,26 @@ async def prompt( elif rewrite_idle: user_text = steer_text user_content = steer_text + elif ( + text_only_prompt + and isinstance(user_content, str) + and not user_text.startswith("/") + ): + # Some ACP clients implement "stop and send" as two protocol calls: + # cancel the active prompt, then submit plain correction text. Keep + # the cancelled request attached so deictic follow-ups ("not that + # file") still have an explicit target. + interrupted_prompt = "" + with state.runtime_lock: + if not state.is_running and state.interrupted_prompt_text: + interrupted_prompt = state.interrupted_prompt_text + state.interrupted_prompt_text = "" + if interrupted_prompt: + user_text = ( + f"{interrupted_prompt}\n\n" + f"User correction/guidance after interrupt: {user_text}" + ) + user_content = user_text # Intercept slash commands — handle locally without calling the LLM. # Slash commands are text-only; if the client included images/resources, @@ -1366,23 +1618,54 @@ async def prompt( await self._send_usage_update(state) return PromptResponse(stop_reason="end_turn") - # If Zed sends another regular prompt while the same ACP session is - # still running, queue it instead of racing two AIAgent loops against - # the same state.history. /steer and /queue are handled above and can - # land immediately. + # If the client sends another regular text prompt while this ACP session + # is running, route it through the core active-turn redirect. Rich media + # and older runtimes retain the proven next-turn queue fallback. + redirected = False + queued_depth: int | None = None with state.runtime_lock: if state.is_running: - queued_text = user_text or "[Image attachment]" - state.queued_prompts.append(queued_text) - depth = len(state.queued_prompts) - if self._conn: - update = acp.update_agent_message_text( - f"Queued for the next turn. ({depth} queued)" + if ( + text_only_prompt + and isinstance(user_content, str) + and getattr( + state.agent, + "_supports_active_turn_redirect", + False, ) - await self._conn.session_update(session_id, update) - return PromptResponse(stop_reason="end_turn") - state.is_running = True - state.current_prompt_text = user_text or "[Image attachment]" + is True + and hasattr(state.agent, "redirect") + ): + try: + redirected = bool(state.agent.redirect(user_content)) + except Exception: + logger.debug( + "ACP active-turn redirect failed for %s", + session_id, + exc_info=True, + ) + if not redirected: + queued_text = user_text or "[Image attachment]" + state.queued_prompts.append(queued_text) + queued_depth = len(state.queued_prompts) + else: + state.is_running = True + state.current_prompt_text = user_text or "[Image attachment]" + + if redirected: + if self._conn: + update = acp.update_agent_message_text( + "Redirected the active turn with your correction." + ) + await self._conn.session_update(session_id, update) + return PromptResponse(stop_reason="end_turn") + if queued_depth is not None: + if self._conn: + update = acp.update_agent_message_text( + f"Queued for the next turn. ({queued_depth} queued)" + ) + await self._conn.session_update(session_id, update) + return PromptResponse(stop_reason="end_turn") logger.info("Prompt on session %s: %s", session_id, user_text[:100]) @@ -1478,7 +1761,16 @@ def _run_agent() -> dict: clear_session_vars, set_session_vars, ) - session_tokens = set_session_vars(session_key=session_id) + # ``cwd`` pins the logical working directory for this context, + # which is what the system prompt's "Current working directory" + # line reports (agent/prompt_builder.py -> resolve_agent_cwd). + # Without it the prompt advertises the global Hermes workspace + # while the tools are rooted at the client's project, so the + # model emits absolute paths under ~/.hermes/workspace and the + # edit silently lands outside the editor's workspace. + session_tokens = set_session_vars( + session_key=session_id, cwd=state.cwd, + ) except Exception: session_tokens = None clear_session_vars = None # type: ignore[assignment] @@ -1756,7 +2048,7 @@ def _handle_slash_command(self, text: str, state: SessionState) -> str | None: "tools": self._cmd_tools, "context": self._cmd_context, "reset": self._cmd_reset, - "compact": self._cmd_compact, + "compress": self._cmd_compress, "steer": self._cmd_steer, "queue": self._cmd_queue, "version": self._cmd_version, @@ -1765,8 +2057,26 @@ def _handle_slash_command(self, text: str, state: SessionState) -> str | None: if handler is None: return None # not a known command — let the LLM handle it - try: + # Slash handlers run on the event-loop thread, OUTSIDE the per-turn + # contextvars.copy_context() that pins the session cwd for the agent + # call. ``/compress`` and ``/model`` reach code that REBUILDS the + # system prompt (agent._build_system_prompt -> resolve_agent_cwd), so + # an unpinned handler bakes the Hermes install tree into the session's + # cached prompt — persisted, and therefore poisoning every later turn + # even though the turn itself is pinned. Pin inside a fresh context so + # the write can't leak into other concurrent ACP sessions and needs no + # teardown. + def _dispatch() -> str | None: + try: + from agent.runtime_cwd import set_session_cwd + + set_session_cwd(state.cwd) + except Exception: + logger.debug("Could not pin ACP session cwd for slash command", exc_info=True) return handler(args, state) + + try: + return contextvars.copy_context().run(_dispatch) except Exception as e: logger.error("Slash command /%s error: %s", cmd, e, exc_info=True) return f"Error executing /{cmd}: {e}" @@ -1826,8 +2136,8 @@ def _cmd_tools(self, args: str, state: SessionState) -> str: return "No tools available." lines = [f"Available tools ({len(tools)}):"] for t in tools: - name = t.get("function", {}).get("name", "?") - desc = t.get("function", {}).get("description", "") + name = (t.get("function") or {}).get("name", "?") + desc = (t.get("function") or {}).get("description", "") # Truncate long descriptions if len(desc) > 80: desc = desc[:77] + "..." @@ -1898,7 +2208,7 @@ def _cmd_context(self, args: str, state: SessionState) -> str: lines.append( f"Compression: due now (threshold ~{threshold_tokens:,}" + (f", {threshold_pct:.0f}%" if threshold_pct else "") - + "). Run /compact." + + "). Run /compress." ) else: lines.append( @@ -1911,9 +2221,12 @@ def _cmd_context(self, args: str, state: SessionState) -> str: lines.append(f"Compression threshold: ~{threshold_tokens:,} tokens") if getattr(agent, "compression_enabled", True) is False: - lines.append("Compression is disabled for this agent.") + lines.append( + "Auto-compaction is disabled (compression.enabled: false); " + "/compress still compresses manually." + ) else: - lines.append("Tip: run /compact to compress manually before the threshold.") + lines.append("Tip: run /compress to compress manually before the threshold.") return "\n".join(lines) @@ -1933,13 +2246,14 @@ def _cmd_reset(self, args: str, state: SessionState) -> str: return "Conversation history cleared. Agent session state reset failed; see logs." return "Conversation history cleared." - def _cmd_compact(self, args: str, state: SessionState) -> str: + def _cmd_compress(self, args: str, state: SessionState) -> str: if not state.history: return "Nothing to compress — conversation is empty." try: agent = state.agent - if not getattr(agent, "compression_enabled", True): - return "Context compression is disabled for this agent." + # No compression_enabled gate: the flag disables *automatic* + # compaction only; manual /compress must keep working (matches + # the CLI /compress and gateway handlers). if not hasattr(agent, "_compress_context"): return "Context compression not available for this agent." @@ -1964,6 +2278,7 @@ def _cmd_compact(self, args: str, state: SessionState) -> str: getattr(agent, "_cached_system_prompt", "") or "", approx_tokens=approx_tokens, task_id=state.session_id, + force=True, ) finally: agent._session_db = original_session_db diff --git a/acp_registry/agent.json b/acp_registry/agent.json deleted file mode 100644 index 1d3752bf15a1..000000000000 --- a/acp_registry/agent.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "hermes-agent", - "name": "Hermes Agent", - "version": "0.19.0", - "description": "Self-improving open-source AI agent by Nous Research with ACP editor integration, persistent memory, skills, and rich tool support.", - "repository": "https://github.com/NousResearch/hermes-agent", - "website": "https://hermes-agent.nousresearch.com/docs/user-guide/features/acp", - "authors": ["Nous Research"], - "license": "MIT", - "distribution": { - "uvx": { - "package": "hermes-agent[acp]==0.19.0", - "args": ["hermes-acp"] - } - } -} diff --git a/acp_registry/icon.svg b/acp_registry/icon.svg deleted file mode 100644 index f42c0daea458..000000000000 --- a/acp_registry/icon.svg +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - diff --git a/agent/account_usage.py b/agent/account_usage.py index 571d18446daf..b7abb1801764 100644 --- a/agent/account_usage.py +++ b/agent/account_usage.py @@ -701,6 +701,18 @@ def redeem_codex_reset_credit( remaining = max(0, available - 1) plural = "s" if remaining != 1 else "" if code == "reset": + # The redeemed reset restores the account's quota upstream — lift any + # persisted pool cooldowns so Hermes doesn't keep the credential + # frozen behind the now-stale ``last_error_reset_at`` (issue #43747). + try: + from hermes_cli.auth import clear_codex_pool_quota_cooldowns + + clear_codex_pool_quota_cooldowns() + except Exception: + logger.debug( + "Failed to clear Codex pool cooldowns after reset redemption", + exc_info=True, + ) return CodexResetRedeemResult( status="reset", message=( diff --git a/agent/agent_init.py b/agent/agent_init.py index a9fb685568ab..0e4a925c05e1 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -28,7 +28,7 @@ import uuid from datetime import datetime from typing import Any, Callable, Dict, List, Optional -from urllib.parse import urlparse, parse_qs, urlunparse +from urllib.parse import parse_qs, urlparse, urlunparse from agent.context_compressor import ContextCompressor from agent.iteration_budget import IterationBudget @@ -48,6 +48,7 @@ ToolGuardrailDecision, ) from hermes_cli.config import cfg_get +from hermes_cli.route_identity import normalize_route_base_url from hermes_cli.timeouts import get_provider_request_timeout from hermes_constants import get_hermes_home from utils import base_url_host_matches, is_truthy_value @@ -68,18 +69,188 @@ def _ra(): return run_agent -def _build_codex_gpt5_autoraise_notice(autoraise: Dict[str, Any]) -> str: +def _moa_reference_output_allowed(agent: Any) -> bool: + """Keep MoA display events off only the machine-readable ``-Q`` surface.""" + return not ( + getattr(agent, "platform", None) == "cli" + and getattr(agent, "tool_progress_mode", "all") == "off" + ) + + +def _relay_moa_reference_event(agent: Any, event: str, **kwargs: Any) -> None: + """Relay MoA display events while preserving the ``-Q`` stdout contract.""" + if not _moa_reference_output_allowed(agent): + return + cb = getattr(agent, "tool_progress_callback", None) + if cb is None: + return + try: + if event == "moa.reference": + cb( + "moa.reference", + str(kwargs.get("label") or ""), + str(kwargs.get("text") or ""), + None, + moa_index=kwargs.get("index"), + moa_count=kwargs.get("count"), + ) + elif event == "moa.aggregating": + cb( + "moa.aggregating", + str(kwargs.get("aggregator") or ""), + None, + None, + moa_ref_count=kwargs.get("ref_count"), + ) + except Exception: + pass + + +def _normalize_route_base_url(base_url: Any) -> str: + """Canonicalize an endpoint URL for model-route identity comparisons.""" + return normalize_route_base_url(base_url) + + +def _provider_default_routes(provider: str) -> set[str]: + """Return known exact default routes for a canonical provider id.""" + routes: set[str] = set() + try: + from hermes_cli.providers import HERMES_OVERLAYS, get_provider + + overlay = HERMES_OVERLAYS.get(provider) + provider_def = get_provider(provider) + for value in ( + getattr(overlay, "base_url_override", ""), + getattr(provider_def, "base_url", ""), + ): + route = _normalize_route_base_url(value) + if route: + routes.add(route) + except Exception: + pass + + try: + from providers import get_provider_profile + + profile = get_provider_profile(provider) + route = _normalize_route_base_url( + getattr(profile, "base_url", "") + ) + if route: + routes.add(route) + except Exception: + pass + + try: + from hermes_cli.auth import PROVIDER_REGISTRY + from hermes_cli.models import normalize_provider as normalize_model_provider + from hermes_cli.providers import normalize_provider as normalize_registry_provider + + for provider_id, config in PROVIDER_REGISTRY.items(): + canonical_id = normalize_registry_provider( + normalize_model_provider(provider_id) + ) + if canonical_id != provider: + continue + route = _normalize_route_base_url( + getattr(config, "inference_base_url", "") + ) + if route: + routes.add(route) + except Exception: + pass + + if provider == "gemini": + routes.update( + f"{route.rstrip('/')}/openai" + for route in list(routes) + ) + return routes + + +def _context_route_mismatch( + configured_base_url: Any, + active_base_url: Any, + configured_provider: Any, + active_provider: Any, + *, + already_normalized: bool = False, +) -> bool: + """Return whether a context pin's configured route differs from runtime.""" + if already_normalized: + configured_route = str(configured_base_url or "") + active_route = str(active_base_url or "") + else: + configured_route = _normalize_route_base_url(configured_base_url) + active_route = _normalize_route_base_url(active_base_url) + if configured_route: + return configured_route != active_route + + configured_provider = str(configured_provider or "").strip() + active_provider = str(active_provider or "").strip() + if not configured_provider: + return False + try: + from hermes_cli.models import normalize_provider as normalize_model_provider + + configured_provider = normalize_model_provider(configured_provider) + active_provider = normalize_model_provider(active_provider) + except Exception: + configured_provider = configured_provider.lower() + active_provider = active_provider.lower() + try: + from hermes_cli.providers import normalize_provider as normalize_registry_provider + + configured_provider = normalize_registry_provider(configured_provider) + active_provider = normalize_registry_provider(active_provider) + except Exception: + pass + + if active_route: + configured_routes = _provider_default_routes(configured_provider) + return not configured_routes or active_route not in configured_routes + return bool( + configured_provider + and active_provider + and configured_provider != active_provider + ) + + +def _normalize_custom_provider_name(value: Any) -> str: + """Mirror runtime normalization for a requested custom-provider identity.""" + return str(value or "").strip().lower().replace(" ", "-") + + +def _custom_provider_runtime_ids(value: Any) -> set[str]: + """Return raw/menu identities that runtime accepts for a configured name.""" + normalized = _normalize_custom_provider_name(value) + if not normalized: + return set() + return {normalized, f"custom:{normalized}"} + + +def _build_codex_gpt5_autoraise_notice( + autoraise: Dict[str, Any], context_length: Optional[int] = None +) -> str: """Build the one-time notice shown when Codex gpt-5.x raises compaction. ``autoraise`` is ``{"model": , "from": , "to": }``. - The same text is printed inline for CLI users and replayed via + ``context_length`` is the live-resolved window from the context compressor + (Codex's /models catalog is authoritative and can change server-side, e.g. + the gpt-5.6 family's 272K → 372K → 272K shifts in July 2026), so the banner + reports what this session actually got rather than a hardcoded cap. The + same text is printed inline for CLI users and replayed via ``status_callback`` for gateway users, so it must be self-contained and include the exact opt-back-out command. """ model = str(autoraise.get("model") or "gpt-5.4/5.5").strip().lower().rsplit("/", 1)[-1] - # gpt-5.3-codex-spark has a native 128K window; the gpt-5.4/5.5/5.6 family - # is capped at 272K by the Codex OAuth backend. - cap = "128K" if model.startswith("gpt-5.3-codex-spark") else "272K" + if isinstance(context_length, int) and context_length > 0: + cap = f"{round(context_length / 1000)}K" + else: + # Static fallback when the resolved window isn't available: + # gpt-5.3-codex-spark has a native 128K window; the gpt-5.4/5.5/5.6 + # family is capped at 272K by the Codex OAuth backend. + cap = "128K" if model.startswith("gpt-5.3-codex-spark") else "272K" from_pct = int(round(autoraise["from"] * 100)) to_pct = int(round(autoraise["to"] * 100)) return ( @@ -284,7 +455,7 @@ def init_agent( command: str = None, args: list[str] | None = None, model: str = "", - max_iterations: int = 90, # Default tool-calling iterations (shared with subagents) + max_iterations: int = 500, # Default tool-calling iterations (shared with subagents) tool_delay: float = 1.0, enabled_toolsets: List[str] = None, disabled_toolsets: List[str] = None, @@ -346,6 +517,7 @@ def init_agent( checkpoint_max_total_size_mb: int = 500, checkpoint_max_file_size_mb: int = 10, pass_session_id: bool = False, + requested_provider: str = None, ): """ Initialize the AI Agent. @@ -354,9 +526,10 @@ def init_agent( base_url (str): Base URL for the model API (optional) api_key (str): API key for authentication (optional, uses env var if not provided) provider (str): Provider identifier (optional; used for telemetry/routing hints) + requested_provider (str): Original provider identity before runtime canonicalization api_mode (str): API mode override: "chat_completions" or "codex_responses" model (str): Model name to use (default: "anthropic/claude-opus-4.6") - max_iterations (int): Maximum number of tool calling iterations (default: 90) + max_iterations (int): Maximum number of tool calling iterations (default: 500) tool_delay (float): Delay between tool calls in seconds (default: 1.0) enabled_toolsets (List[str]): Only enable tools from these toolsets (optional) disabled_toolsets (List[str]): Disable tools from these toolsets (optional) @@ -434,6 +607,11 @@ def init_agent( agent.base_url = base_url or "" provider_name = provider.strip().lower() if isinstance(provider, str) and provider.strip() else None agent.provider = provider_name or "" + agent.requested_provider = ( + requested_provider.strip().lower() + if isinstance(requested_provider, str) and requested_provider.strip() + else agent.provider + ) agent._credential_pool = credential_pool agent.acp_command = acp_command or command agent.acp_args = list(acp_args or args or []) @@ -467,6 +645,13 @@ def init_agent( # AWS Bedrock — auto-detect from provider name or base URL # (bedrock-runtime..amazonaws.com). agent.api_mode = "bedrock_converse" + elif agent.provider in {"nous", "nous-portal", "nousresearch"}: + # Portal is dual-wire: anthropic/* → Messages, everything else → + # chat_completions. Callers that already pass api_mode win above; + # this covers direct AIAgent construction without a resolved runtime. + from hermes_cli.providers import nous_api_mode + + agent.api_mode = nous_api_mode(agent.model) else: agent.api_mode = "chat_completions" @@ -586,6 +771,8 @@ def init_agent( agent._execution_thread_id: int | None = None # Set at run_conversation() start agent._interrupt_thread_signal_pending = False agent._client_lock = threading.RLock() + agent._model_request_active = threading.Event() + agent._supports_active_turn_redirect = True # /steer mechanism — inject a user note into the next tool result # without interrupting the agent. Unlike interrupt(), steer() does @@ -597,6 +784,13 @@ def init_agent( agent._pending_steer: Optional[str] = None agent._pending_steer_lock = threading.Lock() + # Active-turn redirect mechanism. A regular follow-up sent while the model + # is generating is different from a hard /stop: preserve the valid turn + # prefix, cancel only the in-flight model request, and rebuild its tail with + # the correction. The loop drains this slot at a role-safe boundary. + agent._pending_redirect: Optional[str] = None + agent._pending_redirect_lock = threading.Lock() + # Concurrent-tool worker thread tracking. `_execute_tool_calls_concurrent` # runs each tool on its own ThreadPoolExecutor worker — those worker # threads have tids distinct from `_execution_thread_id`, so @@ -636,9 +830,10 @@ def init_agent( # Anthropic prompt caching: auto-enabled for Claude models on native # Anthropic, OpenRouter, and third-party gateways that speak the # Anthropic protocol (``api_mode == 'anthropic_messages'``). Reduces - # input costs by ~75% on multi-turn conversations. Uses system_and_3 - # strategy (4 breakpoints). See ``_anthropic_prompt_cache_policy`` - # for the layout-vs-transport decision. + # input costs by ~75% on multi-turn conversations. Uses four breakpoints: + # the static system prefix, full system prompt, and last two messages + # (falling back to system-and-3 when no static prefix is available). See + # ``_anthropic_prompt_cache_policy`` for the layout-vs-transport decision. agent._use_prompt_caching, agent._use_native_cache_layout = ( agent._anthropic_prompt_cache_policy() ) @@ -763,6 +958,12 @@ def init_agent( agent._stream_writer_tls = threading.local() agent._stream_writer_dropped = 0 + # Displayed reasoning text streamed during the current model response, + # captured only when a surface consumed it via a reasoning callback. Used + # by active-turn redirect to checkpoint what the user actually saw without + # ever persisting hidden provider reasoning. + agent._current_streamed_reasoning_text = "" + # Optional current-turn user-message override used when the API-facing # user message intentionally differs from the persisted transcript # (e.g. CLI voice mode adds a temporary prefix for the live call only). @@ -869,49 +1070,20 @@ def init_agent( elif isinstance(effective_key, str) and len(effective_key) > 12: print(f"🔑 Using token: {effective_key[:8]}...{effective_key[-4:]}") elif agent.provider == "moa": - from agent.moa_loop import MoAClient + from agent.moa_loop import build_moa_facade agent.api_mode = "chat_completions" - # Route reference-model outputs to the agent's tool_progress_callback so + # build_moa_facade wires the reference relay that routes + # reference-model outputs to the agent's tool_progress_callback so # every surface that already consumes it (CLI spinner/scrollback, TUI, - # desktop, gateway) can show each reference's answer as a labelled block - # before the aggregator acts. The facade emits "moa.reference" and - # "moa.aggregating" events; we forward them through the same callback - # the tool lifecycle uses. Best-effort and cache-safe — these are - # display-only events, they never touch the message history. - def _moa_reference_relay(event: str, **kwargs: Any) -> None: - cb = getattr(agent, "tool_progress_callback", None) - if cb is None: - return - try: - if event == "moa.reference": - label = str(kwargs.get("label") or "") - text = str(kwargs.get("text") or "") - idx = kwargs.get("index") - count = kwargs.get("count") - cb( - "moa.reference", - label, - text, - None, - moa_index=idx, - moa_count=count, - ) - elif event == "moa.aggregating": - cb( - "moa.aggregating", - str(kwargs.get("aggregator") or ""), - None, - None, - moa_ref_count=kwargs.get("ref_count"), - ) - except Exception: - pass - - agent.client = MoAClient( - agent.model or "default", - reference_callback=_moa_reference_relay, - ) + # desktop, gateway) can show each reference's answer as a labelled + # block before the aggregator acts. The facade emits "moa.reference", + # "moa.progress", "moa.phase", and "moa.aggregating" events, forwarded + # through the same callback the tool lifecycle uses. Best-effort and + # cache-safe — display-only events, they never touch the message + # history. The factory is shared with the fallback-restore/recovery + # paths so a restored facade keeps emitting these events (#53802). + agent.client = build_moa_facade(agent, agent.model) agent._client_kwargs = {} agent.api_key = api_key or "moa-virtual-provider" agent.base_url = "moa://local" @@ -1177,6 +1349,13 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: print("⚠️ Warning: API key appears invalid or missing") except Exception as e: raise RuntimeError(f"Failed to initialize OpenAI client: {e}") + + # Keep a stable identity for the pool entry that supplied this runtime. + # OAuth refreshes can replace the runtime token before a failed request is + # recovered, so the mutable API-key value alone cannot reliably attribute + # the failure to its source entry. + from agent.agent_runtime_helpers import sync_credential_pool_entry_id + sync_credential_pool_entry_id(agent) # Provider fallback chain — ordered list of backup providers tried # when the primary is exhausted (rate-limit, overload, connection @@ -1321,6 +1500,9 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: # Cached system prompt -- built once per session, only rebuilt on compression agent._cached_system_prompt: Optional[str] = None + # Cross-session-stable prefix of the cached prompt. It remains separate + # from the persisted string and is used only to place an early cache marker. + agent._cached_system_prompt_static: Optional[str] = None # Filesystem checkpoint manager (transparent — not a tool) from tools.checkpoint_manager import CheckpointManager @@ -1425,7 +1607,14 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: agent._memory_nudge_interval = 10 agent._turns_since_memory = 0 agent._iters_since_skill = 0 - if not skip_memory: + # A flush/background agent may pass skip_memory=True to avoid spinning up an + # external memory *provider*, but if the caller also explicitly enables the + # "memory" toolset it still needs the built-in file-backed store — otherwise + # the memory tool dispatches with store=None and every call fails (#65429). + # So the built-in store is created unless memory is globally disabled, while + # the external-provider block below stays gated on skip_memory. + _memory_toolset_requested = "memory" in (agent.enabled_toolsets or []) + if not skip_memory or _memory_toolset_requested: try: mem_config = _agent_cfg.get("memory", {}) agent._memory_enabled = mem_config.get("memory_enabled", False) @@ -1645,6 +1834,89 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: compression_enabled = str(_compression_cfg.get("enabled", True)).lower() in {"true", "1", "yes"} compression_target_ratio = float(_compression_cfg.get("target_ratio", 0.20)) compression_protect_last = int(_compression_cfg.get("protect_last_n", 20)) + # Minimum REAL (actionable) user messages guaranteed to survive in the + # uncompressed tail (compression.min_tail_user_messages). Default 1 + # preserves current behavior exactly — the existing single-user tail + # anchor. Values > 1 extend the guarantee to the last N actionable + # user turns. Booleans rejected (bool subclasses int), non-int-like + # values fall back to 1, floor at 1. + _raw_min_tail_users = _compression_cfg.get("min_tail_user_messages", 1) + if isinstance(_raw_min_tail_users, bool): + compression_min_tail_users = 1 + elif isinstance(_raw_min_tail_users, int): + compression_min_tail_users = _raw_min_tail_users + elif isinstance(_raw_min_tail_users, float): + compression_min_tail_users = ( + int(_raw_min_tail_users) if _raw_min_tail_users.is_integer() else 1 + ) + else: + try: + compression_min_tail_users = int(str(_raw_min_tail_users).strip()) + except (TypeError, ValueError): + compression_min_tail_users = 1 + if compression_min_tail_users < 1: + compression_min_tail_users = 1 + # Cap on compression retry rounds before a turn gives up with "max + # compression attempts reached" (compression.max_attempts). Hardcoding 3 + # strands sessions that legitimately need more rounds — e.g. a restart + # history reload whose incompressible tool schemas keep the request + # estimate above the threshold even though the messages compress fine + # (the #62605 failure class). Default 3 preserves current behavior, so + # an unset key is behavior-neutral; validated >= 1, hard-capped at 10, + # and any non-int-like value falls back to 3. Booleans are rejected + # (bool subclasses int, so int(True) would silently become 1) and + # fractional floats are rejected rather than truncated — "4.7 attempts" + # is a config mistake, not a request for 4. + _raw_max_attempts = _compression_cfg.get("max_attempts", 3) + if isinstance(_raw_max_attempts, bool): + compression_max_attempts = 3 + elif isinstance(_raw_max_attempts, int): + compression_max_attempts = _raw_max_attempts + elif isinstance(_raw_max_attempts, float): + compression_max_attempts = ( + int(_raw_max_attempts) if _raw_max_attempts.is_integer() else 3 + ) + else: + try: + compression_max_attempts = int(str(_raw_max_attempts).strip()) + except (TypeError, ValueError): + compression_max_attempts = 3 + if compression_max_attempts < 1: + compression_max_attempts = 3 + compression_max_attempts = min(compression_max_attempts, 10) + + def _parse_prune_int(raw, default): + # Same parser semantics as compression.max_attempts above: reject + # booleans (bool subclasses int — YAML `true` would coerce to 1), + # reject fractional floats rather than truncating them, accept + # integral floats and numeric strings, fall back to the default on + # anything else. + if isinstance(raw, bool): + return default + if isinstance(raw, int): + return raw + if isinstance(raw, float): + return int(raw) if raw.is_integer() else default + try: + return int(str(raw).strip()) + except (TypeError, ValueError): + return default + + # Opt-in proactive tool-result prune trigger (0 = disabled — the + # default, so an unset key is behavior-neutral). Negative values are + # treated as disabled rather than erroring. + compression_proactive_prune_tokens = max( + 0, _parse_prune_int(_compression_cfg.get("proactive_prune_tokens", 0), 0) + ) + compression_proactive_prune_min_chars = _parse_prune_int( + _compression_cfg.get("proactive_prune_min_result_chars", 8000), 8000 + ) + compression_proactive_prune_min_reclaim = max( + 0, + _parse_prune_int( + _compression_cfg.get("proactive_prune_min_reclaim_tokens", 4096), 4096 + ), + ) # protect_first_n is the number of non-system messages to protect at # the head, in addition to the system prompt (which is always # implicitly protected by the compressor). Floor at 0 — a value of @@ -1657,13 +1929,40 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: compression_abort_on_summary_failure = str( _compression_cfg.get("abort_on_summary_failure", False) ).lower() in {"true", "1", "yes"} + # Per-model threshold overrides: keys are substring-matched against the + # model name (longest match wins). Empty dict = use the global threshold + # for all models (backward compatible). + _raw_model_thresholds = _compression_cfg.get("model_thresholds", {}) + if isinstance(_raw_model_thresholds, dict): + compression_model_thresholds = { + str(k): float(v) for k, v in _raw_model_thresholds.items() + if isinstance(v, (int, float)) and not isinstance(v, bool) + } + else: + compression_model_thresholds = {} + # Absolute token cap: when set, compression triggers at the lower of + # the ratio-based threshold and this absolute count. Clamped to the + # model's context length at apply-time so a cap above the window is + # a no-op (ratio-based threshold wins). + compression_threshold_tokens = _compression_cfg.get("threshold_tokens") + if compression_threshold_tokens is not None: + try: + compression_threshold_tokens = int(compression_threshold_tokens) + if compression_threshold_tokens <= 0: + compression_threshold_tokens = None + except (TypeError, ValueError): + compression_threshold_tokens = None # In-place compaction: when True, compress_context() rewrites the message # list + rebuilds the system prompt WITHOUT rotating the session id (no # parent_session_id chain, no `name #N` renumber). See #38763 and # agent/conversation_compression.py. Consumed by compress_context(), not the # compressor, so it rides on the agent. + # Default True must match DEFAULT_CONFIG["compression"]["in_place"] + # (#38763). default=False here previously flipped agents into rotation + # mode whenever the merged config omitted the key (partial configs, + # load_config failure → {}), re-arming the pre-lease drift abort. compression_in_place = is_truthy_value( - _compression_cfg.get("in_place"), default=False + _compression_cfg.get("in_place"), default=True ) codex_app_server_auto_compaction = str( _compression_cfg.get("codex_app_server_auto", "native") or "native" @@ -1675,6 +1974,12 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: codex_app_server_auto_compaction, ) codex_app_server_auto_compaction = "native" + # Opt-in idle compaction: compact a session up front when it resumes after + # this many seconds of inactivity (0 = disabled). Time-based, so it + # complements the size-based threshold above. Consumed by build_turn_context(). + compression_idle_compact_after_seconds = max( + 0, int(_compression_cfg.get("idle_compact_after_seconds", 0)) + ) # Read optional explicit context_length override for the auxiliary # compression model. Custom endpoints often cannot report this via @@ -1745,8 +2050,9 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: ) _config_context_length = None - # Resolve custom_providers list once for reuse below (startup - # context-length override and plugin context-engine init). + # Resolve custom_providers once before route-scoping a global context pin: + # a named custom provider may keep its base URL only in this list rather + # than repeating it under ``model``. try: from hermes_cli.config import get_compatible_custom_providers _custom_providers = get_compatible_custom_providers(_agent_cfg) @@ -1755,6 +2061,163 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: if not isinstance(_custom_providers, list): _custom_providers = [] + # ``model.context_length`` describes the configured default model. A + # process launched directly with ``--model`` / ``-m`` has already replaced + # ``agent.model`` before this initializer loads config, so carrying the + # default model's explicit window into that different runtime is stale. The + # live switch/fallback paths already clear this override; keep direct-start + # overrides consistent with them and let provider metadata resolve the + # active model's window instead. + if _config_context_length is not None and isinstance(_model_cfg, dict): + _configured_default_model = str(_model_cfg.get("default") or "").strip() + _configured_default_runtime_model = _configured_default_model + _active_runtime_model = agent.model + if _configured_default_model: + try: + from hermes_cli.model_normalize import normalize_model_for_provider + + _configured_default_runtime_model = normalize_model_for_provider( + _configured_default_model, agent.provider + ) + _active_runtime_model = normalize_model_for_provider( + agent.model, agent.provider + ) + except Exception: + pass + _configured_provider = str(_model_cfg.get("provider") or "").strip() + _configured_base_url = _normalize_route_base_url( + _model_cfg.get("base_url") + ) + _configured_provider_norm = _normalize_custom_provider_name( + _configured_provider + ) + _custom_provider_candidate = bool(_configured_provider_norm) + _runtime_first_provider_ids = { + "auto", + "moa", + "vertex", + "google-vertex", + "vertex-ai", + "gcp-vertex", + "vertexai", + } + if _configured_provider_norm in _runtime_first_provider_ids: + _custom_provider_candidate = False + elif ( + _custom_provider_candidate + and _configured_provider_norm != "custom" + and not _configured_provider_norm.startswith("custom:") + ): + try: + from hermes_cli.auth import resolve_provider as resolve_auth_provider + + _resolved_auth_provider = resolve_auth_provider( + _configured_provider_norm + ) + _custom_provider_candidate = ( + str(_resolved_auth_provider or "").strip().lower() + != _configured_provider_norm + ) + except Exception: + pass + if not _configured_base_url and _custom_provider_candidate: + _configured_custom_provider = _normalize_custom_provider_name( + _configured_provider + ) + _user_providers = _agent_cfg.get("providers") + _disabled_custom_provider_ids: set[str] = set() + if isinstance(_user_providers, dict): + from hermes_cli.config import is_provider_enabled + + for _provider_key, _provider_entry in _user_providers.items(): + if not isinstance(_provider_entry, dict): + continue + _entry_name = str( + _provider_entry.get("name") or "" + ).strip() + _entry_provider_ids = _custom_provider_runtime_ids( + _provider_key + ) | _custom_provider_runtime_ids(_entry_name) + if not is_provider_enabled(_provider_entry): + _disabled_custom_provider_ids.update( + provider_id + for provider_id in _entry_provider_ids + if provider_id + ) + continue + if _configured_custom_provider not in _entry_provider_ids: + continue + _configured_base_url = _normalize_route_base_url( + _provider_entry.get("api") + or _provider_entry.get("url") + or _provider_entry.get("base_url") + ) + if _configured_base_url: + break + if not _configured_base_url: + for _provider_entry in _custom_providers: + if not isinstance(_provider_entry, dict): + continue + _entry_name = str( + _provider_entry.get("name") or "" + ).strip() + _entry_provider_key = str( + _provider_entry.get("provider_key") or "" + ).strip().lower() + _entry_provider_ids = _custom_provider_runtime_ids( + _entry_name + ) | _custom_provider_runtime_ids(_entry_provider_key) + if ( + _entry_provider_key + and _custom_provider_runtime_ids(_entry_provider_key) + & _disabled_custom_provider_ids + ): + continue + if _configured_custom_provider not in _entry_provider_ids: + continue + _configured_base_url = _normalize_route_base_url( + _provider_entry.get("base_url") + ) + if _configured_base_url: + break + _active_route_url = str(agent.base_url or "") + _requested_route_url = str(base_url or "") + if "?" in _requested_route_url.split("#", 1)[0]: + try: + _requested_parts = urlparse(_requested_route_url) + _requested_without_query = urlunparse( + _requested_parts._replace(query="") + ) + if _normalize_route_base_url( + _requested_without_query + ) == _normalize_route_base_url(_active_route_url): + _active_route_url = _requested_route_url + except (TypeError, ValueError): + pass + _active_base_url = _normalize_route_base_url(_active_route_url) + _route_mismatch = _context_route_mismatch( + _configured_base_url, + _active_base_url, + _configured_provider, + agent.provider, + already_normalized=True, + ) + _model_mismatch = bool( + _configured_default_runtime_model + and _configured_default_runtime_model != _active_runtime_model + ) + if _model_mismatch or _route_mismatch: + _ra().logger.debug( + "Ignoring model.context_length=%s for startup runtime %s at %s " + "(configured default is %s at %s)", + _config_context_length, + agent.model, + _active_base_url or agent.provider, + _configured_default_model, + _configured_base_url or _model_cfg.get("provider"), + ) + _config_context_length = None + # Store for reuse by _check_compression_model_feasibility (auxiliary # compression model context-length detection needs the same list). agent._custom_providers = _custom_providers @@ -1777,11 +2240,11 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: # Surface a clear warning if the user set a context_length but it # wasn't a valid positive int — the helper silently skips those. if _config_context_length is None: - _target = agent.base_url.rstrip("/") if agent.base_url else "" + _target = _normalize_route_base_url(agent.base_url) for _cp_entry in _custom_providers: if not isinstance(_cp_entry, dict): continue - _cp_url = (_cp_entry.get("base_url") or "").rstrip("/") + _cp_url = _normalize_route_base_url(_cp_entry.get("base_url")) if _target and _cp_url == _target: _cp_models = _cp_entry.get("models", {}) if isinstance(_cp_models, dict): @@ -1894,6 +2357,16 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: provider=agent.provider, custom_providers=_custom_providers, ) + # Per-model threshold overrides are part of the explicit + # context-engine contract: assign them BEFORE the initial + # update_model() call so the first resolution (which derives + # threshold_percent/threshold_tokens for the initial model) already + # sees the overrides. Assigning after update_model() left the initial + # model on the engine's global threshold until the first /model + # switch. Engines that override update_model() own their own policy + # and may ignore the attribute. + if compression_model_thresholds: + agent.context_compressor.model_thresholds = compression_model_thresholds agent.context_compressor.update_model( model=agent.model, context_length=_plugin_ctx_len, @@ -1920,6 +2393,12 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: api_mode=agent.api_mode, abort_on_summary_failure=compression_abort_on_summary_failure, max_tokens=agent.max_tokens, + model_thresholds=compression_model_thresholds, + threshold_tokens_cap=compression_threshold_tokens, + proactive_prune_tokens=compression_proactive_prune_tokens, + proactive_prune_min_result_chars=compression_proactive_prune_min_chars, + proactive_prune_min_reclaim_tokens=compression_proactive_prune_min_reclaim, + min_tail_user_messages=compression_min_tail_users, ) _bind_session_state = getattr(agent.context_compressor, "bind_session_state", None) if callable(_bind_session_state): @@ -1930,6 +2409,10 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: agent.compression_enabled = compression_enabled agent.compression_in_place = compression_in_place agent.codex_app_server_auto_compaction = codex_app_server_auto_compaction + agent.max_compression_attempts = compression_max_attempts + agent.compression_idle_compact_after_seconds = ( + compression_idle_compact_after_seconds + ) # Reject models whose context window is below the minimum required # for reliable tool-calling workflows (64K tokens). @@ -2114,7 +2597,7 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: # autoraised model) updates the marker state and re-notifies once. The # config display gate (compression.codex_gpt55_autoraise_notice) still # suppresses the banner entirely without disabling the threshold autoraise. - _autoraise = getattr(agent, "_compression_threshold_autoraised", None) + _autoraise = getattr(agent, "_compression_threshold_autoraised", None) or {} _show_autoraise_notice = ( bool(_autoraise) and compression_enabled @@ -2130,14 +2613,21 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: _active_threshold_pct = getattr( agent.context_compressor, "threshold_percent", compression_threshold ) - print(f"📊 Context limit: {agent.context_compressor.context_length:,} tokens (compress at {int(_active_threshold_pct*100)}% = {agent.context_compressor.threshold_tokens:,})") + _cap_note = "" + _cap = getattr(agent.context_compressor, "threshold_tokens_cap", None) + if _cap and _cap > 0: + _cap_note = f" (capped at {_cap:,} tokens)" + print(f"📊 Context limit: {agent.context_compressor.context_length:,} tokens (compress at {int(_active_threshold_pct*100)}% = {agent.context_compressor.threshold_tokens:,}{_cap_note})") else: print(f"📊 Context limit: {agent.context_compressor.context_length:,} tokens (auto-compression disabled)") # Notice with the exact opt-back-out command. Printed inline at startup # for CLI users; gateway users get the same text replayed via # _compression_warning on turn 1 (set below). if _show_autoraise_notice: - print(_build_codex_gpt5_autoraise_notice(_autoraise)) + print(_build_codex_gpt5_autoraise_notice( + _autoraise, + context_length=getattr(agent.context_compressor, "context_length", None), + )) # Check immediately so CLI users see the warning at startup. # Gateway status_callback is not yet wired, so any warning is stored @@ -2147,7 +2637,10 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: # above only reaches the CLI, so stash the same text here to be replayed # through status_callback on the first turn (Telegram/Discord/Slack/etc.). if _show_autoraise_notice: - agent._compression_warning = _build_codex_gpt5_autoraise_notice(_autoraise) + agent._compression_warning = _build_codex_gpt5_autoraise_notice( + _autoraise, + context_length=getattr(agent.context_compressor, "context_length", None), + ) # Mark shown so repeated inits in this profile (e.g. every gateway message) # stay silent. Recorded once, whether the notice went to the CLI print or @@ -2170,6 +2663,7 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: agent._primary_runtime = { "model": agent.model, "provider": agent.provider, + "requested_provider": agent.requested_provider, "base_url": agent.base_url, "api_mode": agent.api_mode, "api_key": getattr(agent, "api_key", ""), diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 263cf1563a27..9fb894287a33 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -850,6 +850,25 @@ def strip_think_blocks(agent, content: str) -> str: +def sync_credential_pool_entry_id(agent) -> None: + """Rebind ``agent._credential_pool_entry_id`` from the current pool + key. + + OAuth refreshes can replace the runtime token before a failed request is + recovered, so the mutable API-key value alone cannot reliably attribute + the failure to its source entry. This resolves the stable pool-entry ID + for the agent's current ``api_key`` and clears it when no pool is bound. + """ + pool = getattr(agent, "_credential_pool", None) + try: + agent._credential_pool_entry_id = ( + pool.entry_id_for_api_key(getattr(agent, "api_key", None)) + if pool is not None + else None + ) + except Exception: + agent._credential_pool_entry_id = None + + def recover_with_credential_pool( agent, *, @@ -922,6 +941,43 @@ def recover_with_credential_pool( ) return False, has_retried_429 + # Attribute the failure to the API key the agent actually dispatched the + # request with, not to pool.current(). The current() pointer is shared, + # mutable state — round-robin select() advances it on every call, and + # concurrent turns or a second process (gateway/dashboard) reloading the + # pool reset it to None — so by the time recovery runs it routinely points + # at a DIFFERENT, healthy entry. Marking that entry exhausted copies this + # request's error/reset time onto it and can take the whole pool offline + # from a single rate-limited key (#43747). ``_swap_credential`` keeps + # ``agent.api_key`` in sync with the entry in use, so it identifies the + # failing entry exactly; fall back to current()'s key only when the agent + # carries no key at all. + _api_key_hint = getattr(agent, "api_key", None) or None + _raw_credential_id = getattr(agent, "_credential_pool_entry_id", None) + _credential_id = ( + _raw_credential_id + if isinstance(_raw_credential_id, str) and _raw_credential_id + else None + ) + if not _api_key_hint: + _cur = pool.current() + if _cur: + _api_key_hint = getattr(_cur, "runtime_api_key", None) + if not _credential_id: + _current_id = getattr(_cur, "id", None) + if isinstance(_current_id, str) and _current_id: + _credential_id = _current_id + + def _rotate_failed_credential(rotate_status: int): + kwargs = { + "status_code": rotate_status, + "error_context": error_context, + "api_key_hint": _api_key_hint, + } + if _credential_id: + kwargs["credential_id"] = _credential_id + return pool.mark_exhausted_and_rotate(**kwargs) + effective_reason = classified_reason if effective_reason is None: if status_code == 402: @@ -952,14 +1008,10 @@ def recover_with_credential_pool( if effective_reason == FailoverReason.billing: rotate_status = status_code if status_code is not None else 402 - next_entry = pool.mark_exhausted_and_rotate( - status_code=rotate_status, - error_context=error_context, - # Runtime credentials can be resolved by a separate pool instance, - # leaving this recovery pool without ``current_id``. Match the key - # that actually failed instead of quarantining a different account. - api_key_hint=getattr(agent, "api_key", None), - ) + # Runtime credentials can be resolved by a separate pool instance, + # leaving this recovery pool without ``current_id``. Match the key + # that actually failed instead of quarantining a different account. + next_entry = _rotate_failed_credential(rotate_status) if next_entry is not None: _ra().logger.info( "Credential %s (billing) — rotated to pool entry %s", @@ -975,7 +1027,21 @@ def recover_with_credential_pool( # rotate immediately. This prevents the "cancel-between-429s" trap # where has_retried_429 (a local var) gets reset on each new prompt, # causing the pool to retry the same exhausted credential forever. - current_entry = pool.current() + # Prefer the entry matching the failing key over the shared current() + # pointer, for the same attribution reason as above. + current_entry = None + if _credential_id: + current_entry = next( + (e for e in pool.entries() if e.id == _credential_id), + None, + ) + if _api_key_hint: + current_entry = current_entry or next( + (e for e in pool.entries() if e.runtime_api_key == _api_key_hint), + None, + ) + if current_entry is None: + current_entry = pool.current() current_last_status = getattr(current_entry, "last_status", None) if current_entry else None if current_last_status == STATUS_EXHAUSTED: _ra().logger.info( @@ -983,7 +1049,7 @@ def recover_with_credential_pool( current_last_status, ) rotate_status = status_code if status_code is not None else 429 - next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context) + next_entry = _rotate_failed_credential(rotate_status) if next_entry is not None: _ra().logger.info( "Credential %s (rate limit, pre-exhausted) — rotated to pool entry %s", @@ -1007,7 +1073,7 @@ def recover_with_credential_pool( if not has_retried_429 and not usage_limit_reached: return False, True rotate_status = status_code if status_code is not None else 429 - next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context) + next_entry = _rotate_failed_credential(rotate_status) if next_entry is not None: _ra().logger.info( "Credential %s (rate limit) — rotated to pool entry %s", @@ -1022,7 +1088,7 @@ def recover_with_credential_pool( # Subscription/entitlement 403s look like auth failures on the wire # but refresh cannot fix them — the OAuth token is already valid, # the account simply lacks the entitlement. Without this guard, - # ``try_refresh_current()`` keeps minting fresh tokens against the + # the refresh path keeps minting fresh tokens against the # same unsubscribed account and the main agent loop spins re-issuing # the same 403 until the user Ctrl+C's. # @@ -1075,9 +1141,16 @@ def recover_with_credential_pool( agent.provider or "provider", ) return False, has_retried_429 - refreshed = pool.try_refresh_current() + # Refresh the entry that supplied the failing key, not current(): + # the shared pointer can reference a different, healthy entry, and + # refreshing it would consume that entry's single-use refresh token + # (or mark it exhausted on failure) for a failure it never had. + refresh_kwargs = {"api_key_hint": _api_key_hint} + if _credential_id: + refresh_kwargs["credential_id"] = _credential_id + refreshed = pool.try_refresh_matching(**refresh_kwargs) if refreshed is not None: - # ``try_refresh_current()`` re-mints a fresh OAuth token and reports + # ``try_refresh_matching()`` re-mints a fresh OAuth token and reports # success even when the upstream keeps rejecting it — a single-entry # pool (common for OAuth/Max subscribers) has nothing to rotate to, # so a bare "refreshed → retry" loop spins forever on the same dead @@ -1105,9 +1178,9 @@ def recover_with_credential_pool( agent._swap_credential(refreshed) return True, has_retried_429 # Refresh failed — rotate to next credential instead of giving up. - # The failed entry is already marked exhausted by try_refresh_current(). + # The failed entry is already marked exhausted by the refresh attempt. rotate_status = status_code if status_code is not None else 401 - next_entry = pool.mark_exhausted_and_rotate(status_code=rotate_status, error_context=error_context) + next_entry = _rotate_failed_credential(rotate_status) if next_entry is not None: _ra().logger.info( "Credential %s (auth refresh failed) — rotated to pool entry %s", @@ -1148,15 +1221,29 @@ def try_recover_primary_transport( if agent._is_openrouter_url(): return False provider_lower = (agent.provider or "").strip().lower() - if provider_lower in {"nous", "nous-research"}: + # Portal OpenAI-wire traffic still rides aggregator retry infra, so one + # more rebuilt OpenAI client won't help. Portal Claude on the native + # Messages route holds a local Anthropic SDK client whose connection + # pool *does* need the rebuild every other anthropic_messages provider + # already gets — don't blanket-skip the dual-wire path. + if ( + provider_lower in {"nous", "nous-portal", "nousresearch"} + and getattr(agent, "api_mode", None) != "anthropic_messages" + ): return False try: - # Close existing client to release stale connections + # Retire the existing client to release stale connections. #70773: + # never hard-close the shared client here — this runs on the + # conversation-loop thread while workers from stale-killed streaming + # attempts may still be unwinding their SSL BIOs on the old pool. + # ``_retire_shared_openai_client`` shuts the sockets down (FD-safe + # from any thread) and defers the FD release to GC, which cannot + # complete until every borrowing thread has unwound. if getattr(agent, "client", None) is not None: try: - agent._close_openai_client( - agent.client, reason="primary_recovery", shared=True, + agent._retire_shared_openai_client( + agent.client, reason="primary_recovery", ) except Exception: pass @@ -1166,6 +1253,7 @@ def try_recover_primary_transport( agent._client_kwargs = dict(rt["client_kwargs"]) agent.model = rt["model"] agent.provider = rt["provider"] + agent.requested_provider = rt.get("requested_provider", agent.provider) agent.base_url = rt["base_url"] agent.api_mode = rt["api_mode"] if hasattr(agent, "_transport_cache"): @@ -1182,6 +1270,14 @@ def try_recover_primary_transport( ) agent._is_anthropic_oauth = rt["is_anthropic_oauth"] agent.client = None + elif (agent.provider or "").strip().lower() == "moa": + # MoA is a virtual provider with empty client_kwargs — rebuilding + # via _create_openai_client would raise "api_key client option + # must be set". Recreate the facade through the shared factory so + # the reference_callback relay survives recovery (#53802). + from agent.moa_loop import build_moa_facade + + agent.client = build_moa_facade(agent, agent.model) else: agent.client = agent._create_openai_client( dict(rt["client_kwargs"]), @@ -1329,6 +1425,7 @@ def restore_primary_runtime(agent) -> bool: # ── Core runtime state ── agent.model = rt["model"] agent.provider = rt["provider"] + agent.requested_provider = rt.get("requested_provider", agent.provider) agent.base_url = rt["base_url"] # setter updates _base_url_lower agent.api_mode = rt["api_mode"] if hasattr(agent, "_transport_cache"): @@ -1344,7 +1441,18 @@ def restore_primary_runtime(agent) -> bool: ) # ── Rebuild client for the primary provider ── - if agent.api_mode == "anthropic_messages": + if agent.provider == "moa": + # MoA is a virtual chat-completions provider. It never has real + # OpenAI client kwargs; restoring it after a fallback must recreate + # the facade, not call OpenAI() with an empty api_key. Use the + # shared factory so the restored facade keeps the reference_callback + # relay wired at init — a bare MoAClient() would silently stop + # emitting moa.reference/moa.aggregating display events (#53802). + from agent.moa_loop import build_moa_facade + + agent.client = build_moa_facade(agent, agent.model) + agent._anthropic_client = None + elif agent.api_mode == "anthropic_messages": from agent.anthropic_adapter import build_anthropic_client agent._anthropic_api_key = rt["anthropic_api_key"] agent._anthropic_base_url = rt["anthropic_base_url"] @@ -1397,6 +1505,7 @@ def restore_primary_runtime(agent) -> bool: pool_matches_primary = False if pool is not None and pool_provider and not pool_matches_primary: agent._credential_pool = None + agent._credential_pool_entry_id = None try: from agent.credential_pool import load_pool @@ -1416,6 +1525,7 @@ def restore_primary_runtime(agent) -> bool: # the pool for its current best entry and swap the live credential in. # When the pool is absent, empty, or the entry has no usable key, we # keep the snapshot key (the existing behavior). Fixes #25205. + agent._credential_pool_entry_id = None pool = getattr(agent, "_credential_pool", None) if pool is not None and pool.has_available(): entry = pool.select() @@ -1793,7 +1903,15 @@ def anthropic_prompt_cache_policy( if is_native_anthropic: return True, True - if (is_openrouter or is_nous_portal) and (is_claude or is_kimi): + # Envelope layout is an OpenAI-wire construct. Portal Claude on the native + # Messages route must fall through to the third-party anthropic_messages + # branch below, which emits inner-block cache_control breakpoints; the + # envelope form would be dropped and serve 0% cache hits. + if ( + (is_openrouter or is_nous_portal) + and (is_claude or is_kimi) + and not is_anthropic_wire + ): return True, False # Nous Portal Qwen (e.g. qwen3.6-plus) takes the same envelope-layout # cache_control path as Portal Claude. Portal proxies to OpenRouter @@ -1957,8 +2075,11 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo from hermes_cli.providers import determine_api_mode # ── Determine api_mode if not provided ── + # Pass model so dual-wire providers (Nous Portal anthropic/* → Messages) + # resolve correctly; without it determine_api_mode falls back to the + # openai_chat overlay default. if not api_mode: - api_mode = determine_api_mode(new_provider, base_url) + api_mode = determine_api_mode(new_provider, base_url, model=new_model) # Defense-in-depth: ensure OpenCode base_url doesn't carry a trailing # /v1 into the anthropic_messages client, which would cause the SDK to @@ -1993,6 +2114,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo for name in ( "model", "provider", + "requested_provider", "base_url", "api_mode", "api_key", @@ -2011,6 +2133,9 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo # restore the original pool (issue #52727: pool reload is part of this # switch and must be reversible on rollback). _snapshot["_credential_pool"] = getattr(agent, "_credential_pool", _MISSING) + _snapshot["_credential_pool_entry_id"] = getattr( + agent, "_credential_pool_entry_id", _MISSING + ) try: # Clear the per-config context_length override so the new model's @@ -2021,6 +2146,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo # ── Swap core runtime fields ── agent.model = new_model agent.provider = new_provider + agent.requested_provider = new_provider # Use the new base_url when provided. When it's empty AND the # provider is actually changing, do NOT fall back to the current # (old provider's) URL — that silently pairs the new provider label @@ -2066,6 +2192,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo # A pool bound to the old provider is worse than no pool: the # recovery guard rejects it and every later 401/429 skips rotation. agent._credential_pool = None + agent._credential_pool_entry_id = None try: from agent.credential_pool import load_pool agent._credential_pool = load_pool(new_provider) @@ -2075,10 +2202,9 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo "continuing without pool rotation this turn", new_provider, _pool_exc, ) - # ── Build new client ── if (new_provider or "").strip().lower() == "moa": - from agent.moa_loop import MoAClient + from agent.moa_loop import build_moa_facade # The MoA virtual provider speaks only chat.completions via the # MoAClient facade — the aggregator's real transport @@ -2095,7 +2221,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo agent.api_key = api_key or "moa-virtual-provider" agent.base_url = "moa://local" agent._client_kwargs = {} - agent.client = MoAClient(agent.model or "default") + agent.client = build_moa_facade(agent, agent.model) elif api_mode == "anthropic_messages": from agent.anthropic_adapter import ( build_anthropic_client, @@ -2171,6 +2297,8 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo reason="switch_model", shared=True, ) + + sync_credential_pool_entry_id(agent) except Exception: # Rollback every mutated field to the pre-swap snapshot so the agent # is left consistent (old model + old provider + old client) and the @@ -2270,6 +2398,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo agent._primary_runtime = { "model": agent.model, "provider": agent.provider, + "requested_provider": agent.requested_provider, "base_url": agent.base_url, "api_mode": agent.api_mode, "api_key": getattr(agent, "api_key", ""), @@ -2504,6 +2633,7 @@ def _execute(next_args: dict) -> Any: _clarify_tool( question=next_args.get("question", ""), choices=next_args.get("choices"), + multi_select=next_args.get("multi_select", False), callback=agent.clarify_callback, ), next_args, diff --git a/agent/anthropic_adapter.py b/agent/anthropic_adapter.py index a7f13ba2d777..0d59d94c9c32 100644 --- a/agent/anthropic_adapter.py +++ b/agent/anthropic_adapter.py @@ -23,7 +23,7 @@ from hermes_constants import get_hermes_home from typing import Any, Dict, List, Optional, Tuple -from utils import base_url_host_matches, normalize_proxy_env_vars +from utils import base_url_host_matches, base_url_hostname, normalize_proxy_env_vars # NOTE: `import anthropic` is deliberately NOT at module top — the SDK pulls # ~220 ms of imports (anthropic.types, anthropic.lib.tools._beta_runner, etc.) @@ -368,7 +368,7 @@ def _detect_claude_code_version() -> str: try: result = _sp.run( [cmd, "--version"], - capture_output=True, text=True, timeout=5, + capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=5, ) if result.returncode == 0 and result.stdout.strip(): # Output is like "2.1.74 (Claude Code)" or just "2.1.74" @@ -546,15 +546,49 @@ def _is_deepseek_anthropic_endpoint(base_url: str | None) -> bool: return "/anthropic" in normalized.rstrip("/").lower() +def _is_nous_portal_endpoint(base_url: str | None) -> bool: + """Return True for Nous Portal's Anthropic Messages route. + + Portal serves its ``anthropic/*`` catalog natively at + ``https://inference-api.nousresearch.com/v1/messages``. Portal-specific + behaviours key off this: Bearer JWT auth, verbatim catalog model ids, + and native thinking-signature replay. + + Trusted hosts only: + + 1. Prod hostname ``inference-api.nousresearch.com`` + 2. The operator-set ``NOUS_INFERENCE_BASE_URL`` hostname (staging/preview) + + Lookalikes such as ``inference-api.nousresearch.com.attacker.test`` are + rejected (hostname match, not substring). + """ + if base_url_host_matches(base_url or "", "inference-api.nousresearch.com"): + return True + try: + from hermes_cli.auth import _nous_inference_env_override + + override = _nous_inference_env_override() + except Exception: + return False + if not override: + return False + # Exact host equality (not subdomain) so the env override can't broaden + # into sibling hosts the operator did not set. + override_host = base_url_hostname(override) + return bool(override_host) and base_url_hostname(base_url or "") == override_host + + def _requires_bearer_auth(base_url: str | None) -> bool: """Return True for Anthropic-compatible providers that require Bearer auth. Some third-party /anthropic endpoints implement Anthropic's Messages API but require Authorization: Bearer instead of Anthropic's native x-api-key header. MiniMax's global and China Anthropic-compatible endpoints, Azure AI - Foundry's Anthropic-style endpoint, and Palantir Foundry's LLM proxy - follow this pattern. + Foundry's Anthropic-style endpoint, Palantir Foundry's LLM proxy, and Nous + Portal's Messages route follow this pattern. """ + if _is_nous_portal_endpoint(base_url): + return True normalized = _normalize_base_url_text(base_url) if not normalized: return False @@ -721,7 +755,11 @@ def _build_anthropic_client_with_bearer_hook( if common_betas: kwargs["default_headers"] = {"anthropic-beta": ",".join(common_betas)} - return _anthropic_sdk.Anthropic(**kwargs) + client = _anthropic_sdk.Anthropic(**kwargs) + # Same env-inference trap as build_anthropic_client: auth_token-only + # construction would otherwise also send ANTHROPIC_API_KEY as X-Api-Key. + client.api_key = None + return client def build_anthropic_client( @@ -850,7 +888,16 @@ def build_anthropic_client( if common_betas: kwargs["default_headers"] = {"anthropic-beta": ",".join(common_betas)} - return _anthropic_sdk.Anthropic(**kwargs) + client = _anthropic_sdk.Anthropic(**kwargs) + # Bearer-only construction leaves ``api_key`` unset, so the SDK fills it + # from ``ANTHROPIC_API_KEY`` (Hermes loads that into the process env from + # ``~/.hermes/.env``). The result is dual auth — + # ``X-Api-Key: sk-ant-…`` *and* ``Authorization: Bearer `` — + # on every Portal / MiniMax / OAuth Messages request. Clear the env-filled + # key whenever we intentionally authenticated via auth_token alone. + if "auth_token" in kwargs and "api_key" not in kwargs: + client.api_key = None + return client def build_anthropic_bedrock_client(region: str): @@ -914,7 +961,7 @@ def _read_claude_code_credentials_from_keychain() -> Optional[Dict[str, Any]]: "-s", "Claude Code-credentials", "-w"], capture_output=True, - text=True, + text=True, encoding='utf-8', errors='replace', timeout=5, stdin=subprocess.DEVNULL, ) @@ -1881,6 +1928,28 @@ def _content_parts_to_anthropic_blocks(parts: Any) -> List[Dict[str, Any]]: return out +_EMPTY_TEXT_PLACEHOLDER = "(empty)" + + +def _safe_text(text: Any) -> str: + """Return ``text`` if it's non-whitespace, else a non-whitespace placeholder. + + The Anthropic Messages API rejects requests where a text content block is + empty or whitespace-only (HTTP 400 "text content blocks must contain + non-whitespace text"). When such a block gets stored in session history — + e.g. produced by context compression — it is replayed verbatim on every + subsequent turn, permanently wedging the session. Coercing to a + non-whitespace placeholder is self-healing: the next API call recovers. + + Mirrors ``bedrock_adapter._safe_text`` (#9486); ref #69512. + """ + if text is None: + return _EMPTY_TEXT_PLACEHOLDER + if not isinstance(text, str): + text = str(text) + return text if text.strip() else _EMPTY_TEXT_PLACEHOLDER + + def _sanitize_replay_block(b: Dict[str, Any]) -> Optional[Dict[str, Any]]: """Strip output-only fields from a stored Anthropic content block so it is valid as REQUEST input on replay. @@ -1898,7 +1967,18 @@ def _sanitize_replay_block(b: Dict[str, Any]) -> Optional[Dict[str, Any]]: return None btype = b.get("type") if btype == "text": - out: Dict[str, Any] = {"type": "text", "text": b.get("text", "")} + text_val = b.get("text", "") + # Bedrock and strict Anthropic-compatible endpoints reject text + # blocks where "text" is empty or whitespace-only (#69512). Drop the + # blank block (the caller relocates any cache_control it carried and + # falls back to a non-whitespace placeholder when nothing survives) + # rather than coercing in place — a coerced "(empty)" block would be + # model-visible noise next to surviving thinking/tool_use blocks. + # Type-safe: captured blocks can carry text=None from an invalid + # upstream payload, which a bare .strip() would crash on. + if not isinstance(text_val, str) or not text_val.strip(): + return None + out: Dict[str, Any] = {"type": "text", "text": text_val} # citations is input-valid ONLY when it's a non-empty list; the SDK # emits citations=None on responses, which the input schema rejects. cits = b.get("citations") @@ -1986,9 +2066,17 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]: parsed_args = {} redacted_input_by_id[_sanitize_tool_id(tc.get("id", ""))] = parsed_args replayed: List[Dict[str, Any]] = [] + _relocated_replay_cache_control = None + _dropped_blank_text = False for b in ordered_blocks: clean = _sanitize_replay_block(b) if clean is None: + if isinstance(b, dict) and b.get("type") == "text": + _dropped_blank_text = True + if isinstance(b, dict) and isinstance(b.get("cache_control"), dict): + # A dropped blank text block can still carry the cache + # breakpoint marker -- relocate it rather than losing it. + _relocated_replay_cache_control = b["cache_control"] continue if clean.get("type") == "tool_use": # Override raw (un-redacted) input with the redacted copy when @@ -1998,20 +2086,90 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]: if redacted is not None: clean["input"] = redacted replayed.append(clean) + # When every text block was blank and nothing cacheable survived + # (e.g. signed thinking + a blank text block, or a SOLE blank + # cache-marked block), emit the non-whitespace placeholder so the + # replayed message stays schema-valid (#69512) and a relocated cache + # marker still has a carrier instead of being silently lost. + _has_cacheable_replay = any( + isinstance(b, dict) and b.get("type") in {"text", "tool_use"} + for b in replayed + ) + if not _has_cacheable_replay and ( + _dropped_blank_text or _relocated_replay_cache_control is not None + ): + replayed.append({"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}) if replayed: + if _relocated_replay_cache_control is not None: + _apply_assistant_cache_control_to_last_cacheable_block( + replayed, _relocated_replay_cache_control + ) _apply_assistant_cache_control_to_last_cacheable_block( replayed, m.get("cache_control") ) + # apply_anthropic_cache_control marks an assistant turn with + # non-empty text by writing cache_control INTO ``content`` (see + # _apply_cache_marker's list branch), not at the top level. This + # branch rebuilds the message from ordered_blocks and never reads + # ``content``, so that marker would be dropped -- and because + # _can_carry_marker already counted this message as a carrier, the + # breakpoint is burned rather than relocated. #56195 covered the + # complementary shape (blank content -> top-level marker); this is + # the interleaved thinking + preamble-text + tool_use shape. + _inline_cc = None + _msg_content = m.get("content") + if isinstance(_msg_content, list): + for _blk in _msg_content: + if isinstance(_blk, dict) and isinstance( + _blk.get("cache_control"), dict + ): + _inline_cc = _blk["cache_control"] + break + if _inline_cc is not None: + _apply_assistant_cache_control_to_last_cacheable_block( + replayed, _inline_cc + ) return {"role": "assistant", "content": replayed} blocks = _extract_preserved_thinking_blocks(m) + # Cache markers dropped along with a blank block are relocated onto the + # last surviving cacheable block below (via + # _apply_assistant_cache_control_to_last_cacheable_block), rather than + # lost -- prompt_caching.py's _apply_cache_marker() sets cache_control + # directly on content[-1] for list content, so if that last part happens + # to be blank text, dropping it silently would lose the breakpoint. + _relocated_cache_control = None if content: if isinstance(content, list): converted_content = _convert_content_to_anthropic(content) if isinstance(converted_content, list): - blocks.extend(converted_content) + # Bedrock and strict Anthropic-compatible endpoints reject + # text blocks where "text" is empty or whitespace-only. The + # ordered-replay path enforces the same invariant via + # _sanitize_replay_block(). Type-safe against ANY invalid + # "text" value from an upstream payload -- None, or a + # truthy non-string like an int -- not just None: checking + # isinstance() first (rather than `blk.get("text") or ""`) + # means a non-string value is treated as blank/invalid + # instead of reaching .strip() and raising AttributeError. + for blk in converted_content: + _blk_text = blk.get("text") if isinstance(blk, dict) else None + if ( + isinstance(blk, dict) + and blk.get("type") == "text" + and (not isinstance(_blk_text, str) or not _blk_text.strip()) + ): + if isinstance(blk.get("cache_control"), dict): + _relocated_cache_control = blk["cache_control"] + continue + blocks.append(blk) else: - blocks.append({"type": "text", "text": str(content)}) + # Scalar (non-list) content: a whitespace-only string is the + # same invalid-payload case as an empty list block -- drop it + # rather than emitting a blank text block. + text_str = str(content) + if text_str.strip(): + blocks.append({"type": "text", "text": text_str}) for tc in m.get("tool_calls", []): if not tc or not isinstance(tc, dict): continue @@ -2027,9 +2185,6 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]: "name": fn.get("name", ""), "input": parsed_args, }) - _apply_assistant_cache_control_to_last_cacheable_block( - blocks, m.get("cache_control") - ) # Kimi's /coding endpoint (Anthropic protocol) requires assistant # tool-call messages to carry reasoning_content when thinking is # enabled server-side. Preserve it as a thinking block so Kimi @@ -2055,10 +2210,26 @@ def _convert_assistant_message(m: Dict[str, Any]) -> Dict[str, Any]: ) if isinstance(reasoning_content, str) and not _already_has_thinking: blocks.insert(0, {"type": "thinking", "thinking": reasoning_content}) - # Anthropic rejects empty assistant content - effective = blocks or content - if not effective or effective == "": - effective = [{"type": "text", "text": "(empty)"}] + # Anthropic rejects empty assistant content. IMPORTANT: fall back only + # to the placeholder, never to the raw `content` variable -- `content` + # is the UNFILTERED original message content, and can itself be exactly + # the blank/whitespace-only payload the filtering above just removed + # (a sole blank text block, or scalar whitespace with no tool_calls). + # `blocks or content` there would silently restore the invalid provider + # payload this function exists to prevent (#69512). + effective = blocks if blocks else [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}] + # Applied here (after the empty-fallback resolution) rather than + # earlier against `blocks` directly, so a cache_control relocated from + # a dropped blank block that was the ONLY block still lands on the + # (empty) placeholder instead of being silently lost when blocks was + # empty at the point the marker would otherwise have been applied. + if _relocated_cache_control is not None: + _apply_assistant_cache_control_to_last_cacheable_block( + effective, _relocated_cache_control + ) + _apply_assistant_cache_control_to_last_cacheable_block( + effective, m.get("cache_control") + ) return {"role": "assistant", "content": effective} @@ -2292,10 +2463,22 @@ def _manage_thinking_signatures( replayed assistant tool-call messages. See hermes-agent#13848 (Kimi) and hermes-agent#16748 (DeepSeek). + Nous Portal's ``/v1/messages`` route is the exception among third-party + hosts: it proxies Claude to Anthropic/Vertex/Bedrock and validates the + same signed thinking blocks. Sticky ``session_id`` keeps a conversation + on one upstream instance so those signatures stay warm — stripping them + here would 400 the first tool-loop turn ("thinking must be passed back"). + Portal therefore takes the native Anthropic replay path below. + Mutates ``result`` in place. """ _THINKING_TYPES = frozenset(("thinking", "redacted_thinking")) - _is_third_party = _is_third_party_anthropic_endpoint(base_url) + # Portal speaks Anthropic's thinking contract end-to-end; do not treat it + # as a signature-blind proxy even though the host is not anthropic.com. + _is_third_party = ( + _is_third_party_anthropic_endpoint(base_url) + and not _is_nous_portal_endpoint(base_url) + ) last_assistant_idx = None for i in range(len(result) - 1, -1, -1): @@ -2412,6 +2595,24 @@ def _evict_old_screenshots(result: List[Dict[str, Any]]) -> None: ] +def _ensure_leading_user_turn(result: List[Dict[str, Any]]) -> None: + """Anthropic requires messages[0] to have role=user. + + After a second context compaction on the auto path the summary can be + emitted as role=assistant with nothing in front of it (the system prompt + lives outside messages[] or is extracted into the separate ``system`` + param), so messages[0] ends up assistant and the Messages API rejects + the request with HTTP 400 — often masked by a misleading + "tool_use ids were found without tool_result blocks" error (#52160). + + Mirror the Bedrock Converse adapter, which unconditionally prepends a + minimal user turn when the first message is not user + (convert_messages_to_converse). + """ + if result and result[0].get("role") != "user": + result.insert(0, {"role": "user", "content": [{"type": "text", "text": " "}]}) + + def convert_messages_to_anthropic( messages: List[Dict], base_url: str | None = None, @@ -2470,6 +2671,7 @@ def convert_messages_to_anthropic( _strip_orphaned_tool_blocks(result) result = _merge_consecutive_roles(result) + _ensure_leading_user_turn(result) _manage_thinking_signatures(result, base_url, model) _evict_old_screenshots(result) @@ -2533,7 +2735,12 @@ def build_anthropic_kwargs( ) anthropic_tools = convert_tools_to_anthropic(tools) if tools else [] - model = normalize_model_name(model, preserve_dots=preserve_dots) + # Nous Portal routes on its own catalog ids (``anthropic/claude-opus-4.8``); + # normalizing to the bare Anthropic slug would make the model unresolvable + # there. Skipping the call preserves the prefix AND the dots, so + # ``preserve_dots`` stays irrelevant for Portal. + if not _is_nous_portal_endpoint(base_url): + model = normalize_model_name(model, preserve_dots=preserve_dots) # effective_max_tokens = output cap for this call (≠ total context window) # Use the resolver helper so non-positive values (negative ints, # fractional floats, NaN, non-numeric) fail locally with a clear error @@ -2772,6 +2979,8 @@ def create_anthropic_message( *, log_prefix: str = "", prefer_stream: bool = True, + on_stream_event=None, + on_response=None, ) -> Any: """Create an Anthropic message, aggregating via stream when available. @@ -2781,6 +2990,20 @@ def create_anthropic_message( crash on ``.content``. Prefer ``messages.stream().get_final_message()`` to match the main turn path, falling back to ``create()`` only for providers that explicitly do not support streaming, such as restricted Bedrock roles. + + ``on_stream_event``: optional callable invoked once per streamed event + (best-effort, exceptions swallowed). Lets callers report forward progress + to liveness watchdogs — e.g. the auxiliary compression path ticking its + progress hook so a slow-but-generating summary model isn't treated as + hung. Only fires on the streaming path; the ``create()`` fallback has no + events to report. + + ``on_response``: optional callable invoked once with the underlying httpx + response before the message is aggregated (best-effort, exceptions + swallowed). Response *headers* carry out-of-band provider state that the + parsed ``Message`` drops — Nous Portal's ``x-nous-credits-*`` balance family + in particular. Only fires on the streaming path, which is the one the main + turn loop takes. """ sanitize_anthropic_kwargs(api_kwargs, log_prefix=log_prefix) @@ -2791,6 +3014,26 @@ def create_anthropic_message( stream_kwargs.pop("stream", None) try: with stream_fn(**stream_kwargs) as stream: + if callable(on_response): + try: + on_response(getattr(stream, "response", None)) + except Exception: + logger.debug( + "%son_response callback failed", + log_prefix, exc_info=True, + ) + if callable(on_stream_event): + # Consume the event stream manually so each event can + # tick the caller's progress callback; get_final_message + # then returns the accumulated snapshot. + for _event in stream: + try: + on_stream_event(_event) + except Exception: + logger.debug( + "%son_stream_event callback failed", + log_prefix, exc_info=True, + ) return stream.get_final_message() except Exception as exc: if not _is_stream_unavailable_error(exc): diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index da93f94486fd..b1e5df6b6338 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -247,6 +247,50 @@ def aux_interrupt_protection(active: bool = True): _aux_interrupt_protection.active = prev +# ── Forward-progress hook for streamed auxiliary calls ─────────────────── +# Long auxiliary calls (context compression is the prime case) are watched by +# wall-clock deadlines in their hosts (gateway session hygiene). A fixed +# deadline punishes SLOW summary models exactly as hard as HUNG ones: a +# reasoning model happily streaming a large summary is killed mid-generation. +# This thread-local hook lets the host observe liveness instead: the wire +# consumers below tick it on every streamed token/SSE event, and the host +# extends its deadline while tokens are moving (see gateway/run.py session +# hygiene + CompressionCommitFence.touch_progress). Thread-local matches the +# call topology — the aux call and its stream consumption run synchronously +# on the thread that installed the hook. +_aux_progress = threading.local() + + +def _notify_aux_progress() -> None: + """Tick the installed forward-progress hook, if any. Never raises.""" + hook = getattr(_aux_progress, "hook", None) + if hook is None: + return + try: + hook() + except Exception: + logger.debug("aux progress hook failed", exc_info=True) + + +def _aux_progress_active() -> bool: + return getattr(_aux_progress, "hook", None) is not None + + +@contextlib.contextmanager +def aux_progress_hook(hook): + """Install *hook* as the current thread's aux forward-progress callback. + + ``hook=None`` is a no-op passthrough so callers can wire it + unconditionally. Re-entrant-safe: restores the previous hook on exit. + """ + prev = getattr(_aux_progress, "hook", None) + _aux_progress.hook = hook if callable(hook) else prev + try: + yield + finally: + _aux_progress.hook = prev + + def _safe_isinstance(obj: Any, maybe_type: Any) -> bool: """Return False instead of raising when a patched symbol is not a type.""" try: @@ -1058,16 +1102,29 @@ def create(self, **kwargs) -> Any: # key in extra_body (not top-level) and GitHub/Copilot Responses opts # out of cache-key routing entirely — for those hosts, skip it here. try: - from agent.transports.codex import _content_cache_key + from agent.transports.codex import ( + _content_cache_key, + _default_prompt_cache_retention_for_request, + ) from utils import base_url_host_matches _host_src = str(getattr(self._client, "base_url", "") or "") _is_xai = base_url_host_matches(_host_src, "x.ai") or base_url_host_matches(_host_src, "api.x.ai") - _is_github = base_url_host_matches(_host_src, "githubcopilot.com") + _is_github = ( + base_url_host_matches(_host_src, "githubcopilot.com") + or base_url_host_matches(_host_src, "models.github.ai") + ) if not _is_xai and not _is_github and "prompt_cache_key" not in resp_kwargs: _cache_key = _content_cache_key(instructions, resp_kwargs.get("tools")) if _cache_key: resp_kwargs["prompt_cache_key"] = _cache_key + if "prompt_cache_retention" not in resp_kwargs: + _cache_retention = _default_prompt_cache_retention_for_request( + model, + _host_src, + ) + if _cache_retention: + resp_kwargs["prompt_cache_retention"] = _cache_retention except Exception: logger.debug( "Codex auxiliary: prompt_cache_key derivation skipped", exc_info=True @@ -1149,6 +1206,10 @@ def _check_cancelled() -> None: def _on_each_event(_event: Any) -> None: # Re-check timeout/cancellation per event, matching the # cadence the old in-line ``_check_cancelled()`` used. + # Each SSE event is also forward progress for hosts watching + # a progress hook (gateway session hygiene): a reasoning + # model streaming a long summary must not look hung. + _notify_aux_progress() _check_cancelled() event_stream = self._client.responses.create(**stream_kwargs) @@ -1301,10 +1362,32 @@ def __init__(self, sync_wrapper: "CodexAuxiliaryClient"): class _AnthropicCompletionsAdapter: """OpenAI-client-compatible adapter for Anthropic Messages API.""" - def __init__(self, real_client: Any, model: str, is_oauth: bool = False): + def __init__( + self, + real_client: Any, + model: str, + is_oauth: bool = False, + base_url: str | None = None, + ): self._client = real_client self._model = model self._is_oauth = is_oauth + # Prefer the caller-supplied URL (AnthropicAuxiliaryClient keeps the + # pre-strip Portal ``.../v1`` form). Only fall back to the SDK + # client's host for Nous Portal — a blanket fallback would flip + # MiniMax/Zhipu/etc. aux adapters from "unknown host = native + # Anthropic" to third-party (stripping thinking signatures). + self._base_url = base_url or None + if not self._base_url: + candidate = str(getattr(real_client, "base_url", "") or "") or None + if candidate: + try: + from agent.anthropic_adapter import _is_nous_portal_endpoint + + if _is_nous_portal_endpoint(candidate): + self._base_url = candidate + except Exception: + pass def create(self, **kwargs) -> Any: from agent.anthropic_adapter import build_anthropic_kwargs, create_anthropic_message @@ -1356,6 +1439,11 @@ def create(self, **kwargs) -> Any: reasoning_config=_reasoning_cfg, tool_choice=normalized_tool_choice, is_oauth=self._is_oauth, + # Portal routes on ``anthropic/`` catalog ids and replays + # signed thinking like native Anthropic; both carve-outs key off + # base_url. Omitting it normalizes the id to a bare Anthropic + # slug and the Portal Messages route cannot resolve it. + base_url=self._base_url, ) # Opus 4.7+ rejects any non-default temperature/top_p/top_k; only set # temperature for models that still accept it. build_anthropic_kwargs @@ -1390,7 +1478,18 @@ def create(self, **kwargs) -> Any: existing = {} anthropic_kwargs["extra_body"] = {**existing, **passthrough} - response = create_anthropic_message(self._client, anthropic_kwargs) + response = create_anthropic_message( + self._client, + anthropic_kwargs, + # Tick the aux forward-progress hook per streamed event so hosts + # watching liveness (gateway session hygiene) don't kill a + # slow-but-generating summary model. No-op when no hook is + # installed (None keeps the fast get_final_message path). + on_stream_event=( + (lambda _event: _notify_aux_progress()) + if _aux_progress_active() else None + ), + ) _transport = get_transport("anthropic_messages") _nr = _transport.normalize_response( response, strip_tool_prefix=self._is_oauth @@ -1438,7 +1537,9 @@ class AnthropicAuxiliaryClient: def __init__(self, real_client: Any, model: str, api_key: str, base_url: str, is_oauth: bool = False): self._real_client = real_client - adapter = _AnthropicCompletionsAdapter(real_client, model, is_oauth=is_oauth) + adapter = _AnthropicCompletionsAdapter( + real_client, model, is_oauth=is_oauth, base_url=base_url, + ) self.chat = _AnthropicChatShim(adapter) self.api_key = api_key self.base_url = base_url @@ -1703,7 +1804,7 @@ def _read_nous_auth() -> Optional[dict]: try: if not _AUTH_JSON_PATH.is_file(): return None - data = json.loads(_AUTH_JSON_PATH.read_text()) + data = json.loads(_AUTH_JSON_PATH.read_text(encoding="utf-8")) if data.get("active_provider") != "nous": return None provider = data.get("providers", {}).get("nous", {}) @@ -2302,6 +2403,62 @@ def _read_main_base_url() -> str: return "" +def _resolve_moa_aggregator(preset_name: Optional[str]) -> Tuple[Optional[str], Optional[str]]: + """Resolve a MoA preset to its aggregator (provider, model) pair. + + "moa" is a virtual provider — the acting model of a preset is its + aggregator slot, and there is no real "moa" HTTP endpoint. Auxiliary + tasks (title generation, compression, vision, commit messages, …) don't + need the reference fan-out, so every aux resolution layer maps + provider="moa"/model= to the aggregator's real provider+model + through this single helper (shared by ``_resolve_auto``, + ``_resolve_task_provider_model``, and ``resolve_provider_client`` so the + preset lookup and validation cannot drift between paths). + + Args: + preset_name: The MoA preset name (usually carried in the "model" + field), or None/"" to resolve the user's default preset. + + Returns: + (aggregator_provider, aggregator_model), or (None, None) when the + preset cannot be resolved (missing config, renamed/deleted preset, + or a malformed aggregator slot). + """ + try: + from hermes_cli.config import load_config + from hermes_cli.moa_config import resolve_moa_preset + + preset = resolve_moa_preset(load_config().get("moa") or {}, preset_name or None) + agg = preset.get("aggregator") or {} + agg_provider = str(agg.get("provider") or "").strip() + agg_model = str(agg.get("model") or "").strip() + if agg_provider and agg_model and agg_provider.lower() != "moa": + return agg_provider, agg_model + except Exception: + logger.debug( + "MoA aggregator resolution failed for preset %r", preset_name, exc_info=True + ) + return None, None + + +def _read_main_model_for_aux() -> str: + """Main model with MoA presets unwrapped to the aggregator's model. + + When the main provider is ``moa``, ``_read_main_model()`` returns a MoA + *preset name* (e.g. "opus-gpt") — never a valid wire model id on any + provider. Auxiliary fallback chains that pre-fill a missing model from + the main model must use this reader instead, so unset aux models default + to the preset's acting (aggregator) model. Returns "" when the main + provider is moa but the preset cannot be resolved — sending nothing is + strictly better than sending a preset name that 400s. + """ + model = _read_main_model() + if (_read_main_provider() or "").strip().lower() == "moa": + _, agg_model = _resolve_moa_aggregator(model) + return agg_model or "" + return model + + def _read_main_api_key_if_same_host(aux_base_url: str) -> str: """Return the main api_key only when *aux_base_url* points at the same host as the main model's base_url. @@ -2377,6 +2534,7 @@ def set_runtime_main( provider: str, model: str, *, + requested_provider: str = "", base_url: str = "", api_key: Any = "", api_mode: str = "", @@ -2392,6 +2550,7 @@ def set_runtime_main( global _RUNTIME_MAIN_AUTH_MODE, _RUNTIME_MAIN_COMPAT_SNAPSHOT runtime = { "provider": (provider or "").strip().lower(), + "requested_provider": (requested_provider or "").strip().lower(), "model": (model or "").strip(), "base_url": (base_url or "").strip(), "api_key": ( @@ -2572,7 +2731,7 @@ def _try_custom_endpoint() -> Tuple[Optional[Any], Optional[str]]: return None, None if custom_base.lower().startswith(_CODEX_AUX_BASE_URL.lower()): return None, None - model = _read_main_model() or "gpt-4o-mini" + model = _read_main_model_for_aux() or "gpt-4o-mini" logger.debug("Auxiliary client: custom endpoint (%s, api_mode=%s)", model, custom_mode or "chat_completions") _clean_base, _dq = _extract_url_query_params(custom_base) _extra = {"default_query": _dq} if _dq else {} @@ -2864,6 +3023,7 @@ def _try_anthropic(explicit_api_key: str = None) -> Tuple[Optional[Any], Optiona } _MAIN_RUNTIME_FIELDS = ("provider", "model", "base_url", "api_key", "api_mode", "auth_mode") +_MAIN_RUNTIME_CONTEXT_FIELDS = _MAIN_RUNTIME_FIELDS + ("requested_provider",) def _normalize_main_runtime(main_runtime: Optional[Dict[str, Any]]) -> Dict[str, Any]: @@ -2886,7 +3046,7 @@ def _normalize_main_runtime(main_runtime: Optional[Dict[str, Any]]) -> Dict[str, if not isinstance(main_runtime, dict): return {} normalized: Dict[str, Any] = {} - for field in _MAIN_RUNTIME_FIELDS: + for field in _MAIN_RUNTIME_CONTEXT_FIELDS: value = main_runtime.get(field) # Preserve a callable api_key (Entra ID bearer provider) unchanged. if field == "api_key" and callable(value) and not isinstance(value, str): @@ -2894,9 +3054,10 @@ def _normalize_main_runtime(main_runtime: Optional[Dict[str, Any]]) -> Dict[str, continue if isinstance(value, str) and value.strip(): normalized[field] = value.strip() - provider = normalized.get("provider") - if isinstance(provider, str): - normalized["provider"] = provider.lower() + for identity_field in ("provider", "requested_provider"): + identity = normalized.get(identity_field) + if isinstance(identity, str): + normalized[identity_field] = identity.lower() return normalized @@ -3587,6 +3748,7 @@ def _retry_same_provider_sync( effective_timeout: float, effective_extra_body: dict, reasoning_config: Optional[dict], + extra_headers: Optional[Dict[str, str]] = None, ) -> Any: if task == "vision": _, retry_client, retry_model = resolve_vision_provider_client( @@ -3622,7 +3784,13 @@ def _retry_same_provider_sync( extra_body=effective_extra_body, reasoning_config=reasoning_config, base_url=retry_base or resolved_base_url, + task=task, ) + # Preserve per-request attribution headers (e.g. Copilot's + # ``x-initiator: user``) across the rebuilt-client retry — dropping them + # here would let a recovery retry silently lose capability gating (#60293). + if extra_headers: + retry_kwargs["extra_headers"] = dict(extra_headers) if _is_anthropic_compat_endpoint(resolved_provider, retry_base): retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"]) return _validate_llm_response( @@ -3646,6 +3814,7 @@ async def _retry_same_provider_async( effective_timeout: float, effective_extra_body: dict, reasoning_config: Optional[dict], + extra_headers: Optional[Dict[str, str]] = None, ) -> Any: if task == "vision": _, retry_client, retry_model = resolve_vision_provider_client( @@ -3681,7 +3850,12 @@ async def _retry_same_provider_async( extra_body=effective_extra_body, reasoning_config=reasoning_config, base_url=retry_base or resolved_base_url, + task=task, ) + # Preserve per-request attribution headers across the rebuilt-client + # retry — see the sync variant above (#60293). + if extra_headers: + retry_kwargs["extra_headers"] = dict(extra_headers) if _is_anthropic_compat_endpoint(resolved_provider, retry_base): retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"]) return _validate_llm_response( @@ -3756,6 +3930,24 @@ def _refresh_provider_credentials(provider: str) -> bool: return False _evict_cached_clients(normalized) return True + if normalized == "vertex": + # Mirrors run_agent.py's _try_refresh_vertex_client_credentials + # for the main conversation loop. Without this branch, an + # auxiliary Vertex client (vision, title generation, reflection, + # context compression, ...) that 401s on its ~1h token expiry + # falls through to the final `return False` below: the stale + # client is never evicted from _client_cache (whose cache key + # ignores the rotating bearer token), so every subsequent + # auxiliary Vertex call keeps 401ing until process restart. + from agent.vertex_adapter import get_vertex_config + + token, base_url = get_vertex_config() + if not isinstance(token, str) or not token.strip(): + return False + if not isinstance(base_url, str) or not base_url.strip(): + return False + _evict_cached_clients(normalized) + return True except Exception as exc: logger.debug("Auxiliary provider credential refresh failed for %s: %s", normalized, exc) return False @@ -3869,7 +4061,7 @@ def _call_fallback_candidate_sync( temperature=temperature, max_tokens=max_tokens, tools=tools, timeout=effective_timeout, extra_body=effective_extra_body, reasoning_config=reasoning_config, - base_url=fb_base) + base_url=fb_base, task=task) try: return _validate_llm_response( fb_client.chat.completions.create(**fb_kwargs), task) @@ -3886,7 +4078,7 @@ def _call_fallback_candidate_sync( tools=tools, timeout=effective_timeout, extra_body=effective_extra_body, reasoning_config=reasoning_config, - base_url=str(getattr(retry_client, "base_url", "") or fb_base)) + base_url=str(getattr(retry_client, "base_url", "") or fb_base), task=task) try: return _validate_llm_response( retry_client.chat.completions.create(**retry_kwargs), task) @@ -3935,7 +4127,7 @@ async def _call_fallback_candidate_async( temperature=temperature, max_tokens=max_tokens, tools=tools, timeout=effective_timeout, extra_body=effective_extra_body, reasoning_config=reasoning_config, - base_url=fb_base) + base_url=fb_base, task=task) try: return _validate_llm_response( await fb_client.chat.completions.create(**fb_kwargs), task) @@ -3953,7 +4145,7 @@ async def _call_fallback_candidate_async( tools=tools, timeout=effective_timeout, extra_body=effective_extra_body, reasoning_config=reasoning_config, - base_url=str(getattr(retry_client, "base_url", "") or fb_base)) + base_url=str(getattr(retry_client, "base_url", "") or fb_base), task=task) try: return _validate_llm_response( await retry_client.chat.completions.create(**retry_kwargs), task) @@ -4024,6 +4216,7 @@ def _try_main_agent_model_fallback( failed_provider: str, task: str = None, reason: str = "error", + failed_model: Optional[str] = None, ) -> Tuple[Optional[Any], Optional[str], str]: """Last-resort fallback to the user's main agent provider + model. @@ -4032,20 +4225,56 @@ def _try_main_agent_model_fallback( layer: if nothing the user asked for can serve the request, try the main chat model before giving up. - Skips when the failed provider already IS the main provider (no point - retrying the same backend that just failed). + ``failed_model`` narrows the same-provider skip to the exact + (provider, model) pair that just failed, mirroring + :func:`_try_configured_fallback_chain`. This matters for self-hosted / + custom endpoints serving several models behind one provider label: the + aux compression model timing out says nothing about the health of the + main agent model deployed on the same URL (real incident: aux + ``glm-5.2`` hung and timed out while main ``macaron-v1-venti`` on the + identical endpoint was serving 448K-token turns fine — the + provider-label skip discarded the one fallback that would have worked). + + - Model-specific runtime failures (timeout, connection, rate limit, + model-incompatible, invalid response) pass ``failed_model``: skip the + main model only when it IS the exact model that failed. + - Provider-wide failures (auth 401, payment 402) and legacy callers + leave ``failed_model`` as None, keeping the whole-provider skip — + the shared credentials/account are broken, so the main model on the + same provider cannot help either. Returns: (client, model, provider_label) or (None, None, "") if no fallback. """ main_provider = (_read_main_provider() or "").strip() main_model = (_read_main_model() or "").strip() + if main_provider.lower() == "moa": + # MoA virtual provider: fall back to the preset's aggregator — the + # acting model — instead of the unreachable "moa"/ pair. + _agg_provider, _agg_model = _resolve_moa_aggregator(main_model) + if not _agg_provider or not _agg_model: + return None, None, "" + main_provider, main_model = _agg_provider, _agg_model if not main_provider or not main_model or main_provider.lower() in {"auto", ""}: return None, None, "" - skip = (failed_provider or "").lower().strip() - if main_provider.lower() == skip: - # The thing that failed IS the main model — nothing to fall back to. + # Identity + scope semantics owned by agent.backend_identity (#72468): + # model-scoped failures skip only the exact deployment that failed; + # provider-wide failures (no failed_model) skip the credential surface. + from agent.backend_identity import ( + BackendIdentity, + FailureScope, + should_skip_candidate, + ) + + skip_model = (failed_model or "").strip().lower() or None + if should_skip_candidate( + BackendIdentity.build(provider=main_provider, model=main_model), + BackendIdentity.build(provider=failed_provider, model=skip_model), + FailureScope.MODEL if skip_model else FailureScope.CREDENTIAL, + ): + # The thing that failed IS the main model (or the failure was + # provider-wide) — nothing to fall back to. return None, None, "" if _is_provider_unhealthy(main_provider): _log_skip_unhealthy(main_provider, task) @@ -4155,6 +4384,7 @@ def _try_configured_fallback_chain( task: str, failed_provider: str, reason: str = "error", + failed_model: Optional[str] = None, ) -> Tuple[Optional[Any], Optional[str], str]: """Try user-configured fallback_chain for a specific auxiliary task. @@ -4162,6 +4392,25 @@ def _try_configured_fallback_chain( entry in order. Each entry must have at least ``provider``; ``model``, ``base_url``, and ``api_key`` are optional. + ``failed_model`` narrows the skip check to the exact (provider, model) + pair that just failed, rather than the whole provider. Without it every + entry sharing the failed provider is skipped (the original behaviour). + Callers pass it only when a sibling model on the same provider could + plausibly recover: + + - Model-specific runtime failures (timeout, connection, rate limit, + model-incompatible, invalid response) pass ``failed_model`` so a + chain that intentionally lists several models under the same provider + — e.g. two more NVIDIA NIM models after the primary NIM model times + out — is not skipped wholesale. Only the exact model that failed is + skipped; the siblings still run instead of jumping straight to the + main-agent-model safety net. + - Provider-wide failures (auth 401, payment 402) and "no client could + be built" callers leave ``failed_model`` as None, keeping the whole + provider skipped — the shared credentials/account behind every model + on that provider are broken, so a sibling can't help and the + main-agent-model safety net should be reached instead. + Returns: (client, model, provider_label) or (None, None, "") if no fallback. """ @@ -4173,7 +4422,24 @@ def _try_configured_fallback_chain( if not chain or not isinstance(chain, list): return None, None, "" - skip = failed_provider.lower().strip() + skip_model = (failed_model or "").strip().lower() or None + # Identity + scope semantics owned by agent.backend_identity (#59561, + # #72468): a failed_model means the failure was model-scoped (timeout / + # connection / rate limit) — only the exact deployment is skipped; no + # failed_model means provider-wide (auth/payment) — the whole credential + # surface is skipped. + from agent.backend_identity import ( + BackendIdentity, + FailureScope, + should_skip_candidate, + ) + + failed_ident = BackendIdentity.build( + provider=failed_provider, model=skip_model, + ) + failure_scope = ( + FailureScope.MODEL if skip_model else FailureScope.CREDENTIAL + ) tried = [] min_ctx = _task_minimum_context_length(task) @@ -4181,9 +4447,20 @@ def _try_configured_fallback_chain( if not isinstance(entry, dict): continue fb_provider = str(entry.get("provider", "")).strip() - if not fb_provider or fb_provider.lower() == skip: + if not fb_provider: + continue + fb_model_raw = str(entry.get("model", "")).strip() + if should_skip_candidate( + BackendIdentity.build( + provider=fb_provider, + model=fb_model_raw, + base_url=str(entry.get("base_url") or ""), + ), + failed_ident, + failure_scope, + ): continue - fb_model = str(entry.get("model", "")).strip() or None + fb_model = fb_model_raw or None label = f"fallback_chain[{i}]({fb_provider})" @@ -4440,26 +4717,17 @@ def _resolve_auto( # model. Resolve the MoA preset to its aggregator slot and continue Step 1 # with that real provider+model. Mirrors the MoA context-length resolution. if main_provider == "moa": - try: - from hermes_cli.config import load_config - from hermes_cli.moa_config import resolve_moa_preset - - _preset = resolve_moa_preset(load_config().get("moa") or {}, main_model) - _agg = _preset.get("aggregator") or {} - _agg_provider = str(_agg.get("provider") or "").strip() - _agg_model = str(_agg.get("model") or "").strip() - if _agg_provider and _agg_model and _agg_provider.lower() != "moa": - main_provider = _agg_provider - main_model = _agg_model - # The MoA virtual runtime carries a non-HTTP base_url - # ("moa://local") and a placeholder api_key; they belong to the - # facade, not the aggregator's real provider. Drop them so the - # aggregator resolves through its own provider credentials. - runtime_base_url = "" - runtime_api_key = "" - runtime_api_mode = "" - except Exception: - logger.debug("MoA aux resolution to aggregator failed", exc_info=True) + _agg_provider, _agg_model = _resolve_moa_aggregator(main_model) + if _agg_provider and _agg_model: + main_provider = _agg_provider + main_model = _agg_model + # The MoA virtual runtime carries a non-HTTP base_url + # ("moa://local") and a placeholder api_key; they belong to the + # facade, not the aggregator's real provider. Drop them so the + # aggregator resolves through its own provider credentials. + runtime_base_url = "" + runtime_api_key = "" + runtime_api_mode = "" if (main_provider and main_model and main_provider not in {"auto", ""}): @@ -4715,6 +4983,27 @@ def resolve_provider_client( # Normalise aliases provider = _normalize_aux_provider(provider) + # MoA virtual provider chokepoint: "moa" is not a real HTTP provider — + # its acting model is the preset's aggregator slot. The two resolver + # layers above (_resolve_auto, _resolve_task_provider_model) already + # unwrap their own paths, but callers that route here directly (vision + # auto-detect, _try_main_agent_model_fallback, get_available_vision_backends, + # plugin code) would otherwise dead-end in the unknown-provider branch. + # ``model`` carries the preset name for moa calls; when the preset can't + # be resolved we leave the call untouched and let the normal + # missing-provider handling produce its diagnostic. + if provider == "moa": + _agg_provider, _agg_model = _resolve_moa_aggregator(model) + if _agg_provider and _agg_model: + original_provider = _agg_provider.strip().lower() + provider = _normalize_aux_provider(_agg_provider) + model = _agg_model + # The moa:// facade endpoint and placeholder key belong to the + # virtual runtime, not the aggregator's real provider. + if explicit_base_url and str(explicit_base_url).lower().startswith("moa://"): + explicit_base_url = None + explicit_api_key = None + # Universal model-resolution fallback for concrete providers. ``auto`` is # intentionally excluded: `_resolve_auto(main_runtime=...)` returns the # model paired with the provider it actually selected. Pre-filling an auto @@ -4735,6 +5024,10 @@ def resolve_provider_client( # the load-bearing step for OAuth providers: an xai-oauth user # with grok-4.3 configured gets grok-4.3 for title generation # instead of silently dropping to whatever Step-2 fallback (#31845). + # When the main provider is MoA, ``_read_main_model_for_aux()`` + # substitutes the preset's aggregator model — the preset NAME is + # never a valid wire model id, so unset aux models default to the + # preset's acting model instead. # # Each provider branch below sees a non-empty ``model`` whenever the # user has *anything* configured — no provider-specific empty-model @@ -4751,7 +5044,7 @@ def resolve_provider_client( # return the actual current runtime model when the caller did not explicitly # request one. (# compression-current-model) if not model and provider != "auto": - model = _get_aux_model_for_provider(provider) or _read_main_model() or model + model = _get_aux_model_for_provider(provider) or _read_main_model_for_aux() or model def _needs_codex_wrap(client_obj, base_url_str: str, model_str: str) -> bool: """Decide if a plain OpenAI client should be wrapped for Responses API. @@ -4835,10 +5128,11 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", # ── Nous Portal (OAuth) ────────────────────────────────────────── if provider == "nous": - # Detect vision tasks: either explicit model override from - # _PROVIDER_VISION_MODELS, or caller passed a known vision model. + # Detect vision tasks: caller flag (strict vision backend), explicit + # model override from _PROVIDER_VISION_MODELS, or a known vision id. _is_vision = ( - model in _PROVIDER_VISION_MODELS.values() + is_vision + or model in _PROVIDER_VISION_MODELS.values() or (model or "").strip().lower() == "mimo-v2-omni" ) client, default = _try_nous(vision=_is_vision) @@ -4847,6 +5141,17 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", "but Nous Portal not configured (run: hermes auth)") return None, None final_model = _normalize_resolved_model(model or default, provider) + # Dual-wire: anthropic/* → /v1/messages, everything else stays on + # /chat/completions. Derive from the catalog id (not a stale + # api_mode=chat_completions) so aux matches the main agent. + from hermes_cli.providers import nous_api_mode + + portal_mode = nous_api_mode(final_model) + api_key_str = str(getattr(client, "api_key", "") or "") + base_url_str = str(getattr(client, "base_url", "") or "") + client = _maybe_wrap_anthropic( + client, final_model, api_key_str, base_url_str, portal_mode, + ) return (_to_async_client(client, final_model, is_vision=is_vision) if async_mode else (client, final_model)) @@ -5025,7 +5330,7 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", model or custom_entry.get("model") or (main_runtime.get("model") if main_runtime else None) - or _read_main_model() + or _read_main_model_for_aux() or "gpt-4o-mini", provider, ) @@ -5260,7 +5565,7 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "", final_model = _normalize_resolved_model( model or (main_runtime.get("model") if main_runtime else None) - or _read_main_model(), + or _read_main_model_for_aux(), provider, ) if provider == "copilot-acp": @@ -5503,7 +5808,10 @@ def _resolve_strict_vision_backend( if provider == "openrouter": return _try_openrouter(model=model) if provider == "nous": - return _try_nous(vision=True) + # Must go through resolve_provider_client so anthropic/* vision + # recommendations wrap onto /v1/messages — _try_nous alone returns + # a bare OpenAI client and the call 404s. + return resolve_provider_client("nous", model, is_vision=True) if provider == "openai-codex": # Route through resolve_provider_client so the caller's explicit # model is used. There is no safe default Codex model (shifting @@ -5628,7 +5936,24 @@ def _finalize(resolved_provider: str, sync_client: Any, default_model: Optional[ # 5. Stop main_provider = str(runtime.get("provider") or _read_main_provider()) main_model = str(runtime.get("model") or _read_main_model()) - if main_provider and main_provider not in {"auto", ""}: + if main_provider.strip().lower() == "moa": + # MoA virtual provider: main_model is a preset NAME, and every + # capability probe below (_PROVIDERS_WITHOUT_VISION, + # _main_model_supports_vision, _resolve_provider_vision_default) + # would run against a provider/model pair that doesn't exist on + # any wire. Unwrap to the preset's aggregator slot first so the + # checks and the eventual client target the real acting model. + _agg_provider, _agg_model = _resolve_moa_aggregator(main_model) + if _agg_provider and _agg_model: + main_provider, main_model = _agg_provider, _agg_model + # Drop the moa:// facade endpoint from the runtime view used + # below — it belongs to the virtual provider, not the + # aggregator's real provider. + runtime = dict(runtime) + runtime["base_url"] = "" + runtime["api_key"] = "" + runtime["api_mode"] = "" + if main_provider and main_provider not in {"auto", "", "moa"}: # A provider-specific vision default wins over the user's chat model: # static overrides (xiaomi/zai) and catalog-backed discovery (the # DeepInfra profile hook) both yield a *known* vision-capable model, @@ -6225,8 +6550,8 @@ def _resolve_task_provider_model( task: str = None, provider: str = None, model: str = None, - base_url: str = None, - api_key: str = None, + base_url: Optional[str] = None, + api_key: Optional[str] = None, ) -> Tuple[str, Optional[str], Optional[str], Optional[str], Optional[str]]: """Determine provider + model for a call. @@ -6269,12 +6594,57 @@ def _resolve_task_provider_model( # which downstream consumers like ContextCompressor accept as the task output. # The provider-side 'auto' is handled in _resolve_auto() via main_runtime # fallback, so dropping cfg_model to None here lets that path do its job. + # + # The explicit `model` kwarg needs the identical normalization: MoA slots + # (agent/moa_loop.py's _slot_runtime) forward a preset's `model:` field as + # this explicit argument rather than through auxiliary. config, so a + # user-configured `model: auto` on a MoA reference/aggregator slot reaches + # this function here, not as cfg_model. Only normalizing cfg_model let that + # literal "auto" slip through via `model or cfg_model` below. + if model and model.lower() == "auto": + model = None if cfg_model and cfg_model.lower() == "auto": cfg_model = None resolved_model = model or cfg_model resolved_api_mode = cfg_api_mode + # MoA virtual provider: an *explicit* `provider: moa` override (either the + # caller-passed `provider` arg or `auxiliary..provider` in + # config.yaml) reaches this function directly — it never goes through + # _resolve_auto(), which only unwraps the *implicit* "main provider is + # moa" case (#53827). Left as-is, "moa" is returned verbatim and + # resolve_provider_client() looks it up in PROVIDER_REGISTRY (which has + # no "moa" entry — it's not a real HTTP provider), falls to the + # unknown-provider dead end, and call_llm surfaces a nonsensical + # "MOA_API_KEY environment variable" error for a provider that was never + # meant to be reached over the wire. Auxiliary tasks don't need the + # reference fan-out — resolve to the preset's aggregator slot instead, + # exactly like the implicit path does (shared helper: _resolve_moa_aggregator). + def _unwrap_moa_provider(prov: str, mdl: Optional[str]) -> Tuple[str, Optional[str]]: + if prov.strip().lower() != "moa": + return prov, mdl + agg_provider, agg_model = _resolve_moa_aggregator(mdl) + if agg_provider and agg_model: + return agg_provider, agg_model + return prov, mdl + + if provider and str(provider).strip().lower() == "moa": + provider, resolved_model = _unwrap_moa_provider(provider, resolved_model) + # The moa:// virtual endpoint (if any explicit base_url/api_key was + # passed alongside provider="moa") belongs to the facade, not the + # aggregator's real provider — drop it so the aggregator resolves + # through its own provider credentials, mirroring _resolve_auto(). + if provider and provider.lower() != "moa": + base_url = None + api_key = None + elif cfg_provider and str(cfg_provider).strip().lower() == "moa": + cfg_provider, cfg_model = _unwrap_moa_provider(cfg_provider, resolved_model) + if cfg_provider and cfg_provider.lower() != "moa": + resolved_model = cfg_model + cfg_base_url = None + cfg_api_key = None + # Convenience aliases for direct API-key endpoints that aren't first-class # providers (e.g. ``provider: openai`` → custom + api.openai.com/v1). # Applied to both explicit args and config-derived values. When the user @@ -6631,6 +7001,7 @@ def _build_call_kwargs( extra_body: Optional[dict] = None, reasoning_config: Optional[dict] = None, base_url: Optional[str] = None, + task: Optional[str] = None, ) -> dict: """Build kwargs for .chat.completions.create() with model/provider adjustments.""" kwargs: Dict[str, Any] = { @@ -6685,11 +7056,38 @@ def _build_call_kwargs( _provider_norm in {"nvidia", "nvidia-nim", "nim", "build-nvidia", "nemotron"} or base_url_host_matches(_effective_base, "integrate.api.nvidia.com") ) + _is_moa = bool(task) and str(task) == "moa_reference" + # Gemini's native generateContent maps max_tokens → maxOutputTokens and, + # when it is omitted, applies a fixed 65,535-token ceiling rather than + # "the model's full budget" (see gemini_native_adapter.build_gemini_request). + # So an explicit cap is both safe and the ONLY way to honor it here — + # dropping max_tokens silently makes MoA's reference_max_tokens a no-op + # for gemini advisors (they run effectively uncapped). + _is_gemini_native = _provider_norm in { + "gemini", "google", "google-gemini", "google-ai-studio", + } + if not _is_gemini_native and _effective_base: + try: + from agent.gemini_native_adapter import is_native_gemini_base_url + _is_gemini_native = is_native_gemini_base_url(_effective_base) + except Exception: + pass + _nous_on_messages = False + if _provider_norm in {"nous", "nous-portal", "nousresearch"}: + from hermes_cli.providers import nous_api_mode + + _nous_on_messages = nous_api_mode(model) == "anthropic_messages" if ( _is_anthropic_compat_endpoint(provider, _effective_base) + or _nous_on_messages or _is_nvidia_nim + or _is_moa + or _is_gemini_native ): - kwargs["max_tokens"] = max_tokens + # Use auxiliary_max_tokens_param() so models that require + # max_completion_tokens (GPT-5 family, Copilot) get the right + # parameter name instead of a hardcoded max_tokens that 400s. + kwargs.update(auxiliary_max_tokens_param(max_tokens, model=model)) if tools: # Defensive dedup: providers like Google Vertex, Azure, and Bedrock @@ -6776,21 +7174,43 @@ def _build_call_kwargs( else: effort = reasoning_config.get("effort") or "medium" merged_extra["reasoning"] = {"enabled": True, "effort": effort} - if provider == "nous" and "tags" not in merged_extra: - merged_extra["tags"] = _nous_portal_tags() + # Portal product tags + sticky session_id. The provider profile usually + # supplies both; this fallback covers profile-load failures and alias + # spellings the profile lookup might miss. session_id keeps aux + # compression/title/vision calls on the same upstream instance as the + # main turn (cache warmth) — tags alone are not enough on /v1/messages. + _provider_for_portal = str(provider or "").strip().lower() + if _provider_for_portal in {"nous", "nous-portal", "nousresearch"}: + if "tags" not in merged_extra: + merged_extra["tags"] = _nous_portal_tags() + if "session_id" not in merged_extra: + try: + from agent.portal_tags import get_conversation_context + + sticky_key = get_conversation_context() + except Exception: + sticky_key = None + if sticky_key: + merged_extra["session_id"] = sticky_key if merged_extra: kwargs["extra_body"] = merged_extra - # Native Anthropic Messages adapters do not consume ``extra_body``. Carry - # the normalized Hermes reasoning config through a private kwarg so the - # adapter can pass it into build_anthropic_kwargs(), where provider-aware - # thinking/output_config projection lives. Do not expose this private kwarg - # to ordinary OpenAI-compatible SDK clients, which would reject it. + # Anthropic Messages adapters translate Hermes reasoning into native + # ``thinking`` via a private kwarg (and strip OpenAI-shaped + # ``extra_body.reasoning``). Do not expose this private kwarg to ordinary + # OpenAI-compatible SDK clients, which would reject it. Portal Claude is + # dual-wire — include it when the catalog id selects /v1/messages. if reasoning_config and isinstance(reasoning_config, dict): provider_norm = str(provider or "").strip().lower() effective_base = base_url or "" + _nous_on_messages = False + if provider_norm in {"nous", "nous-portal", "nousresearch"}: + from hermes_cli.providers import nous_api_mode + + _nous_on_messages = nous_api_mode(model) == "anthropic_messages" if ( provider_norm == "anthropic" + or _nous_on_messages or _endpoint_speaks_anthropic_messages(effective_base) or _is_anthropic_compat_endpoint(provider_norm, effective_base) ): @@ -6906,6 +7326,346 @@ def _obj_get(obj: Any, key: str, default: Any = None) -> Any: return value +# ── Streamed aggregation for progress-hooked auxiliary calls ───────────── +# When a forward-progress hook is installed (aux_progress_hook — today only +# by context compression), the primary chat.completions attempt is upgraded +# to a streamed request that is aggregated back into a complete response. +# Two effects, both deliberate: +# 1. The configured ``timeout`` becomes an INTER-CHUNK idle timeout instead +# of a total budget (httpx applies the read timeout per stream read), so +# a slow-but-generating summary model is never killed mid-generation +# while tokens are moving — only a genuinely silent connection dies. +# 2. Every arriving chunk ticks the progress hook, letting outer watchdogs +# (gateway session hygiene) extend their deadlines on liveness instead +# of guessing with a fixed wall clock. +# A total ceiling still bounds the pathological 1-token-per-idle-window +# stream; see _aux_stream_total_ceiling(). + +_AUX_STREAM_CEILING_FLOOR_SECONDS = 600.0 +_AUX_STREAM_CEILING_MULTIPLIER = 4.0 + + +def _aux_stream_total_ceiling(effective_timeout: Optional[float]) -> float: + """Absolute wall-clock bound for a progress-hooked streamed aux call. + + Generous by design — the idle timeout is the real guard; this only stops + a degenerate stream that trickles one token per idle window forever. + """ + try: + timeout = float(effective_timeout) if effective_timeout is not None else 0.0 + except (TypeError, ValueError): + timeout = 0.0 + return max(_AUX_STREAM_CEILING_FLOOR_SECONDS, + _AUX_STREAM_CEILING_MULTIPLIER * timeout) + + +def _client_streams_internally(client: Any) -> bool: + """Wire adapters that consume a stream inside .create() already tick the + progress hook themselves (Codex per SSE event, Anthropic per stream + event); Bedrock's Converse shim cannot stream at all. None of them + accept chat-completions ``stream=True`` semantics from us.""" + return isinstance(client, ( + CodexAuxiliaryClient, + AnthropicAuxiliaryClient, + BedrockAuxiliaryClient, + )) + + +def _is_streaming_rejected_error(exc: Exception) -> bool: + """Provider explicitly refused a streamed chat.completions request.""" + err = str(exc).lower() + if "stream_options" in err: + return True + return "stream" in err and ( + "not supported" in err + or "unsupported" in err + or "not allowed" in err + or "disabled" in err + ) + + +def _provider_requires_stream(provider: str, base_url: Optional[str]) -> bool: + """Detect providers that only accept streaming (non-stream = HTTP 400). + + Some OpenAI-compatible endpoints reject non-streaming chat requests + outright — e.g. Tencent Copilot returns + ``{"code": 11101, "msg": "Non-stream chat request is currently not + supported"}``. The main conversation loop already streams, so interactive + chat works; auxiliary tasks (title generation, compression, web extract) + used the non-streaming path and failed on every call. When this returns + True the auxiliary client sends ``stream=True`` and aggregates the chunks + itself (see :func:`_aggregate_chat_stream`). Credit @kudi88 (PR #60686). + + Beyond the known-host list, users can mark ANY custom endpoint as + stream-only via ``auxiliary.stream_only_base_urls`` in config.yaml + (list of substrings matched against the endpoint URL). + """ + _url = str(base_url or "").lower() + if not _url: + return False + # Tencent Copilot — "Non-stream chat request is currently not supported" + if base_url_host_matches(_url, "copilot.tencent.com"): + return True + try: + from hermes_cli.config import load_config + aux_cfg = (load_config() or {}).get("auxiliary", {}) + markers = aux_cfg.get("stream_only_base_urls") or [] + if isinstance(markers, (list, tuple)): + for marker in markers: + if isinstance(marker, str) and marker.strip() and marker.strip().lower() in _url: + return True + except Exception: + # Config read is best-effort; never break an aux call over it. + pass + return False + + +def _create_with_progress( + client: Any, + kwargs: Dict[str, Any], + task: Optional[str] = None, + *, + force_stream: bool = False, +) -> Any: + """chat.completions.create() that streams when a progress hook is active + or the provider only accepts streamed requests. + + Behavior is byte-for-byte identical to a plain ``create(**kwargs)`` when + neither trigger applies (every existing caller/task) or when the client's + wire adapter streams internally. With a hook + a chunk-capable client, + the request is sent with ``stream=True`` and aggregated, ticking the hook + per chunk — so the configured ``timeout`` acts per stream read (idle) + rather than as a total budget, and outer liveness watchdogs see tokens + moving. ``force_stream=True`` (stream-only providers such as Tencent + Copilot — credit @kudi88, PR #60686) takes the same streamed path even + without a hook. Providers that reject the streamed request fall back to + the plain non-streaming call — except under ``force_stream``, where a + stream-only provider rejects the plain call by definition, so the + original error is surfaced to the normal recovery chains instead. + """ + _notify_aux_progress() # request dispatched counts as progress + if (not _aux_progress_active() and not force_stream) or _client_streams_internally(client): + return client.chat.completions.create(**kwargs) + + total_ceiling = _aux_stream_total_ceiling(kwargs.get("timeout")) + stream_kwargs = dict(kwargs) + stream_kwargs["stream"] = True + stream_kwargs["stream_options"] = {"include_usage": True} + try: + chunks = client.chat.completions.create(**stream_kwargs) + except Exception as exc: + # Genuine provider failures (auth, credit, rate limit, network) are + # not streaming's fault — surface them unchanged so the existing + # recovery chains (credential refresh, pool rotation, provider + # fallback) see the same error they would on a plain call. + if ( + force_stream + or _is_transient_transport_error(exc) + or _is_auth_error(exc) + or _is_payment_error(exc) + or _is_rate_limit_error(exc) + ): + raise + # Anything else may be a streaming-specific rejection (explicit + # "stream not supported", stream_options 400, or an idiosyncratic + # 4xx). Retry non-streaming once; if the request itself is bad the + # plain call reproduces the real error for the normal except-chains. + logger.debug( + "Auxiliary %s: streamed request failed (%s); retrying " + "non-streaming", task or "call", exc, + ) + return client.chat.completions.create(**kwargs) + + # Some shims (MoA virtual provider under quiet mode, defensive adapters) + # return a complete response even when stream=True was requested. + if hasattr(chunks, "choices"): + _notify_aux_progress() + return chunks + return _aggregate_chat_stream( + chunks, model=str(kwargs.get("model") or ""), total_ceiling=total_ceiling, + ) + + +def _aggregate_chat_stream( + chunks: Any, + *, + model: str = "", + total_ceiling: Optional[float] = None, +) -> Any: + """Consume a chat.completions chunk stream into a complete response. + + Ticks the thread-local aux progress hook on every chunk. Raises + TimeoutError when *total_ceiling* seconds elapse before the stream + finishes — phrased with "timed out" so existing timeout classification + (``_is_timeout_error``) treats it exactly like a request timeout. + Accumulation is shared with the async mirror via + :class:`_ChatStreamAccumulator`. + """ + acc = _ChatStreamAccumulator(model=model, total_ceiling=total_ceiling) + try: + for chunk in chunks: + acc.feed(chunk) + finally: + close_fn = getattr(chunks, "close", None) + if callable(close_fn): + try: + close_fn() + except Exception: + pass + return acc.finish() + + +class _ChatStreamAccumulator: + """Shared per-chunk accumulation for sync and async stream aggregation. + + Mirrors :func:`_aggregate_chat_stream`'s chunk handling so the async + consumer below cannot drift from the sync one (same content/reasoning/ + tool-call delta reassembly, same "timed out" ceiling phrasing). + """ + + def __init__(self, model: str = "", total_ceiling: Optional[float] = None): + self._started = time.monotonic() + self._total_ceiling = total_ceiling + self.content_parts: List[str] = [] + self.reasoning_parts: List[str] = [] + self.tool_calls_acc: Dict[int, Dict[str, Any]] = {} + self.finish_reason = None + self.usage = None + self.resp_id = "" + self.resp_model = model or "" + + def feed(self, chunk: Any) -> None: + _notify_aux_progress() + if ( + self._total_ceiling is not None + and (time.monotonic() - self._started) >= self._total_ceiling + ): + raise TimeoutError( + f"Auxiliary streamed call timed out after {self._total_ceiling:.0f}s " + "total ceiling (stream still open but over budget)" + ) + self.resp_id = getattr(chunk, "id", None) or self.resp_id + self.resp_model = getattr(chunk, "model", None) or self.resp_model + chunk_usage = getattr(chunk, "usage", None) + if chunk_usage: + self.usage = chunk_usage + choices = getattr(chunk, "choices", None) or [] + if not choices: + return + choice = choices[0] + self.finish_reason = getattr(choice, "finish_reason", None) or self.finish_reason + delta = getattr(choice, "delta", None) + if delta is None: + return + piece = getattr(delta, "content", None) + if piece: + self.content_parts.append(piece) + reasoning_piece = ( + getattr(delta, "reasoning", None) + or getattr(delta, "reasoning_content", None) + ) + if reasoning_piece and isinstance(reasoning_piece, str): + self.reasoning_parts.append(reasoning_piece) + for tc in (getattr(delta, "tool_calls", None) or []): + idx = getattr(tc, "index", 0) or 0 + acc = self.tool_calls_acc.setdefault( + idx, {"id": "", "name": "", "arguments": []} + ) + if getattr(tc, "id", None): + acc["id"] = tc.id + fn = getattr(tc, "function", None) + if fn is not None: + if getattr(fn, "name", None): + acc["name"] = fn.name + if getattr(fn, "arguments", None): + acc["arguments"].append(fn.arguments) + + def finish(self) -> Any: + tool_calls = None + if self.tool_calls_acc: + tool_calls = [ + SimpleNamespace( + id=acc["id"], + type="function", + function=SimpleNamespace( + name=acc["name"], + arguments="".join(acc["arguments"]), + ), + ) + for _idx, acc in sorted(self.tool_calls_acc.items()) + ] + message = SimpleNamespace( + role="assistant", + content="".join(self.content_parts), + tool_calls=tool_calls, + reasoning="".join(self.reasoning_parts) or None, + ) + choice = SimpleNamespace( + index=0, + message=message, + finish_reason=self.finish_reason or "stop", + ) + return SimpleNamespace( + id=self.resp_id, + model=self.resp_model, + object="chat.completion", + choices=[choice], + usage=self.usage, + ) + + +async def _aggregate_chat_stream_async( + chunks: Any, + *, + model: str = "", + total_ceiling: Optional[float] = None, +) -> Any: + """Async mirror of :func:`_aggregate_chat_stream` (``async for`` consumer). + + The AsyncOpenAI stream contract is an async iterator — consuming it with + the sync helper raises. Same accumulation and ceiling semantics via + :class:`_ChatStreamAccumulator`. + """ + acc = _ChatStreamAccumulator(model=model, total_ceiling=total_ceiling) + try: + async for chunk in chunks: + acc.feed(chunk) + finally: + close_fn = getattr(chunks, "close", None) or getattr(chunks, "aclose", None) + if callable(close_fn): + try: + result = close_fn() + if inspect.isawaitable(result): + await result + except Exception: + pass + return acc.finish() + + +async def _acreate_with_stream( + client: Any, + kwargs: Dict[str, Any], + task: Optional[str] = None, +) -> Any: + """Async chat.completions.create() for stream-only providers. + + Sends ``stream=True`` and aggregates the async chunk stream into a + complete response (credit @kudi88, PR #60686 — async contract fixed to + ``async for`` and tool-call deltas preserved per sweeper review). + """ + total_ceiling = _aux_stream_total_ceiling(kwargs.get("timeout")) + stream_kwargs = dict(kwargs) + stream_kwargs["stream"] = True + stream_kwargs["stream_options"] = {"include_usage": True} + chunks = await client.chat.completions.create(**stream_kwargs) + # Defensive: shims may hand back a complete response despite stream=True. + if hasattr(chunks, "choices"): + return chunks + return await _aggregate_chat_stream_async( + chunks, model=str(kwargs.get("model") or ""), total_ceiling=total_ceiling, + ) + + def call_llm( task: str = None, *, @@ -6921,6 +7681,7 @@ def call_llm( timeout: float = None, extra_body: dict = None, reasoning_config: Optional[dict] = None, + extra_headers: Optional[Dict[str, str]] = None, api_mode: str = None, stream: bool = False, stream_options: dict = None, @@ -6946,6 +7707,9 @@ def call_llm( extra_body: Additional request body fields. reasoning_config: Optional Hermes reasoning config for direct model calls such as MoA reference/aggregator slots. + extra_headers: Additional per-request HTTP headers. These override + client-level defaults for providers that gate capabilities on + request attribution (for example Copilot's ``x-initiator``). stream: When True, return the raw SDK streaming iterator instead of a validated complete response. The caller is responsible for consuming chunks (and for any fallback). Used by the MoA aggregator so its @@ -7058,7 +7822,9 @@ def call_llm( temperature=temperature, max_tokens=max_tokens, tools=tools, timeout=effective_timeout, extra_body=effective_extra_body, reasoning_config=reasoning_config, - base_url=_base_info or resolved_base_url) + base_url=_base_info or resolved_base_url, task=task) + if extra_headers: + kwargs["extra_headers"] = dict(extra_headers) # Convert image blocks for Anthropic-compatible endpoints (e.g. MiniMax) _client_base = str(getattr(client, "base_url", "") or "") @@ -7100,7 +7866,13 @@ def call_llm( # for the transient retry every auxiliary task shares. (PR #16587) try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task, + _create_with_progress( + client, kwargs, task, + force_stream=_provider_requires_stream( + resolved_provider, _base_info or resolved_base_url, + ), + ), + task, provider=resolved_provider, base_url=_base_info) except Exception as transient_err: if not _is_transient_transport_error(transient_err): @@ -7133,7 +7905,13 @@ def call_llm( time.sleep(_backoff) try: return _validate_llm_response( - client.chat.completions.create(**kwargs), task) + _create_with_progress( + client, kwargs, task, + force_stream=_provider_requires_stream( + resolved_provider, _base_info or resolved_base_url, + ), + ), + task) except Exception as retry_transient: if not _is_transient_transport_error(retry_transient): raise @@ -7312,6 +8090,7 @@ def call_llm( effective_timeout=effective_timeout, effective_extra_body=effective_extra_body, reasoning_config=reasoning_config, + extra_headers=extra_headers, ) # ── Same-provider credential-pool recovery ───────────────────── @@ -7355,6 +8134,7 @@ def call_llm( effective_timeout=effective_timeout, effective_extra_body=effective_extra_body, reasoning_config=reasoning_config, + extra_headers=extra_headers, ) except Exception as retry2_err: # The rotated key also hit a quota/auth wall. Mark it @@ -7448,6 +8228,15 @@ def call_llm( logger.info("Auxiliary %s: %s on %s (%s), trying fallback", task or "call", reason, resolved_provider, first_err) + # Narrow the configured-chain skip to the exact model that + # failed ONLY for model-specific failures. Auth (401) and + # payment (402) errors are provider-wide — the credentials or + # account behind every model on that provider are the same — so + # a sibling model can't recover; keep skipping the whole + # provider so the main-agent-model safety net is still reached. + _chain_failed_model = ( + None if reason in ("auth error", "payment error") else final_model + ) # Fallback order (#26882, #26803): # 1. User-configured fallback_chain (per-task) if set # 2. For auto: top-level main fallback_providers/fallback_model @@ -7456,7 +8245,8 @@ def call_llm( fb_client, fb_model, fb_label = (None, None, "") if is_auto: fb_client, fb_model, fb_label = _try_configured_fallback_chain( - task, resolved_provider or "auto", reason=reason) + task, resolved_provider or "auto", reason=reason, + failed_model=_chain_failed_model) if fb_client is None: fb_client, fb_model, fb_label = _try_main_fallback_chain( task, resolved_provider or "auto", reason=reason) @@ -7465,10 +8255,12 @@ def call_llm( resolved_provider, task, reason=reason) else: fb_client, fb_model, fb_label = _try_configured_fallback_chain( - task, resolved_provider or "auto", reason=reason) + task, resolved_provider or "auto", reason=reason, + failed_model=_chain_failed_model) if fb_client is None: fb_client, fb_model, fb_label = _try_main_agent_model_fallback( - resolved_provider, task, reason=reason) + resolved_provider, task, reason=reason, + failed_model=_chain_failed_model) if fb_client is not None: fb_resp = _call_fallback_candidate_sync( @@ -7674,7 +8466,7 @@ async def async_call_llm( temperature=temperature, max_tokens=max_tokens, tools=tools, timeout=effective_timeout, extra_body=effective_extra_body, reasoning_config=reasoning_config, - base_url=_client_base or resolved_base_url) + base_url=_client_base or resolved_base_url, task=task) # Convert image blocks for Anthropic-compatible endpoints (e.g. MiniMax) if _is_anthropic_compat_endpoint(resolved_provider, _client_base): @@ -7684,9 +8476,25 @@ async def async_call_llm( # Retry ONCE on the same provider for a transient transport blip # before the except-chain escalates to fallback — see call_llm() # for the rationale. (PR #16587) + _force_stream_async = ( + _provider_requires_stream( + resolved_provider, _client_base or resolved_base_url, + ) + and not isinstance(client, ( + AsyncCodexAuxiliaryClient, + AsyncAnthropicAuxiliaryClient, + AsyncBedrockAuxiliaryClient, + )) + ) + + async def _acreate(_kwargs: Dict[str, Any]) -> Any: + if _force_stream_async: + return await _acreate_with_stream(client, _kwargs, task) + return await client.chat.completions.create(**_kwargs) + try: return _validate_llm_response( - await client.chat.completions.create(**kwargs), task, + await _acreate(kwargs), task, provider=resolved_provider, base_url=_client_base) except Exception as transient_err: if not _is_transient_transport_error(transient_err): @@ -7707,7 +8515,7 @@ async def async_call_llm( task or "call", transient_err, ) return _validate_llm_response( - await client.chat.completions.create(**kwargs), task) + await _acreate(kwargs), task) except Exception as first_err: if "temperature" in kwargs and _is_unsupported_temperature_error(first_err): retry_kwargs = dict(kwargs) @@ -7966,6 +8774,15 @@ async def async_call_llm( logger.info("Auxiliary %s (async): %s on %s (%s), trying fallback", task or "call", reason, resolved_provider, first_err) + # Narrow the configured-chain skip to the exact model that + # failed ONLY for model-specific failures. Auth (401) and + # payment (402) errors are provider-wide — the credentials or + # account behind every model on that provider are the same — so + # a sibling model can't recover; keep skipping the whole + # provider so the main-agent-model safety net is still reached. + _chain_failed_model = ( + None if reason in ("auth error", "payment error") else final_model + ) # Fallback order (#26882, #26803): # 1. User-configured fallback_chain (per-task) if set # 2. For auto: top-level main fallback_providers/fallback_model @@ -7974,7 +8791,8 @@ async def async_call_llm( fb_client, fb_model, fb_label = (None, None, "") if is_auto: fb_client, fb_model, fb_label = _try_configured_fallback_chain( - task, resolved_provider or "auto", reason=reason) + task, resolved_provider or "auto", reason=reason, + failed_model=_chain_failed_model) if fb_client is None: fb_client, fb_model, fb_label = _try_main_fallback_chain( task, resolved_provider or "auto", reason=reason) @@ -7983,10 +8801,12 @@ async def async_call_llm( resolved_provider, task, reason=reason) else: fb_client, fb_model, fb_label = _try_configured_fallback_chain( - task, resolved_provider or "auto", reason=reason) + task, resolved_provider or "auto", reason=reason, + failed_model=_chain_failed_model) if fb_client is None: fb_client, fb_model, fb_label = _try_main_agent_model_fallback( - resolved_provider, task, reason=reason) + resolved_provider, task, reason=reason, + failed_model=_chain_failed_model) if fb_client is not None: # Convert sync fallback client to async diff --git a/agent/backend_identity.py b/agent/backend_identity.py new file mode 100644 index 000000000000..7a7e9efb6bfe --- /dev/null +++ b/agent/backend_identity.py @@ -0,0 +1,204 @@ +"""Single owner for backend identity and failure-scoped skip decisions. + +Every fallback / dedup / skip / quarantine decision in Hermes ultimately asks +one question: **"is this candidate the same backend as the one that failed, +along the axis that failure invalidated?"** Before this module, that +question was re-implemented inline at six call sites across four subsystems, +each comparing whatever string was locally convenient (provider label, +provider+model, base_url+model, ...). Each incident fixed one site while the +others kept the bug: #22548 (same-shim aliases), #70893 (xai-oauth vs xai — +same host, distinct credential), #59561 (aux chain skipped sibling models), +#72468 (aux main-model safety net, same bug three weeks later), #62984 / +#54250 / #57584 (dedup ignoring base_url strands multi-endpoint pools). + +The root insight: "provider" conflates three independent identity axes, and +each failure class invalidates a different one: + +* **credential surface** — auth 401 / payment 402 kill everything sharing the + credential (every model, every host reached with that key/token). +* **endpoint** — DNS failure / connection refused kill everything behind the + URL, regardless of model or credential. +* **model deployment** — timeout / overload / rate limit / model-incompatible + kill ONE model's deployment. A sibling model behind the same URL is an + independent deployment (real incident: aux ``glm-5.2`` hung and timed out + while main ``macaron-v1-venti`` on the identical endpoint was serving + 448K-token turns). + +Call sites should build :class:`BackendIdentity` values, classify the failure +with :func:`classify_failure_scope`, and ask :func:`should_skip_candidate`. +Do not re-implement any comparison inline — extend THIS module instead. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from enum import Enum +from typing import Optional + +logger = logging.getLogger(__name__) + + +class FailureScope(Enum): + """Which identity axis a failure invalidates.""" + + #: Timeout, overload/429, connection blip, model-incompatible, invalid + #: response: evidence against ONE model deployment only. + MODEL = "model" + #: Auth 401 / payment 402: evidence against the shared credential — + #: every model reached with it is equally dead. + CREDENTIAL = "credential" + #: DNS / connection-refused / unreachable host: evidence against the + #: endpoint — every model behind the URL is equally dead. + ENDPOINT = "endpoint" + + +#: Reason strings already used by auxiliary_client's except-chain, mapped to +#: scopes. Unknown reasons default to MODEL — the least-invalidating scope — +#: so an unrecognized failure never over-skips viable candidates. +_REASON_SCOPES = { + "auth error": FailureScope.CREDENTIAL, + "payment error": FailureScope.CREDENTIAL, + "rate limit": FailureScope.MODEL, + "model incompatible with route": FailureScope.MODEL, + "invalid provider response": FailureScope.MODEL, + "connection error": FailureScope.MODEL, + "timeout": FailureScope.MODEL, +} + + +def classify_failure_scope(reason: Optional[str]) -> FailureScope: + """Map a human-readable failure reason to the identity axis it kills.""" + return _REASON_SCOPES.get((reason or "").strip().lower(), FailureScope.MODEL) + + +def _norm_provider(value: Optional[str]) -> str: + return (value or "").strip().lower() + + +def _norm_model(value: Optional[str]) -> str: + return (value or "").strip().lower() + + +def _norm_base_url(value: Optional[str]) -> str: + return (value or "").strip().rstrip("/").lower() + + +@dataclass(frozen=True) +class BackendIdentity: + """Normalized identity of one (provider, model, endpoint) deployment. + + Empty fields mean "unknown" — comparisons treat an unknown axis as + non-distinguishing (it can neither prove sameness nor difference on its + own; the remaining axes decide). + """ + + provider: str = "" + model: str = "" + base_url: str = "" + + @classmethod + def build( + cls, + provider: Optional[str] = None, + model: Optional[str] = None, + base_url: Optional[str] = None, + ) -> "BackendIdentity": + return cls( + provider=_norm_provider(provider), + model=_norm_model(model), + base_url=_norm_base_url(base_url), + ) + + +def _both_first_class(a: BackendIdentity, b: BackendIdentity) -> bool: + """True when both providers are distinct registered first-class providers. + + Two different registry providers have distinct credential surfaces even + when they share an inference host (xai-oauth vs xai, openai-codex vs + openai-api) — #70893. Custom/shim aliases are NOT in the registry, so + two aliases pointing at one URL still count as the same backend (#22548). + """ + if not a.provider or not b.provider or a.provider == b.provider: + return False + try: + from hermes_cli.auth import PROVIDER_REGISTRY + + return a.provider in PROVIDER_REGISTRY and b.provider in PROVIDER_REGISTRY + except Exception: + return False + + +def same_credential_surface(a: BackendIdentity, b: BackendIdentity) -> bool: + """Do two identities share the credential a 401/402 just invalidated? + + Conservative on purpose: an unprovable axis must answer "different" + (try the candidate — worst case one wasted RTT) rather than "same" + (skip — worst case stranded failover). Two distinct custom labels at + one URL may carry different per-entry api_keys, so a shared URL alone + never proves a shared credential; it is only used as a weak signal + when a provider label is missing entirely. + """ + if a.provider and b.provider: + # Same label = same configured credential. Different labels = + # different credential config (first-class registry providers + # explicitly so — #70893; custom entries can each carry their own + # api_key, so sameness is unprovable and we must not skip). + return a.provider == b.provider + # Provider unknown on a side: same explicit URL is the best signal left. + return bool(a.base_url and a.base_url == b.base_url) + + +def same_endpoint(a: BackendIdentity, b: BackendIdentity) -> bool: + """Do two identities sit behind the endpoint that just went unreachable?""" + if a.base_url and b.base_url: + return a.base_url == b.base_url + # An unknown base_url inherits the provider default → same provider + # label implies the same default endpoint. + return bool(a.provider and a.provider == b.provider) + + +def same_deployment(a: BackendIdentity, b: BackendIdentity) -> bool: + """Are these the exact same model deployment (the thing a timeout kills)? + + Provider+model must match; the base_url axis distinguishes only when BOTH + sides carry an explicit URL (#62984: same provider+model on two different + explicit URLs is two deployments — a pool). A side with an unknown URL + inherits the provider default and cannot prove difference. + """ + if not (a.provider and b.provider and a.provider == b.provider): + # Same-host different-label shims: same URL + same model IS the same + # deployment even when the alias labels differ (#22548) — unless both + # labels are first-class registry providers (#70893). + if ( + a.base_url + and a.base_url == b.base_url + and a.model + and a.model == b.model + and not _both_first_class(a, b) + ): + return True + return False + if not (a.model and b.model and a.model == b.model): + return False + if a.base_url and b.base_url and a.base_url != b.base_url: + return False # distinct explicit endpoints — a pool, not a dup + return True + + +def should_skip_candidate( + candidate: BackendIdentity, + failed: BackendIdentity, + scope: FailureScope = FailureScope.MODEL, +) -> bool: + """THE skip predicate: would trying ``candidate`` just repeat the failure? + + True when the candidate is the same backend as ``failed`` along the axis + ``scope`` says the failure invalidated. Every fallback/dedup/skip site + must call this instead of comparing labels inline. + """ + if scope is FailureScope.CREDENTIAL: + return same_credential_surface(candidate, failed) + if scope is FailureScope.ENDPOINT: + return same_endpoint(candidate, failed) + return same_deployment(candidate, failed) diff --git a/agent/background_review.py b/agent/background_review.py index c2ea87bd94e2..a0dbd4a99e28 100644 --- a/agent/background_review.py +++ b/agent/background_review.py @@ -209,7 +209,10 @@ def _digest_history(messages_snapshot: List[Dict], tail: int = 24) -> List[Dict] "conversation for skills the user loaded via /skill-name or you " "read via skill_view. If any of them covers the territory of the " "new learning, PATCH that one first. It is the skill that was in " - "play, so it's the right one to extend.\n" + "play, so it's the right one to extend — but only if it is " + "curator-managed. Bundled, hub, pinned, and user-owned skills are " + "off-limits to you no matter how relevant (see Protected skills " + "below); for those, fall through to the next option.\n" " 2. UPDATE AN EXISTING UMBRELLA (via skills_list + skill_view). " "If no loaded skill fits but an existing class-level skill does, " "patch it. Add a subsection, a pitfall, or broaden a trigger.\n" @@ -251,10 +254,18 @@ def _digest_history(messages_snapshot: List[Dict], tail: int = 24) -> List[Dict] "Protected skills (DO NOT edit these):\n" " • Bundled skills (shipped with Hermes, e.g. 'hermes-agent').\n" " • Hub-installed skills (installed via 'hermes skills install').\n" - "Pinned skills (marked via 'hermes curator pin') CAN be improved — " - "pin only blocks deletion/archive/consolidation by the curator, not " - "content updates. Patch them when a pitfall or missing step turns up, " - "same as any other agent-created skill.\n" + " • Skills in skills.external_dirs (externally owned).\n" + " • PINNED skills (marked via 'hermes curator pin'). You are an " + "autonomous no-user-present actor, so pin blocks your writes too — " + "content updates included. Only the user, in a foreground session, " + "can change a pinned skill.\n" + " • USER-OWNED skills — anything not curator-managed. A skill the " + "user hand-wrote, installed by URL, or asked a foreground agent to " + "create is theirs, not yours; your writes to it WILL be refused. " + "This includes skills that were loaded or consulted this session: " + "being in play does not make one yours to edit. If such a skill is " + "wrong or outdated, say so in your reply and recommend " + "'hermes curator adopt ' — do not try to patch it.\n" "If the only skills that need updating are protected, say\n" "'Nothing to save.' and stop.\n\n" "Do NOT capture (these become persistent self-imposed constraints " @@ -309,7 +320,9 @@ def _digest_history(messages_snapshot: List[Dict], tail: int = 24) -> List[Dict] " 1. UPDATE A CURRENTLY-LOADED SKILL. Check what skills were " "loaded via /skill-name or skill_view in the conversation. If one " "of them covers the learning, PATCH it first. It was in play; " - "it's the right place.\n" + "it's the right place — provided it is curator-managed. Protected " + "and user-owned skills are off-limits however relevant; fall " + "through when one of those is the best fit.\n" " 2. UPDATE AN EXISTING UMBRELLA (skills_list + skill_view to " "find the right one). Patch it.\n" " 3. ADD A SUPPORT FILE under an existing umbrella via " @@ -337,10 +350,15 @@ def _digest_history(messages_snapshot: List[Dict], tail: int = 24) -> List[Dict] "Protected skills (DO NOT edit these):\n" " • Bundled skills (shipped with Hermes, e.g. 'hermes-agent').\n" " • Hub-installed skills (installed via 'hermes skills install').\n" - "Pinned skills (marked via 'hermes curator pin') CAN be improved — " - "pin only blocks deletion/archive/consolidation by the curator, not " - "content updates. Patch them when a pitfall or missing step turns up, " - "same as any other agent-created skill.\n" + " • Skills in skills.external_dirs (externally owned).\n" + " • PINNED skills (marked via 'hermes curator pin'). Pin blocks " + "autonomous writes entirely — content updates included — because no " + "user is present to consent. Only a foreground session can change one.\n" + " • USER-OWNED skills — anything not curator-managed (hand-written, " + "URL-installed, or created by a foreground agent at the user's " + "request). Your writes to these WILL be refused, including to skills " + "loaded or consulted this session. If one is wrong, say so in your " + "reply and recommend 'hermes curator adopt ' instead.\n" "If the only skills that need updating are protected, say\n" "'Nothing to save.' and stop.\n\n" "Do NOT capture as skills (these become persistent self-imposed " diff --git a/agent/battery.py b/agent/battery.py new file mode 100644 index 000000000000..a1c0f32fa4d1 --- /dev/null +++ b/agent/battery.py @@ -0,0 +1,131 @@ +"""System-battery read-out for the CLI/TUI status bar. + +Reads the host battery through ``psutil`` (already a Hermes dependency) and +exposes a compact, colour-coded label. Everything degrades to "unavailable" +when there is no battery (desktops, servers, VMs) or when the read fails, so +callers can render the result unconditionally and simply show nothing. + +The status bar repaints often (every keystroke and on a ~1s idle refresh), so +:func:`read_battery` memoises the last reading for a few seconds instead of +hitting ``psutil`` on every frame. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import Optional + + +@dataclass(frozen=True) +class BatteryStatus: + """A single battery reading. + + ``available`` is False on machines without a battery (or when the read + failed). ``percent`` is clamped to 0-100. ``plugged`` is True when on AC + power, False on battery, and None when the platform can't tell. + """ + + available: bool + percent: Optional[int] = None + plugged: Optional[bool] = None + + @property + def charging(self) -> bool: + return bool(self.plugged) + + +UNAVAILABLE = BatteryStatus(available=False) + +# Colour buckets, mirroring the status-bar context styles but inverted (a full +# battery is "good", an empty one is "critical"). +CATEGORY_GOOD = "good" +CATEGORY_WARN = "warn" +CATEGORY_BAD = "bad" +CATEGORY_CRITICAL = "critical" +CATEGORY_DIM = "dim" + +_CACHE_TTL_SECONDS = 8.0 +_cache: Optional[tuple[float, BatteryStatus]] = None + + +def _read_battery_uncached() -> BatteryStatus: + try: + import psutil + except Exception: + return UNAVAILABLE + + # ``sensors_battery`` is missing on some platforms/builds of psutil. + reader = getattr(psutil, "sensors_battery", None) + if reader is None: + return UNAVAILABLE + + try: + batt = reader() + except Exception: + return UNAVAILABLE + + if batt is None: + return UNAVAILABLE + + percent: Optional[int] = None + raw_percent = getattr(batt, "percent", None) + if raw_percent is not None: + try: + percent = max(0, min(100, int(round(float(raw_percent))))) + except (TypeError, ValueError): + percent = None + + plugged = getattr(batt, "power_plugged", None) + if plugged is not None: + plugged = bool(plugged) + + return BatteryStatus(available=True, percent=percent, plugged=plugged) + + +def read_battery(use_cache: bool = True) -> BatteryStatus: + """Return the current battery status (cached for a few seconds).""" + global _cache + if use_cache and _cache is not None: + ts, cached = _cache + if time.monotonic() - ts < _CACHE_TTL_SECONDS: + return cached + + status = _read_battery_uncached() + _cache = (time.monotonic(), status) + return status + + +def clear_cache() -> None: + """Drop the memoised reading (used by tests).""" + global _cache + _cache = None + + +def battery_category(status: BatteryStatus) -> str: + """Bucket a reading into a colour category: good/warn/bad/critical/dim.""" + if not status.available or status.percent is None: + return CATEGORY_DIM + # On AC power the level isn't a concern — always read as healthy. + if status.charging: + return CATEGORY_GOOD + pct = status.percent + if pct <= 10: + return CATEGORY_CRITICAL + if pct <= 20: + return CATEGORY_BAD + if pct <= 50: + return CATEGORY_WARN + return CATEGORY_GOOD + + +def battery_glyph(status: BatteryStatus) -> str: + """Return the leading glyph: a bolt while charging, else a battery.""" + return "\u26a1" if status.charging else "\U0001f50b" # ⚡ / 🔋 + + +def format_battery(status: BatteryStatus) -> str: + """Return a compact label like ``🔋 82%`` / ``⚡ 82%`` (empty if N/A).""" + if not status.available or status.percent is None: + return "" + return f"{battery_glyph(status)} {status.percent}%" diff --git a/agent/bedrock_adapter.py b/agent/bedrock_adapter.py index c8cff3f76e13..c399081619ff 100644 --- a/agent/bedrock_adapter.py +++ b/agent/bedrock_adapter.py @@ -433,6 +433,29 @@ def _model_supports_tool_use(model_id: str) -> bool: return not any(pattern in model_lower for pattern in _NON_TOOL_CALLING_PATTERNS) +# --------------------------------------------------------------------------- +# Prompt-cache capability detection (Converse API cachePoint) +# --------------------------------------------------------------------------- +# Claude on Bedrock already gets prompt caching through the AnthropicBedrock +# SDK path (see is_anthropic_bedrock_model / runtime_provider.py's dual-path +# routing) — it never reaches build_converse_kwargs unless bearer-token auth +# forces the Converse path (#28156). This allowlist covers the Converse API +# itself: sending an unsupported model a cachePoint block raises a +# ValidationException, so — like _model_supports_tool_use but inverted — +# unknown models default to NOT receiving cache markers until confirmed. +# Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html +_CACHE_POINT_PATTERNS = [ + "anthropic.claude", # bearer-token fallback path + "amazon.nova", +] + + +def _model_supports_prompt_cache(model_id: str) -> bool: + """Return True if the model accepts a Converse API cachePoint block.""" + model_lower = model_id.lower() + return any(pattern in model_lower for pattern in _CACHE_POINT_PATTERNS) + + def is_anthropic_bedrock_model(model_id: str) -> bool: """Return True if the model is an Anthropic Claude model on Bedrock. @@ -764,14 +787,22 @@ def normalize_converse_response(response: Dict) -> SimpleNamespace: reasoning_content="\n\n".join(reasoning_parts) if reasoning_parts else None, ) - # Build usage stats + # Build usage stats. Converse's inputTokens excludes cache read/write + # tokens (unlike OpenAI's prompt_tokens, which includes them) — restore + # the OpenAI-style "total includes cache" convention here so downstream + # normalize_usage() can subtract them back out consistently, and surface + # the Anthropic-named fields it already falls back to for cache reads. usage_data = response.get("usage", {}) + input_tokens = usage_data.get("inputTokens", 0) + cache_read_tokens = usage_data.get("cacheReadInputTokens", 0) + cache_write_tokens = usage_data.get("cacheWriteInputTokens", 0) + output_tokens = usage_data.get("outputTokens", 0) usage = SimpleNamespace( - prompt_tokens=usage_data.get("inputTokens", 0), - completion_tokens=usage_data.get("outputTokens", 0), - total_tokens=( - usage_data.get("inputTokens", 0) + usage_data.get("outputTokens", 0) - ), + prompt_tokens=input_tokens + cache_read_tokens + cache_write_tokens, + completion_tokens=output_tokens, + total_tokens=input_tokens + cache_read_tokens + cache_write_tokens + output_tokens, + cache_read_input_tokens=cache_read_tokens, + cache_creation_input_tokens=cache_write_tokens, ) finish_reason = _converse_stop_reason_to_openai(stop_reason) @@ -936,6 +967,8 @@ def stream_converse_with_callbacks( usage_data = { "inputTokens": meta_usage.get("inputTokens", 0), "outputTokens": meta_usage.get("outputTokens", 0), + "cacheReadInputTokens": meta_usage.get("cacheReadInputTokens", 0), + "cacheWriteInputTokens": meta_usage.get("cacheWriteInputTokens", 0), } # Flush remaining text @@ -949,12 +982,16 @@ def stream_converse_with_callbacks( reasoning_content="\n\n".join(reasoning_parts) if reasoning_parts else None, ) + input_tokens = usage_data.get("inputTokens", 0) + cache_read_tokens = usage_data.get("cacheReadInputTokens", 0) + cache_write_tokens = usage_data.get("cacheWriteInputTokens", 0) + output_tokens = usage_data.get("outputTokens", 0) usage = SimpleNamespace( - prompt_tokens=usage_data.get("inputTokens", 0), - completion_tokens=usage_data.get("outputTokens", 0), - total_tokens=( - usage_data.get("inputTokens", 0) + usage_data.get("outputTokens", 0) - ), + prompt_tokens=input_tokens + cache_read_tokens + cache_write_tokens, + completion_tokens=output_tokens, + total_tokens=input_tokens + cache_read_tokens + cache_write_tokens + output_tokens, + cache_read_input_tokens=cache_read_tokens, + cache_creation_input_tokens=cache_write_tokens, ) finish_reason = _converse_stop_reason_to_openai(stop_reason) @@ -993,6 +1030,7 @@ def build_converse_kwargs( Converts OpenAI-format inputs to Converse API parameters. """ system_prompt, converse_messages = convert_messages_to_converse(messages) + cache_enabled = _model_supports_prompt_cache(model) kwargs: Dict[str, Any] = { "modelId": model, @@ -1003,6 +1041,8 @@ def build_converse_kwargs( } if system_prompt: + if cache_enabled: + system_prompt = system_prompt + [{"cachePoint": {"type": "default"}}] kwargs["system"] = system_prompt from agent.anthropic_adapter import _forbids_sampling_params @@ -1026,6 +1066,8 @@ def build_converse_kwargs( # Strip tools for known non-tool-calling models and warn the user. # Ref: PR #7920 feedback from @ptlally, pattern from PR #4346. if _model_supports_tool_use(model): + if cache_enabled: + converse_tools = converse_tools + [{"cachePoint": {"type": "default"}}] kwargs["toolConfig"] = {"tools": converse_tools} else: logger.warning( @@ -1033,6 +1075,14 @@ def build_converse_kwargs( "The agent will operate in text-only mode.", model ) + if cache_enabled and len(converse_messages) >= 2: + # Checkpoint everything up to (not including) the newest turn, so the + # marker survives unchanged across requests as only the tail grows — + # mirroring the Anthropic system_and_3 strategy in prompt_caching.py. + content = converse_messages[-2].get("content") + if isinstance(content, list) and content: + content.append({"cachePoint": {"type": "default"}}) + if guardrail_config: kwargs["guardrailConfig"] = guardrail_config diff --git a/agent/billing_links.py b/agent/billing_links.py new file mode 100644 index 000000000000..1e9320ebb45a --- /dev/null +++ b/agent/billing_links.py @@ -0,0 +1,124 @@ +"""Provider-agnostic billing/credit recovery links. + +Maps a billing-classified failure onto a recovery link + label. *Detection* +is not done here — that is :mod:`agent.error_classifier` +(``FailoverReason.billing``), the single source of truth for "credit wall vs. +rate limit / auth / transport". The resulting :class:`BillingBlock` rides the +turn result and the gateway ``message.complete`` event so every surface (CLI, +TUI, desktop) renders one structured signal instead of re-parsing error text. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Optional + +from utils import base_url_host_matches + + +@dataclass +class BillingBlock: + """Structured billing-wall descriptor shared across every surface. + + ``is_nous`` is the routing bit: Nous has a first-class in-app billing surface + (desktop Settings → Billing, TUI/CLI ``/topup``), so surfaces prefer that over + ``billing_url``; third-party providers have no in-app flow, so ``billing_url`` + is the deep link the user actually needs. + """ + + provider: str + provider_label: str + model: str + billing_url: Optional[str] + is_nous: bool + message: str + + def to_dict(self) -> dict: + return asdict(self) + + +@dataclass(frozen=True) +class _Provider: + label: str + url: str + slugs: tuple[str, ...] + hosts: tuple[str, ...] = () + + +# Single source of truth: internal slug(s) + base_url host(s) → billing page. +# Curated "add credits / manage billing" landing pages, not marketing homes. +# Hosts back the OpenAI-compatible fallback where the slug is a generic bucket +# (e.g. "openai_compatible") but base_url reveals the real upstream. An unknown +# provider degrades to a readable label with no invented URL. +_PROVIDERS: tuple[_Provider, ...] = ( + _Provider("OpenAI", "https://platform.openai.com/settings/organization/billing", ("openai",), ("api.openai.com",)), + _Provider("Anthropic", "https://console.anthropic.com/settings/billing", ("anthropic",), ("api.anthropic.com",)), + _Provider("OpenRouter", "https://openrouter.ai/settings/credits", ("openrouter",), ("openrouter.ai",)), + _Provider("xAI", "https://console.x.ai/team/default/billing", ("xai", "xai-oauth"), ("api.x.ai",)), + _Provider("DeepSeek", "https://platform.deepseek.com/top_up", ("deepseek",), ("api.deepseek.com",)), + _Provider("Groq", "https://console.groq.com/settings/billing", ("groq",), ("api.groq.com",)), + _Provider("Mistral", "https://console.mistral.ai/billing", ("mistral",), ("api.mistral.ai",)), + _Provider("Together AI", "https://api.together.ai/settings/billing", ("together",), ("api.together.ai", "api.together.xyz")), + _Provider("Fireworks AI", "https://fireworks.ai/account/billing", ("fireworks",), ("fireworks.ai",)), + _Provider("Perplexity", "https://www.perplexity.ai/settings/api", ("perplexity",), ("perplexity.ai",)), + _Provider("Google AI", "https://aistudio.google.com/app/billing", ("google", "gemini"), ("generativelanguage.googleapis.com",)), + _Provider("Cohere", "https://dashboard.cohere.com/billing", ("cohere",)), + _Provider("Moonshot AI", "https://platform.moonshot.ai/console/pay", ("moonshot",)), + _Provider("NVIDIA", "https://build.nvidia.com/settings/billing", ("nvidia",)), +) + +_BY_SLUG: dict[str, _Provider] = {slug: p for p in _PROVIDERS for slug in p.slugs} + + +def is_nous_inference_route(provider: str, base_url: str) -> bool: + """True when the failing route is the Nous-managed inference gateway.""" + if (provider or "").strip().lower() == "nous": + return True + return base_url_host_matches(str(base_url or ""), "inference-api.nousresearch.com") + + +def _nous_billing_url() -> Optional[str]: + """Best-effort Nous portal billing URL (text-surface fallback; Nous prefers the in-app flow).""" + try: + from hermes_cli.nous_account import nous_portal_billing_url + + return nous_portal_billing_url(None) + except Exception: + return "https://portal.nousresearch.com/billing" + + +def _resolve_provider_link(slug: str, base_url: str) -> tuple[str, Optional[str]]: + """Resolve ``(label, url)``: exact slug → base_url host → readable-label fallback.""" + hit = _BY_SLUG.get(slug) + if hit: + return hit.label, hit.url + + base = str(base_url or "") + for p in _PROVIDERS: + if any(base_url_host_matches(base, host) for host in p.hosts): + return p.label, p.url + + return slug.replace("_", " ").replace("-", " ").strip().title() or "your provider", None + + +def build_billing_block( + *, + provider: str, + base_url: str, + model: str, + message: str = "", +) -> BillingBlock: + """Build the billing descriptor for a billing-classified failure. + + ``message`` is the guidance already assembled by the agent loop + (:func:`agent.conversation_loop._billing_or_entitlement_message`), carried + through unchanged so every surface shows identical copy. + """ + slug = (provider or "").strip().lower() + model = (model or "").strip() + + if is_nous_inference_route(slug, base_url): + return BillingBlock(slug or "nous", "Nous Portal", model, _nous_billing_url(), True, message or "") + + label, url = _resolve_provider_link(slug, base_url) + return BillingBlock(slug, label, model, url, False, message or "") diff --git a/agent/billing_view.py b/agent/billing_view.py index 0e9930fd3ea6..29e8068c2275 100644 --- a/agent/billing_view.py +++ b/agent/billing_view.py @@ -1,4 +1,4 @@ -"""Surface-agnostic core for the Phase 2b terminal-billing screens. +"""Surface-agnostic core for the Phase 2b Remote Spending screens. One fetch/parse per concern, consumed identically by the CLI handler (``cli.py::_show_billing``), the TUI JSON-RPC methods @@ -107,6 +107,22 @@ def display(self) -> str: return f"{self.masked} — {label}" if label else self.masked +@dataclass(frozen=True) +class PaymentMethodInfo: + """The payment method on file. `kind` is "card", "link", or "unknown" + — anything else is normalised to "unknown" at parse time, so consumers + only ever see fields that belong to the kind they are looking at.""" + + kind: str + brand: Optional[str] = None + last4: Optional[str] = None + wallet: Optional[str] = None + email: Optional[str] = None + resolved_via: Optional[str] = None + #: What the server called it, when we did not recognise the kind. + raw_kind: Optional[str] = None + + @dataclass(frozen=True) class MonthlyCap: limit_usd: Optional[Decimal] = None @@ -150,6 +166,7 @@ class BillingState: min_usd: Optional[Decimal] = None max_usd: Optional[Decimal] = None card: Optional[CardInfo] = None + payment_method: Optional[PaymentMethodInfo] = None monthly_cap: Optional[MonthlyCap] = None auto_reload: Optional[AutoReload] = None portal_url: Optional[str] = None @@ -201,6 +218,41 @@ def _parse_card(raw: Any) -> Optional[CardInfo]: return CardInfo(brand=brand, last4=last4, resolved_via=resolved_via) +def _parse_payment_method(raw: Any) -> Optional[PaymentMethodInfo]: + if not isinstance(raw, dict): + return None + kind = raw.get("kind") + if not isinstance(kind, str): + return None + + def _optional_string(key: str) -> Optional[str]: + value = raw.get(key) + return value if isinstance(value, str) else None + + resolved_via = _optional_string("resolvedVia") + brand = _optional_string("brand") + last4 = _optional_string("last4") + # Settle the kind here, the way _parse_card settles a card, so nothing + # downstream has to re-check which fields this kind is allowed to have. + if kind == "card" and brand and last4: + return PaymentMethodInfo( + kind="card", + brand=brand, + last4=last4, + wallet=_optional_string("wallet"), + resolved_via=resolved_via, + ) + if kind == "link": + return PaymentMethodInfo( + kind="link", + email=_optional_string("email"), + resolved_via=resolved_via, + ) + return PaymentMethodInfo( + kind="unknown", raw_kind=kind, resolved_via=resolved_via + ) + + def _parse_monthly_cap(raw: Any) -> Optional[MonthlyCap]: if not isinstance(raw, dict): return None @@ -274,6 +326,7 @@ def billing_state_from_payload( min_usd=parse_money(bounds.get("minUsd")), max_usd=parse_money(bounds.get("maxUsd")), card=_parse_card(payload.get("card")), + payment_method=_parse_payment_method(payload.get("paymentMethod")), monthly_cap=_parse_monthly_cap(payload.get("monthlyCap")), auto_reload=_parse_auto_reload(payload.get("autoReload")), portal_url=portal_url, diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index b2e5c8653a48..b0c3faaf6f73 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -188,6 +188,31 @@ def _provider_preferences_for_agent(agent) -> Dict[str, Any]: return preferences +def _merge_nous_portal_messages_extra_body(agent, anthropic_kwargs: dict) -> dict: + """Merge Portal ``tags`` / ``session_id`` onto an Anthropic Messages kwargs dict. + + The Nous provider profile is only consulted by the OpenAI-wire transport; + anthropic_messages callers must merge it themselves. Passes ``session_id`` + only — not ``provider_preferences`` (those become a top-level ``provider`` + routing object on the OpenAI wire). Never blocks a turn on tagging. + """ + if getattr(agent, "provider", None) not in {"nous", "nous-portal", "nousresearch"}: + return anthropic_kwargs + try: + from providers import get_provider_profile + + nous_profile = get_provider_profile("nous") + if nous_profile is not None: + anthropic_kwargs.setdefault("extra_body", {}).update( + nous_profile.build_extra_body( + session_id=getattr(agent, "session_id", None) + ) + ) + except Exception as exc: # noqa: BLE001 — never block a turn on tagging + logger.debug("Nous Portal extra_body merge failed: %s", exc) + return anthropic_kwargs + + def _env_float(name: str, default: float) -> float: try: return float(os.getenv(name, str(default))) @@ -433,26 +458,56 @@ def _dispatch_nonstreaming_api_request(agent, api_kwargs: dict, *, make_client): def should_use_direct_api_call(agent) -> bool: - """Whether a cron OpenAI-wire request should skip the interrupt worker. - - Issue #62151 is specific to OpenRouter's chat-completions path inside the - gateway cron thread stack. Keep native/Codex/Bedrock/MoA transports on their - established workers: their cancellation and client ownership differ, and - the report provides no evidence that those paths share the pre-HTTP wedge. + """Whether an OpenAI-wire request should skip the interrupt worker. + + Two nested-pool contexts wedge before the socket opens when the request + is pushed onto yet another daemon worker thread: + + - Gateway cron turns (#62151): gateway asyncio loop → cron thread → + interrupt worker. Fixed by running inline. + - Delegated children (#60203): gateway loop → async-delegation executor + (module-lifetime daemon pool) → per-child timeout executor → interrupt + worker. Same fingerprint after multi-day gateway uptime — children hang + at their FIRST API call with zero stale-detector output (the worker + never reaches dispatch), all providers, restart cures it. The cron fix + originally excluded delegation "for lack of evidence"; #60203 is that + evidence. + + Running inline drops the deepest thread layer (whose only job is + interactive-interrupt responsiveness). Interrupts still work: the inline + path registers ``agent._active_request_abort``, which ``interrupt()`` + invokes cross-thread to shut the active sockets — the same mechanism the + async-delegation stall monitor (#72227) relies on. + + Keep native/Codex/Bedrock/MoA transports on their established workers: + their cancellation and client ownership differ. """ - return ( - getattr(agent, "platform", None) == "cron" - and getattr(agent, "api_mode", None) == "chat_completions" - and getattr(agent, "provider", None) != "moa" - ) + if getattr(agent, "api_mode", None) != "chat_completions": + return False + if getattr(agent, "provider", None) == "moa": + return False + if getattr(agent, "platform", None) == "cron": + return True + # Delegated child (delegate_task sync or background) — detected via the + # execution ContextVar set by _run_single_child, with the agent's own + # platform stamp as a fallback for callers that bypass the runner. + try: + from agent.delegation_context import is_delegated_child_context + + if is_delegated_child_context(): + return True + except Exception: + pass + return getattr(agent, "platform", None) == "subagent" def direct_api_call(agent, api_kwargs: dict): """Run a non-streaming LLM call inline on the conversation thread. - Used when ``should_use_direct_api_call`` is True. Skips the interrupt worker - (whose only job is interactive-interrupt responsiveness, which this context - does not have) so the nested-pool deadlock (#62151) cannot occur. Because the + Used when ``should_use_direct_api_call`` is True (cron turns and + delegated children). Skips the interrupt worker (whose only job is + interactive-interrupt responsiveness, which these contexts do not have) + so the nested-pool deadlock (#62151, #60203) cannot occur. Because the request runs in-flight normally, the per-request OpenAI client's own httpx timeout (provider ``request_timeout_seconds`` / ``HERMES_API_TIMEOUT``) bounds a genuinely hung provider — the same bound interactive calls already rely on. @@ -463,7 +518,7 @@ def direct_api_call(agent, api_kwargs: dict): request_client_lock = threading.Lock() def _abort_active_request(reason: str) -> None: - """Abort the inline request from cron's watchdog/interrupt thread.""" + """Abort the inline request from a watchdog/interrupt thread.""" with request_client_lock: request_client = request_client_holder["client"] if request_client is not None: @@ -993,7 +1048,7 @@ def build_api_kwargs(agent, api_messages: list) -> dict: ephemeral_out = getattr(agent, "_ephemeral_max_output_tokens", None) if ephemeral_out is not None: agent._ephemeral_max_output_tokens = None # consume immediately - return _transport.build_kwargs( + anthropic_kwargs = _transport.build_kwargs( model=agent.model, messages=anthropic_messages, tools=tools_for_api, @@ -1006,6 +1061,12 @@ def build_api_kwargs(agent, api_messages: list) -> dict: fast_mode=(agent.request_overrides or {}).get("speed") == "fast", drop_context_1m_beta=bool(getattr(agent, "_oauth_1m_beta_disabled", False)), ) + # Nous Portal reads ``tags`` and ``session_id`` as top-level body fields + # on its Messages route the same way it does on /chat/completions, but + # the profile hook that produces them is only consulted by the + # OpenAI-wire transport. Merge them here so Messages traffic keeps + # product attribution and sticky routing. + return _merge_nous_portal_messages_extra_body(agent, anthropic_kwargs) # AWS Bedrock native Converse API — bypasses the OpenAI client entirely. # The adapter handles message/tool conversion and boto3 calls directly. @@ -1076,6 +1137,7 @@ def build_api_kwargs(agent, api_messages: list) -> dict: tools=tools_for_api, reasoning_config=agent.reasoning_config, session_id=getattr(agent, "session_id", None), + base_url=agent.base_url, max_tokens=agent.max_tokens, timeout=agent._resolved_api_call_timeout(), request_overrides=agent.request_overrides, @@ -1598,29 +1660,28 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool ) return agent._try_activate_fallback(reason) - # Skip entries that resolve to the current (provider, model) — falling - # back to the same backend that just failed loops the failure. Compare - # base_url too so two distinct custom_providers entries pointing at the - # same shim/proxy URL also dedup. See issue #22548. - current_provider = (getattr(agent, "provider", "") or "").strip().lower() - current_model = (getattr(agent, "model", "") or "").strip() - current_base_url = str(getattr(agent, "base_url", "") or "").rstrip("/").lower() - fb_base_url_for_dedup = (fb.get("base_url") or "").strip().rstrip("/").lower() - if fb_provider == current_provider and fb_model == current_model: - logger.warning( - "Fallback skip: chain entry %s/%s matches current provider/model", - fb_provider, fb_model, - ) - return agent._try_activate_fallback(reason) - if ( - fb_base_url_for_dedup - and current_base_url - and fb_base_url_for_dedup == current_base_url - and fb_model == current_model - ): + # Skip entries that resolve to the same backend that just failed — + # falling back to it loops the failure. Identity semantics (which axes + # distinguish two backends, shim aliases, first-class credential + # surfaces, multi-endpoint pools) are owned by agent.backend_identity — + # see #22548, #70893, #62984. Do not re-implement comparisons here. + from agent.backend_identity import BackendIdentity, should_skip_candidate + + current_ident = BackendIdentity.build( + provider=getattr(agent, "provider", ""), + model=getattr(agent, "model", ""), + base_url=str(getattr(agent, "base_url", "") or ""), + ) + fb_ident = BackendIdentity.build( + provider=fb_provider, + model=fb_model, + base_url=(fb.get("base_url") or ""), + ) + if should_skip_candidate(fb_ident, current_ident): logger.warning( - "Fallback skip: chain entry base_url %s matches current backend", - fb_base_url_for_dedup, + "Fallback skip: chain entry %s/%s resolves to the same backend " + "as the current one (%s)", + fb_provider, fb_model, current_ident.base_url or current_ident.provider, ) return agent._try_activate_fallback(reason) @@ -1671,6 +1732,14 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool _fb_is_azure = agent._is_azure_openai_url(fb_base_url) if fb_provider == "openai-codex": fb_api_mode = "codex_responses" + elif fb_provider in {"nous", "nous-portal", "nousresearch"}: + # Portal is dual-wire: anthropic/* must land on /v1/messages. + # resolve_provider_client still returns an OpenAI client for + # Nous; the anthropic_messages branch below rebuilds the native + # client from that credential + base_url. + from hermes_cli.providers import nous_api_mode + + fb_api_mode = nous_api_mode(fb_model) elif ( fb_provider == "anthropic" or fb_base_url.rstrip("/").lower().endswith("/anthropic") @@ -1712,6 +1781,7 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool agent._config_context_length = None agent.model = fb_model agent.provider = fb_provider + agent.requested_provider = fb_provider agent.base_url = fb_base_url agent.api_mode = fb_api_mode if hasattr(agent, "_transport_cache"): @@ -1738,6 +1808,7 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool fb_provider, fb_model, _pool_provider, ) agent._credential_pool = None + agent._credential_pool_entry_id = None if getattr(agent, "_credential_pool", None) is None: try: from agent.credential_pool import load_pool @@ -1800,6 +1871,9 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool # not only after a later credential-rotation rebuild. agent._replace_primary_openai_client(reason="fallback_timeout_apply") + from agent.agent_runtime_helpers import sync_credential_pool_entry_id + sync_credential_pool_entry_id(agent) + # Re-evaluate prompt caching for the new provider/model agent._use_prompt_caching, agent._use_native_cache_layout = ( agent._anthropic_prompt_cache_policy( @@ -1945,7 +2019,17 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: for internal_key in [k for k in api_msg if isinstance(k, str) and k.startswith("_")]: api_msg.pop(internal_key, None) if _needs_sanitize: - agent._sanitize_tool_calls_for_strict_api(api_msg, model=agent.model) + # In MoA mode, agent.model is the virtual preset name, + # not the actual aggregator model. Resolve the real + # aggregator model so Gemini preserves thought_signature. + _sanitize_model = agent.model + if agent.provider == "moa": + _moa_client = getattr(agent, "client", None) + if _moa_client is not None: + _agg_slot = getattr(_moa_client, "last_aggregator_slot", None) + if _agg_slot and _agg_slot.get("model"): + _sanitize_model = _agg_slot["model"] + agent._sanitize_tool_calls_for_strict_api(api_msg, model=_sanitize_model) api_messages.append(api_msg) effective_system = agent._cached_system_prompt or "" @@ -2082,7 +2166,9 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: _ant_kw = _tsum.build_kwargs(model=agent.model, messages=api_messages, tools=None, max_tokens=agent.max_tokens, reasoning_config=agent.reasoning_config, is_oauth=agent._is_anthropic_oauth, - preserve_dots=agent._anthropic_preserve_dots()) + preserve_dots=agent._anthropic_preserve_dots(), + base_url=getattr(agent, "_anthropic_base_url", None)) + _ant_kw = _merge_nous_portal_messages_extra_body(agent, _ant_kw) summary_response = agent._anthropic_messages_create(_ant_kw) _summary_result = _tsum.normalize_response(summary_response, strip_tool_prefix=agent._is_anthropic_oauth) final_response = (_summary_result.content or "").strip() @@ -2112,7 +2198,9 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: _ant_kw2 = _tretry.build_kwargs(model=agent.model, messages=api_messages, tools=None, is_oauth=agent._is_anthropic_oauth, max_tokens=agent.max_tokens, reasoning_config=agent.reasoning_config, - preserve_dots=agent._anthropic_preserve_dots()) + preserve_dots=agent._anthropic_preserve_dots(), + base_url=getattr(agent, "_anthropic_base_url", None)) + _ant_kw2 = _merge_nous_portal_messages_extra_body(agent, _ant_kw2) retry_response = agent._anthropic_messages_create(_ant_kw2) _retry_result = _tretry.normalize_response(retry_response, strip_tool_prefix=agent._is_anthropic_oauth) final_response = (_retry_result.content or "").strip() @@ -2462,7 +2550,11 @@ def _on_reasoning(text): request_client_holder = {"client": None, "diag": None, "owner_tid": None} # Transport kind of the registered request client — see the non-streaming # variant. Routes _close_request_client_once to anthropic vs openai abort/ - # close helpers (#67142). + # close helpers (#67142). ``kind="stream"`` registers a per-request + # *stream handle* instead of a client — used under the MoA facade, whose + # singleton client has no per-request sockets to abort + # (_abort_request_openai_client is a no-op on it), so interrupts must + # close the stream object itself (#57354). request_client_kind = {"value": "openai"} request_client_lock = threading.Lock() # Request-local cancellation flag — see interruptible_api_call for the full @@ -2482,6 +2574,44 @@ def _set_request_client(client, *, kind: str = "openai"): request_client_holder["owner_tid"] = threading.get_ident() return client + def _stream_close_callable(stream): + close = getattr(stream, "close", None) + if callable(close): + return close + response = getattr(stream, "response", None) + close = getattr(response, "close", None) + if callable(close): + return close + return None + + def _set_request_stream_handle(stream): + # Register the per-request *stream* under kind="stream" so an + # interrupt closes the stream handle itself. Under the MoA facade the + # registered "client" is the shared facade singleton whose + # per-request abort helpers are no-ops, leaving the underlying HTTP + # stream open until the provider drained it (#57354). + if _stream_close_callable(stream) is None: + return stream + with request_client_lock: + request_client_holder["client"] = stream + request_client_kind["value"] = "stream" + request_client_holder["owner_tid"] = threading.get_ident() + return stream + + def _close_request_stream_handle(stream, reason: str) -> None: + close = _stream_close_callable(stream) + if close is None: + return + try: + close() + logger.info("Streaming response handle closed (%s)", reason) + except Exception as exc: + logger.debug( + "Streaming response handle close failed (%s): %s", + reason, + exc, + ) + def _close_request_client_once(reason: str) -> None: # See #29507 explanation in the non-streaming variant above. A # stranger thread (the interrupt-check / stale-stream detector loop) @@ -2489,9 +2619,15 @@ def _close_request_client_once(reason: str) -> None: # so the worker thread retains ownership of the FD release. with request_client_lock: request_client = request_client_holder.get("client") + request_kind = request_client_kind.get("value", "openai") owner_tid = request_client_holder.get("owner_tid") + # A registered stream handle (kind="stream", MoA facade path) is + # safe to close from any thread — closing IS the abort — so the + # stranger-thread ownership carve-out only applies to real + # per-request clients (#57354). stranger_thread = ( - request_client is not None + request_kind != "stream" + and request_client is not None and owner_tid is not None and owner_tid != threading.get_ident() ) @@ -2500,8 +2636,9 @@ def _close_request_client_once(reason: str) -> None: request_client_holder["owner_tid"] = None if request_client is None: return - kind = request_client_kind.get("value", "openai") - if kind == "anthropic_messages": + if request_kind == "stream": + _close_request_stream_handle(request_client, reason) + elif request_kind == "anthropic_messages": if stranger_thread: agent._abort_request_anthropic_client(request_client, reason=reason) else: @@ -2680,6 +2817,11 @@ def _call_chat_completions(stream_attempt_id: int): _diag = agent._stream_diag_init() request_client_holder["diag"] = _diag stream = request_client.chat.completions.create(**stream_kwargs) + if agent.provider == "moa": + # The MoA facade is a shared singleton — abort/close of the + # registered client is a no-op, so register the stream handle + # itself for interrupt teardown (#57354). + stream = _set_request_stream_handle(stream) # Claim the delta sink for THIS attempt (#65991). If a prior attempt's # stream is somehow still alive (a stale-stream reconnect whose socket # abort raced), this claim supersedes it so its late chunks are fenced @@ -3379,13 +3521,9 @@ def _call(): # already worker-owned-closed by _close_request_client_once # above; the next attempt builds a fresh one. The shared # _anthropic_client is never closed from inside a request. - if agent.api_mode != "anthropic_messages": - try: - agent._replace_primary_openai_client( - reason="stream_mid_tool_retry_pool_cleanup" - ) - except Exception: - pass + # #70773: same FD-recycle corruption vector for OpenAI. + # The shared client will be replaced lazily by + # _ensure_primary_openai_client on the next attempt. continue # SSE error events from proxies (e.g. OpenRouter sends @@ -3444,13 +3582,9 @@ def _call(): # above; next attempt builds fresh), so the shared # _anthropic_client is never closed from inside a # request — only the OpenAI-wire primary is refreshed. - if agent.api_mode != "anthropic_messages": - try: - agent._replace_primary_openai_client( - reason="stream_retry_pool_cleanup" - ) - except Exception: - pass + # #70773: same FD-recycle corruption vector for OpenAI. + # The shared client will be replaced lazily by + # _ensure_primary_openai_client on the next attempt. continue # Retries exhausted. Log the final failure with # full diagnostic detail (chain, headers, @@ -3693,10 +3827,15 @@ def _call(): # FD-recycle corruption vector. Nothing further is needed. pass else: - try: - agent._replace_primary_openai_client(reason="stale_stream_pool_cleanup") - except Exception: - pass + # #70773: same FD-recycle corruption vector as #67142. + # The shared OpenAI client's connection pool must NOT be + # closed from this watchdog/poll thread — worker threads + # from previous stale-killed attempts may still be + # unwinding their SSL BIOs. The request-local client is + # already closed above via _close_request_client_once. + # The shared client will be replaced lazily by + # _ensure_primary_openai_client on the next request. + pass # Reset the timer so we don't kill repeatedly while # the inner thread processes the closure. last_chunk_time["t"] = time.time() diff --git a/agent/codex_responses_adapter.py b/agent/codex_responses_adapter.py index bce372ebb5da..ee75f4190e6d 100644 --- a/agent/codex_responses_adapter.py +++ b/agent/codex_responses_adapter.py @@ -912,7 +912,8 @@ def _preflight_codex_api_kwargs( allowed_keys = { "model", "instructions", "input", "tools", "store", "reasoning", "include", "max_output_tokens", "temperature", - "tool_choice", "parallel_tool_calls", "prompt_cache_key", "service_tier", + "tool_choice", "parallel_tool_calls", "prompt_cache_key", + "prompt_cache_retention", "service_tier", "extra_headers", "extra_body", "timeout", } normalized: Dict[str, Any] = { @@ -950,8 +951,13 @@ def _preflight_codex_api_kwargs( if isinstance(temperature, (int, float)): normalized["temperature"] = float(temperature) - # Pass through tool_choice, parallel_tool_calls, prompt_cache_key - for passthrough_key in ("tool_choice", "parallel_tool_calls", "prompt_cache_key"): + # Pass through cache routing/retention and tool-dispatch hints. + for passthrough_key in ( + "tool_choice", + "parallel_tool_calls", + "prompt_cache_key", + "prompt_cache_retention", + ): val = api_kwargs.get(passthrough_key) if val is not None: normalized[passthrough_key] = val diff --git a/agent/codex_runtime.py b/agent/codex_runtime.py index 91c2af3e995f..da3bc4f95695 100644 --- a/agent/codex_runtime.py +++ b/agent/codex_runtime.py @@ -702,6 +702,16 @@ def run_codex_app_server_turn( except Exception: pass agent._codex_session = None + _user_interrupted = bool( + getattr(agent, "_interrupt_requested", False) + ) + _interrupt_message = ( + getattr(agent, "_interrupt_message", None) + if _user_interrupted + else None + ) + if _user_interrupted: + agent.clear_interrupt() return { "final_response": ( f"Codex app-server turn failed: {exc}. " @@ -711,9 +721,27 @@ def run_codex_app_server_turn( "api_calls": 0, "completed": False, "partial": True, + "interrupted": _user_interrupted, + **( + {"interrupt_message": _interrupt_message} + if _interrupt_message + else {} + ), "error": str(exc), } + # This runtime bypasses the normal conversation-loop finalizer. Mirror its + # interrupt handoff/cleanup so a hard stop cannot poison the next turn and a + # message-bearing compatibility interrupt can still be replayed by callers. + _user_interrupted = bool( + turn.interrupted and getattr(agent, "_interrupt_requested", False) + ) + _interrupt_message = ( + getattr(agent, "_interrupt_message", None) if _user_interrupted else None + ) + if _user_interrupted: + agent.clear_interrupt() + # If the turn signalled the underlying client is wedged (deadline # blown, post-tool watchdog tripped, OAuth refresh died, subprocess # exited), retire the session so the next turn respawns codex @@ -750,12 +778,27 @@ def run_codex_app_server_turn( # the already-flushed user turn). See gateway/run.py agent_persisted. if getattr(agent, "_session_db", None) is not None: try: - agent._flush_messages_to_session_db(messages) + _codex_flush_ok = agent._flush_messages_to_session_db(messages) except Exception: - logger.debug( + _codex_flush_ok = False + logger.warning( "codex app-server projected-message flush failed", exc_info=True, ) + if _codex_flush_ok is False: + # Unlike the chat-completions loop (which fails closed BEFORE + # projection — see conversation_loop session_persistence_failed), + # codex output has already streamed to the user by the time this + # flush runs, so there is nothing left to withhold. We cannot + # flip agent_persisted=False either: the gateway fallback write + # would re-INSERT the already-flushed user turn (#860/#42039). + # Surface the durability gap loudly instead of a silent debug. + logger.warning( + "codex app-server turn was delivered but could NOT be " + "persisted to the session DB (session=%s) — this turn " + "will be missing after restart/resume", + getattr(agent, "session_id", None), + ) # Counter ticks for the agent-improvement loop. @@ -819,6 +862,12 @@ def run_codex_app_server_turn( "api_calls": api_calls, "completed": not turn.interrupted and turn.error is None, "partial": turn.interrupted or turn.error is not None, + "interrupted": _user_interrupted, + **( + {"interrupt_message": _interrupt_message} + if _interrupt_message + else {} + ), "error": turn.error, # The codex app-server runtime IS an early-return path that bypasses # conversation_loop, but we flush the projected assistant/tool messages diff --git a/agent/coding_context.py b/agent/coding_context.py index db38ab3daa8a..fabbdb48e079 100644 --- a/agent/coding_context.py +++ b/agent/coding_context.py @@ -55,13 +55,12 @@ import logging import os import re -import subprocess import tempfile from dataclasses import dataclass from pathlib import Path from typing import Any, Optional -from hermes_cli._subprocess_compat import IS_WINDOWS, windows_hide_flags +from hermes_cli._subprocess_compat import bounded_git_probe logger = logging.getLogger("hermes.coding_context") @@ -521,30 +520,46 @@ def toolset_selection(self, config: Optional[dict[str, Any]] = None) -> Optional return None return [self.profile.toolset, *_enabled_mcp_servers(config)] - def system_blocks(self) -> list[str]: - """Stable system-prompt blocks for this posture (brief + workspace). + def system_prompt_parts(self) -> tuple[list[str], list[str], list[str]]: + """Return prefix, workspace, and trailing posture blocks separately. The operating brief carries a model-family edit-format nudge appended to it (one cached string, not a separate block) so the model is steered toward the `patch` mode it handles best — see ``_edit_format_line``. + + The three lists preserve the historical flat prompt order: the brief, + the live workspace snapshot, then configured operator instructions. + Prompt assembly can therefore put a cache boundary before the snapshot + without changing the persisted system-prompt bytes. """ if not self.is_coding: - return [] - blocks: list[str] = [] + return [], [], [] + prefix: list[str] = [] + workspace_parts: list[str] = [] + trailing: list[str] = [] if self.profile.guidance: brief = self.profile.guidance edit_line = _edit_format_line(self.model) if edit_line: brief = f"{brief}\n{edit_line}" - blocks.append(brief) + prefix.append(brief) workspace = build_coding_workspace_block(self.cwd) if workspace: - blocks.append(workspace) + workspace_parts.append(workspace) # Operator instructions ride their own block so the brief (block 0) stays # byte-stable and cache-keyed independently of user config. if self.instructions: - blocks.append(f"Operator instructions (from config):\n{self.instructions}") - return blocks + trailing.append(f"Operator instructions (from config):\n{self.instructions}") + return prefix, workspace_parts, trailing + + def system_blocks(self) -> list[str]: + """Return posture blocks in their historical display order. + + ``system_prompt_parts`` is the cache-aware API. This compatibility + helper retains the public flat list for callers outside prompt assembly. + """ + prefix, workspace, trailing = self.system_prompt_parts() + return [*prefix, *workspace, *trailing] def compact_skill_categories(self) -> frozenset[str]: """Skill categories to demote to names-only in the prompt's skill index. @@ -645,6 +660,19 @@ def coding_system_blocks( ).system_blocks() +def coding_system_prompt_parts( + *, + platform: Optional[str] = None, + cwd: Optional[str | Path] = None, + config: Optional[dict[str, Any]] = None, + model: Optional[str] = None, +) -> tuple[list[str], list[str], list[str]]: + """Return coding prefix, workspace snapshot, and trailing guidance.""" + return resolve_runtime_mode( + platform=platform, cwd=cwd, config=config, model=model + ).system_prompt_parts() + + def coding_compact_skill_categories( *, platform: Optional[str] = None, @@ -689,18 +717,14 @@ def _enabled_mcp_servers(config: Optional[dict[str, Any]]) -> list[str]: def _git(cwd: Path, *args: str) -> str: - _popen_kwargs = {"creationflags": windows_hide_flags()} if IS_WINDOWS else {} - try: - out = subprocess.run( - ["git", "-C", str(cwd), *args], - capture_output=True, - text=True, - timeout=_GIT_TIMEOUT, - **_popen_kwargs, - ) - except (OSError, subprocess.SubprocessError): - return "" - return out.stdout.strip() if out.returncode == 0 else "" + """``git -C `` → stripped stdout, or ``""`` on any failure. + + Uses the shared :func:`bounded_git_probe` so the post-kill cleanup is bounded + on Windows — a plain ``subprocess.run(timeout=...)`` here deadlocked the agent + turn inside ``build_coding_workspace_block`` when a killed git left a suspended + descendant holding the pipe handles (issue #66037). + """ + return bounded_git_probe(["git", "-C", str(cwd), *args], timeout=_GIT_TIMEOUT) def _parse_status(porcelain: str) -> tuple[dict[str, str], dict[str, int]]: diff --git a/agent/context_breakdown.py b/agent/context_breakdown.py index 0e2eb772f2ff..4527c5dca220 100644 --- a/agent/context_breakdown.py +++ b/agent/context_breakdown.py @@ -154,3 +154,207 @@ def compute_session_context_breakdown( "estimated_total": estimated_total, "model": getattr(agent, "model", "") or "", } + + +# ── /context rendering (CLI + gateway) ────────────────────────────────────── +# +# Pure text renderers over the payload above. The CLI shows a glyph block-grid +# plus a category table; the gateway uses the same table without the grid +# (proportional monospace is not guaranteed on messaging platforms). + +_CATEGORY_GLYPHS = { + "system_prompt": "■", + "tool_definitions": "▣", + "rules": "▩", + "skills": "▤", + "mcp": "▥", + "subagent_definitions": "▦", + "memory": "▧", + "conversation": "▨", +} +_FREE_GLYPH = "·" +_GRID_COLUMNS = 20 +_GRID_ROWS = 5 # 100 cells → 1 cell per percent of the context window + +# Human-readable tables cap the expanded listings; nothing is dropped from +# the underlying data. +_DETAILS_TABLE_LIMIT = 15 + + +def _bytes_to_tokens(size: Optional[int]) -> Optional[int]: + if size is None: + return None + return (int(size) + 3) // 4 + + +def compute_context_details(agent: Any) -> Dict[str, Any]: + """Expanded per-skill / per-toolset cost listing for ``/context all``. + + Reuses the ``hermes prompt-size`` attribution mechanism (PR #66656): + per-skill index-line bytes parsed from the live ```` + block, and per-toolset schema bytes attributed via the tool registry's + canonical tool→toolset map. Byte figures are converted to the same + chars/4 token heuristic the categories above use. + """ + from hermes_cli.prompt_size import ( + _compute_skills_breakdown, + _compute_toolsets_breakdown, + ) + from agent.system_prompt import build_system_prompt_parts + + parts = build_system_prompt_parts(agent) + stable = parts.get("stable", "") or "" + skills_match = _SKILLS_BLOCK_RE.search(stable) + skills_block = skills_match.group(0) if skills_match else "" + + skills: List[Dict[str, Any]] = [] + if skills_block: + for entry in _compute_skills_breakdown(skills_block): + skills.append({ + "name": entry.get("name", ""), + "index_tokens": _bytes_to_tokens(entry.get("index_line_bytes")) or 0, + "skill_md_tokens": _bytes_to_tokens(entry.get("skill_md_bytes")), + }) + + toolsets: List[Dict[str, Any]] = [] + tools = list(getattr(agent, "tools", None) or []) + if tools: + for group in _compute_toolsets_breakdown(tools): + toolsets.append({ + "toolset": group.get("toolset", ""), + "tool_count": int(group.get("tool_count", 0) or 0), + "schema_tokens": _bytes_to_tokens(group.get("json_bytes")) or 0, + }) + + return {"skills": skills, "toolsets": toolsets} + + +def render_context_grid(payload: Dict[str, Any]) -> List[str]: + """Render the payload as a Claude Code-style glyph block grid. + + 100 cells (5×20), each one percent of the model context window. Categories + fill in declaration order; the remainder renders as free space. + """ + context_max = int(payload.get("context_max") or 0) + categories = payload.get("categories") or [] + total_cells = _GRID_COLUMNS * _GRID_ROWS + + cells: List[str] = [] + if context_max > 0: + for cat in categories: + tokens = int(cat.get("tokens") or 0) + n = round(tokens / context_max * total_cells) + if tokens > 0 and n == 0: + n = 1 # never render a nonzero category as invisible + glyph = _CATEGORY_GLYPHS.get(str(cat.get("id") or ""), "▪") + cells.extend([glyph] * n) + cells = cells[:total_cells] + cells.extend([_FREE_GLYPH] * (total_cells - len(cells))) + + return [ + " ".join(cells[row * _GRID_COLUMNS:(row + 1) * _GRID_COLUMNS]) + for row in range(_GRID_ROWS) + ] + + +def render_context_category_lines(payload: Dict[str, Any]) -> List[str]: + """Render the 'Estimated usage by category' table as plain-text lines.""" + categories = payload.get("categories") or [] + context_max = int(payload.get("context_max") or 0) + estimated_total = int(payload.get("estimated_total") or 0) + denom = context_max or estimated_total + + lines = ["Estimated usage by category"] + if not categories: + lines.append(" (no data yet — send a message first)") + return lines + + width = max(len(str(cat.get("label") or "")) for cat in categories) + width = max(width, len("Free space")) + for cat in categories: + tokens = int(cat.get("tokens") or 0) + glyph = _CATEGORY_GLYPHS.get(str(cat.get("id") or ""), "▪") + pct = tokens / denom * 100 if denom else 0.0 + label = str(cat.get("label") or cat.get("id") or "") + lines.append(f"{glyph} {label:<{width}} {tokens:>9,} tokens {pct:>5.1f}%") + if context_max > 0: + free = max(0, context_max - estimated_total) + pct = free / context_max * 100 + lines.append(f"{_FREE_GLYPH} {'Free space':<{width}} {free:>9,} tokens {pct:>5.1f}%") + return lines + + +def render_context_details_lines(details: Dict[str, Any]) -> List[str]: + """Render the expanded ``/context all`` per-skill / per-toolset tables.""" + lines: List[str] = [] + + toolsets = details.get("toolsets") or [] + if toolsets: + lines.append("Toolsets by schema cost (largest first)") + for group in toolsets[:_DETAILS_TABLE_LIMIT]: + lines.append( + f" {group['toolset']:<24} {group['tool_count']:>3} tools" + f" {group['schema_tokens']:>8,} tokens" + ) + remaining = len(toolsets) - _DETAILS_TABLE_LIMIT + if remaining > 0: + lines.append(f" … and {remaining} more") + + skills = details.get("skills") or [] + if skills: + if lines: + lines.append("") + lines.append("Skills by cost (index = always-on; SKILL.md = cost when loaded)") + for entry in skills[:_DETAILS_TABLE_LIMIT]: + name = str(entry.get("name") or "") + if len(name) > 28: + name = name[:27] + "…" + md = entry.get("skill_md_tokens") + md_str = f"{md:>8,}" if md is not None else f"{'n/a':>8}" + lines.append( + f" {name:<28} index {entry['index_tokens']:>6,}" + f" SKILL.md {md_str} tokens" + ) + remaining = len(skills) - _DETAILS_TABLE_LIMIT + if remaining > 0: + lines.append(f" … and {remaining} more") + + return lines + + +def render_context_breakdown_lines( + payload: Dict[str, Any], + *, + details: Optional[Dict[str, Any]] = None, + grid: bool = True, +) -> List[str]: + """Render the full /context view as plain-text lines. + + ``grid=True`` (CLI) prepends the glyph block grid; the gateway passes + ``grid=False`` and keeps its own gauge. ``details`` (from + :func:`compute_context_details`) appends the expanded listings. + """ + lines: List[str] = [] + if grid: + lines.extend(render_context_grid(payload)) + lines.append("") + lines.extend(render_context_category_lines(payload)) + + context_max = int(payload.get("context_max") or 0) + context_used = int(payload.get("context_used") or 0) + if context_max > 0: + pct = int(payload.get("context_percent") or 0) + lines.append("") + lines.append( + f"Context window: {context_used:,} / {context_max:,} tokens ({pct}%)" + ) + + if details is not None: + detail_lines = render_context_details_lines(details) + if detail_lines: + lines.append("") + lines.extend(detail_lines) + else: + lines.append("") + lines.append("Use /context all for per-skill and per-toolset costs.") + return lines diff --git a/agent/context_compressor.py b/agent/context_compressor.py index a16ec913461d..73fa1e36f215 100644 --- a/agent/context_compressor.py +++ b/agent/context_compressor.py @@ -22,6 +22,7 @@ import sqlite3 import re import time +import uuid from typing import Any, Dict, List, Optional from agent.auxiliary_client import call_llm, _is_connection_error, aux_interrupt_protection @@ -31,13 +32,23 @@ MINIMUM_CONTEXT_LENGTH, get_model_context_length, estimate_messages_tokens_rough, + estimate_tokens_rough, ) from agent.redact import redact_sensitive_text from agent.turn_context import drop_stale_api_content +from tools.todo_tool import TODO_INJECTION_HEADER logger = logging.getLogger(__name__) +def _safe_int(value: Any) -> int | None: + """Best-effort integer coercion for telemetry fields.""" + try: + return int(value) + except (TypeError, ValueError): + return None + + _SUMMARY_PERMANENT_QUOTA_MARKERS: tuple[str, ...] = ( "insufficient_quota", "quota exceeded", @@ -79,9 +90,6 @@ def _is_summary_access_or_quota_error(exc: Exception) -> bool: HISTORICAL_TASK_HEADING = "## Historical Task Snapshot" -HISTORICAL_IN_PROGRESS_HEADING = "## Historical In-Progress State" -HISTORICAL_PENDING_ASKS_HEADING = "## Historical Pending User Asks" -HISTORICAL_REMAINING_WORK_HEADING = "## Historical Remaining Work" SUMMARY_PREFIX = ( @@ -96,9 +104,7 @@ def _is_summary_access_or_quota_error(exc: Exception) -> bool: "Topic overlap with the summary does NOT mean you should resume its " "task: even on similar topics, the latest user message WINS. Treat ONLY " "the latest message as the active task and discard stale items from " - f"'{HISTORICAL_TASK_HEADING}' / '{HISTORICAL_IN_PROGRESS_HEADING}' / " - f"'{HISTORICAL_PENDING_ASKS_HEADING}' / " - f"'{HISTORICAL_REMAINING_WORK_HEADING}' entirely — do not 'wrap up' or " + f"'{HISTORICAL_TASK_HEADING}' entirely — do not 'wrap up' or " "'finish' work described there unless the latest message explicitly " "asks for it. " "Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll " @@ -130,8 +136,20 @@ def _is_summary_access_or_quota_error(exc: Exception) -> bool: # poisoning every subsequent request in the session — a bare key like # "is_compressed_summary" would reach the wire and trip exactly that. COMPRESSED_SUMMARY_METADATA_KEY = "_compressed_summary" +COMPRESSED_SUMMARY_HAS_USER_TURN_KEY = "_compressed_summary_has_user_turn" _DB_PERSISTED_MARKER = "_db_persisted" +_NO_USER_TASK_SENTINEL = "None. This session contains no user-authored turns." +COMPRESSION_CONTINUATION_USER_CONTENT = ( + "Continue from the compressed conversation context above. " + "This marker exists because no human user turn was available." +) +_LEGACY_COMPRESSION_CONTINUATION_USER_CONTENT = ( + "Continue from the compressed conversation context above. " + "This marker exists because the compacted transcript contained " + "no preserved user turn." +) + def _fresh_compaction_message_copy(msg: Dict[str, Any]) -> Dict[str, Any]: """Copy a message for compaction assembly without persistence markers. @@ -196,8 +214,45 @@ def _strip_persistence_markers(messages: List[Dict[str, Any]]) -> None: # stale directive it carried (e.g. "resume exactly from Active Task") survives # embedded in the body and keeps hijacking replies. Keep newest-first; entries # are matched literally. Add a frozen copy here whenever SUMMARY_PREFIX changes. +# NEVER mutate or reorder an existing entry — each one is the exact wire text a +# shipped build persisted, so editing it silently un-normalizes every summary +# written by that build generation; prepend only. tests/agent/ +# test_summary_prefix_semantics.py byte-pins every entry to enforce this. _HISTORICAL_SUMMARY_PREFIXES = ( - # Jul 2026 (#65848 class): identical to the current prefix except it + # Pre-#69619: identical to the current prefix except the stale-item + # discard clause named all four historical headings (the three + # section headers removed by #69619 were still in the template). + # Summaries persisted by builds immediately before #69619 carry this + # exact text and must remain detectable/strippable on resume. + "[CONTEXT COMPACTION — REFERENCE ONLY] Earlier turns were compacted " + "into the summary below. This is a handoff from a previous context " + "window — treat it as background reference, NOT as active instructions. " + "Do NOT answer questions or fulfill requests mentioned in this summary; " + "they were already addressed. " + "Respond ONLY to the latest user message that appears AFTER this " + "summary — that message is the single source of truth for what to do " + "right now. " + "Topic overlap with the summary does NOT mean you should resume its " + "task: even on similar topics, the latest user message WINS. Treat ONLY " + "the latest message as the active task and discard stale items from " + "'## Historical Task Snapshot' / '## Historical In-Progress State' / " + "'## Historical Pending User Asks' / " + "'## Historical Remaining Work' entirely — do not 'wrap up' or " + "'finish' work described there unless the latest message explicitly " + "asks for it. " + "Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll " + "back', 'just verify', 'don't do that anymore', 'never mind', a new " + "topic) must immediately end any in-flight work described in the " + "summary; do not re-surface it in later turns. " + "IMPORTANT: Your persistent memory (MEMORY.md, USER.md) in the system " + "prompt is ALWAYS authoritative and active — never ignore or deprioritize " + "memory content due to this compaction note. " + "None of the above restricts HOW you work: your tools remain fully " + "active — keep calling them normally for the active task (edit files, " + "run commands, search) instead of merely narrating what you would do. " + "The current session state (files, config, etc.) may reflect work " + "described here — avoid repeating it:", + # Jul 2026 (#65848 class): identical to the pre-#69619 prefix except it # lacked the explicit "tools remain fully active" clause — the strong # REFERENCE ONLY framing bled into general tool-use suppression # (observed: 7 consecutive narration-only turns immediately after a @@ -213,9 +268,9 @@ def _strip_persistence_markers(messages: List[Dict[str, Any]]) -> None: "Topic overlap with the summary does NOT mean you should resume its " "task: even on similar topics, the latest user message WINS. Treat ONLY " "the latest message as the active task and discard stale items from " - f"'{HISTORICAL_TASK_HEADING}' / '{HISTORICAL_IN_PROGRESS_HEADING}' / " - f"'{HISTORICAL_PENDING_ASKS_HEADING}' / " - f"'{HISTORICAL_REMAINING_WORK_HEADING}' entirely — do not 'wrap up' or " + "'## Historical Task Snapshot' / '## Historical In-Progress State' / " + "'## Historical Pending User Asks' / " + "'## Historical Remaining Work' entirely — do not 'wrap up' or " "'finish' work described there unless the latest message explicitly " "asks for it. " "Reverse signals in the latest message (e.g. 'stop', 'undo', 'roll " @@ -265,6 +320,12 @@ def _strip_persistence_markers(messages: List[Dict[str, Any]]) -> None: "config, etc.) may reflect work described here — avoid repeating it:", ) +# Restart handoff detection should be early and bounded: it needs to catch the +# restored protected head plus a small cluster of already-stacked handoff/ack +# turns, but it must not treat arbitrary summary-looking live-tail messages as +# proof that this is a resumed compacted session. +_RESTART_HANDOFF_PROBE_EXTRA_MESSAGES = 4 + # Minimum tokens for the summary output _MIN_SUMMARY_TOKENS = 2000 # Proportion of compressed content to allocate for summary @@ -274,9 +335,238 @@ def _strip_persistence_markers(messages: List[Dict[str, Any]]) -> None: # itself a context-pressure source and slows every compaction. _SUMMARY_TOKENS_CEILING = 10_000 +# Aggregate cap on the serialized turn block fed to the summarizer prompt +# (chars). Per-message truncation (_CONTENT_MAX / _TOOL_ARGS_MAX) alone is +# not enough: a compression window with hundreds of already-truncated turns +# can still produce a multi-hundred-KB prompt that blows past slow auxiliary +# backends' context limits or timeouts (Codex Responses fallback paths +# especially). 160K chars ≈ 40K tokens — comfortably inside every supported +# aux model's window while leaving room for the template + previous summary. +# Applied AFTER per-message truncation, with head+tail retention and an +# explicit omitted-middle marker (see _bound_summary_input). This is a +# prompt-side bound only — NEVER add a max_tokens wire cap on the summary +# call (see the no-wire-cap contract test in +# test_compression_small_ctx_threshold_floor.py). +_SUMMARY_INPUT_MAX_CHARS = 160_000 + # Placeholder used when pruning old tool results _PRUNED_TOOL_PLACEHOLDER = "[Old tool output cleared to save context space]" +# Ghost-skill defense (#32106): when compaction reduces an old ``skill_view`` +# result to a 1-line metadata summary, the model still believes the skill is +# loaded even though its instructions are gone. The marker below is the ONE +# canonical prune signal — ``_skill_pruned_marker()`` builds it and every +# presence check matches against the same string, so the emit side and the +# check side can never drift apart (the original PR #44166 emitted +# ``[SKILL_PRUNED:`` but presence-checked ``[SKILL_PRUNED]``, making +# re-injection fire even when the marker had survived). +SKILL_PRUNED_MARKER_PREFIX = "[SKILL_PRUNED:" +# skill_view results at or below this size stay verbatim in pruned +# summaries — small skills are cheap to keep and their loss is unlikely to +# ghost the model. Shared by the emit site and the summarizer-input scan. +_SKILL_VIEW_PRUNE_MIN_CHARS = 5000 +# Cap for the deterministic marker re-injection list — keeps a very long +# session from growing an unbounded "## Pruned Skills" block in every +# iterative summary update. Newest-referenced skills win. +_MAX_PRUNED_SKILL_MARKERS = 20 + + +def _skill_pruned_marker(skill_name: str) -> str: + """Return the canonical prune marker for *skill_name*. + + Used verbatim by BOTH the emit sites (tool-result summarization, + summary re-injection) and the survival check in + ``_reinject_pruned_skill_markers`` — one string, no drift. + """ + return ( + f"{SKILL_PRUNED_MARKER_PREFIX} content lost in compression; " + f"reload with skill_view(name='{skill_name}')]" + ) + + +# Matches the canonical marker and captures the skill name. Anchored on the +# shared prefix constant so a wording change to the marker body updates the +# emit helper and this extractor together. +_SKILL_PRUNED_MARKER_RE = re.compile( + re.escape(SKILL_PRUNED_MARKER_PREFIX) + + r"[^\]]*?reload with skill_view\(name='([^']+)'\)" +) + + +def _extract_pruned_skill_names(text: str) -> list[str]: + """Return skill names referenced by prune markers in *text*, in order.""" + names: list[str] = [] + for match in _SKILL_PRUNED_MARKER_RE.finditer(text or ""): + name = match.group(1) + if name not in names: + names.append(name) + return names + + +def _collect_ghosted_skill_names(turns: List[Dict[str, Any]]) -> list[str]: + """Skill names whose instructions are about to be lost in compaction. + + Covers BOTH shapes a compacted middle window can carry: + + - a ``skill_view`` result already demoted by Phase-1 pruning — the + canonical ``[SKILL_PRUNED: ...]`` marker is in the row content; + - a RAW ``skill_view`` body that was never demoted (it sat inside the + protected tail of an earlier prune, then aged into the compression + window). The summarizer will paraphrase the instructions away, which + is exactly the ghost-skill failure — so it needs a marker too. + """ + names: list[str] = [] + + def _add(name: str) -> None: + if name and name not in names: + names.append(name) + + call_id_to_skill: dict[str, str] = {} + for idx, skill in _skill_view_call_sites(turns): + msg = turns[idx] + for tc in msg.get("tool_calls") or []: + tc_fn = tc.get("function", {}) if isinstance(tc, dict) else getattr(tc, "function", None) + tc_name = tc_fn.get("name", "") if isinstance(tc_fn, dict) else getattr(tc_fn, "name", "") + if tc_name != "skill_view": + continue + cid = tc.get("id", "") if isinstance(tc, dict) else (getattr(tc, "id", "") or "") + if cid: + call_id_to_skill[cid] = skill + for msg in turns: + content = msg.get("content") + text = content if isinstance(content, str) else _content_text_for_contains(content) + for name in _extract_pruned_skill_names(text): + _add(name) + if ( + msg.get("role") == "tool" + and isinstance(content, str) + and len(content) > _SKILL_VIEW_PRUNE_MIN_CHARS + ): + skill = call_id_to_skill.get(str(msg.get("tool_call_id") or "")) + if skill: + _add(skill) + return names + + +_PRUNED_SKILLS_SECTION_HEADING = "## Pruned Skills" + + +def _reinject_pruned_skill_markers(summary: str, skill_names: list[str]) -> str: + """Deterministically restore prune markers the summarizer dropped. + + ``skill_names`` was extracted from the summarizer INPUT before the LLM + call. For every skill whose canonical marker (``_skill_pruned_marker``) + is absent from the model's output, append it under a ``## Pruned + Skills`` section. Presence is checked against the SAME canonical string + the emit sites produce — a paraphrased or renamed marker counts as + dropped and is restored (the original PR checked the literal + ``[SKILL_PRUNED]``, which never matches the emitted ``[SKILL_PRUNED:`` + form, so it duplicated markers that HAD survived). + + The appended block is plain body text: it never carries a handoff + prefix, the merged-summary delimiter, or a start-of-content scaffolding + marker, so ``classify_summary_content`` / todo-snapshot flag handling + are unaffected. The block is routed through ``_redact_compaction_text`` + like every other compaction-boundary text. + """ + if not skill_names: + return summary + missing = [ + name for name in skill_names + if _skill_pruned_marker(name) not in summary + ] + if not missing: + return summary + lines = [_skill_pruned_marker(name) for name in missing] + block = ( + "\n\n" + _PRUNED_SKILLS_SECTION_HEADING + "\n" + + "\n".join(lines) + + "\n(The listed skills' instructions were pruned during context " + "compression. Reload with the skill_view call in each marker before " + "relying on that skill; one reload per skill is enough — ignore any " + "older markers for the same skill.)" + ) + return summary + _redact_compaction_text(block) + + +# A skill_view call within this many trailing messages counts as "just +# loaded": its full instruction body must survive the Phase-1 prune even when +# the token-budget boundary would otherwise demote it (#32106). Distinct from +# the protected-tail boundary, which is token-based and can land immediately +# after a bulky just-loaded skill body. +_SKILL_PRUNE_RECENT_WINDOW = 10 + + +def _skill_view_call_sites( + messages: List[Dict[str, Any]], +) -> list[tuple[int, str]]: + """Yield ``(message_index, skill_name)`` for every skill_view tool call.""" + sites: list[tuple[int, str]] = [] + for i, msg in enumerate(messages): + if msg.get("role") != "assistant": + continue + for tc in msg.get("tool_calls") or []: + if isinstance(tc, dict): + fn = tc.get("function", {}) + name = fn.get("name", "") if isinstance(fn, dict) else "" + args_str = fn.get("arguments", "") if isinstance(fn, dict) else "" + else: + fn = getattr(tc, "function", None) + name = getattr(fn, "name", "") if fn else "" + args_str = getattr(fn, "arguments", "") if fn else "" + if name != "skill_view" or not isinstance(args_str, str) or not args_str: + continue + try: + args = json.loads(args_str) + except (json.JSONDecodeError, TypeError): + continue + if isinstance(args, dict): + skill = args.get("name", "") + if isinstance(skill, str) and skill: + sites.append((i, skill)) + return sites + + +def _collect_protected_skill_names( + messages: List[Dict[str, Any]], prune_boundary: int, +) -> set[str]: + """Skill names whose skill_view bodies must survive Phase-1 demotion. + + A skill is protected (lower-cased set) when any of these hold: + + - its most recent ``skill_view`` call sits within the last + ``_SKILL_PRUNE_RECENT_WINDOW`` messages (just loaded / just reloaded); + - its most recent ``skill_view`` call sits inside the protected tail + (at or after *prune_boundary*); + - its name is mentioned in a user message inside the protected tail + (the user is actively steering work that depends on it). + + Protection applies to the ordinary Phase-1/2 prune only. The Pass-4 + pressure demotion deliberately ignores it: when the protected region + itself exceeds the soft budget, exempting skill bodies would recreate + the #61932 dead-end shape. + """ + total = len(messages) + if not total: + return set() + recent_start = max(0, total - _SKILL_PRUNE_RECENT_WINDOW) + tail_start = max(0, prune_boundary) + tail_user_texts: list[str] = [] + for msg in messages[tail_start:]: + if msg.get("role") != "user": + continue + content = msg.get("content") + if isinstance(content, str) and content: + tail_user_texts.append(content.lower()) + protected: set[str] = set() + for idx, skill in _skill_view_call_sites(messages): + key = skill.lower() + if idx >= recent_start or idx >= tail_start: + protected.add(key) + elif any(key in text for text in tail_user_texts): + protected.add(key) + return protected + # Chars per token rough estimate _CHARS_PER_TOKEN = 4 # Flat token cost per attached image part. Real cost varies by provider and @@ -295,6 +585,7 @@ def _strip_persistence_markers(messages: List[Dict[str, Any]]) -> None: # only meant to preserve continuity anchors from the dropped window, not to # become another unbounded transcript copy after the LLM summarizer failed. _FALLBACK_SUMMARY_MAX_CHARS = 8_000 +_FALLBACK_PREVIOUS_SUMMARY_MAX_CHARS = 3_000 _FALLBACK_TURN_MAX_CHARS = 700 _AUTO_FOCUS_MAX_TURNS = 3 _AUTO_FOCUS_TURN_MAX_CHARS = 260 @@ -305,6 +596,11 @@ def _strip_persistence_markers(messages: List[Dict[str, Any]]) -> None: # high for small/light tails, but using all 20 as a hard floor here would bring # back the old large-tool-output case where nothing can be compacted. _MAX_TAIL_MESSAGE_FLOOR = 8 +# Under context pressure (protected-tail tool bodies alone exceed the soft +# tail budget), demote large completed tool/file outputs even inside the +# protected region — but always keep this many trailing messages verbatim so +# the active user ask / latest tool pair remain readable. Issue #61932. +_PRESSURE_KEEP_RECENT_MESSAGES = 3 # Models with context windows below this get their compression threshold # floored at ``_SMALL_CTX_THRESHOLD_PERCENT`` (raise-only — an explicitly @@ -328,6 +624,27 @@ def _strip_persistence_markers(messages: List[Dict[str, Any]]) -> None: ) +def _redact_compaction_text(text: Any) -> str: + """Redact text that crosses a compaction summary boundary. + + Compaction summaries persist across sessions and are re-injected into + every subsequent summarizer prompt, so this boundary uses strict mode: + + - ``force=True`` — deliberately overrides ``security.redact_secrets: + false``. That opt-out targets *live tool output* (e.g. working on the + redactor itself); a summary is a persistence boundary where a leaked + credential keeps re-entering prompts indefinitely. + - ``redact_url_credentials=True`` — OAuth callback codes, magic-link + tokens, and URL userinfo never need to survive summarization the way + they must survive live navigation flows. + """ + return redact_sensitive_text( + text or "", + force=True, + redact_url_credentials=True, + ) + + def _dedupe_append(items: list[str], value: str, *, limit: int) -> None: value = value.strip() if value and value not in items and len(items) < limit: @@ -436,11 +753,15 @@ def _estimate_msg_budget_tokens(msg: dict) -> int: compaction re-fires continuously (#55572). Accounting-only: replay fields are never mutated or pruned here. """ - content_len = _content_length_for_budget(msg.get("content") or "") - tokens = content_len // _CHARS_PER_TOKEN + 10 # +10 for role/key overhead + content = msg.get("content") or "" + if isinstance(content, str): + tokens = estimate_tokens_rough(content) + 10 # +10 for role/key overhead + else: + content_len = _content_length_for_budget(content) + tokens = content_len // _CHARS_PER_TOKEN + 10 for tc in msg.get("tool_calls") or []: if isinstance(tc, dict): - tokens += len(str(tc)) // _CHARS_PER_TOKEN + tokens += estimate_tokens_rough(str(tc)) for key in _REPLAY_BUDGET_KEYS: tokens += _serialized_length_for_budget(msg.get(key)) // _CHARS_PER_TOKEN return tokens @@ -817,7 +1138,19 @@ def _summarize_tool_result_unguarded(tool_name: str, tool_args: str, tool_conten code_preview += "..." return f"[execute_code] `{code_preview}` ({line_count} lines output)" - if tool_name in {"skill_view", "skills_list", "skill_manage"}: + if tool_name == "skill_view": + name = args.get("name", "?") + if content_len > _SKILL_VIEW_PRUNE_MIN_CHARS: + # Ghost-skill defense (#32106): a metadata-only summary makes the + # model believe the skill is still loaded. The canonical marker + # tells it the instructions are gone AND how to get them back. + return ( + f"[skill_view] name={name} ({content_len:,} chars) " + + _skill_pruned_marker(str(name)) + ) + return f"[skill_view] name={name} ({content_len:,} chars)" + + if tool_name in {"skills_list", "skill_manage"}: name = args.get("name", "?") return f"[{tool_name}] name={name} ({content_len:,} chars)" @@ -856,6 +1189,32 @@ def _summarize_tool_result_unguarded(tool_name: str, tool_args: str, tool_conten return f"[{tool_name}]{first_arg} ({content_len:,} chars result)" +def resolve_model_threshold( + model: str, + model_thresholds: dict[str, float] | None, + default: float, +) -> float: + """Resolve the effective compression threshold for a given model. + + ``model_thresholds`` maps substring keys to override fractions. The + longest matching key wins (so ``glm-5.2-1M`` beats ``glm-5.2`` when the + model is ``glm-5.2-1M``). When no override matches, or when + ``model_thresholds`` is empty/None, ``default`` is returned unchanged. + + This is a module-level helper so plugin context engines (e.g. LCM) can + import and reuse the same resolution logic as the built-in compressor. + """ + if not model_thresholds or not model: + return default + best_key = "" + for key in model_thresholds: + if key in model and len(key) > len(best_key): + best_key = key + if best_key: + return float(model_thresholds[best_key]) + return default + + class ContextCompressor(ContextEngine): """Default context engine — compresses conversation context via lossy summarization. @@ -877,6 +1236,7 @@ def on_session_reset(self) -> None: self._context_probed = False self._context_probe_persistable = False self._previous_summary = None + self._summary_has_user_turn = None self._last_summary_error = None self._consecutive_timeout_failures = 0 self._last_summary_dropped_count = 0 @@ -885,6 +1245,7 @@ def on_session_reset(self) -> None: self._last_aux_model_failure_model = None self._last_compression_savings_pct = 100.0 self._ineffective_compression_count = 0 + self._anti_thrash_recovery_deadline = 0.0 self._fallback_compression_streak = 0 self._verify_compaction_cleared_threshold = False self._last_compression_made_progress = False @@ -896,6 +1257,102 @@ def on_session_reset(self) -> None: self.last_compression_rough_tokens = 0 self.last_rough_tokens_when_real_prompt_fit = 0 self.awaiting_real_usage_after_compression = False + self._last_compression_telemetry = None + self._active_compression_telemetry = None + self._compression_telemetry_seed = None + + def _begin_compression_telemetry( + self, + *, + current_tokens: int | None, + attempt_id: str | None = None, + session_id: str | None = None, + trigger_source: str | None = None, + ) -> Dict[str, Any]: + """Initialize content-free per-attempt compression telemetry.""" + seed = getattr(self, "_compression_telemetry_seed", None) + if isinstance(seed, dict): + attempt_id = attempt_id or seed.get("attempt_id") + session_id = session_id or seed.get("session_id") + trigger_source = trigger_source or seed.get("trigger_source") + telemetry: Dict[str, Any] = { + "event": "compression_attempt", + "attempt_id": attempt_id or uuid.uuid4().hex, + "session_id": session_id or "", + "trigger_source": trigger_source or "unknown", + "main_provider": self.provider or "", + "main_model": self.model or "", + "main_context_limit": _safe_int(self.context_length), + "current_estimated_tokens": _safe_int(current_tokens), + "effective_threshold": _safe_int(self.threshold_tokens), + "protected_head_tokens": None, + "protected_tail_tokens": None, + "middle_window_tokens": None, + "aux_prompt_tokens": None, + "aux_output_reservation": None, + "aux_provider": "", + "aux_model": "", + "effective_aux_context": None, + "fit_margin": None, + "chunking": False, + "chunk_count": 0, + "total_duration_ms": None, + "aux_call_duration_ms": None, + "fallback_used": False, + "commit_status": "unknown", + "split_status": "unknown", + "failure_class": None, + } + self._active_compression_telemetry = telemetry + self._last_compression_telemetry = telemetry + return telemetry + + def _record_compression_regions( + self, + *, + head_messages: List[Dict[str, Any]], + middle_messages: List[Dict[str, Any]], + tail_messages: List[Dict[str, Any]], + ) -> None: + telemetry = getattr(self, "_active_compression_telemetry", None) + if not isinstance(telemetry, dict): + return + telemetry["protected_head_tokens"] = estimate_messages_tokens_rough(head_messages) + telemetry["middle_window_tokens"] = estimate_messages_tokens_rough(middle_messages) + telemetry["protected_tail_tokens"] = estimate_messages_tokens_rough(tail_messages) + + def _record_aux_compression_call( + self, + *, + prompt_messages: List[Dict[str, Any]], + max_tokens: int | None, + duration_ms: int, + aux_provider: str | None = None, + aux_model: str | None = None, + effective_aux_context: int | None = None, + ) -> None: + telemetry = getattr(self, "_active_compression_telemetry", None) + if not isinstance(telemetry, dict): + return + telemetry["aux_prompt_tokens"] = estimate_messages_tokens_rough(prompt_messages) + telemetry["aux_output_reservation"] = _safe_int(max_tokens) + if aux_provider: + telemetry["aux_provider"] = aux_provider + if aux_model: + telemetry["aux_model"] = aux_model + if effective_aux_context is not None: + telemetry["effective_aux_context"] = _safe_int(effective_aux_context) + if ( + telemetry["effective_aux_context"] is not None + and telemetry["aux_prompt_tokens"] is not None + ): + telemetry["fit_margin"] = ( + telemetry["effective_aux_context"] + - telemetry["aux_prompt_tokens"] + - (telemetry["aux_output_reservation"] or 0) + ) + previous = telemetry.get("aux_call_duration_ms") or 0 + telemetry["aux_call_duration_ms"] = previous + max(0, int(duration_ms)) def on_session_end(self, session_id: str, messages: List[Dict[str, Any]]) -> None: """Clear all per-session compaction state at a real session boundary. @@ -917,6 +1374,7 @@ def on_session_end(self, session_id: str, messages: List[Dict[str, Any]]) -> Non surface the moment the owning session ends. """ self._previous_summary = None + self._summary_has_user_turn = None self._last_summary_error = None self._consecutive_timeout_failures = 0 self._last_summary_dropped_count = 0 @@ -925,6 +1383,7 @@ def on_session_end(self, session_id: str, messages: List[Dict[str, Any]]) -> Non self._last_aux_model_failure_model = None self._last_compression_savings_pct = 100.0 self._ineffective_compression_count = 0 + self._anti_thrash_recovery_deadline = 0.0 self._fallback_compression_streak = 0 self._verify_compaction_cleared_threshold = False self._last_compression_made_progress = False @@ -937,6 +1396,9 @@ def on_session_end(self, session_id: str, messages: List[Dict[str, Any]]) -> Non self.last_compression_rough_tokens = 0 self.last_rough_tokens_when_real_prompt_fit = 0 self.awaiting_real_usage_after_compression = False + self._last_compression_telemetry = None + self._active_compression_telemetry = None + self._compression_telemetry_seed = None def bind_session_state(self, session_db: Any = None, session_id: str = "") -> None: """Bind the current session row so durable cooldowns can round-trip.""" @@ -947,8 +1409,11 @@ def bind_session_state(self, session_db: Any = None, session_id: str = "") -> No self._last_summary_error = None self._consecutive_timeout_failures = 0 self._fallback_compression_streak = 0 + self._ineffective_compression_count = 0 + self._anti_thrash_recovery_deadline = 0.0 self.get_active_compression_failure_cooldown() self._load_fallback_compression_streak() + self._load_ineffective_compression_count() def on_session_start(self, session_id: str, **kwargs) -> None: """Bind session-scoped compression state for a new or resumed session.""" @@ -957,6 +1422,7 @@ def on_session_start(self, session_id: str, **kwargs) -> None: old_session_id = kwargs.get("old_session_id") session_db = kwargs.get("session_db", getattr(self, "_session_db", None)) previous_fallback_streak = self._fallback_compression_streak + previous_ineffective_count = self._ineffective_compression_count if boundary_reason == "compression" and old_session_id: getter = getattr(session_db, "get_compression_fallback_streak", None) if callable(getter): @@ -971,12 +1437,37 @@ def on_session_start(self, session_id: str, **kwargs) -> None: "compression parent fallback streak lookup failed (non-sqlite): %s", exc, ) + count_getter = getattr( + session_db, "get_compression_ineffective_count", None, + ) + if callable(count_getter): + try: + stored_count = count_getter(old_session_id) + if isinstance(stored_count, (int, float, str)): + previous_ineffective_count = max(0, int(stored_count)) + except (TypeError, ValueError, sqlite3.Error) as exc: + logger.debug( + "compression parent ineffective count lookup failed: %s", exc, + ) + except Exception as exc: + logger.debug( + "compression parent ineffective count lookup failed (non-sqlite): %s", + exc, + ) self.bind_session_state(session_db, session_id) if boundary_reason == "compression": # Rotation creates a fresh child row before this callback. Preserve # the logical conversation's streak until boundary bookkeeping # persists the updated value onto the child row. self._fallback_compression_streak = previous_fallback_streak + # Same for the anti-thrash strike counter — but unlike the streak, + # no later boundary bookkeeping writes it, so persist the carried + # value onto the (fresh) child row now. Otherwise a restart between + # rotation and the next real-usage verdict would silently disarm + # an armed guard (#54923). + if self._ineffective_compression_count != previous_ineffective_count: + self._ineffective_compression_count = previous_ineffective_count + self._persist_ineffective_compression_count() def _load_fallback_compression_streak(self) -> None: session_db = getattr(self, "_session_db", None) @@ -1010,6 +1501,59 @@ def _persist_fallback_compression_streak(self) -> None: except Exception as exc: logger.debug("compression fallback streak persist failed (non-sqlite): %s", exc) + def _load_ineffective_compression_count(self) -> None: + """Load the durable anti-thrash strike count for the bound session. + + A fresh compressor on a resumed session starts with + ``compression_count == 0`` and, historically, an in-memory-only + ineffective counter — so a guard armed (1 strike) or tripped + (2 strikes) before a process restart silently disarmed, and a + near-threshold session could re-compact once per restart forever + (#54923). The counter now round-trips through the session row like + the failure cooldown and the fallback streak. + """ + session_db = getattr(self, "_session_db", None) + session_id = getattr(self, "_session_id", "") + getter = getattr(session_db, "get_compression_ineffective_count", None) + if not session_id or not callable(getter): + return + try: + stored_count = getter(session_id) + self._ineffective_compression_count = max( + 0, + int(stored_count) + if isinstance(stored_count, (int, float, str)) + else 0, + ) + except (TypeError, ValueError, sqlite3.Error) as exc: + logger.debug("compression ineffective count lookup failed: %s", exc) + except Exception as exc: + logger.debug("compression ineffective count lookup failed (non-sqlite): %s", exc) + + def _persist_ineffective_compression_count(self) -> None: + session_db = getattr(self, "_session_db", None) + session_id = getattr(self, "_session_id", "") + setter = getattr(session_db, "set_compression_ineffective_count", None) + if not session_id or not callable(setter): + return + try: + setter(session_id, self._ineffective_compression_count) + except sqlite3.Error as exc: + logger.debug("compression ineffective count persist failed: %s", exc) + except Exception as exc: + logger.debug("compression ineffective count persist failed (non-sqlite): %s", exc) + + def _record_ineffective_compression_verdict(self, count: int) -> None: + """Set the anti-thrash strike counter, keeping the durable copy in sync. + + Persists only on change so the reset issued by every ordinary fitting + response (already-zero -> zero) never costs a DB write. + """ + if count == self._ineffective_compression_count: + return + self._ineffective_compression_count = count + self._persist_ineffective_compression_count() + def record_completed_compaction(self, *, used_fallback: bool = False) -> None: """Record one completed boundary and its summary quality.""" self._verify_compaction_cleared_threshold = True @@ -1163,17 +1707,19 @@ def update_model( self.provider = provider self.api_mode = api_mode self.context_length = context_length - # Re-apply the small-context threshold floor for the NEW window, - # starting from the originally-configured percent (not the possibly - # floored live value) so a small -> large switch drops back to the - # configured threshold and a large -> small switch gains the floor. - # Guard with getattr: compressors unpickled/constructed before this - # attribute existed fall back to the live value. - _configured_pct = getattr( - self, "_configured_threshold_percent", self.threshold_percent, + # Re-resolve per-model threshold for the NEW model, then re-apply the + # small-context threshold floor. Starting from _config_threshold_percent + # (the raw config value) so a switch from a model with an override to + # one without correctly falls back to the global threshold. + _config_pct = getattr( + self, "_config_threshold_percent", self.threshold_percent, ) + _new_base = resolve_model_threshold( + model, self.model_thresholds, _config_pct, + ) + self._base_threshold_percent = _new_base self.threshold_percent = self._effective_threshold_percent( - context_length, _configured_pct, + context_length, _new_base, ) # max_tokens=None here means "caller didn't specify" → keep the existing # output reservation. A switch that genuinely changes the output budget @@ -1183,6 +1729,11 @@ def update_model( self.threshold_tokens = self._compute_threshold_tokens( context_length, self.threshold_percent, self.max_tokens, ) + # Re-apply the absolute token cap so it survives model switches + # and fallback activations. The cap is a first-class config value + # stored on the compressor instance, not a one-time post-construction + # patch — this is why update_model() must re-apply it. + self._apply_threshold_tokens_cap() # Recalculate token budgets for the new context length so the # compressor stays calibrated after a model switch (e.g. 200K → 32K). target_tokens = int(self.threshold_tokens * self.summary_target_ratio) @@ -1211,7 +1762,10 @@ def update_model( self.last_rough_tokens_when_real_prompt_fit = 0 self.last_compression_rough_tokens = 0 self.awaiting_real_usage_after_compression = False - self._ineffective_compression_count = 0 + # Strikes were judged against the PREVIOUS threshold; a recomputed + # trigger invalidates them. Keep the durable copy in sync so a + # restart doesn't resurrect strikes this recalibration just voided. + self._record_ineffective_compression_verdict(0) if runtime_changed: self._fallback_compression_streak = 0 self._persist_fallback_compression_streak() @@ -1228,6 +1782,15 @@ def update_model( # rationale as the gpt-5.5/Codex 85% autoraise. _MIN_CTX_TRIGGER_RATIO = 0.85 + # Anti-thrash recovery window (#14694): once the ineffective/fallback + # breaker trips, automatic compaction stays blocked for this long, then + # ONE probe attempt is allowed (counters drop to 1 strike, so another + # ineffective pass re-trips immediately). Long enough that a genuinely + # incompressible session isn't compacting in a loop; short enough that a + # session which has since grown real compressible material recovers well + # before it rides into the provider's hard context limit. + _ANTI_THRASH_RECOVERY_SECONDS = 300.0 + @staticmethod def _coerce_max_tokens(value: Any) -> int | None: """Normalize a max_tokens value to a positive int or None. @@ -1245,6 +1808,36 @@ def _coerce_max_tokens(value: Any) -> int | None: return None return ivalue if ivalue > 0 else None + @staticmethod + def _coerce_threshold_tokens_cap(value: Any) -> int | None: + """Normalize a threshold_tokens cap to a positive int or None. + + None means "no absolute cap — use the ratio-based threshold only". + Non-numeric or non-positive values are treated as None so a bad + config value never silently caps the threshold at zero. + """ + if value is None: + return None + try: + ivalue = int(value) + except (TypeError, ValueError): + return None + return ivalue if ivalue > 0 else None + + def _apply_threshold_tokens_cap(self) -> None: + """Apply the absolute token cap if configured. + + After ``threshold_tokens`` is (re)computed from the ratio-based + percent, clamp it to the cap so compression never fires later + than the user's preferred absolute token count. The cap itself + is clamped to the current context length so a cap larger than + the model's window is a no-op (the ratio-based threshold wins). + """ + if self.threshold_tokens_cap is not None and self.threshold_tokens_cap > 0: + _effective_cap = min(self.threshold_tokens_cap, self.context_length) + if _effective_cap < self.threshold_tokens: + self.threshold_tokens = _effective_cap + @staticmethod def _effective_threshold_percent( context_length: int, threshold_percent: float, @@ -1319,15 +1912,67 @@ def __init__( api_mode: str = "", abort_on_summary_failure: bool = False, max_tokens: int | None = None, + model_thresholds: dict[str, float] | None = None, + threshold_tokens_cap: Any = None, + proactive_prune_tokens: int = 0, + proactive_prune_min_result_chars: int = 8000, + proactive_prune_min_reclaim_tokens: int = 4096, + min_tail_user_messages: int = 1, ): self.model = model self.base_url = base_url self.api_key = api_key self.provider = provider self.api_mode = api_mode - self.threshold_percent = threshold_percent + # Per-model threshold overrides (longest substring match wins). + # Stored as a plain dict; resolved in _resolve_threshold(), then the + # small-context floor is applied on top. + self.model_thresholds = model_thresholds or {} + # _config_threshold_percent is the raw config value (before per-model + # override or small-context floor). Used as the fallback when switching + # to a model with no matching override. + self._config_threshold_percent = threshold_percent + # Resolve per-model override first, then apply the small-context floor. + self._base_threshold_percent = resolve_model_threshold( + model, self.model_thresholds, threshold_percent, + ) + self.threshold_percent = self._base_threshold_percent + # Absolute token cap from config (compression.threshold_tokens). When + # set, the effective trigger point is min(ratio-based threshold, cap) + # so compression never fires later than the user's preferred token + # count regardless of which model is active. Applied in __init__ and + # re-applied in update_model() so it survives model switches/fallbacks. + self.threshold_tokens_cap = self._coerce_threshold_tokens_cap( + threshold_tokens_cap, + ) self.protect_first_n = protect_first_n self.protect_last_n = protect_last_n + # Proactive tool-result pruning (cost-oriented; runs INDEPENDENTLY of the + # full-compression trigger, via prune_tool_results_only()). 0 = disabled. + self.proactive_prune_tokens = int(proactive_prune_tokens or 0) + # Floor the summarize threshold at 200 chars (matching + # _prune_old_tool_results' dedup floor). Below ~200 a generated summary + # can be longer than the floor it replaces, so Pass 2 would re-summarize + # its own output every turn (corrupting it and never converging); a + # negative value would strip every non-tail tool result outright. A + # configured 0 keeps the 8000 default via `or`. Keep the floor well above + # typical summary length (default 8000) to stay idempotent. + self.proactive_prune_min_result_chars = max( + 200, int(proactive_prune_min_result_chars or 8000) + ) + # Minimum estimated token reclaim before a proactive prune COMMITS. + # Every commit rewrites messages the provider has already seen, which + # invalidates the prompt-cache prefix from the earliest rewritten + # message forward. Without this gate a busy tool loop would re-fire + # the prune nearly every iteration (each new tool pair ages an old one + # out of the protected tail), breaking the cache per turn. Requiring a + # meaningful batch of reclaimable tokens makes fires episodic and + # amortized — the same way full compression is the one sanctioned + # cache break. 0 disables the gate (commit any non-zero prune). + self.proactive_prune_min_reclaim_tokens = max( + 0, int(proactive_prune_min_reclaim_tokens or 0) + ) + self.min_tail_user_messages = min_tail_user_messages self.summary_target_ratio = max(0.10, min(summary_target_ratio, 0.80)) self.quiet_mode = quiet_mode # Output-token reservation: the provider carves max_tokens out of the @@ -1355,9 +2000,11 @@ def __init__( # resolved and BEFORE threshold_tokens is derived. The pre-floor # value is kept so update_model() can re-derive for a new window # (switching small -> large must drop back to the configured value). + # Note: _base_threshold_percent already has the per-model override + # applied, so the floor stacks on top of any model-specific threshold. self._configured_threshold_percent = self.threshold_percent self.threshold_percent = self._effective_threshold_percent( - self.context_length, self.threshold_percent, + self.context_length, self._base_threshold_percent, ) threshold_percent = self.threshold_percent # Floor: never compress below MINIMUM_CONTEXT_LENGTH tokens even if @@ -1369,6 +2016,9 @@ def __init__( self.threshold_tokens = self._compute_threshold_tokens( self.context_length, threshold_percent, self.max_tokens, ) + # Apply absolute token cap (compression.threshold_tokens) — takes + # the lower of the ratio-based threshold and the cap. + self._apply_threshold_tokens_cap() self.compression_count = 0 # Derive token budgets: ratio is relative to the threshold, not total context @@ -1403,9 +2053,19 @@ def __init__( # Stores the previous compaction summary for iterative updates self._previous_summary: Optional[str] = None + # Provenance for the rolling summary. A compaction handoff can carry + # role="user" solely to satisfy provider alternation, so role alone + # cannot prove that a human-authored turn ever existed. + self._summary_has_user_turn: Optional[bool] = None # Anti-thrashing: track whether last compression was effective self._last_compression_savings_pct: float = 100.0 self._ineffective_compression_count: int = 0 + # Monotonic deadline after which a tripped anti-thrash guard grants + # one probation probe (#14694). 0.0 = clock not armed. Armed lazily on + # the first blocked evaluation; deliberately NOT durable, so a process + # restart with a persisted tripped counter (#69872) waits a full fresh + # window before probing (#54923: restart must never disarm a guard). + self._anti_thrash_recovery_deadline: float = 0.0 # Consecutive completed deterministic-fallback boundaries. Unlike the # real-usage effectiveness counter, ordinary fitting responses must not # reset this breaker; only a healthy completed summary does. @@ -1455,6 +2115,9 @@ def __init__( # succeeded. Silent recovery would hide the broken config. self._last_aux_model_failure_error: Optional[str] = None self._last_aux_model_failure_model: Optional[str] = None + self._last_compression_telemetry: Optional[Dict[str, Any]] = None + self._active_compression_telemetry: Optional[Dict[str, Any]] = None + self._compression_telemetry_seed: Optional[Dict[str, Any]] = None def update_from_response(self, usage: Dict[str, Any]): """Update tracked token usage from API response.""" @@ -1471,7 +2134,7 @@ def update_from_response(self, usage: Dict[str, Any]): # when this response was not immediately after compaction. The # independent fallback streak is boundary-scoped and survives # ordinary fitting responses during context regrowth. - self._ineffective_compression_count = 0 + self._record_ineffective_compression_verdict(0) else: self.last_rough_tokens_when_real_prompt_fit = 0 @@ -1493,7 +2156,9 @@ def update_from_response(self, usage: Dict[str, Any]): # per compaction. if self._verify_compaction_cleared_threshold: if self.last_prompt_tokens >= self.threshold_tokens: - self._ineffective_compression_count += 1 + self._record_ineffective_compression_verdict( + self._ineffective_compression_count + 1, + ) if not self.quiet_mode: logger.warning( "Compaction did not clear the threshold: %d real " @@ -1505,13 +2170,23 @@ def update_from_response(self, usage: Dict[str, Any]): self._ineffective_compression_count, ) else: - self._ineffective_compression_count = 0 + self._record_ineffective_compression_verdict(0) # Consume the pending-verification flag once real usage arrives, whether # or not prompt_tokens was reported, so a usage-less response can't leave # it armed for a later, unrelated reading. self._verify_compaction_cleared_threshold = False self.awaiting_real_usage_after_compression = False + def snapshot_preflight_display_tokens(self) -> int: + """Capture the display token count before a speculative preflight seed.""" + return self.last_prompt_tokens + + def rollback_interrupted_preflight_display_tokens(self, snapshot: int) -> None: + """Restore a speculative display seed without touching compaction state.""" + if self.awaiting_real_usage_after_compression and self.last_prompt_tokens == -1: + return + self.last_prompt_tokens = snapshot + def should_defer_preflight_to_real_usage(self, rough_tokens: int) -> bool: """Return True when a high rough preflight estimate is known-noisy. @@ -1557,23 +2232,83 @@ def should_defer_preflight_to_real_usage(self, rough_tokens: int) -> bool: def should_compress(self, prompt_tokens: int = None) -> bool: """Check if context exceeds the compression threshold. + Returns ``True`` when compression should run now. For the caller-facing + *reason* (e.g. why compression is skipped while still over threshold), + see :meth:`should_compress_info`, which returns a ``(bool, reason)`` + tuple without changing the decision logic here. + + Includes anti-thrashing protection: if the last two compressions + each saved less than 10%, skip compression to avoid infinite loops + where each pass removes only 1-2 messages. + """ + decision, _reason = self.should_compress_info(prompt_tokens) + return decision + + def should_compress_info( + self, prompt_tokens: int = None + ) -> "tuple[bool, str | None]": + """Check if context exceeds the compression threshold. + + Returns a ``(should_compress, reason)`` tuple instead of a bare bool so + callers can tell *why* compression is skipped when it is skipped while + the context is already over threshold. ``reason`` is ``None`` unless + compression is needed but blocked: + + * ``"cooldown:"`` — the summary LLM is recovering from a + recent 429/transient failure; compression is deferred to avoid the + freeze loop described in #11529. + * ``"ineffective"`` — anti-thrashing has backed off because the last + two compressions each saved <10%. + + When ``reason`` is non-``None`` the session is over its compression + threshold yet cannot shrink — callers should surface a warning so the + user knows the model may silently stop answering (the context keeps + growing until it hits the hard provider limit). Without this signal an + over-threshold session fails opaquely. + Includes anti-thrashing protection: if the last two compressions each saved less than 10%, skip compression to avoid infinite loops where each pass removes only 1-2 messages. """ tokens = prompt_tokens if prompt_tokens is not None else self.last_prompt_tokens if tokens < self.threshold_tokens: - return False - return not self._automatic_compression_blocked() + return False, None + if self._automatic_compression_blocked(): + return False, self._compression_block_reason() or "blocked" + return True, None + + def _compression_block_reason(self) -> "str | None": + """Return a human-readable reason for the current automatic-compaction + block, derived from the same in-memory state that + :meth:`_automatic_compression_blocked_locally` evaluates. + + * ``"cooldown:"`` — the summary LLM is recovering from a + recent 429/transient failure; compression is deferred to avoid the + freeze loop described in #11529. + * ``"ineffective"`` — anti-thrashing has backed off (the last two + compressions each saved <10%, or the fallback streak tripped). + * ``None`` — no block active. + """ + _cooldown_remaining = self._summary_failure_cooldown_until - time.monotonic() + if _cooldown_remaining > 0: + return f"cooldown:{_cooldown_remaining:.0f}" + if ( + self._ineffective_compression_count >= 2 + or self._fallback_compression_streak >= 2 + ): + return "ineffective" + return None def _refresh_durable_guards(self) -> None: - """Re-read durable cooldown + fallback-streak state from the DB. + """Re-read durable cooldown + breaker state from the DB. Cheap, best-effort, and only called when a gate is about to say "blocked": another agent on the same session may have cleared the - durable rows (successful boundary, forced retry) after this - compressor was bound, and a fallback streak has no timer — without - a re-read the stale in-memory snapshot blocks forever. + durable rows (successful boundary, forced retry, a real usage + reading that dipped below the threshold) after this compressor was + bound, and neither the fallback streak nor the ineffective-strike + counter has a timer — without a re-read the stale in-memory + snapshot blocks forever. """ try: self.get_active_compression_failure_cooldown(refresh=True) @@ -1583,26 +2318,22 @@ def _refresh_durable_guards(self) -> None: self._load_fallback_compression_streak() except Exception as exc: logger.debug("compression fallback-streak refresh failed: %s", exc) + try: + self._load_ineffective_compression_count() + except Exception as exc: + logger.debug("compression ineffective-count refresh failed: %s", exc) def _automatic_compression_blocked(self) -> bool: """Return whether automatic compaction is in cooldown or tripped.""" if not self._automatic_compression_blocked_locally(): return False # Blocked on the in-memory snapshot. Durable guard rows may have - # been cleared by another agent since bind_session_state(); refresh - # and re-evaluate so a stale local block cannot outlive the durable - # state that justified it. The unblocked hot path above never pays - # for the DB reads. - if ( - self._summary_failure_cooldown_until <= time.monotonic() - and self._fallback_compression_streak < 2 - ): - # Blocked solely by the in-memory ineffective-compression - # counter, which is not durable — there is nothing in the DB - # that could unblock it, so skip the refresh (otherwise this - # branch would re-read the DB on every gate check for the rest - # of the session). - return True + # been cleared by another agent since bind_session_state() — a + # successful boundary, a forced retry, or a real usage reading + # below the threshold (which zeroes the durable ineffective + # counter) — so refresh and re-evaluate before letting a stale + # local block outlive the durable state that justified it. The + # unblocked hot path above never pays for the DB reads. self._refresh_durable_guards() return self._automatic_compression_blocked_locally() @@ -1625,21 +2356,66 @@ def _automatic_compression_blocked_locally(self) -> bool: _cooldown_remaining, ) return True - # Anti-thrashing: back off if recent compressions were ineffective + # Anti-thrashing: back off if recent compressions were ineffective. + # The back-off must not be permanent (#14694): the tripped state was + # judged against the transcript as it existed THEN (e.g. a middle + # region too small to matter), but the conversation keeps growing and + # can accumulate plenty of compressible material later. Without a + # recovery path the session never auto-compacts again and rides into + # the provider's hard context limit. Recovery is a probation probe: + # after _ANTI_THRASH_RECOVERY_SECONDS of continuous block, allow ONE + # attempt by dropping the tripped counter(s) to 1 strike (persisted, + # so sibling agents on the same session row unblock too). If the probe + # is ineffective again the very next verdict re-trips the guard, so + # the worst case in the truly-incompressible state is one compaction + # attempt per recovery window — bounded, not thrash. + # + # The clock is armed lazily on the first BLOCKED evaluation rather + # than persisted at trip time: a fresh process that loads a durable + # tripped counter (#69872) therefore starts a full window blocked, + # preserving the restart-must-not-disarm contract (#54923). if ( self._ineffective_compression_count >= 2 or self._fallback_compression_streak >= 2 ): + _now = time.monotonic() + if self._anti_thrash_recovery_deadline <= 0.0: + self._anti_thrash_recovery_deadline = ( + _now + self._ANTI_THRASH_RECOVERY_SECONDS + ) + elif _now >= self._anti_thrash_recovery_deadline: + self._anti_thrash_recovery_deadline = 0.0 + if self._ineffective_compression_count >= 2: + self._record_ineffective_compression_verdict(1) + if self._fallback_compression_streak >= 2: + self._fallback_compression_streak = 1 + self._persist_fallback_compression_streak() + if not self.quiet_mode: + logger.info( + "Anti-thrashing recovery: %.0fs elapsed since the " + "guard tripped — allowing one compaction probe " + "(ineffective=%d fallback=%d).", + self._ANTI_THRASH_RECOVERY_SECONDS, + self._ineffective_compression_count, + self._fallback_compression_streak, + ) + return False if not self.quiet_mode: logger.warning( "Compression skipped — repeated compaction attempts did not " "restore healthy context. ineffective=%d fallback=%d. " - "Consider /new to start fresh, or /compress for " - "focused compression.", + "Auto-compaction will retry once in %.0fs. Consider /new " + "to start fresh, or /compress for focused " + "compression.", self._ineffective_compression_count, self._fallback_compression_streak, + max(0.0, self._anti_thrash_recovery_deadline - _now), ) return True + # Guard not tripped (counters were cleared by an effective compaction + # or a fitting real-usage reading) — disarm any pending recovery clock + # so a LATER trip starts its own full window. + self._anti_thrash_recovery_deadline = 0.0 return False # ------------------------------------------------------------------ @@ -1649,6 +2425,7 @@ def _automatic_compression_blocked_locally(self) -> bool: def _prune_old_tool_results( self, messages: List[Dict[str, Any]], protect_tail_count: int, protect_tail_tokens: int | None = None, + min_prune_chars: int = 200, ) -> tuple[List[Dict[str, Any]], int]: """Replace old tool result contents with informative 1-line summaries. @@ -1665,7 +2442,14 @@ def _prune_old_tool_results( fall within ``protect_tail_tokens`` (when provided) OR the last ``protect_tail_count`` messages (backward-compatible default). When both are given, the token budget takes priority and the message - count acts as a hard minimum floor. + count acts as a hard minimum floor — capped at + ``_MAX_TAIL_MESSAGE_FLOOR`` so a default ``protect_last_n=20`` cannot + freeze a whole run of bulky tool outputs against pruning. + + When the protected region itself still exceeds the soft tail budget + (``protect_tail_tokens * 1.5``), a pressure pass demotes large + completed tool/file outputs *inside* that region while keeping a + short recent floor verbatim (issue #61932). Returns (pruned_messages, pruned_count). """ @@ -1693,10 +2477,17 @@ def _prune_old_tool_results( # Determine the prune boundary if protect_tail_tokens is not None and protect_tail_tokens > 0: - # Token-budget approach: walk backward accumulating tokens + # Token-budget approach: walk backward accumulating tokens. + # Cap the message-count floor the same way tail-cut does so a + # default protect_last_n=20 cannot lock a bulky recent tool run + # outside the compressible / prunable window (#61932). accumulated = 0 boundary = len(result) - min_protect = min(protect_tail_count, len(result)) + min_protect = min( + protect_tail_count, + len(result), + _MAX_TAIL_MESSAGE_FLOOR, + ) for i in range(len(result) - 1, -1, -1): msg = result[i] msg_tokens = _estimate_msg_budget_tokens(msg) @@ -1744,54 +2535,72 @@ def _prune_old_tool_results( else: content_hashes[h] = (i, msg.get("tool_call_id", "?")) - # Pass 2: Replace old tool results with informative summaries - for i in range(prune_boundary): - msg = result[i] + # Ghost-skill defense (#32106): skills just loaded (or actively + # referenced in the protected tail) keep their full skill_view + # bodies through the ordinary prune passes. Without this, a skill + # loaded moments before a compaction can be demoted to metadata + # while the model still believes its instructions are in context. + protected_skills = _collect_protected_skill_names(result, prune_boundary) + + def _demote_tool_result_at(idx: int, *, spare_protected_skills: bool = True) -> bool: + """Replace a bulky tool result at ``idx`` with a 1-line summary. + + Returns True when the message was modified. + """ + nonlocal pruned + msg = result[idx] if msg.get("role") != "tool": - continue + return False content = msg.get("content", "") - # Multimodal content (base64 screenshots etc.): strip the image - # payload — keep a lightweight text placeholder in its place. - # Without this, an old computer_use screenshot (~1MB base64 + - # ~1500 real tokens) survives every compression pass forever. if isinstance(content, list): stripped = _strip_image_parts_from_parts(content) if stripped is not None: - result[i] = {**msg, "content": stripped} + result[idx] = {**msg, "content": stripped} pruned += 1 - continue + return True + return False if isinstance(content, dict) and content.get("_multimodal"): summary = content.get("text_summary") or "[screenshot removed to save context]" - result[i] = {**msg, "content": f"[screenshot removed] {summary[:200]}"} + result[idx] = {**msg, "content": f"[screenshot removed] {summary[:200]}"} pruned += 1 - continue + return True if not isinstance(content, str): - continue + return False if not content or content == _PRUNED_TOOL_PLACEHOLDER: - continue - # Skip already-deduplicated or previously-summarized results + return False if content.startswith("[Duplicate tool output"): - continue - # Only prune if the content is substantial (>200 chars) - if len(content) > 200: - call_id = msg.get("tool_call_id", "") - tool_name, tool_args = call_id_to_tool.get(call_id, ("unknown", "")) - summary = _summarize_tool_result(tool_name, tool_args, content) - result[i] = {**msg, "content": summary} - pruned += 1 + return False + # Already replaced by a prior prune/pressure pass (1-line summary). + if content.startswith("[") and " chars)" in content and len(content) < 400: + return False + if content.startswith("[screenshot removed"): + return False + # Only prune if the content is substantial (default >200 chars; the + # proactive path raises this floor via min_prune_chars). + if len(content) <= min_prune_chars: + return False + call_id = msg.get("tool_call_id", "") + tool_name, tool_args = call_id_to_tool.get(call_id, ("unknown", "")) + if spare_protected_skills and tool_name == "skill_view" and protected_skills: + # Just-loaded / actively-referenced skills survive verbatim + # (#32106). Pass-4 pressure demotion overrides this. + try: + _args = json.loads(tool_args) if tool_args else {} + except (json.JSONDecodeError, TypeError): + _args = {} + _skill = _args.get("name", "") if isinstance(_args, dict) else "" + if isinstance(_skill, str) and _skill.lower() in protected_skills: + return False + summary = _summarize_tool_result(tool_name, tool_args, content) + result[idx] = {**msg, "content": summary} + pruned += 1 + return True - # Pass 3: Truncate large tool_call arguments in assistant messages - # outside the protected tail. write_file with 50KB content, for - # example, survives pruning entirely without this. - # - # The shrinking is done inside the parsed JSON structure so the - # result remains valid JSON — otherwise downstream providers 400 - # on every subsequent turn until the broken call falls out of - # the window. See ``_truncate_tool_call_args_json`` docstring. - for i in range(prune_boundary): - msg = result[i] + def _truncate_tool_call_args_at(idx: int) -> bool: + """Shrink large tool_call argument payloads at ``idx``.""" + msg = result[idx] if msg.get("role") != "assistant" or not msg.get("tool_calls"): - continue + return False new_tcs = [] modified = False for tc in msg["tool_calls"]: @@ -1804,10 +2613,170 @@ def _prune_old_tool_results( modified = True new_tcs.append(tc) if modified: - result[i] = {**msg, "tool_calls": new_tcs} + result[idx] = {**msg, "tool_calls": new_tcs} + return modified + + # Pass 2: Replace old tool results with informative summaries + for i in range(max(0, prune_boundary)): + _demote_tool_result_at(i) + + # Pass 3: Truncate large tool_call arguments in assistant messages + # outside the protected tail. write_file with 50KB content, for + # example, survives pruning entirely without this. + # + # The shrinking is done inside the parsed JSON structure so the + # result remains valid JSON — otherwise downstream providers 400 + # on every subsequent turn until the broken call falls out of + # the window. See ``_truncate_tool_call_args_json`` docstring. + for i in range(max(0, prune_boundary)): + _truncate_tool_call_args_at(i) + + # Pass 4 (issue #61932): protected-tail pressure demotion. + # After multiple in-place compactions the transcript can be short + # enough that nearly every remaining message sits inside the + # protected floor, yet those messages are huge completed tool / + # file outputs. Summarizing the (empty) middle does nothing and + # preflight ends in "Cannot compress further". Demote bulky tool + # bodies *inside* the protected region until the protected tail + # fits the soft budget, always keeping a short recent floor + # verbatim so the active ask stays readable. + if protect_tail_tokens is not None and protect_tail_tokens > 0 and result: + soft_ceiling = int(protect_tail_tokens * 1.5) + keep_recent = min(_PRESSURE_KEEP_RECENT_MESSAGES, len(result)) + demote_end = len(result) - keep_recent + + def _protected_region_tokens() -> int: + start = max(0, prune_boundary) + return sum( + _estimate_msg_budget_tokens(result[i]) + for i in range(start, len(result)) + ) + + if demote_end > prune_boundary and _protected_region_tokens() > soft_ceiling: + pressure_hits = 0 + for i in range(max(0, prune_boundary), demote_end): + # Pressure passes override the just-loaded-skill guard: + # when the protected region itself blows the soft budget, + # sparing skill bodies would recreate the #61932 dead-end. + if _demote_tool_result_at(i, spare_protected_skills=False): + pressure_hits += 1 + if _truncate_tool_call_args_at(i): + pressure_hits += 1 + if _protected_region_tokens() <= soft_ceiling: + break + # If the short recent floor itself is still dominated by a + # stack of huge tool bodies, demote every protected tool + # result except the single most recent one. The active + # user message (usually the last row) stays untouched. + if _protected_region_tokens() > soft_ceiling: + last_tool_idx = None + for i in range(len(result) - 1, -1, -1): + if result[i].get("role") == "tool": + last_tool_idx = i + break + for i in range(max(0, prune_boundary), len(result)): + if last_tool_idx is not None and i == last_tool_idx: + continue + if result[i].get("role") == "tool": + if _demote_tool_result_at(i, spare_protected_skills=False): + pressure_hits += 1 + elif result[i].get("role") == "assistant": + if _truncate_tool_call_args_at(i): + pressure_hits += 1 + # Absolute last resort: even the newest tool body can + # be larger than the soft budget alone (one 200KB file + # read). Summarize it so compression can still reclaim + # enough headroom to continue the session. + if ( + last_tool_idx is not None + and last_tool_idx >= prune_boundary + and _protected_region_tokens() > soft_ceiling + ): + if _demote_tool_result_at( + last_tool_idx, spare_protected_skills=False + ): + pressure_hits += 1 + if pressure_hits and not self.quiet_mode: + logger.info( + "Pre-compression pressure demotion: reclaimed protected-tail " + "tool output (%d change(s); protected region now ~%s tokens, " + "soft ceiling %s)", + pressure_hits, + f"{_protected_region_tokens():,}", + f"{soft_ceiling:,}", + ) return result, pruned + def prune_tool_results_only( + self, messages: List[Dict[str, Any]], current_tokens: int | None = None, + ) -> tuple[List[Dict[str, Any]], int]: + """Deterministic, no-LLM tool-result prune for the cost-oriented path. + + Runs the Phase-1 prune (``_prune_old_tool_results``) WITHOUT the + compression summary phase, gated on ``proactive_prune_tokens`` rather + than the (much higher) full-compression threshold. On large-window + models ``should_compress()`` (≈50% of the window) rarely fires, so old + tool outputs otherwise ride in history and are re-sent verbatim on every + subsequent turn; this reclaims them early with no quality-risky LLM + summarization. + + Protects the recent tail by message COUNT (``protect_last_n``), never by + ``tail_token_budget`` — the latter is derived from the 50% compression + threshold (≈100K tokens on a 1M window) and would protect the entire + session, pruning nothing. + + ``_prune_old_tool_results`` runs all three deterministic passes: + (1) dedup byte-identical tool results — keeps the newest full copy and + back-references older exact duplicates ANYWHERE in the list (including + the protected tail), so no unique content is ever lost; (2) summarize + non-tail tool results larger than ``min_prune_chars``; (3) truncate + oversized tool_call arguments on non-tail assistant messages. Only + pass (2)'s floor is raised by ``proactive_prune_min_result_chars``; + passes (1) and (3) keep their own fixed floors. The recent-tail + protection applies to passes (2) and (3); pass (1) is tail-agnostic by + design because dedup is lossless. + + PROMPT-CACHE CONTRACT: a committed prune rewrites message bodies the + provider has already seen, invalidating the cached prefix from the + earliest rewritten message forward — exactly like a compression + boundary. To keep that break episodic rather than per-turn, the prune + only COMMITS when the estimated reclaim meets + ``proactive_prune_min_reclaim_tokens`` (measured on the actual pruned + output, not guessed up front). Below the gate the INPUT list object is + returned unchanged — the standard no-op caller contract (callers gate + bookkeeping on ``result is not input``). + + Returns ``(messages, 0)`` — the input object — when disabled, below + the trigger, or when the reclaim gate rejects the commit. + """ + if self.proactive_prune_tokens <= 0: + return messages, 0 + if current_tokens is not None and current_tokens < self.proactive_prune_tokens: + return messages, 0 + # Nothing to reclaim until there are messages outside the protected tail. + if len(messages) <= self.protect_last_n + self._protect_head_size(messages) + 1: + return messages, 0 + pruned_msgs, pruned_count = self._prune_old_tool_results( + messages, + protect_tail_count=self.protect_last_n, + protect_tail_tokens=None, + min_prune_chars=self.proactive_prune_min_result_chars, + ) + if not pruned_count: + # Standard no-op contract: hand back the INPUT object so callers + # can gate bookkeeping on `result is not input`. + return messages, 0 + # Measured-savings gate (prompt-cache hysteresis): only commit when + # the prune reclaims a meaningful batch of tokens. Estimated on the + # real before/after messages so dedup + arg truncation count too. + if self.proactive_prune_min_reclaim_tokens > 0: + before = sum(_estimate_msg_budget_tokens(m) for m in messages) + after = sum(_estimate_msg_budget_tokens(m) for m in pruned_msgs) + if (before - after) < self.proactive_prune_min_reclaim_tokens: + return messages, 0 + return pruned_msgs, pruned_count + # ------------------------------------------------------------------ # Summarization # ------------------------------------------------------------------ @@ -1831,6 +2800,10 @@ def _compute_summary_budget(self, turns_to_summarize: List[Dict[str, Any]]) -> i _CONTENT_TAIL = 1500 # chars kept from the end _TOOL_ARGS_MAX = 1500 # tool call argument chars _TOOL_ARGS_HEAD = 1200 # kept from the start of tool args + # Aggregate cap over the whole serialized block, applied AFTER the + # per-message limits above. Alias of the module-level constant (which + # carries the full rationale) so subclasses/tests can override per-class. + _SUMMARY_INPUT_MAX_CHARS = _SUMMARY_INPUT_MAX_CHARS def _serialize_for_summary(self, turns: List[Dict[str, Any]]) -> str: """Serialize conversation turns into labeled text for the summarizer. @@ -1867,7 +2840,7 @@ def _serialize_for_summary(self, turns: List[Dict[str, Any]]) -> str: elif isinstance(part, str): text_parts.append(part) content = "\n".join(text_parts) - content = redact_sensitive_text(content or "") + content = _redact_compaction_text(content or "") content = _MEDIA_DIRECTIVE_RE.sub("[media attachment]", content) # Strip inline reasoning blocks (, , etc.) from # assistant content before it reaches the summarizer. Reasoning @@ -1900,7 +2873,7 @@ def _serialize_for_summary(self, turns: List[Dict[str, Any]]) -> str: if isinstance(tc, dict): fn = tc.get("function", {}) name = fn.get("name", "?") - args = redact_sensitive_text(fn.get("arguments", "")) + args = _redact_compaction_text(fn.get("arguments", "")) # Truncate long arguments but keep enough for context if len(args) > self._TOOL_ARGS_MAX: args = args[:self._TOOL_ARGS_HEAD] + "..." @@ -1943,7 +2916,7 @@ def _build_static_fallback_summary( last_dropped_turns: list[str] = [] def _compact_fallback_turn(value: Any) -> str: - text = redact_sensitive_text(_content_text_for_contains(value)) + text = _redact_compaction_text(_content_text_for_contains(value)) text = re.sub(r"\bgh[pousr]_[A-Za-z0-9_]{8,}\b", "[REDACTED]", text) text = re.sub(r"\s+", " ", text).strip() if len(text) > _FALLBACK_TURN_MAX_CHARS: @@ -1975,7 +2948,7 @@ def _collect_paths_from_jsonish(obj: Any) -> None: if msg.get("role") == "assistant" and msg.get("tool_calls"): for tc in msg.get("tool_calls") or []: name, raw_args = _extract_tool_call_name_and_args(tc) - args = redact_sensitive_text(raw_args) + args = _redact_compaction_text(raw_args) call_id = _extract_tool_call_id(tc) if call_id: call_id_to_tool[call_id] = (name, args) @@ -1990,6 +2963,9 @@ def _collect_paths_from_jsonish(obj: Any) -> None: role = msg.get("role", "unknown") text = _compact_fallback_turn(msg.get("content")) _collect_path_mentions(text, relevant_files) + synthetic_user = ( + role == "user" and self._is_synthetic_compression_user_turn(msg) + ) turn_text = text turn_tool_names: list[str] = [] @@ -2000,12 +2976,13 @@ def _collect_paths_from_jsonish(obj: Any) -> None: if turn_tool_names: prefix = "tool calls: " + ", ".join(turn_tool_names[:6]) turn_text = f"{prefix}; {turn_text}" if turn_text else prefix - _remember_dropped_turn(str(role).upper(), turn_text) + turn_label = "INTERNAL CONTEXT" if synthetic_user else str(role).upper() + _remember_dropped_turn(turn_label, turn_text) if len(text) > 600: text = text[:420].rstrip() + " ... " + text[-160:].lstrip() - if role == "user" and text: + if role == "user" and text and not synthetic_user: user_asks.append(text) elif role == "assistant": tool_names: list[str] = [] @@ -2051,13 +3028,21 @@ def _bullets(items: list[str], limit: int = 8) -> str: active_task = ( f"User asked: {user_asks[-1]!r}" if user_asks - else "Unknown from deterministic fallback." + else _NO_USER_TASK_SENTINEL ) previous_summary_note = "" if self._previous_summary: + previous_summary = redact_sensitive_text(self._previous_summary.strip()) + if len(previous_summary) > _FALLBACK_PREVIOUS_SUMMARY_MAX_CHARS: + previous_summary = ( + previous_summary[: _FALLBACK_PREVIOUS_SUMMARY_MAX_CHARS - 45].rstrip() + + "\n...[previous summary snapshot truncated]" + ) previous_summary_note = ( - "\n\nPrevious compaction summary was present and should still be treated as " - "background continuity context, but the latest LLM summary update failed." + "\n\n## Previous Summary Snapshot\n" + f"{previous_summary}\n\n" + "The previous compaction summary above remains background " + "continuity context because the latest LLM summary update failed." ) reason_text = f" Summary failure reason: {reason}." if reason else "" @@ -2078,12 +3063,6 @@ def _bullets(items: list[str], limit: int = 8) -> str: ## Active State Unknown from deterministic fallback. Inspect current repository/session state if needed. -{HISTORICAL_IN_PROGRESS_HEADING} -Unknown from deterministic fallback — the latest user ask is recorded once under -"{HISTORICAL_TASK_HEADING}" above as historical context only. Do NOT treat it as an -unfulfilled instruction to re-answer; verify current state and continue from the -protected recent messages after this summary. - ## Blocked {_bullets(blockers, limit=5)} @@ -2093,27 +3072,62 @@ def _bullets(items: list[str], limit: int = 8) -> str: ## Resolved Questions None recoverable from deterministic fallback. -{HISTORICAL_PENDING_ASKS_HEADING} -None recoverable from deterministic fallback. (The latest user ask is preserved once -under "{HISTORICAL_TASK_HEADING}" as historical context — it is NOT necessarily -outstanding.) - ## Relevant Files {_bullets(relevant_files, limit=12)} -{HISTORICAL_REMAINING_WORK_HEADING} -Continue from the most recent unfulfilled user ask and protected tail messages. Verify state with tools before making claims. - ## Last Dropped Turns {_bullets(last_dropped_turns, limit=8)} ## Critical Context Summary generation was unavailable, so this is a best-effort deterministic fallback for {len(turns_to_summarize)} compacted message(s).{reason_text}""" - summary = self._with_summary_prefix(redact_sensitive_text(body.strip())) + # Ghost-skill defense (#32106): the fallback's per-turn truncation + # (``_FALLBACK_TURN_MAX_CHARS``) routinely cuts [SKILL_PRUNED: ...] + # markers out of the compacted turns. Re-derive the ghosted skills + # from the raw turn contents and re-inject deterministically, + # exactly like the LLM-summary path. + _pruned_names = _collect_ghosted_skill_names(turns_to_summarize) + del _pruned_names[_MAX_PRUNED_SKILL_MARKERS:] + summary = self._with_summary_prefix(_redact_compaction_text(body.strip())) if len(summary) > _FALLBACK_SUMMARY_MAX_CHARS: summary = summary[: _FALLBACK_SUMMARY_MAX_CHARS - 42].rstrip() + "\n...[fallback summary truncated]" + # Re-inject AFTER the size cap: the markers live at the end of the + # body, exactly where the truncation above cuts. + summary = _reinject_pruned_skill_markers(summary, _pruned_names) return summary + @classmethod + def _bound_summary_input(cls, content: str) -> str: + """Cap total summarizer input while preserving beginning and recent tail. + + Per-message truncation alone is not enough for very long sessions: a + compression window with hundreds of messages can still produce a huge + single prompt that slow auxiliary backends time out on. Keep both edges + because the beginning often has task setup and the tail has the most + recent state; explicitly mark the omitted middle so the summarizer knows + context was intentionally compressed before it saw the prompt. + """ + if len(content) <= cls._SUMMARY_INPUT_MAX_CHARS: + return content + + marker_template = ( + "\n\n...[summary input truncated: omitted " + "{omitted:,} chars from the middle to keep compression prompt bounded]...\n\n" + ) + # Estimate once, then rebuild with the exact omitted span after the + # head/tail split is known. The second marker can differ by a few chars + # if the comma-formatted number changes width, so recompute once. + marker = marker_template.format(omitted=len(content)) + remaining = max(cls._SUMMARY_INPUT_MAX_CHARS - len(marker), 0) + head_chars = int(remaining * 0.45) + tail_chars = remaining - head_chars + omitted = max(len(content) - head_chars - tail_chars, 0) + marker = marker_template.format(omitted=omitted) + remaining = max(cls._SUMMARY_INPUT_MAX_CHARS - len(marker), 0) + head_chars = int(remaining * 0.45) + tail_chars = remaining - head_chars + tail = content[-tail_chars:].lstrip() if tail_chars else "" + return content[:head_chars].rstrip() + marker + tail + def _fallback_to_main_for_compression(self, e: Exception, reason: str) -> None: """Switch from a separate ``summary_model`` back to the main model. @@ -2138,6 +3152,10 @@ def _fallback_to_main_for_compression(self, e: Exception, reason: str) -> None: _err_text = _err_text[:217].rstrip() + "..." self._last_aux_model_failure_error = _err_text self._last_aux_model_failure_model = self.summary_model + telemetry = getattr(self, "_active_compression_telemetry", None) + if isinstance(telemetry, dict): + telemetry["fallback_used"] = True + telemetry["failure_class"] = telemetry.get("failure_class") or "aux_model_fallback" self.summary_model = "" # empty = use main model self._clear_compression_failure_cooldown() # no cooldown — retry immediately @@ -2172,8 +3190,34 @@ def _generate_summary( ) return None + # Strict-redact prompt inputs that bypass _serialize_for_summary: + # a manual `/compress ` string, and a previous summary that + # may predate compaction redaction (resumed from a persisted + # handoff message written before this boundary existed). + if focus_topic: + focus_topic = _redact_compaction_text(focus_topic) + if self._previous_summary: + self._previous_summary = _redact_compaction_text(self._previous_summary) + summary_budget = self._compute_summary_budget(turns_to_summarize) content_to_summarize = self._serialize_for_summary(turns_to_summarize) + # P2 ghost-skill defense (#32106): [SKILL_PRUNED: ...] markers entering + # the summarizer are prompt INPUT only — LLMs routinely paraphrase them + # into vague prose ("some skills were loaded"), which erases the reload + # instruction. Collect the ghosted skills deterministically BEFORE the + # call (both already-pruned marker rows AND raw skill_view bodies whose + # instructions are about to be summarized away); + # ``_reinject_pruned_skill_markers`` restores any marker the model + # dropped AFTER the call. Markers already carried by the previous + # summary must survive iterative rewrites the same way. Collection + # walks the turn LIST, so the serialized input bound below cannot + # hide a marker in its omitted middle. + _pruned_skill_names = _collect_ghosted_skill_names(turns_to_summarize) + for _name in _extract_pruned_skill_names(self._previous_summary or ""): + if _name not in _pruned_skill_names: + _pruned_skill_names.append(_name) + del _pruned_skill_names[_MAX_PRUNED_SKILL_MARKERS:] + content_to_summarize = self._bound_summary_input(content_to_summarize) _sanitized_memory_context = sanitize_memory_context(memory_context) _serialized_memory_context = json.dumps( _sanitized_memory_context, @@ -2194,6 +3238,9 @@ def _generate_summary( if _sanitized_memory_context else "" ) + has_user_turn = getattr(self, "_summary_has_user_turn", None) + if has_user_turn is None: + has_user_turn = self._transcript_has_real_user_turn(turns_to_summarize) # Current date for temporal anchoring (see ## Temporal Anchoring below). # Date-only granularity matches system_prompt.py:337 (PR #20451) and the @@ -2211,18 +3258,86 @@ def _generate_summary( # Preamble shared by both first-compaction and iterative-update prompts. # Keep the wording deliberately plain: Azure/OpenAI-compatible content # filters have flagged stronger "injection" / "do not respond" framing. + if has_user_turn: + _language_and_provenance_rule = ( + "Write the summary in the same language the user was using in the " + "conversation — do not translate or switch to English. " + ) + _historical_task_instructions = """[THE SINGLE MOST IMPORTANT FIELD. Capture the user's most recent unfulfilled +input verbatim — the exact words they used. This includes: +- Explicit task assignments ("") +- Questions awaiting an answer ("") +- Decisions awaiting input ("